diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 0000000000..7cdcd9123e --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,51 @@ +--- +name: lint + +on: + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name || github.sha }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: yarn + - name: install typescript + shell: bash + run: yarn global add typescript@4.9.5 + - name: install dependencies + run: yarn install --frozen-lockfile --network-concurrency 1 + - name: Fetch base for Prettier + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: git fetch --no-tags origin "$BASE_REF" --depth=50 + - name: Prettier (changed files) + run: scripts/format-changed.sh + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + cache: pip + - name: Install python deps + run: pip install flake8 + - name: Lint Javascript + run: yarn run --silent lint -- --max-warnings 0 + - name: Lint Markdown + run: yarn run --silent lint_md + - name: Lint python + run: flake8 $(git ls-files "*.py") + - name: Lint Yaml + run: yamllint -c yamllint.yml $(git ls-files "*.yml") diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 2a13f2ee93..1d98d04766 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -34,26 +34,26 @@ env: ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }} azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >- ${{ secrets.AZURE_STORAGE_ENDPOINT }} - azuretest_AZURE_BLOB_ENDPOINT: "${{ secrets.AZURE_STORAGE_ENDPOINT }}" - b2backend_B2_ACCOUNT_ID: "${{ secrets.B2BACKEND_B2_ACCOUNT_ID }}" + azuretest_AZURE_BLOB_ENDPOINT: '${{ secrets.AZURE_STORAGE_ENDPOINT }}' + b2backend_B2_ACCOUNT_ID: '${{ secrets.B2BACKEND_B2_ACCOUNT_ID }}' b2backend_B2_STORAGE_ACCESS_KEY: >- ${{ secrets.B2BACKEND_B2_STORAGE_ACCESS_KEY }} - GOOGLE_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}" - GOOGLE_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}" - AWS_S3_BACKEND_ACCESS_KEY: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY }}" - AWS_S3_BACKEND_SECRET_KEY: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY }}" - AWS_S3_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY_2 }}" - AWS_S3_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY_2 }}" - AWS_GCP_BACKEND_ACCESS_KEY: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }}" - AWS_GCP_BACKEND_SECRET_KEY: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}" - AWS_GCP_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY_2 }}" - AWS_GCP_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY_2 }}" - b2backend_B2_STORAGE_ENDPOINT: "${{ secrets.B2BACKEND_B2_STORAGE_ENDPOINT }}" - gcpbackend2_GCP_SERVICE_EMAIL: "${{ secrets.GCP2_SERVICE_EMAIL }}" - gcpbackend2_GCP_SERVICE_KEY: "${{ secrets.GCP2_SERVICE_KEY }}" + GOOGLE_SERVICE_EMAIL: '${{ secrets.GCP_SERVICE_EMAIL }}' + GOOGLE_SERVICE_KEY: '${{ secrets.GCP_SERVICE_KEY }}' + AWS_S3_BACKEND_ACCESS_KEY: '${{ secrets.AWS_S3_BACKEND_ACCESS_KEY }}' + AWS_S3_BACKEND_SECRET_KEY: '${{ secrets.AWS_S3_BACKEND_SECRET_KEY }}' + AWS_S3_BACKEND_ACCESS_KEY_2: '${{ secrets.AWS_S3_BACKEND_ACCESS_KEY_2 }}' + AWS_S3_BACKEND_SECRET_KEY_2: '${{ secrets.AWS_S3_BACKEND_SECRET_KEY_2 }}' + AWS_GCP_BACKEND_ACCESS_KEY: '${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }}' + AWS_GCP_BACKEND_SECRET_KEY: '${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}' + AWS_GCP_BACKEND_ACCESS_KEY_2: '${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY_2 }}' + AWS_GCP_BACKEND_SECRET_KEY_2: '${{ secrets.AWS_GCP_BACKEND_SECRET_KEY_2 }}' + b2backend_B2_STORAGE_ENDPOINT: '${{ secrets.B2BACKEND_B2_STORAGE_ENDPOINT }}' + gcpbackend2_GCP_SERVICE_EMAIL: '${{ secrets.GCP2_SERVICE_EMAIL }}' + gcpbackend2_GCP_SERVICE_KEY: '${{ secrets.GCP2_SERVICE_KEY }}' gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey - gcpbackend_GCP_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}" - gcpbackend_GCP_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}" + gcpbackend_GCP_SERVICE_EMAIL: '${{ secrets.GCP_SERVICE_EMAIL }}' + gcpbackend_GCP_SERVICE_KEY: '${{ secrets.GCP_SERVICE_KEY }}' gcpbackendmismatch_GCP_SERVICE_EMAIL: >- ${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_EMAIL }} gcpbackendmismatch_GCP_SERVICE_KEY: >- @@ -63,9 +63,9 @@ env: gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey # Configs - ENABLE_LOCAL_CACHE: "true" - REPORT_TOKEN: "report-token-1" - REMOTE_MANAGEMENT_DISABLE: "1" + ENABLE_LOCAL_CACHE: 'true' + REPORT_TOKEN: 'report-token-1' + REMOTE_MANAGEMENT_DISABLE: '1' # https://github.com/git-lfs/git-lfs/issues/5749 GIT_CLONE_PROTECTION_ACTIVE: 'false' AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE: '1' @@ -74,85 +74,60 @@ permissions: contents: read packages: write -jobs: - lint: - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: yarn - - name: install typescript - shell: bash - run: yarn global add typescript@4.9.5 - - name: install dependencies - run: yarn install --frozen-lockfile --network-concurrency 1 - - uses: actions/setup-python@v5 - with: - python-version: '3.9' - cache: pip - - name: Install python deps - run: pip install flake8 - - name: Lint Javascript - run: yarn run --silent lint -- --max-warnings 0 - - name: Lint Markdown - run: yarn run --silent lint_md - - name: Lint python - run: flake8 $(git ls-files "*.py") - - name: Lint Yaml - run: yamllint -c yamllint.yml $(git ls-files "*.yml") +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name || github.sha }} + cancel-in-progress: true +jobs: unit-tests: runs-on: ubuntu-24.04 steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: yarn - - name: install typescript - shell: bash - run: yarn global add typescript@4.9.5 - - name: install dependencies - run: yarn install --frozen-lockfile --network-concurrency 1 - - name: Unit Coverage - run: | - set -ex - yarn run cover test - yarn run cover test_legacy_location - env: - S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json - NO_PROXY: "test.scality.com,scality.com" - - name: Upload test results to Codecov - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: '**/junit/*junit*.xml' - flags: unit - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - directory: ./coverage/test - flags: unit - if: ${{ !cancelled() }} - - name: preparing junit files for upload - run: | - mkdir -p artifacts/junit - find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";" - if: always() - - name: Upload files to artifacts - uses: scality/action-artifacts@v4 - with: - method: upload - url: https://artifacts.scality.net - user: ${{ secrets.ARTIFACTS_USER }} - password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: artifacts - if: always() + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: yarn + - name: install typescript + shell: bash + run: yarn global add typescript@4.9.5 + - name: install dependencies + run: yarn install --frozen-lockfile --network-concurrency 1 + - name: Unit Coverage + run: | + set -ex + yarn run cover test + yarn run cover test_legacy_location + env: + S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json + NO_PROXY: 'test.scality.com,scality.com' + - name: Upload test results to Codecov + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: '**/junit/*junit*.xml' + flags: unit + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + directory: ./coverage/test + flags: unit + if: ${{ !cancelled() }} + - name: preparing junit files for upload + run: | + mkdir -p artifacts/junit + find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";" + if: always() + - name: Upload files to artifacts + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ secrets.ARTIFACTS_USER }} + password: ${{ secrets.ARTIFACTS_PASSWORD }} + source: artifacts + if: always() build: runs-on: ubuntu-latest @@ -209,8 +184,7 @@ jobs: push: true context: images/federation provenance: false - build-args: - CLOUDSERVER_VERSION=${{ github.sha }} + build-args: CLOUDSERVER_VERSION=${{ github.sha }} tags: | ghcr.io/${{ github.repository }}:${{ github.sha }}-federation labels: | @@ -254,58 +228,58 @@ jobs: ENABLE_NULL_VERSION_COMPAT_MODE: true # needed with mongodb backend JOB_NAME: ${{ github.job }} steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Login to Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - name: Setup CI environment - uses: ./.github/actions/setup-ci - - name: Setup CI services - run: docker compose --profile sproxyd --profile mongo up -d - working-directory: .github/docker - - name: Run multiple backend test - run: |- - set -o pipefail; - bash wait_for_local_port.bash 8000 40 - bash wait_for_local_port.bash 81 40 - yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log - env: - S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json - - name: Cleanup and upload coverage - uses: ./.github/actions/cleanup-and-coverage - with: - profiles: > - sproxyd - mongo - codecov-token: ${{ secrets.CODECOV_TOKEN }} - if: always() - - name: Upload test results to Codecov - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: '**/junit/*junit*.xml' - flags: multiple-backend - if: always() && !cancelled() - - name: Upload logs to artifacts - uses: scality/action-artifacts@v4 - with: - method: upload - url: https://artifacts.scality.net - user: ${{ secrets.ARTIFACTS_USER }} - password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts - if: always() - + - name: Checkout + uses: actions/checkout@v4 + - name: Login to Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + - name: Setup CI environment + uses: ./.github/actions/setup-ci + - name: Setup CI services + run: docker compose --profile sproxyd --profile mongo up -d + working-directory: .github/docker + - name: Run multiple backend test + run: |- + set -o pipefail; + bash wait_for_local_port.bash 8000 40 + bash wait_for_local_port.bash 81 40 + yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log + env: + S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json + - name: Cleanup and upload coverage + uses: ./.github/actions/cleanup-and-coverage + with: + profiles: > + sproxyd + mongo + codecov-token: ${{ secrets.CODECOV_TOKEN }} + if: always() + - name: Upload test results to Codecov + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: '**/junit/*junit*.xml' + flags: multiple-backend + if: always() && !cancelled() + - name: Upload logs to artifacts + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ secrets.ARTIFACTS_USER }} + password: ${{ secrets.ARTIFACTS_PASSWORD }} + source: /tmp/artifacts + if: always() + mongo-v0-ft-tests: runs-on: ubuntu-24.04 needs: build env: S3BACKEND: mem - MPU_TESTING: "yes" + MPU_TESTING: 'yes' S3METADATA: mongodb S3KMS: file S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json @@ -315,56 +289,56 @@ jobs: JOB_NAME: ${{ github.job }} S3_SERVER_ACCESS_LOGS_MODE: ENABLED steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup CI environment - uses: ./.github/actions/setup-ci - - name: Setup server access logs file and directory - shell: bash - run: | - set -exu - sudo mkdir -p /logs - sudo chmod 0777 /logs - sudo touch /logs/server-access.log && sudo chmod 0666 /logs/server-access.log - - name: Setup CI services - run: docker compose --profile mongo up -d - working-directory: .github/docker - - name: Run functional tests - run: |- - set -o pipefail; - bash wait_for_local_port.bash 8000 40 - yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log - env: - S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json - - name: Cleanup and upload coverage - uses: ./.github/actions/cleanup-and-coverage - with: - profiles: mongo - codecov-token: ${{ secrets.CODECOV_TOKEN }} - if: always() - - name: Upload test results to Codecov - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: '**/junit/*junit*.xml' - flags: mongo-v0-ft-tests - if: always() && !cancelled() - - name: Upload logs to artifacts - uses: scality/action-artifacts@v4 - with: - method: upload - url: https://artifacts.scality.net - user: ${{ secrets.ARTIFACTS_USER }} - password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts - if: always() + - name: Checkout + uses: actions/checkout@v4 + - name: Setup CI environment + uses: ./.github/actions/setup-ci + - name: Setup server access logs file and directory + shell: bash + run: | + set -exu + sudo mkdir -p /logs + sudo chmod 0777 /logs + sudo touch /logs/server-access.log && sudo chmod 0666 /logs/server-access.log + - name: Setup CI services + run: docker compose --profile mongo up -d + working-directory: .github/docker + - name: Run functional tests + run: |- + set -o pipefail; + bash wait_for_local_port.bash 8000 40 + yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log + env: + S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json + - name: Cleanup and upload coverage + uses: ./.github/actions/cleanup-and-coverage + with: + profiles: mongo + codecov-token: ${{ secrets.CODECOV_TOKEN }} + if: always() + - name: Upload test results to Codecov + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: '**/junit/*junit*.xml' + flags: mongo-v0-ft-tests + if: always() && !cancelled() + - name: Upload logs to artifacts + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ secrets.ARTIFACTS_USER }} + password: ${{ secrets.ARTIFACTS_PASSWORD }} + source: /tmp/artifacts + if: always() mongo-v1-ft-tests: runs-on: ubuntu-24.04 needs: build env: S3BACKEND: mem - MPU_TESTING: "yes" + MPU_TESTING: 'yes' S3METADATA: mongodb S3KMS: file S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json @@ -376,50 +350,50 @@ jobs: JOB_NAME: ${{ github.job }} S3_SERVER_ACCESS_LOGS_MODE: ENABLED steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup CI environment - uses: ./.github/actions/setup-ci - - name: Setup server access logs file and directory - shell: bash - run: | - set -exu - sudo mkdir -p /logs - sudo chmod 0777 /logs - sudo touch /logs/server-access.log && sudo chmod 0666 /logs/server-access.log - - name: Setup CI services - run: docker compose --profile mongo up -d - working-directory: .github/docker - - name: Run functional tests - run: |- - set -o pipefail; - bash wait_for_local_port.bash 8000 40 - yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log - yarn run ft_mixed_bucket_format_version | tee /tmp/artifacts/${{ github.job }}/mixed-tests.log - env: - S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json - - name: Cleanup and upload coverage - uses: ./.github/actions/cleanup-and-coverage - with: - profiles: mongo - codecov-token: ${{ secrets.CODECOV_TOKEN }} - if: always() - - name: Upload test results to Codecov - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: '**/junit/*junit*.xml' - flags: mongo-v1-ft-tests - if: always() && !cancelled() - - name: Upload logs to artifacts - uses: scality/action-artifacts@v4 - with: - method: upload - url: https://artifacts.scality.net - user: ${{ secrets.ARTIFACTS_USER }} - password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts - if: always() + - name: Checkout + uses: actions/checkout@v4 + - name: Setup CI environment + uses: ./.github/actions/setup-ci + - name: Setup server access logs file and directory + shell: bash + run: | + set -exu + sudo mkdir -p /logs + sudo chmod 0777 /logs + sudo touch /logs/server-access.log && sudo chmod 0666 /logs/server-access.log + - name: Setup CI services + run: docker compose --profile mongo up -d + working-directory: .github/docker + - name: Run functional tests + run: |- + set -o pipefail; + bash wait_for_local_port.bash 8000 40 + yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log + yarn run ft_mixed_bucket_format_version | tee /tmp/artifacts/${{ github.job }}/mixed-tests.log + env: + S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json + - name: Cleanup and upload coverage + uses: ./.github/actions/cleanup-and-coverage + with: + profiles: mongo + codecov-token: ${{ secrets.CODECOV_TOKEN }} + if: always() + - name: Upload test results to Codecov + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: '**/junit/*junit*.xml' + flags: mongo-v1-ft-tests + if: always() && !cancelled() + - name: Upload logs to artifacts + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ secrets.ARTIFACTS_USER }} + password: ${{ secrets.ARTIFACTS_PASSWORD }} + source: /tmp/artifacts + if: always() # All tests use non federation images @@ -439,57 +413,57 @@ jobs: S3VAULT: mem CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} - MPU_TESTING: "yes" - ENABLE_NULL_VERSION_COMPAT_MODE: "${{ matrix.enable-null-compat }}" + MPU_TESTING: 'yes' + ENABLE_NULL_VERSION_COMPAT_MODE: '${{ matrix.enable-null-compat }}' JOB_NAME: ${{ matrix.job-name }} S3_SERVER_ACCESS_LOGS_MODE: ENABLED steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup CI environment - uses: ./.github/actions/setup-ci - - name: Setup matrix job artifacts directory - shell: bash - run: | - set -exu - mkdir -p /tmp/artifacts/${{ matrix.job-name }}/ - - name: Setup server access logs file and directory - shell: bash - run: | - set -exu - sudo mkdir -p /logs - sudo chmod 0777 /logs - sudo touch /logs/server-access.log && sudo chmod 0666 /logs/server-access.log - - name: Setup CI services - run: docker compose up -d - working-directory: .github/docker - - name: Run file ft tests - run: |- - set -o pipefail; - bash wait_for_local_port.bash 8000 40 - yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log - - name: Cleanup and upload coverage - uses: ./.github/actions/cleanup-and-coverage - with: - codecov-token: ${{ secrets.CODECOV_TOKEN }} - flags: ${{ matrix.job-name }} - if: always() - - name: Upload test results to Codecov - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: '**/junit/*junit*.xml' - flags: ${{ matrix.job-name }} - if: always() && !cancelled() - - name: Upload logs to artifacts - uses: scality/action-artifacts@v4 - with: - method: upload - url: https://artifacts.scality.net - user: ${{ secrets.ARTIFACTS_USER }} - password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts - if: always() + - name: Checkout + uses: actions/checkout@v4 + - name: Setup CI environment + uses: ./.github/actions/setup-ci + - name: Setup matrix job artifacts directory + shell: bash + run: | + set -exu + mkdir -p /tmp/artifacts/${{ matrix.job-name }}/ + - name: Setup server access logs file and directory + shell: bash + run: | + set -exu + sudo mkdir -p /logs + sudo chmod 0777 /logs + sudo touch /logs/server-access.log && sudo chmod 0666 /logs/server-access.log + - name: Setup CI services + run: docker compose up -d + working-directory: .github/docker + - name: Run file ft tests + run: |- + set -o pipefail; + bash wait_for_local_port.bash 8000 40 + yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log + - name: Cleanup and upload coverage + uses: ./.github/actions/cleanup-and-coverage + with: + codecov-token: ${{ secrets.CODECOV_TOKEN }} + flags: ${{ matrix.job-name }} + if: always() + - name: Upload test results to Codecov + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: '**/junit/*junit*.xml' + flags: ${{ matrix.job-name }} + if: always() && !cancelled() + - name: Upload logs to artifacts + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ secrets.ARTIFACTS_USER }} + password: ${{ secrets.ARTIFACTS_PASSWORD }} + source: /tmp/artifacts + if: always() # Configure and run as Integration run S3C tests s3c-ft-tests: @@ -514,7 +488,7 @@ jobs: S3METADATA: scality S3VAULT: scality CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }} - MPU_TESTING: "yes" + MPU_TESTING: 'yes' DEFAULT_BUCKET_KEY_FORMAT: ${{ matrix.vformat }} ENABLE_NULL_VERSION_COMPAT_MODE: ${{ matrix.enable-null-compat }} COMPOSE_FILE: docker-compose.yaml:docker-compose.sse.yaml @@ -524,102 +498,102 @@ jobs: S3_TESTVAL_OWNERCANONICALID: 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be S3_SERVER_ACCESS_LOGS_MODE: ENABLED steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Login to Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - name: Setup CI environment - uses: ./.github/actions/setup-ci - - name: Setup matrix job artifacts directory - shell: bash - run: | - set -exu - mkdir -p /tmp/artifacts/${{ matrix.job-name }}/ - - name: Setup server access logs file and directory - shell: bash - run: | - set -exu - sudo mkdir -p /logs - sudo chmod 0777 /logs - sudo touch /logs/server-access.log && sudo chmod 0666 /logs/server-access.log - - name: Modify md-config.json for vformat - run: | - sed -i 's/\("METADATA_NEW_BUCKETS_VFORMAT":\s*\)"[^"]*"/\1"${{ matrix.vformat }}"/' .github/docker/md-config.json - - name: Copy S3C config - run: cp .github/docker/config.s3c.json tests/functional/sse-kms-migration/config.json - - name: Setup CI services - run: docker compose up -d --quiet-pull redis sproxyd metadata-standalone vault cloudserver-sse-before-migration - working-directory: .github/docker - - name: Wait for services to be ready - run: |- - set -o pipefail; - bash wait_for_local_port.bash 81 40 - bash wait_for_local_port.bash 9000 40 - bash wait_for_local_port.bash 8000 40 - - name: Setup vault credentials like S3C Integration - run: ./setup-s3c.sh - working-directory: .github/docker - - name: Set config files env variables - run: |- - echo "S3_CONFIG_FILE=${{ github.workspace }}/.github/docker/config.s3c.json" >> $GITHUB_ENV - echo "S3_LOCATION_FILE=${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json" >> $GITHUB_ENV - - name: Run cloudserver-object tests - run: |- - set -o pipefail; - yarn run ft_awssdk_objects_misc | tee /tmp/artifacts/${{ matrix.job-name }}/ft_awssdk_objects_misc.log - - name: Run cloudserver-version tests - run: |- - set -o pipefail; - yarn run ft_awssdk_versioning | tee /tmp/artifacts/${{ matrix.job-name }}/ft_awssdk_versioning.log - - name: Run cloudserver-bucket tests - run: |- - set -o pipefail; - yarn run ft_awssdk_buckets | tee /tmp/artifacts/${{ matrix.job-name }}/ft_awssdk_buckets.log - - name: Run cloudserver-routes (metadata) tests - run: |- - set -o pipefail; - yarn run ft_node_routes | tee /tmp/artifacts/${{ matrix.job-name }}/ft_node_routes.log - - name: Run backbeat route tests - run: |- - set -o pipefail; - yarn run ft_route_backbeat | tee /tmp/artifacts/${{ matrix.job-name }}/ft_route_backbeat.log - - name: Run backbeat tests - run: |- - set -o pipefail; - yarn run ft_backbeat | tee /tmp/artifacts/${{ matrix.job-name }}/ft_backbeat.log - - name: Run healthchecks tests - run: |- - set -o pipefail; - yarn run ft_healthchecks | tee /tmp/artifacts/${{ matrix.job-name }}/ft_healthchecks.log - - name: Teardown CI services - run: docker compose down redis sproxyd metadata-standalone vault cloudserver-sse-before-migration - working-directory: .github/docker - - name: Cleanup and upload coverage - uses: ./.github/actions/cleanup-and-coverage - with: - codecov-token: ${{ secrets.CODECOV_TOKEN }} - flags: ${{ matrix.job-name }} - if: always() - - name: Upload test results to Codecov - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: '**/junit/*junit*.xml' - flags: ${{ matrix.job-name }} - if: always() && !cancelled() - - name: Upload logs to artifacts - uses: scality/action-artifacts@v4 - with: - method: upload - url: https://artifacts.scality.net - user: ${{ secrets.ARTIFACTS_USER }} - password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts - if: always() + - name: Checkout + uses: actions/checkout@v4 + - name: Login to Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + - name: Setup CI environment + uses: ./.github/actions/setup-ci + - name: Setup matrix job artifacts directory + shell: bash + run: | + set -exu + mkdir -p /tmp/artifacts/${{ matrix.job-name }}/ + - name: Setup server access logs file and directory + shell: bash + run: | + set -exu + sudo mkdir -p /logs + sudo chmod 0777 /logs + sudo touch /logs/server-access.log && sudo chmod 0666 /logs/server-access.log + - name: Modify md-config.json for vformat + run: | + sed -i 's/\("METADATA_NEW_BUCKETS_VFORMAT":\s*\)"[^"]*"/\1"${{ matrix.vformat }}"/' .github/docker/md-config.json + - name: Copy S3C config + run: cp .github/docker/config.s3c.json tests/functional/sse-kms-migration/config.json + - name: Setup CI services + run: docker compose up -d --quiet-pull redis sproxyd metadata-standalone vault cloudserver-sse-before-migration + working-directory: .github/docker + - name: Wait for services to be ready + run: |- + set -o pipefail; + bash wait_for_local_port.bash 81 40 + bash wait_for_local_port.bash 9000 40 + bash wait_for_local_port.bash 8000 40 + - name: Setup vault credentials like S3C Integration + run: ./setup-s3c.sh + working-directory: .github/docker + - name: Set config files env variables + run: |- + echo "S3_CONFIG_FILE=${{ github.workspace }}/.github/docker/config.s3c.json" >> $GITHUB_ENV + echo "S3_LOCATION_FILE=${{ github.workspace }}/tests/locationConfig/locationConfigS3C.json" >> $GITHUB_ENV + - name: Run cloudserver-object tests + run: |- + set -o pipefail; + yarn run ft_awssdk_objects_misc | tee /tmp/artifacts/${{ matrix.job-name }}/ft_awssdk_objects_misc.log + - name: Run cloudserver-version tests + run: |- + set -o pipefail; + yarn run ft_awssdk_versioning | tee /tmp/artifacts/${{ matrix.job-name }}/ft_awssdk_versioning.log + - name: Run cloudserver-bucket tests + run: |- + set -o pipefail; + yarn run ft_awssdk_buckets | tee /tmp/artifacts/${{ matrix.job-name }}/ft_awssdk_buckets.log + - name: Run cloudserver-routes (metadata) tests + run: |- + set -o pipefail; + yarn run ft_node_routes | tee /tmp/artifacts/${{ matrix.job-name }}/ft_node_routes.log + - name: Run backbeat route tests + run: |- + set -o pipefail; + yarn run ft_route_backbeat | tee /tmp/artifacts/${{ matrix.job-name }}/ft_route_backbeat.log + - name: Run backbeat tests + run: |- + set -o pipefail; + yarn run ft_backbeat | tee /tmp/artifacts/${{ matrix.job-name }}/ft_backbeat.log + - name: Run healthchecks tests + run: |- + set -o pipefail; + yarn run ft_healthchecks | tee /tmp/artifacts/${{ matrix.job-name }}/ft_healthchecks.log + - name: Teardown CI services + run: docker compose down redis sproxyd metadata-standalone vault cloudserver-sse-before-migration + working-directory: .github/docker + - name: Cleanup and upload coverage + uses: ./.github/actions/cleanup-and-coverage + with: + codecov-token: ${{ secrets.CODECOV_TOKEN }} + flags: ${{ matrix.job-name }} + if: always() + - name: Upload test results to Codecov + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: '**/junit/*junit*.xml' + flags: ${{ matrix.job-name }} + if: always() && !cancelled() + - name: Upload logs to artifacts + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ secrets.ARTIFACTS_USER }} + password: ${{ secrets.ARTIFACTS_PASSWORD }} + source: /tmp/artifacts + if: always() utapi-v2-tests: runs-on: ubuntu-24.04 @@ -633,39 +607,39 @@ jobs: MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} JOB_NAME: ${{ github.job }} steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup CI environment - uses: ./.github/actions/setup-ci - - name: Setup CI services - run: docker compose up -d - working-directory: .github/docker - - name: Run file utapi v2 tests - run: |- - set -ex -o pipefail; - bash wait_for_local_port.bash 8000 40 - yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log - - name: Cleanup and upload coverage - uses: ./.github/actions/cleanup-and-coverage - with: - codecov-token: ${{ secrets.CODECOV_TOKEN }} - if: always() - - name: Upload test results to Codecov - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: '**/junit/*junit*.xml' - flags: utapi-v2-tests - if: always() && !cancelled() - - name: Upload logs to artifacts - uses: scality/action-artifacts@v4 - with: - method: upload - url: https://artifacts.scality.net - user: ${{ secrets.ARTIFACTS_USER }} - password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts - if: always() + - name: Checkout + uses: actions/checkout@v4 + - name: Setup CI environment + uses: ./.github/actions/setup-ci + - name: Setup CI services + run: docker compose up -d + working-directory: .github/docker + - name: Run file utapi v2 tests + run: |- + set -ex -o pipefail; + bash wait_for_local_port.bash 8000 40 + yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log + - name: Cleanup and upload coverage + uses: ./.github/actions/cleanup-and-coverage + with: + codecov-token: ${{ secrets.CODECOV_TOKEN }} + if: always() + - name: Upload test results to Codecov + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: '**/junit/*junit*.xml' + flags: utapi-v2-tests + if: always() && !cancelled() + - name: Upload logs to artifacts + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ secrets.ARTIFACTS_USER }} + password: ${{ secrets.ARTIFACTS_PASSWORD }} + source: /tmp/artifacts + if: always() sur-tests: runs-on: ubuntu-24.04 @@ -673,10 +647,10 @@ jobs: strategy: matrix: inflights: - - name: "With Inflights" - value: "true" - - name: "Without Inflights" - value: "false" + - name: 'With Inflights' + value: 'true' + - name: 'Without Inflights' + value: 'false' env: S3METADATA: mongodb S3BACKEND: mem @@ -689,41 +663,41 @@ jobs: MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} JOB_NAME: ${{ github.job }} steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup CI environment - uses: ./.github/actions/setup-ci - - name: Setup CI services - run: docker compose --profile mongo up -d - working-directory: .github/docker - - name: Run SUR-related tests - run: |- - set -ex -o pipefail; - bash wait_for_local_port.bash 8000 40 - yarn run test_sur | tee /tmp/artifacts/${{ github.job }}/tests.log - - name: Cleanup and upload coverage - uses: ./.github/actions/cleanup-and-coverage - with: - profiles: mongo - codecov-token: ${{ secrets.CODECOV_TOKEN }} - flags: ${{ github.job }}${{ matrix.inflights.value == 'true' && '-inflights' || '' }} - if: always() - - name: Upload test results to Codecov - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: '**/junit/*junit*.xml' - flags: ${{ github.job }}${{ matrix.inflights.value == 'true' && '-inflights' || '' }} - if: always() && !cancelled() - - name: Upload logs to artifacts - uses: scality/action-artifacts@v4 - with: - method: upload - url: https://artifacts.scality.net - user: ${{ secrets.ARTIFACTS_USER }} - password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts - if: always() + - name: Checkout + uses: actions/checkout@v4 + - name: Setup CI environment + uses: ./.github/actions/setup-ci + - name: Setup CI services + run: docker compose --profile mongo up -d + working-directory: .github/docker + - name: Run SUR-related tests + run: |- + set -ex -o pipefail; + bash wait_for_local_port.bash 8000 40 + yarn run test_sur | tee /tmp/artifacts/${{ github.job }}/tests.log + - name: Cleanup and upload coverage + uses: ./.github/actions/cleanup-and-coverage + with: + profiles: mongo + codecov-token: ${{ secrets.CODECOV_TOKEN }} + flags: ${{ github.job }}${{ matrix.inflights.value == 'true' && '-inflights' || '' }} + if: always() + - name: Upload test results to Codecov + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: '**/junit/*junit*.xml' + flags: ${{ github.job }}${{ matrix.inflights.value == 'true' && '-inflights' || '' }} + if: always() && !cancelled() + - name: Upload logs to artifacts + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ secrets.ARTIFACTS_USER }} + password: ${{ secrets.ARTIFACTS_PASSWORD }} + source: /tmp/artifacts + if: always() kmip-ft-tests: runs-on: ubuntu-24.04 @@ -731,7 +705,7 @@ jobs: env: S3BACKEND: file S3VAULT: mem - MPU_TESTING: "yes" + MPU_TESTING: 'yes' CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }} MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }} @@ -746,44 +720,44 @@ jobs: S3KMIP_CERT: /tmp/ssl-kmip/kmip-client-cert.pem S3KMIP_CA: /tmp/ssl-kmip/kmip-ca.pem steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup CI environment - uses: ./.github/actions/setup-ci - - name: Copy KMIP certs - run: cp -r ./certs /tmp/ssl-kmip - working-directory: .github/pykmip - - name: Setup CI services - run: docker compose --profile pykmip up -d - working-directory: .github/docker - - name: Run file KMIP tests - run: |- - set -ex -o pipefail; - bash wait_for_local_port.bash 8000 40 - bash wait_for_local_port.bash 5696 40 - yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log - - name: Cleanup and upload coverage - uses: ./.github/actions/cleanup-and-coverage - with: - profiles: pykmip - codecov-token: ${{ secrets.CODECOV_TOKEN }} - if: always() - - name: Upload test results to Codecov - uses: codecov/test-results-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: '**/junit/*junit*.xml' - flags: kmip-ft-tests - if: always() && !cancelled() - - name: Upload logs to artifacts - uses: scality/action-artifacts@v4 - with: - method: upload - url: https://artifacts.scality.net - user: ${{ secrets.ARTIFACTS_USER }} - password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts - if: always() + - name: Checkout + uses: actions/checkout@v4 + - name: Setup CI environment + uses: ./.github/actions/setup-ci + - name: Copy KMIP certs + run: cp -r ./certs /tmp/ssl-kmip + working-directory: .github/pykmip + - name: Setup CI services + run: docker compose --profile pykmip up -d + working-directory: .github/docker + - name: Run file KMIP tests + run: |- + set -ex -o pipefail; + bash wait_for_local_port.bash 8000 40 + bash wait_for_local_port.bash 5696 40 + yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log + - name: Cleanup and upload coverage + uses: ./.github/actions/cleanup-and-coverage + with: + profiles: pykmip + codecov-token: ${{ secrets.CODECOV_TOKEN }} + if: always() + - name: Upload test results to Codecov + uses: codecov/test-results-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: '**/junit/*junit*.xml' + flags: kmip-ft-tests + if: always() && !cancelled() + - name: Upload logs to artifacts + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ secrets.ARTIFACTS_USER }} + password: ${{ secrets.ARTIFACTS_PASSWORD }} + source: /tmp/artifacts + if: always() kmip-cluster-ft-tests: runs-on: ubuntu-latest @@ -798,54 +772,54 @@ jobs: COMPOSE_FILE: docker-compose.yaml:docker-compose.sse.yaml S3KMS: kmip steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - name: Setup CI environment - uses: ./.github/actions/setup-ci - - name: Install tcpdump to analyze traffic on kmip cluster interfaces - run: sudo apt-get update && sudo apt-get install -y tcpdump - - name: Copy KMIP certs - run: cp -r ./certs /tmp/ssl-kmip - working-directory: .github/pykmip - - name: Merge config.json and kmip-cluster config - run: | - jq -s ' - .[0] * .[1] - ' \ - configs/base.json \ - configs/kmip-cluster.json \ - > config.json - working-directory: tests/functional/sse-kms-migration - - name: Setup CI services - run: docker compose up -d --quiet-pull redis pykmip cloudserver-sse-before-migration - working-directory: .github/docker - - name: Run file KMIP cluster tests - shell: bash # for pipefail - env: - # Functional tests needs access to the running config to use the same - # KMS kmip cluster config - # yarn run does a cd into the test folder - # absolute path to override default root config - S3_CONFIG_FILE: ${{ github.workspace }}/tests/functional/sse-kms-migration/config.json - S3KMS: kmip - S3_END_TO_END: true # to use the default credentials profile and not vault profile - run: |- - set -ex -o pipefail; - bash wait_for_local_port.bash 8000 40 - bash wait_for_local_port.bash 5696 40 - yarn run ft_kmip_cluster | tee /tmp/artifacts/${{ github.job }}/tests.log - - name: Upload logs to artifacts - uses: scality/action-artifacts@v4 - with: - method: upload - url: https://artifacts.scality.net - user: ${{ secrets.ARTIFACTS_USER }} - password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts - if: always() + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + - name: Setup CI environment + uses: ./.github/actions/setup-ci + - name: Install tcpdump to analyze traffic on kmip cluster interfaces + run: sudo apt-get update && sudo apt-get install -y tcpdump + - name: Copy KMIP certs + run: cp -r ./certs /tmp/ssl-kmip + working-directory: .github/pykmip + - name: Merge config.json and kmip-cluster config + run: | + jq -s ' + .[0] * .[1] + ' \ + configs/base.json \ + configs/kmip-cluster.json \ + > config.json + working-directory: tests/functional/sse-kms-migration + - name: Setup CI services + run: docker compose up -d --quiet-pull redis pykmip cloudserver-sse-before-migration + working-directory: .github/docker + - name: Run file KMIP cluster tests + shell: bash # for pipefail + env: + # Functional tests needs access to the running config to use the same + # KMS kmip cluster config + # yarn run does a cd into the test folder + # absolute path to override default root config + S3_CONFIG_FILE: ${{ github.workspace }}/tests/functional/sse-kms-migration/config.json + S3KMS: kmip + S3_END_TO_END: true # to use the default credentials profile and not vault profile + run: |- + set -ex -o pipefail; + bash wait_for_local_port.bash 8000 40 + bash wait_for_local_port.bash 5696 40 + yarn run ft_kmip_cluster | tee /tmp/artifacts/${{ github.job }}/tests.log + - name: Upload logs to artifacts + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ secrets.ARTIFACTS_USER }} + password: ${{ secrets.ARTIFACTS_PASSWORD }} + source: /tmp/artifacts + if: always() # This test with the final yarn run ft_sse_arn covers more code than the kmip tests sse-kms-migration-tests: @@ -890,7 +864,7 @@ jobs: CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}-testcoverage VAULT_IMAGE: ghcr.io/scality/vault:7.70.32 KMS_IMAGE: nsmithuk/local-kms:3.11.7 - MPU_TESTING: "yes" + MPU_TESTING: 'yes' JOB_NAME: >- sse-kms-migration-${{ matrix.kms.provider @@ -901,130 +875,130 @@ jobs: }} COMPOSE_FILE: docker-compose.yaml:docker-compose.sse.yaml steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - name: Login to GitHub Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ github.token }} - - name: Setup CI environment - uses: ./.github/actions/setup-ci - - name: Copy KMIP certs - run: cp -r ./certs /tmp/ssl-kmip - working-directory: .github/pykmip - - name: Setup matrix job artifacts directory - shell: bash - run: | - set -exu - mkdir -p /tmp/artifacts/${{ env.JOB_NAME }}/ - - name: Copy base config - run: cp configs/base.json config.json - working-directory: tests/functional/sse-kms-migration - - name: Setup CI services (with old cloudserver image before sse migration) - run: docker compose up -d --quiet-pull redis vault-sse-before-migration cloudserver-sse-before-migration - working-directory: .github/docker - env: - CLOUDSERVER_IMAGE: ${{ env.CLOUDSERVER_IMAGE_BEFORE_SSE_MIGRATION }} - VAULT_IMAGE: ${{ env.VAULT_IMAGE_BEFORE_SSE_MIGRATION }} - - name: Wait for services vault and s3 - run: |- - bash wait_for_local_port.bash 8500 40 - bash wait_for_local_port.bash 8000 40 - - name: Ensure old version of cloudserver and vault is used - run: |- - ./ensure-version.sh cloudserver-sse-before-migration ${{ env.CLOUDSERVER_VERSION_BEFORE }} - ./ensure-version.sh vault-sse-before-migration ${{ env.VAULT_VERSION_BEFORE }} - working-directory: .github/docker - - name: Create vault account and keys - run: |- - export PATH="$PATH:$(pwd)/node_modules/vaultclient/bin/" - vaultclient --config .github/docker/admin.json delete-account --name test || true - vaultclient --config .github/docker/admin.json create-account --name test --email test@scality.com - vaultclient --config .github/docker/admin.json generate-account-access-key --name test --accesskey TESTAK00000000000000 --secretkey TESTSK0000000000000000000000000000000000 - vaultclient --config .github/docker/admin.json get-account --account-name test - - name: Run SSE before migration tests (setup buckets and objects) - shell: bash # for pipefail - env: - # absolute path to override default root config - S3_CONFIG_FILE: ${{ github.workspace }}/tests/functional/sse-kms-migration/config.json - S3KMS: file - run: yarn run ft_sse_before_migration | tee /tmp/artifacts/${{ env.JOB_NAME }}/beforeMigration.log - - name: Merge config.json files for options - run: | - jq -s ' - .[0] * .[1] * .[2] * - { kmsHideScalityArn: ${{ matrix.opts.kmsHideScalityArn }} } * - { globalEncryptionEnabled: ${{ matrix.opts.globalEncryptionEnabled }} } - ' \ - configs/base.json \ - configs/${{ matrix.kms.provider }}.json \ - configs/sseMigration.json \ - > config.json - working-directory: tests/functional/sse-kms-migration - - name: Replace old cloudserver image with current one - run: |- - docker compose down cloudserver-sse-before-migration vault-sse-before-migration - docker compose up -d --quiet-pull ${{ matrix.kms.container }} vault-sse-migration cloudserver-sse-migration - working-directory: .github/docker - env: - S3KMS: ${{ matrix.kms.provider }} # S3 - KMS_BACKEND: ${{ matrix.kms.provider == 'aws' && 'aws' || '' }} # vault only supports aws - - name: Wait for services kms vault and s3 - run: |- - bash wait_for_local_port.bash ${{ matrix.kms.port }} 40 - bash wait_for_local_port.bash 8500 40 - bash wait_for_local_port.bash 8000 40 - - name: Ensure latest version of cloudserver and vault is used - run: |- - ./ensure-version.sh cloudserver-sse-migration `jq -r .version ../../package.json` - ./ensure-version.sh vault-sse-migration ${{ env.VAULT_VERSION_CURRENT }} - working-directory: .github/docker - - name: Run SSE migration tests - shell: bash # for pipefail - env: - # Functional tests needs access to the running config to use the same - # KMS provider and sseMigration - # absolute path to override default root config - S3_CONFIG_FILE: ${{ github.workspace }}/tests/functional/sse-kms-migration/config.json - S3KMS: ${{ matrix.kms.provider }} - run: yarn run ft_sse_migration | tee /tmp/artifacts/${{ env.JOB_NAME }}/migration.log - - name: Run SSE arnPrefix tests - shell: bash # for pipefail - env: - # Functional tests needs access to the running config to use the same - # KMS provider and sseMigration - # absolute path to override default root config - S3_CONFIG_FILE: ${{ github.workspace }}/tests/functional/sse-kms-migration/config.json - S3KMS: ${{ matrix.kms.provider }} - run: yarn run ft_sse_arn | tee /tmp/artifacts/${{ env.JOB_NAME }}/arnPrefix.log - - name: Print docker compose logs - run: |- - docker compose logs \ - cloudserver-sse-before-migration \ - cloudserver-sse-migration \ - vault-sse-before-migration \ - vault-sse-migration \ - ${{ matrix.kms.container == 'localkms' && 'localkms' || '' }} - # pykmip logs are already uploaded to artifacts, but not localkms - working-directory: .github/docker - if: failure() - - name: Remove empty artifact files to simplify viewing artifacts - run: find /tmp/artifacts/${{ env.JOB_NAME }}/ -size 0 -delete - if: always() - - name: Upload logs to artifacts - uses: scality/action-artifacts@v4 - with: - method: upload - url: https://artifacts.scality.net - user: ${{ secrets.ARTIFACTS_USER }} - password: ${{ secrets.ARTIFACTS_PASSWORD }} - source: /tmp/artifacts - if: always() + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + - name: Login to GitHub Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + - name: Setup CI environment + uses: ./.github/actions/setup-ci + - name: Copy KMIP certs + run: cp -r ./certs /tmp/ssl-kmip + working-directory: .github/pykmip + - name: Setup matrix job artifacts directory + shell: bash + run: | + set -exu + mkdir -p /tmp/artifacts/${{ env.JOB_NAME }}/ + - name: Copy base config + run: cp configs/base.json config.json + working-directory: tests/functional/sse-kms-migration + - name: Setup CI services (with old cloudserver image before sse migration) + run: docker compose up -d --quiet-pull redis vault-sse-before-migration cloudserver-sse-before-migration + working-directory: .github/docker + env: + CLOUDSERVER_IMAGE: ${{ env.CLOUDSERVER_IMAGE_BEFORE_SSE_MIGRATION }} + VAULT_IMAGE: ${{ env.VAULT_IMAGE_BEFORE_SSE_MIGRATION }} + - name: Wait for services vault and s3 + run: |- + bash wait_for_local_port.bash 8500 40 + bash wait_for_local_port.bash 8000 40 + - name: Ensure old version of cloudserver and vault is used + run: |- + ./ensure-version.sh cloudserver-sse-before-migration ${{ env.CLOUDSERVER_VERSION_BEFORE }} + ./ensure-version.sh vault-sse-before-migration ${{ env.VAULT_VERSION_BEFORE }} + working-directory: .github/docker + - name: Create vault account and keys + run: |- + export PATH="$PATH:$(pwd)/node_modules/vaultclient/bin/" + vaultclient --config .github/docker/admin.json delete-account --name test || true + vaultclient --config .github/docker/admin.json create-account --name test --email test@scality.com + vaultclient --config .github/docker/admin.json generate-account-access-key --name test --accesskey TESTAK00000000000000 --secretkey TESTSK0000000000000000000000000000000000 + vaultclient --config .github/docker/admin.json get-account --account-name test + - name: Run SSE before migration tests (setup buckets and objects) + shell: bash # for pipefail + env: + # absolute path to override default root config + S3_CONFIG_FILE: ${{ github.workspace }}/tests/functional/sse-kms-migration/config.json + S3KMS: file + run: yarn run ft_sse_before_migration | tee /tmp/artifacts/${{ env.JOB_NAME }}/beforeMigration.log + - name: Merge config.json files for options + run: | + jq -s ' + .[0] * .[1] * .[2] * + { kmsHideScalityArn: ${{ matrix.opts.kmsHideScalityArn }} } * + { globalEncryptionEnabled: ${{ matrix.opts.globalEncryptionEnabled }} } + ' \ + configs/base.json \ + configs/${{ matrix.kms.provider }}.json \ + configs/sseMigration.json \ + > config.json + working-directory: tests/functional/sse-kms-migration + - name: Replace old cloudserver image with current one + run: |- + docker compose down cloudserver-sse-before-migration vault-sse-before-migration + docker compose up -d --quiet-pull ${{ matrix.kms.container }} vault-sse-migration cloudserver-sse-migration + working-directory: .github/docker + env: + S3KMS: ${{ matrix.kms.provider }} # S3 + KMS_BACKEND: ${{ matrix.kms.provider == 'aws' && 'aws' || '' }} # vault only supports aws + - name: Wait for services kms vault and s3 + run: |- + bash wait_for_local_port.bash ${{ matrix.kms.port }} 40 + bash wait_for_local_port.bash 8500 40 + bash wait_for_local_port.bash 8000 40 + - name: Ensure latest version of cloudserver and vault is used + run: |- + ./ensure-version.sh cloudserver-sse-migration `jq -r .version ../../package.json` + ./ensure-version.sh vault-sse-migration ${{ env.VAULT_VERSION_CURRENT }} + working-directory: .github/docker + - name: Run SSE migration tests + shell: bash # for pipefail + env: + # Functional tests needs access to the running config to use the same + # KMS provider and sseMigration + # absolute path to override default root config + S3_CONFIG_FILE: ${{ github.workspace }}/tests/functional/sse-kms-migration/config.json + S3KMS: ${{ matrix.kms.provider }} + run: yarn run ft_sse_migration | tee /tmp/artifacts/${{ env.JOB_NAME }}/migration.log + - name: Run SSE arnPrefix tests + shell: bash # for pipefail + env: + # Functional tests needs access to the running config to use the same + # KMS provider and sseMigration + # absolute path to override default root config + S3_CONFIG_FILE: ${{ github.workspace }}/tests/functional/sse-kms-migration/config.json + S3KMS: ${{ matrix.kms.provider }} + run: yarn run ft_sse_arn | tee /tmp/artifacts/${{ env.JOB_NAME }}/arnPrefix.log + - name: Print docker compose logs + run: |- + docker compose logs \ + cloudserver-sse-before-migration \ + cloudserver-sse-migration \ + vault-sse-before-migration \ + vault-sse-migration \ + ${{ matrix.kms.container == 'localkms' && 'localkms' || '' }} + # pykmip logs are already uploaded to artifacts, but not localkms + working-directory: .github/docker + if: failure() + - name: Remove empty artifact files to simplify viewing artifacts + run: find /tmp/artifacts/${{ env.JOB_NAME }}/ -size 0 -delete + if: always() + - name: Upload logs to artifacts + uses: scality/action-artifacts@v4 + with: + method: upload + url: https://artifacts.scality.net + user: ${{ secrets.ARTIFACTS_USER }} + password: ${{ secrets.ARTIFACTS_PASSWORD }} + source: /tmp/artifacts + if: always() cleanup-gcp-buckets: runs-on: ubuntu-24.04 @@ -1035,16 +1009,16 @@ jobs: - file-ft-tests if: always() steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: yarn - - name: Install dependencies - run: yarn install --frozen-lockfile --network-concurrency 1 - - name: Delete stale GCP CI buckets - run: yarn run cleanup_gcp_buckets - env: - AWS_GCP_BACKEND_ACCESS_KEY: ${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }} - AWS_GCP_BACKEND_SECRET_KEY: ${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }} + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: yarn + - name: Install dependencies + run: yarn install --frozen-lockfile --network-concurrency 1 + - name: Delete stale GCP CI buckets + run: yarn run cleanup_gcp_buckets + env: + AWS_GCP_BACKEND_ACCESS_KEY: ${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }} + AWS_GCP_BACKEND_SECRET_KEY: ${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }} diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000000..49284af952 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,4 @@ +coverage/ +localData/ +localMetadata/ +junit/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1d04b33459..53da7c19c3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,7 @@ # Contributing rules Please follow the -[Contributing Guidelines]( -https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md). +[Contributing Guidelines](https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md). ## Development @@ -15,3 +14,16 @@ yarn dev This will start the developement server, watching for changes and restarting as needed. + +### Formatting + +Prettier is enforced in CI for changed files, including `package.json`. +Since `yarn add`/`yarn remove` may rewrite `package.json` with 2-space +indentation, run formatting before pushing dependency changes: + +```bash +yarn format +``` + +When reviewing PRs with indentation-only changes, enable **Hide whitespace** in +the GitHub file diff view to make functional changes easier to spot. diff --git a/package.json b/package.json index 9f0dfe709e..74c841042e 100644 --- a/package.json +++ b/package.json @@ -1,148 +1,152 @@ { - "name": "@zenko/cloudserver", - "version": "9.3.6", - "description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol", - "main": "index.js", - "engines": { - "node": ">=22" - }, - "repository": "scality/S3", - "keywords": [ - "s3", - "cloud", - "server" - ], - "author": "Scality Inc.", - "license": "Apache-2.0", - "bugs": { - "url": "https://github.com/scality/S3/issues" - }, - "homepage": "https://github.com/scality/S3#readme", - "dependencies": { - "@aws-sdk/client-iam": "^3.930.0", - "@aws-sdk/client-s3": "^3.1013.0", - "@aws-sdk/client-sts": "^3.930.0", - "@aws-sdk/credential-providers": "^3.864.0", - "@aws-sdk/middleware-retry": "^3.374.0", - "@aws-sdk/protocol-http": "^3.374.0", - "@aws-sdk/s3-request-presigner": "^3.901.0", - "@aws-sdk/signature-v4": "^3.374.0", - "@azure/storage-blob": "^12.28.0", - "@hapi/joi": "^17.1.1", - "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/Arsenal#8.3.8", - "async": "2.6.4", - "bucketclient": "scality/bucketclient#8.2.7", - "bufferutil": "^4.0.8", - "commander": "^12.1.0", - "cron-parser": "^4.9.0", - "diskusage": "^1.2.0", - "google-auto-auth": "^0.10.1", - "http-proxy": "npm:http-proxy-3@^1.21.0", - "http-proxy-agent": "^7.0.2", - "https-proxy-agent": "^7.0.5", - "level-mem": "^5.0.1", - "moment": "^2.30.1", - "mongodb": "^6.11.0", - "node-fetch": "^2.6.0", - "node-forge": "^1.3.1", - "npm-run-all": "^4.1.5", - "prom-client": "^15.1.3", - "request": "^2.88.2", - "scubaclient": "git+https://github.com/scality/scubaclient.git#1.1.4", - "sql-where-parser": "^2.2.1", - "utapi": "github:scality/utapi#8.2.4", - "utf-8-validate": "^6.0.5", - "utf8": "^3.0.0", - "uuid": "^11.0.3", - "vaultclient": "scality/vaultclient#8.5.3", - "werelogs": "scality/werelogs#8.2.2", - "ws": "^8.18.0", - "xml2js": "^0.6.2" - }, - "devDependencies": { - "@eslint/compat": "^1.2.2", - "@scality/cloudserverclient": "1.0.7", - "@scality/eslint-config-scality": "scality/Guidelines#8.3.0", - "eslint": "^9.14.0", - "eslint-plugin-import": "^2.31.0", - "eslint-plugin-mocha": "^10.5.0", - "express": "^4.21.1", - "ioredis": "^5.4.1", - "istanbul": "^0.4.5", - "istanbul-api": "^3.0.0", - "lolex": "^6.0.0", - "mocha": "^11.7.5", - "mocha-junit-reporter": "^2.2.1", - "mocha-multi-reporters": "^1.5.1", - "node-mocks-http": "^1.16.1", - "nodemon": "^3.1.10", - "nyc": "^15.1.0", - "pino-pretty": "^13.1.3", - "sinon": "^13.0.1", - "tv4": "^1.3.0" - }, - "resolutions": { - "jsonwebtoken": "^9.0.0", - "nan": "v2.22.0", - "fast-xml-parser": "^5.5.6" - }, - "mocha": { - "recursive": true, - "timeout": 40000 - }, - "scripts": { - "cloudserver": "S3METADATA=mongodb npm-run-all --parallel start_dataserver start_s3server", - "dev": "nodemon --exec \"yarn run start\" | pino-pretty -c -S -m message -i \"pid,hostname,name,authn,authz,address,clientPort,clientIP,elapsed_ms\"", - "ft_awssdk": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/ --exit", - "ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/ --exit", - "ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/bucket --exit", - "ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/legacy test/object test/service test/support --exit", - "ft_awssdk_versioning": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/versioning/ --exit", - "ft_awssdk_external_backends": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/multipleBackend --exit", - "ft_mixed_bucket_format_version": "cd tests/functional/metadata && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json MixedVersionFormat.js --exit", - "ft_management": "cd tests/functional/report && yarn test", - "ft_backbeat": "cd tests/functional/backbeat && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", - "ft_node": "cd tests/functional/raw-node && yarn test", - "ft_node_routes": "cd tests/functional/raw-node && yarn run test-routes", - "ft_route_backbeat": "cd tests/multipleBackend/routes && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 routeBackbeat.js routeBackbeatForReplication.js --exit", - "ft_gcp": "cd tests/functional/raw-node && yarn run test-gcp", - "cleanup_gcp_buckets": "node .github/scripts/cleanupOldGCPBuckets.js", - "ft_healthchecks": "cd tests/functional/healthchecks && yarn test", - "ft_s3cmd": "cd tests/functional/s3cmd && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", - "ft_s3curl": "cd tests/functional/s3curl && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", - "ft_util": "cd tests/functional/utilities && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", - "ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_healthchecks ft_management ft_util ft_backbeat", - "ft_search": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 90000 test/mdSearch --exit", - "ft_kmip": "cd tests/functional/kmip && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", - "ft_kmip_cluster": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 300000 load.js --exit", - "ft_sse_cleanup": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 cleanup.js --exit", - "ft_sse_before_migration": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 cleanup.js beforeMigration.js --exit", - "ft_sse_migration": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 migration.js --exit", - "ft_sse_arn": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 cleanup.js arnPrefix.js --exit", - "lint": "eslint $(git ls-files '*.js')", - "lint_md": "mdlint $(git ls-files '*.md')", - "mem_backend": "S3BACKEND=mem node index.js", - "start": "npm-run-all --parallel start_dmd start_s3server", - "start_mongo": "yarn run cloudserver", - "start_mdserver": "node mdserver.js", - "start_dataserver": "node dataserver.js", - "start_pfsserver": "node pfsserver.js", - "start_s3server": "node index.js", - "start_dmd": "npm-run-all --parallel start_mdserver start_dataserver", - "start_utapi": "node lib/utapi/utapi.js", - "start_secure_channel_proxy": "node bin/secure_channel_proxy.js", - "start_metrics_server": "node bin/metrics_server.js", - "utapi_replay": "node lib/utapi/utapiReplay.js", - "utapi_reindex": "node lib/utapi/utapiReindex.js", - "management_agent": "node managementAgent.js", - "test": "NO_PROXY=\"test.scality.com,scality.com\" CI=true S3BACKEND=mem S3QUOTA=scuba mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit --exit", - "test_versionid_base62": "S3_VERSION_ID_ENCODING_TYPE=base62 CI=true S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit/api --exit", - "test_legacy_location": "CI=true S3QUOTA=scuba S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit --exit", - "test_utapi_v2": "mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/utapi --exit", - "test_sur": "mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/sur --exit", - "multiple_backend_test": "CI=true S3BACKEND=mem S3METADATA=mem S3DATA=multiple mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 20000 --recursive tests/multipleBackend --exit", - "cover": "nyc --clean --silent yarn run", - "postcover": "nyc report --report-dir ./coverage/test --reporter=lcov" - } + "name": "@zenko/cloudserver", + "version": "9.3.6", + "description": "Zenko CloudServer, an open-source Node.js implementation of a server handling the Amazon S3 protocol", + "main": "index.js", + "engines": { + "node": ">=22" + }, + "repository": "scality/S3", + "keywords": [ + "s3", + "cloud", + "server" + ], + "author": "Scality Inc.", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/scality/S3/issues" + }, + "homepage": "https://github.com/scality/S3#readme", + "dependencies": { + "@aws-sdk/client-iam": "^3.930.0", + "@aws-sdk/client-s3": "^3.1013.0", + "@aws-sdk/client-sts": "^3.930.0", + "@aws-sdk/credential-providers": "^3.864.0", + "@aws-sdk/middleware-retry": "^3.374.0", + "@aws-sdk/protocol-http": "^3.374.0", + "@aws-sdk/s3-request-presigner": "^3.901.0", + "@aws-sdk/signature-v4": "^3.374.0", + "@azure/storage-blob": "^12.28.0", + "@hapi/joi": "^17.1.1", + "@smithy/node-http-handler": "^3.0.0", + "arsenal": "git+https://github.com/scality/Arsenal#8.3.8", + "async": "2.6.4", + "bucketclient": "scality/bucketclient#8.2.7", + "bufferutil": "^4.0.8", + "commander": "^12.1.0", + "cron-parser": "^4.9.0", + "diskusage": "^1.2.0", + "google-auto-auth": "^0.10.1", + "http-proxy": "npm:http-proxy-3@^1.21.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.5", + "level-mem": "^5.0.1", + "moment": "^2.30.1", + "mongodb": "^6.11.0", + "node-fetch": "^2.6.0", + "node-forge": "^1.3.1", + "npm-run-all": "^4.1.5", + "prom-client": "^15.1.3", + "request": "^2.88.2", + "scubaclient": "git+https://github.com/scality/scubaclient.git#1.1.4", + "sql-where-parser": "^2.2.1", + "utapi": "github:scality/utapi#8.2.4", + "utf-8-validate": "^6.0.5", + "utf8": "^3.0.0", + "uuid": "^11.0.3", + "vaultclient": "scality/vaultclient#8.5.3", + "werelogs": "scality/werelogs#8.2.2", + "ws": "^8.18.0", + "xml2js": "^0.6.2" + }, + "devDependencies": { + "@eslint/compat": "^1.2.2", + "@scality/cloudserverclient": "1.0.7", + "@scality/eslint-config-scality": "scality/Guidelines#8.3.1", + "eslint": "^9.14.0", + "eslint-plugin-import": "^2.31.0", + "eslint-plugin-mocha": "^10.5.0", + "express": "^4.21.1", + "ioredis": "^5.4.1", + "istanbul": "^0.4.5", + "istanbul-api": "^3.0.0", + "lolex": "^6.0.0", + "mocha": "^11.7.5", + "mocha-junit-reporter": "^2.2.1", + "mocha-multi-reporters": "^1.5.1", + "node-mocks-http": "^1.16.1", + "nodemon": "^3.1.10", + "nyc": "^15.1.0", + "pino-pretty": "^13.1.3", + "prettier": "^3.4.2", + "sinon": "^13.0.1", + "tv4": "^1.3.0" + }, + "resolutions": { + "jsonwebtoken": "^9.0.0", + "nan": "v2.22.0", + "fast-xml-parser": "^5.5.6" + }, + "mocha": { + "recursive": true, + "timeout": 40000 + }, + "scripts": { + "cloudserver": "S3METADATA=mongodb npm-run-all --parallel start_dataserver start_s3server", + "dev": "nodemon --exec \"yarn run start\" | pino-pretty -c -S -m message -i \"pid,hostname,name,authn,authz,address,clientPort,clientIP,elapsed_ms\"", + "ft_awssdk": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/ --exit", + "ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/ --exit", + "ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/bucket --exit", + "ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/legacy test/object test/service test/support --exit", + "ft_awssdk_versioning": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/versioning/ --exit", + "ft_awssdk_external_backends": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/multipleBackend --exit", + "ft_mixed_bucket_format_version": "cd tests/functional/metadata && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json MixedVersionFormat.js --exit", + "ft_management": "cd tests/functional/report && yarn test", + "ft_backbeat": "cd tests/functional/backbeat && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", + "ft_node": "cd tests/functional/raw-node && yarn test", + "ft_node_routes": "cd tests/functional/raw-node && yarn run test-routes", + "ft_route_backbeat": "cd tests/multipleBackend/routes && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 routeBackbeat.js routeBackbeatForReplication.js --exit", + "ft_gcp": "cd tests/functional/raw-node && yarn run test-gcp", + "cleanup_gcp_buckets": "node .github/scripts/cleanupOldGCPBuckets.js", + "ft_healthchecks": "cd tests/functional/healthchecks && yarn test", + "ft_s3cmd": "cd tests/functional/s3cmd && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", + "ft_s3curl": "cd tests/functional/s3curl && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", + "ft_util": "cd tests/functional/utilities && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", + "ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_healthchecks ft_management ft_util ft_backbeat", + "ft_search": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 90000 test/mdSearch --exit", + "ft_kmip": "cd tests/functional/kmip && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js --exit", + "ft_kmip_cluster": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 300000 load.js --exit", + "ft_sse_cleanup": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 cleanup.js --exit", + "ft_sse_before_migration": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 cleanup.js beforeMigration.js --exit", + "ft_sse_migration": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 migration.js --exit", + "ft_sse_arn": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 cleanup.js arnPrefix.js --exit", + "lint": "eslint $(git ls-files '*.js')", + "lint_md": "mdlint $(git ls-files '*.md')", + "format:changed": "scripts/format-changed.sh", + "format": "prettier --write .", + "format:check": "prettier --check .", + "mem_backend": "S3BACKEND=mem node index.js", + "start": "npm-run-all --parallel start_dmd start_s3server", + "start_mongo": "yarn run cloudserver", + "start_mdserver": "node mdserver.js", + "start_dataserver": "node dataserver.js", + "start_pfsserver": "node pfsserver.js", + "start_s3server": "node index.js", + "start_dmd": "npm-run-all --parallel start_mdserver start_dataserver", + "start_utapi": "node lib/utapi/utapi.js", + "start_secure_channel_proxy": "node bin/secure_channel_proxy.js", + "start_metrics_server": "node bin/metrics_server.js", + "utapi_replay": "node lib/utapi/utapiReplay.js", + "utapi_reindex": "node lib/utapi/utapiReindex.js", + "management_agent": "node managementAgent.js", + "test": "NO_PROXY=\"test.scality.com,scality.com\" CI=true S3BACKEND=mem S3QUOTA=scuba mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit --exit", + "test_versionid_base62": "S3_VERSION_ID_ENCODING_TYPE=base62 CI=true S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit/api --exit", + "test_legacy_location": "CI=true S3QUOTA=scuba S3_LOCATION_FILE=tests/locationConfig/locationConfigLegacy.json S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit --exit", + "test_utapi_v2": "mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/utapi --exit", + "test_sur": "mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/sur --exit", + "multiple_backend_test": "CI=true S3BACKEND=mem S3METADATA=mem S3DATA=multiple mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 20000 --recursive tests/multipleBackend --exit", + "cover": "nyc --clean --silent yarn run", + "postcover": "nyc report --report-dir ./coverage/test --reporter=lcov" + } } diff --git a/prettier.config.cjs b/prettier.config.cjs new file mode 100644 index 0000000000..fa91d66520 --- /dev/null +++ b/prettier.config.cjs @@ -0,0 +1,2 @@ +// Reuse the shared Prettier rules from the eslint-config-scality package +module.exports = require('@scality/eslint-config-scality/prettier.config.cjs'); diff --git a/scripts/format-changed.sh b/scripts/format-changed.sh new file mode 100755 index 0000000000..7c3c0e6de4 --- /dev/null +++ b/scripts/format-changed.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Purpose: +# - Run Prettier in CI only on files changed in the current branch/PR. +# - Avoid blocking PRs because of legacy formatting in untouched files. +# +# It works by: +# 1) Resolving a diff base from BASE_SHA, BASE_REF, or DEFAULT_BASE. +# 2) Ensuring that base commit exists locally (fetch shallow history when needed). +# 3) Listing changed files between base..HEAD, keeping only supported extensions. +# 4) Running `prettier --check` on that filtered list. +# 5) Exiting 0 when no supported files changed. + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +BASE_SHA="${BASE_SHA:-}" +BASE_REF="${BASE_REF:-${GITHUB_BASE_REF:-}}" +DEFAULT_BASE="${DEFAULT_BASE:-origin/main}" + +if [[ -n "$BASE_SHA" ]]; then + # Preferred in PR CI: exact base commit SHA + DIFF_BASE="$BASE_SHA" +elif [[ -n "$BASE_REF" ]]; then + # Fallback: base branch name (e.g. main) + DIFF_BASE="origin/${BASE_REF#origin/}" +else + # Local default when no CI vars are provided + DIFF_BASE="$DEFAULT_BASE" +fi + +if ! git rev-parse --verify "${DIFF_BASE}^{commit}" >/dev/null 2>&1; then + if [[ -n "$BASE_REF" ]]; then + git fetch --no-tags --prune --depth=50 origin "${BASE_REF#origin/}" >/dev/null 2>&1 || true + else + git fetch --no-tags --prune --depth=50 origin "${DIFF_BASE#origin/}" >/dev/null 2>&1 || true + fi +fi + +if ! git rev-parse --verify "${DIFF_BASE}^{commit}" >/dev/null 2>&1; then + echo "::error::Base ref ${DIFF_BASE} not found; cannot run Prettier check." >&2 + exit 1 +fi + +mapfile -t CHANGED < <(git diff --name-only "${DIFF_BASE}..HEAD" --diff-filter=ACMRT \ + | grep -E '\.(js|cjs|mjs|ts|tsx|json|ya?ml|md)$' || true) + +if [[ ${#CHANGED[@]} -eq 0 ]]; then + echo "No supported files changed; skipping Prettier check." + exit 0 +fi + +echo "Running Prettier check on ${#CHANGED[@]} file(s):" +printf ' - %s\n' "${CHANGED[@]}" + +yarn run --silent prettier --check "${CHANGED[@]}" diff --git a/yarn.lock b/yarn.lock index ea038ef5d9..7729d9993a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3444,9 +3444,9 @@ JSONStream "^1.3.5" fast-xml-parser "^4.3.2" -"@scality/eslint-config-scality@scality/Guidelines#8.3.0": - version "8.3.0" - resolved "https://codeload.github.com/scality/Guidelines/tar.gz/666b90495dc7e9a401a37ba4d58c7eba89db90ac" +"@scality/eslint-config-scality@scality/Guidelines#8.3.1": + version "8.3.1" + resolved "https://codeload.github.com/scality/Guidelines/tar.gz/a538b6b58e821fd8af9fa277c35f7268c23d57d8" dependencies: commander "11.1.0" markdownlint "0.31.1" @@ -10492,6 +10492,11 @@ prelude-ls@~1.1.2: resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" integrity sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w== +prettier@^3.4.2: + version "3.8.1" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.8.1.tgz#edf48977cf991558f4fcbd8a3ba6015ba2a3a173" + integrity sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg== + proc-log@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/proc-log/-/proc-log-5.0.0.tgz#e6c93cf37aef33f835c53485f314f50ea906a9d8"