From dc92972579a9a2cc940b7951759b1e463fd0ee92 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 25 Jun 2026 08:51:41 +0200 Subject: [PATCH 1/8] CLDSRV-932: validate x-amz-content-sha256 in non-streaming methods --- .../apiUtils/integrity/validateChecksums.js | 122 +++++++++++++ .../functional/raw-node/test/xAmzChecksum.js | 7 +- .../functional/raw-node/utils/makeRequest.js | 4 +- .../apiUtils/integrity/validateChecksums.js | 162 ++++++++++++++++++ 4 files changed, 292 insertions(+), 3 deletions(-) diff --git a/lib/api/apiUtils/integrity/validateChecksums.js b/lib/api/apiUtils/integrity/validateChecksums.js index ea42aac27b..22f5f0ca40 100644 --- a/lib/api/apiUtils/integrity/validateChecksums.js +++ b/lib/api/apiUtils/integrity/validateChecksums.js @@ -5,6 +5,7 @@ const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt'); const { errors: ArsenalErrors, errorInstances } = require('arsenal'); const { config } = require('../../../Config'); const { combinePartCrcs } = require('./crcCombine'); +const { supportedSignatureChecksums, unsupportedSignatureChecksums } = require('../../../../constants'); const defaultChecksumData = Object.freeze({ algorithm: 'crc64nvme', isTrailer: false, expected: undefined }); @@ -45,6 +46,21 @@ const errCopyChecksumAlgoNotSupported = errorInstances.InvalidRequest.customizeD '[CRC32, CRC32C, CRC64NVME, SHA1, SHA256]', ); +const errContentSHA256Mismatch = errorInstances.XAmzContentSHA256Mismatch; + +const errMissingContentSHA256 = errorInstances.InvalidRequest.customizeDescription( + 'Missing required header for this request: x-amz-content-sha256', +); + +const errInvalidContentSHA256 = errorInstances.InvalidArgument.customizeDescription( + 'x-amz-content-sha256 must be UNSIGNED-PAYLOAD, STREAMING-UNSIGNED-PAYLOAD-TRAILER, ' + + 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD, STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER, ' + + 'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD, STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER ' + + 'or a valid sha256 value.', +); + +const sha256HexRegex = /^[0-9a-f]{64}$/i; + // Methods that validate BOTH Content-MD5 and x-amz-checksum-* against the // buffered request body. For these, x-amz-checksum-* is the body digest. const checksumedMethods = Object.freeze({ @@ -98,6 +114,9 @@ const ChecksumError = Object.freeze({ MPUTypeWithoutAlgo: 'MPUTypeWithoutAlgo', MPUInvalidCombination: 'MPUInvalidCombination', CopyChecksumAlgoNotSupported: 'CopyChecksumAlgoNotSupported', + ContentSHA256Missing: 'ContentSHA256Missing', + ContentSHA256Invalid: 'ContentSHA256Invalid', + ContentSHA256Mismatch: 'ContentSHA256Mismatch', }); const base64Regex = /^[A-Za-z0-9+/]*={0,2}$/; @@ -368,6 +387,90 @@ function validateContentMd5(headers, body) { return null; } +// Classification of an x-amz-content-sha256 header value, returned by +// parseContentSHA256. +const ContentSHA256Type = Object.freeze({ + Skip: 0, // not SigV4 header auth; do not enforce + Absent: 1, // SigV4 header auth but header missing + Unsigned: 2, // UNSIGNED-PAYLOAD + Streaming: 3, // a STREAMING-* token + HexSHA256: 4, // a hex sha256 of the payload + Invalid: 5, // anything else (malformed) +}); + +/** + * parseContentSHA256 - Classify x-amz-content-sha256 (the SigV4 payload hash) + * for a request. The header is only meaningful for SigV4 header-authenticated + * requests, so anything else (SigV2, presigned/query, anonymous) returns + * { type: ContentSHA256Type.Skip } and is not enforced. Otherwise the header + * value is classified. + * + * @param {object} headers - http request headers + * @return {object} { type, value } where type is a ContentSHA256Type and value + * is the raw header value (null when not sent). Streaming results also carry + * a { supported } boolean. + */ +function parseContentSHA256(headers) { + const value = headers['x-amz-content-sha256'] ?? null; + const authHeader = headers.authorization; + if (typeof authHeader !== 'string' || !authHeader.startsWith('AWS4')) { + return { type: ContentSHA256Type.Skip, value }; + } + if (value === null) { + return { type: ContentSHA256Type.Absent, value }; + } + if (value === 'UNSIGNED-PAYLOAD') { + return { type: ContentSHA256Type.Unsigned, value }; + } + if (supportedSignatureChecksums.has(value)) { + return { type: ContentSHA256Type.Streaming, value, supported: true }; + } + if (unsupportedSignatureChecksums.has(value)) { + return { type: ContentSHA256Type.Streaming, value, supported: false }; + } + if (sha256HexRegex.test(value)) { + return { type: ContentSHA256Type.HexSHA256, value }; + } + return { type: ContentSHA256Type.Invalid, value }; +} + +/** + * validateXAmzContentSHA256 - Validate the SHA256 of a SigV4 request. + * + * @param {object} headers - http request headers + * @param {Buffer} body - buffered http request body + * @return {object|null} - { error: ChecksumError } on missing/invalid/mismatch, + * else null + */ +function validateXAmzContentSHA256(headers, body) { + const parsed = parseContentSHA256(headers); + switch (parsed.type) { + case ContentSHA256Type.Absent: // required for SigV4 header auth + return { error: ChecksumError.ContentSHA256Missing }; + case ContentSHA256Type.Invalid: + return { error: ChecksumError.ContentSHA256Invalid, details: { value: parsed.value } }; + case ContentSHA256Type.HexSHA256: { + const computed = crypto.createHash('sha256').update(body).digest('hex'); + if (computed !== parsed.value.toLowerCase()) { + return { + error: ChecksumError.ContentSHA256Mismatch, + details: { calculated: computed, expected: parsed.value }, + }; + } + return null; + } + // Skip (non-SigV4-header auth), Unsigned and Streaming are not a literal + // payload hash, so there is nothing to validate here. + case ContentSHA256Type.Skip: + return null; + case ContentSHA256Type.Unsigned: + return null; + case ContentSHA256Type.Streaming: + return null; + } + return null; +} + /** * validateChecksumsNoChunking - Validate the checksums of a request. * @param {object} headers - http headers @@ -448,6 +551,12 @@ function arsenalErrorFromChecksumError(err) { ); case ChecksumError.CopyChecksumAlgoNotSupported: return errCopyChecksumAlgoNotSupported; + case ChecksumError.ContentSHA256Missing: + return errMissingContentSHA256; + case ChecksumError.ContentSHA256Invalid: + return errInvalidContentSHA256; + case ChecksumError.ContentSHA256Mismatch: + return errContentSHA256Mismatch; default: return ArsenalErrors.BadDigest; } @@ -547,6 +656,15 @@ function md5OnlyValidationFunc(request, body, log) { * @return {object} - error */ async function validateMethodChecksumNoChunking(request, body, log) { + const contentSHA256Err = validateXAmzContentSHA256(request.headers, body); + if (contentSHA256Err) { + log.debug('failed x-amz-content-sha256 validation', { + method: request.apiMethod, + ...contentSHA256Err.details, + }); + return arsenalErrorFromChecksumError(contentSHA256Err); + } + if (config.integrityChecks[request.apiMethod] === false) { return null; } @@ -738,6 +856,10 @@ module.exports = { ChecksumError, defaultChecksumData, validateChecksumsNoChunking, + ContentSHA256Type, + parseContentSHA256, + errInvalidContentSHA256, + validateXAmzContentSHA256, validateMethodChecksumNoChunking, getChecksumDataFromHeaders, arsenalErrorFromChecksumError, diff --git a/tests/functional/raw-node/test/xAmzChecksum.js b/tests/functional/raw-node/test/xAmzChecksum.js index 14df75dabc..57a0d59e2a 100644 --- a/tests/functional/raw-node/test/xAmzChecksum.js +++ b/tests/functional/raw-node/test/xAmzChecksum.js @@ -1,9 +1,12 @@ const assert = require('assert'); +const crypto = require('crypto'); const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); const bucket = 'xxx'; const objectKey = 'key'; const objData = Buffer.alloc(1, 'a'); +// SigV4 requires x-amz-content-sha256 to be the hex-encoded sha256 of the body. +const objDataSha256Hex = crypto.createHash('sha256').update(objData).digest('hex'); const authCredentials = { accessKey: 'accessKey1', @@ -147,7 +150,7 @@ describe('Test x-amz-checksums', () => { { method: method.HTTPMethod, headers: { - 'x-amz-content-sha256': 'ypeBEsobvcr6wjGzmiPcTaeG7/gUfE5yuYB3ha/uSLs=', + 'x-amz-content-sha256': objDataSha256Hex, 'content-length': objData.length, ...headers, }, @@ -283,7 +286,7 @@ describe('Test x-amz-checksums', () => { { method: method.HTTPMethod, headers: { - 'x-amz-content-sha256': 'ypeBEsobvcr6wjGzmiPcTaeG7/gUfE5yuYB3ha/uSLs=', + 'x-amz-content-sha256': objDataSha256Hex, 'content-length': objData.length, 'x-amz-sdk-checksum-algorithm': algo.name, [`x-amz-checksum-${algo.name.toLowerCase()}`]: algo.objDataDigest, diff --git a/tests/functional/raw-node/utils/makeRequest.js b/tests/functional/raw-node/utils/makeRequest.js index 6759160c2e..bfad8e774b 100644 --- a/tests/functional/raw-node/utils/makeRequest.js +++ b/tests/functional/raw-node/utils/makeRequest.js @@ -139,8 +139,10 @@ function makeRequest(params, callback) { // decode path because signing code re-encodes it req.path = _decodeURI(encodedPath); if (authCredentials && !params.GCP) { + // Pass an explicit payload (never undefined) so generateV4Headers signs + // the real body for POST instead of falling back to the querystring. auth.client.generateV4Headers(req, queryObj || '', - authCredentials.accessKey, authCredentials.secretKey, 's3', undefined, undefined, requestBody); + authCredentials.accessKey, authCredentials.secretKey, 's3', undefined, undefined, requestBody || ''); } // restore original URL-encoded path req.path = savedPath; diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index 1e62ad0e41..0dd415f30f 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -5,6 +5,9 @@ const { DummyRequestLogger } = require('../../../helpers'); const { validateChecksumsNoChunking, ChecksumError, + ContentSHA256Type, + parseContentSHA256, + validateXAmzContentSHA256, validateMethodChecksumNoChunking, checksumedMethods, getChecksumDataFromHeaders, @@ -1298,3 +1301,162 @@ describe('getCopyObjectChecksumAlgorithm', () => { } }); }); + +const sigV4Auth = 'AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/s3/aws4_request, ' + + 'SignedHeaders=host, Signature=abc'; + +describe('parseContentSHA256', () => { + // build SigV4 header-auth headers carrying the given x-amz-content-sha256 + const v4 = value => ({ authorization: sigV4Auth, 'x-amz-content-sha256': value }); + + it('should return Skip for non-SigV4 header auth, capturing the value', () => { + assert.deepStrictEqual( + parseContentSHA256({ authorization: 'AWS AKID:sig', 'x-amz-content-sha256': 'abc' }), + { type: ContentSHA256Type.Skip, value: 'abc' }); + }); + + it('should return Skip with null value when there is no auth header', () => { + assert.deepStrictEqual(parseContentSHA256({}), { type: ContentSHA256Type.Skip, value: null }); + }); + + it('should return Absent when SigV4 header auth but the header is missing', () => { + assert.deepStrictEqual(parseContentSHA256({ authorization: sigV4Auth }), + { type: ContentSHA256Type.Absent, value: null }); + }); + + it('should return Unsigned for UNSIGNED-PAYLOAD', () => { + assert.deepStrictEqual(parseContentSHA256(v4('UNSIGNED-PAYLOAD')), + { type: ContentSHA256Type.Unsigned, value: 'UNSIGNED-PAYLOAD' }); + }); + + it('should return Streaming (supported) for a supported streaming token', () => { + const tok = 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'; + assert.deepStrictEqual(parseContentSHA256(v4(tok)), + { type: ContentSHA256Type.Streaming, value: tok, supported: true }); + }); + + it('should return Streaming (not supported) for an unsupported streaming token', () => { + const tok = 'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD'; + assert.deepStrictEqual(parseContentSHA256(v4(tok)), + { type: ContentSHA256Type.Streaming, value: tok, supported: false }); + }); + + it('should return HexSHA256 for a hex sha256, preserving the raw value', () => { + const hex = 'a'.repeat(64); + assert.deepStrictEqual(parseContentSHA256(v4(hex)), + { type: ContentSHA256Type.HexSHA256, value: hex }); + }); + + it('should return HexSHA256 for an uppercase hex sha256', () => { + const hex = 'A'.repeat(64); + assert.deepStrictEqual(parseContentSHA256(v4(hex)), + { type: ContentSHA256Type.HexSHA256, value: hex }); + }); + + it('should return Invalid for a malformed value', () => { + assert.deepStrictEqual(parseContentSHA256(v4('xxx')), + { type: ContentSHA256Type.Invalid, value: 'xxx' }); + }); +}); + +describe('validateXAmzContentSHA256', () => { + const body = 'Hello, World!'; + const correctHex = crypto.createHash('sha256').update(body).digest('hex'); + const wrongHex = crypto.createHash('sha256').update('other').digest('hex'); + + it('should return null when the hash matches the body', () => { + assert.ifError(validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': correctHex }, body)); + }); + + it('should return null for an uppercase hash that matches (case-insensitive)', () => { + assert.ifError(validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': correctHex.toUpperCase() }, body)); + }); + + it('should return ContentSHA256Mismatch with calculated/expected details on mismatch', () => { + const result = validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, body); + assert.strictEqual(result.error, ChecksumError.ContentSHA256Mismatch); + assert.strictEqual(result.details.expected, wrongHex); + assert.strictEqual(result.details.calculated, correctHex); + }); + + it('should return ContentSHA256Missing when the header is absent', () => { + assert.deepStrictEqual(validateXAmzContentSHA256({ authorization: sigV4Auth }, body), + { error: ChecksumError.ContentSHA256Missing }); + }); + + it('should return ContentSHA256Invalid for a malformed value', () => { + const result = validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': 'xxx' }, body); + assert.strictEqual(result.error, ChecksumError.ContentSHA256Invalid); + assert.strictEqual(result.details.value, 'xxx'); + }); + + it('should return null for UNSIGNED-PAYLOAD', () => { + assert.ifError(validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }, body)); + }); + + it('should return null for a streaming token', () => { + assert.ifError(validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }, body)); + }); + + it('should return null (skip) for non-SigV4 auth even with a wrong hash', () => { + assert.ifError(validateXAmzContentSHA256( + { authorization: 'AWS AKID:sig', 'x-amz-content-sha256': wrongHex }, body)); + }); + + describe('mapped through arsenalErrorFromChecksumError', () => { + it('should map mismatch to XAmzContentSHA256Mismatch (400)', () => { + const result = validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, body); + const err = arsenalErrorFromChecksumError(result); + assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); + assert.strictEqual(err.code, 400); + }); + + it('should map invalid to InvalidArgument (400)', () => { + const result = validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': 'xxx' }, body); + const err = arsenalErrorFromChecksumError(result); + assert.strictEqual(err.message, 'InvalidArgument'); + assert.strictEqual(err.code, 400); + }); + + it('should map missing to InvalidRequest (400)', () => { + const result = validateXAmzContentSHA256({ authorization: sigV4Auth }, body); + const err = arsenalErrorFromChecksumError(result); + assert.strictEqual(err.message, 'InvalidRequest'); + assert.strictEqual(err.code, 400); + }); + }); +}); + +describe('validateMethodChecksumNoChunking x-amz-content-sha256', () => { + const body = 'Hello, World!'; + const correctHex = crypto.createHash('sha256').update(body).digest('hex'); + const correctMd5 = crypto.createHash('md5').update(body).digest('base64'); + + it('should reject a wrong x-amz-content-sha256 with XAmzContentSHA256Mismatch', async () => { + const wrongHex = crypto.createHash('sha256').update('other').digest('hex'); + const request = { + apiMethod: 'bucketPutCors', + headers: { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, + }; + const result = await validateMethodChecksumNoChunking(request, body, new DummyRequestLogger()); + assert.strictEqual(result.message, 'XAmzContentSHA256Mismatch'); + assert.strictEqual(result.code, 400); + }); + + it('should accept a matching x-amz-content-sha256', async () => { + const request = { + apiMethod: 'bucketPutCors', + headers: { authorization: sigV4Auth, 'x-amz-content-sha256': correctHex, 'content-md5': correctMd5 }, + }; + const result = await validateMethodChecksumNoChunking(request, body, new DummyRequestLogger()); + assert.ifError(result); + }); +}); From 72fa6112a2e41b6af48460ea74ffcade1f1d5e8c Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Fri, 26 Jun 2026 00:52:32 +0200 Subject: [PATCH 2/8] CLDSRV-932: replace validateChecksumHeaders with validatePayloadProtocol --- .../object/validateChecksumHeaders.js | 33 -------- .../object/validatePayloadProtocol.js | 32 ++++++++ lib/api/objectPut.js | 8 +- lib/api/objectPutPart.js | 8 +- .../api/apiUtils/validateChecksumHeaders.js | 75 ------------------- .../api/apiUtils/validatePayloadProtocol.js | 45 +++++++++++ 6 files changed, 85 insertions(+), 116 deletions(-) delete mode 100644 lib/api/apiUtils/object/validateChecksumHeaders.js create mode 100644 lib/api/apiUtils/object/validatePayloadProtocol.js delete mode 100644 tests/unit/api/apiUtils/validateChecksumHeaders.js create mode 100644 tests/unit/api/apiUtils/validatePayloadProtocol.js diff --git a/lib/api/apiUtils/object/validateChecksumHeaders.js b/lib/api/apiUtils/object/validateChecksumHeaders.js deleted file mode 100644 index d2a50395a3..0000000000 --- a/lib/api/apiUtils/object/validateChecksumHeaders.js +++ /dev/null @@ -1,33 +0,0 @@ -const { errorInstances } = require('arsenal'); - -const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require('../../../../constants'); - -function validateChecksumHeaders(headers) { - // If the x-amz-trailer header is present the request is using one of the - // trailing checksum algorithms, which are not supported. - if (headers['x-amz-trailer'] !== undefined && - headers['x-amz-content-sha256'] !== 'STREAMING-UNSIGNED-PAYLOAD-TRAILER') { - return errorInstances.BadRequest.customizeDescription('signed trailing checksum is not supported'); - } - - const signatureChecksum = headers['x-amz-content-sha256']; - if (signatureChecksum === undefined) { - return null; - } - - if (supportedSignatureChecksums.has(signatureChecksum)) { - return null; - } - - // If the value is not one of the possible checksum algorithms - // the only other valid value is the actual sha256 checksum of the payload. - // Do a simple sanity check of the length to guard against future algos. - // If the value is an unknown algo, then it will fail checksum validation. - if (!unsupportedSignatureChecksums.has(signatureChecksum) && signatureChecksum.length === 64) { - return null; - } - - return errorInstances.BadRequest.customizeDescription('unsupported checksum algorithm'); -} - -module.exports = validateChecksumHeaders; diff --git a/lib/api/apiUtils/object/validatePayloadProtocol.js b/lib/api/apiUtils/object/validatePayloadProtocol.js new file mode 100644 index 0000000000..19134b6278 --- /dev/null +++ b/lib/api/apiUtils/object/validatePayloadProtocol.js @@ -0,0 +1,32 @@ +const { errorInstances } = require('arsenal'); + +const { parseContentSHA256, ContentSHA256Type, errInvalidContentSHA256 } = require('../integrity/validateChecksums'); + +/** + * Validate the SigV4 payload protocol selected by x-amz-content-sha256. + * + * @param {object} headers - http request headers + * @return {ArsenalError|null} - error if the protocol is unsupported/malformed, else null + */ +function validatePayloadProtocol(headers) { + const parsed = parseContentSHA256(headers); + switch (parsed.type) { + case ContentSHA256Type.Skip: // not SigV4 header auth; header not meaningful + return null; + case ContentSHA256Type.Absent: + return null; + case ContentSHA256Type.Unsigned: + return null; + case ContentSHA256Type.HexSHA256: + return null; + case ContentSHA256Type.Streaming: + return parsed.supported + ? null + : errorInstances.BadRequest.customizeDescription(`${parsed.value} is not supported`); + case ContentSHA256Type.Invalid: + return errInvalidContentSHA256; + } + return null; +} + +module.exports = validatePayloadProtocol; diff --git a/lib/api/objectPut.js b/lib/api/objectPut.js index 590fd7f1e5..8bdce061ed 100644 --- a/lib/api/objectPut.js +++ b/lib/api/objectPut.js @@ -18,7 +18,7 @@ const monitoring = require('../utilities/monitoringHandler'); const { validatePutVersionId } = require('./apiUtils/object/coldStorage'); const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); -const validateChecksumHeaders = require('./apiUtils/object/validateChecksumHeaders'); +const validatePayloadProtocol = require('./apiUtils/object/validatePayloadProtocol'); const writeContinue = require('../utilities/writeContinue'); const { overheadField } = require('../../constants'); @@ -119,9 +119,9 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) { )); } - const checksumHeaderErr = validateChecksumHeaders(headers); - if (checksumHeaderErr) { - return callback(checksumHeaderErr); + const payloadProtocolErr = validatePayloadProtocol(headers); + if (payloadProtocolErr) { + return callback(payloadProtocolErr); } log.trace('owner canonicalID to send to data', { canonicalID }); diff --git a/lib/api/objectPutPart.js b/lib/api/objectPutPart.js index d8c2d560cb..780c1e9b8e 100644 --- a/lib/api/objectPutPart.js +++ b/lib/api/objectPutPart.js @@ -18,7 +18,7 @@ const { BackendInfo } = models; const writeContinue = require('../utilities/writeContinue'); const { parseObjectEncryptionHeaders } = require('./apiUtils/bucket/bucketEncryption'); -const validateChecksumHeaders = require('./apiUtils/object/validateChecksumHeaders'); +const validatePayloadProtocol = require('./apiUtils/object/validatePayloadProtocol'); const { getChecksumDataFromHeaders, arsenalErrorFromChecksumError } = require('./apiUtils/integrity/validateChecksums'); const { validateQuotas } = require('./apiUtils/quotas/quotaUtils'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); @@ -76,9 +76,9 @@ function objectPutPart(authInfo, request, streamingV4Params, log, cb) { return cb(errors.EntityTooLarge); } - const checksumHeaderErr = validateChecksumHeaders(request.headers); - if (checksumHeaderErr) { - return cb(checksumHeaderErr); + const payloadProtocolErr = validatePayloadProtocol(request.headers); + if (payloadProtocolErr) { + return cb(payloadProtocolErr); } // Note: Part sizes cannot be less than 5MB in size except for the last. diff --git a/tests/unit/api/apiUtils/validateChecksumHeaders.js b/tests/unit/api/apiUtils/validateChecksumHeaders.js deleted file mode 100644 index 6b1f7dbbf6..0000000000 --- a/tests/unit/api/apiUtils/validateChecksumHeaders.js +++ /dev/null @@ -1,75 +0,0 @@ -const assert = require('assert'); - -const validateChecksumHeaders = require('../../../../lib/api/apiUtils/object/validateChecksumHeaders'); -const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require('../../../../constants'); - -const passingCases = [ - { - description: 'should return null if no checksum headers are present', - headers: {}, - }, - { - description: 'should return null if UNSIGNED-PAYLOAD is used', - headers: { - 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - }, - }, - { - description: 'should return null if a sha256 checksum is used', - headers: { - 'x-amz-content-sha256': 'thisIs64CharactersLongAndThatsAllWeCheckFor1234567890abcdefghijk', - }, - }, -]; - -supportedSignatureChecksums.forEach(checksum => { - passingCases.push({ - description: `should return null if ${checksum} is used`, - headers: { - 'x-amz-content-sha256': checksum, - }, - }); -}); - -const failingCases = [ - { - description: 'should return BadRequest if a trailing checksum is used', - headers: { - 'x-amz-trailer': 'test', - }, - }, - { - description: 'should return BadRequest if an unknown algo is used', - headers: { - 'x-amz-content-sha256': 'UNSUPPORTED-CHECKSUM', - }, - }, -]; - -unsupportedSignatureChecksums.forEach(checksum => { - failingCases.push({ - description: `should return BadRequest if ${checksum} is used`, - headers: { - 'x-amz-content-sha256': checksum, - }, - }); -}); - - -describe('validateChecksumHeaders', () => { - passingCases.forEach(testCase => { - it(testCase.description, () => { - const result = validateChecksumHeaders(testCase.headers); - assert.ifError(result); - }); - }); - - failingCases.forEach(testCase => { - it(testCase.description, () => { - const result = validateChecksumHeaders(testCase.headers); - assert(result instanceof Error, 'Expected an error to be returned'); - assert.strictEqual(result.is.BadRequest, true); - assert.strictEqual(result.code, 400); - }); - }); -}); diff --git a/tests/unit/api/apiUtils/validatePayloadProtocol.js b/tests/unit/api/apiUtils/validatePayloadProtocol.js new file mode 100644 index 0000000000..7e72c88dcd --- /dev/null +++ b/tests/unit/api/apiUtils/validatePayloadProtocol.js @@ -0,0 +1,45 @@ +const assert = require('assert'); + +const validatePayloadProtocol = require('../../../../lib/api/apiUtils/object/validatePayloadProtocol'); +const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require('../../../../constants'); + +// validatePayloadProtocol only validates x-amz-content-sha256 for SigV4 +// header-authenticated requests (Authorization: "AWS4-..."). +const sigV4 = 'AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/s3/aws4_request, ' + + 'SignedHeaders=host, Signature=abc'; +const sigV2 = 'AWS AKID:signature'; +const validSha256Hex = 'a'.repeat(64); + +// build SigV4 header-auth headers, merging any extras +const v4 = extra => Object.assign({ authorization: sigV4 }, extra); + +describe('validatePayloadProtocol', () => { + it('should return null for a valid hex sha256', () => + assert.ifError(validatePayloadProtocol(v4({ 'x-amz-content-sha256': validSha256Hex })))); + + supportedSignatureChecksums.forEach(protocol => { + it(`should return null for supported protocol ${protocol}`, () => + assert.ifError(validatePayloadProtocol(v4({ 'x-amz-content-sha256': protocol })))); + }); + + it('should return null for non-SigV4 header auth, even with an invalid value', () => + assert.ifError(validatePayloadProtocol({ authorization: sigV2, 'x-amz-content-sha256': 'BAD' }))); + + unsupportedSignatureChecksums.forEach(protocol => { + it(`should return BadRequest for unsupported protocol ${protocol}`, () => { + const err = validatePayloadProtocol(v4({ 'x-amz-content-sha256': protocol })); + assert(err instanceof Error, 'expected an error'); + assert.strictEqual(err.message, 'BadRequest'); + assert.strictEqual(err.code, 400); + assert.match(err.description, /is not supported/); + }); + }); + + it('should return InvalidArgument for a non-hex x-amz-content-sha256', () => { + const err = validatePayloadProtocol(v4({ 'x-amz-content-sha256': 'BAD' })); + assert(err instanceof Error, 'expected an error'); + assert.strictEqual(err.message, 'InvalidArgument'); + assert.strictEqual(err.code, 400); + assert.match(err.description, /x-amz-content-sha256 must be/); + }); +}); From 29cff04f4657ab736228c237f18568ce1cd409be Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Fri, 26 Jun 2026 04:33:37 +0200 Subject: [PATCH 3/8] CLDSRV-932: validate x-amz-content-sha256 in streaming methods (PutObject, UploadPart) --- lib/api/apiUtils/object/prepareStream.js | 21 +++- lib/api/apiUtils/object/storeObject.js | 18 ++- .../streamingV4/ContentSHA256Transform.js | 40 +++++++ .../unit/api/apiUtils/object/prepareStream.js | 75 ++++++++++++ tests/unit/api/apiUtils/object/storeObject.js | 64 +++++++++++ tests/unit/auth/ContentSHA256Transform.js | 107 ++++++++++++++++++ 6 files changed, 320 insertions(+), 5 deletions(-) create mode 100644 lib/auth/streamingV4/ContentSHA256Transform.js create mode 100644 tests/unit/auth/ContentSHA256Transform.js diff --git a/lib/api/apiUtils/object/prepareStream.js b/lib/api/apiUtils/object/prepareStream.js index c267936b83..66895fba6f 100644 --- a/lib/api/apiUtils/object/prepareStream.js +++ b/lib/api/apiUtils/object/prepareStream.js @@ -1,6 +1,8 @@ const V4Transform = require('../../../auth/streamingV4/V4Transform'); const TrailingChecksumTransform = require('../../../auth/streamingV4/trailingChecksumTransform'); const ChecksumTransform = require('../../../auth/streamingV4/ChecksumTransform'); +const ContentSHA256Transform = require('../../../auth/streamingV4/ContentSHA256Transform'); +const { parseContentSHA256, ContentSHA256Type } = require('../integrity/validateChecksums'); const { errors, errorInstances, jsutil } = require('arsenal'); const { unsupportedSignatureChecksums } = require('../../../../constants'); @@ -103,20 +105,33 @@ function prepareStream(request, streamingV4Params, checksums, log, errCb) { }; } - const onStreamError = secondary ? jsutil.once(errCb) : errCb; + const parsedContentSHA256 = parseContentSHA256(request.headers); + const shouldValidateContentSHA256 = parsedContentSHA256.type === ContentSHA256Type.HexSHA256; + const onStreamError = (secondary || shouldValidateContentSHA256) ? jsutil.once(errCb) : errCb; + let contentSHA256Stream = null; let secondaryChecksumStream = null; let stream = request; + if (shouldValidateContentSHA256) { + contentSHA256Stream = new ContentSHA256Transform(parsedContentSHA256.value, log); + contentSHA256Stream.on('error', onStreamError); + stream = stream.pipe(contentSHA256Stream); + } if (secondary) { secondaryChecksumStream = new ChecksumTransform( secondary.algorithm, secondary.expected, secondary.isTrailer, log); secondaryChecksumStream.on('error', onStreamError); - stream = request.pipe(secondaryChecksumStream); + stream = stream.pipe(secondaryChecksumStream); } const primaryStream = new ChecksumTransform(primary.algorithm, primary.expected, primary.isTrailer, log); primaryStream.on('error', onStreamError); - return { error: null, stream: stream.pipe(primaryStream), secondaryChecksumStream }; + return { + error: null, + stream: stream.pipe(primaryStream), + secondaryChecksumStream, + contentSHA256Stream, + }; } } } diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js index ae4db49165..06dd669052 100644 --- a/lib/api/apiUtils/object/storeObject.js +++ b/lib/api/apiUtils/object/storeObject.js @@ -100,10 +100,24 @@ function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, }; // stream is always the primary (end of pipe, stored checksum). - // secondaryChecksumStream is upstream and only validated. - const { secondaryChecksumStream } = checksumedStream; + // secondaryChecksumStream and contentSHA256Stream are upstream and + // only validated. + const { secondaryChecksumStream, contentSHA256Stream } = checksumedStream; const doValidate = () => { + // Validate the SigV4 payload hash (x-amz-content-sha256) first. + if (contentSHA256Stream) { + const contentErr = contentSHA256Stream.validateChecksum(); + if (contentErr) { + log.debug('failed x-amz-content-sha256 validation', { error: contentErr }); + return data.batchDelete([dataRetrievalInfo], null, null, log, deleteErr => { + if (deleteErr) { + log.error('dataStore failed to delete old data', { error: deleteErr }); + } + return cbOnce(arsenalErrorFromChecksumError(contentErr)); + }); + } + } // Validate the secondary (checked-only) checksum first. if (secondaryChecksumStream) { const secondaryErr = secondaryChecksumStream.validateChecksum(); diff --git a/lib/auth/streamingV4/ContentSHA256Transform.js b/lib/auth/streamingV4/ContentSHA256Transform.js new file mode 100644 index 0000000000..a7201db46f --- /dev/null +++ b/lib/auth/streamingV4/ContentSHA256Transform.js @@ -0,0 +1,40 @@ +const crypto = require('crypto'); +const { Transform } = require('stream'); +const { ChecksumError } = require('../../api/apiUtils/integrity/validateChecksums'); + +/** + * Computes the sha256 of the streamed body to verify it against a literal + * x-amz-content-sha256 header (the SigV4 payload hash) via validateChecksum(). + */ +class ContentSHA256Transform extends Transform { + constructor(expectedDigest, log) { + super({}); + this.log = log; + this.expectedDigest = (expectedDigest || '').toLowerCase(); + this.hash = crypto.createHash('sha256'); + this.digest = undefined; + } + + validateChecksum() { + if (this.digest !== this.expectedDigest) { + return { + error: ChecksumError.ContentSHA256Mismatch, + details: { calculated: this.digest, expected: this.expectedDigest }, + }; + } + return null; + } + + _transform(chunk, encoding, callback) { + const input = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + this.hash.update(input); + callback(null, input); + } + + _flush(callback) { + this.digest = this.hash.digest('hex'); + callback(); + } +} + +module.exports = ContentSHA256Transform; diff --git a/tests/unit/api/apiUtils/object/prepareStream.js b/tests/unit/api/apiUtils/object/prepareStream.js index d0492615e5..05bcc628c4 100644 --- a/tests/unit/api/apiUtils/object/prepareStream.js +++ b/tests/unit/api/apiUtils/object/prepareStream.js @@ -1,8 +1,10 @@ const assert = require('assert'); +const crypto = require('crypto'); const { errors } = require('arsenal'); const { prepareStream } = require('../../../../../lib/api/apiUtils/object/prepareStream'); const ChecksumTransform = require('../../../../../lib/auth/streamingV4/ChecksumTransform'); +const ContentSHA256Transform = require('../../../../../lib/auth/streamingV4/ContentSHA256Transform'); const { DummyRequestLogger } = require('../../../helpers'); const DummyRequest = require('../../../DummyRequest'); const { defaultChecksumData } = require('../../../../../lib/api/apiUtils/integrity/validateChecksums'); @@ -10,6 +12,13 @@ const { defaultChecksumData } = require('../../../../../lib/api/apiUtils/integri const log = new DummyRequestLogger(); const defaultChecksums = { primary: defaultChecksumData, secondary: null }; +// A literal payload hash is only verified for SigV4 header-authenticated +// requests, so these tests carry an AWS4 Authorization header. +const sigV4Auth = 'AWS4-HMAC-SHA256 Credential=AK/20210101/us-east-1/s3/aws4_request, ' + + 'SignedHeaders=host, Signature=abc'; +const bodyData = 'the streamed body'; +const bodyHex = crypto.createHash('sha256').update(Buffer.from(bodyData)).digest('hex'); + function makeRequest(headers, body) { return new DummyRequest({ headers }, body != null ? Buffer.from(body) : undefined); } @@ -256,4 +265,70 @@ describe('prepareStream', () => { result.stream.emit('error', errors.InternalError); }); }); + + describe('default (literal sha256 payload hash)', () => { + it('should return a ContentSHA256Transform for a SigV4 header-auth request with a hex value', () => { + const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': bodyHex }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert.strictEqual(result.error, null); + assert(result.stream instanceof ChecksumTransform); + assert(result.contentSHA256Stream instanceof ContentSHA256Transform); + }); + + it('should return null contentSHA256Stream for UNSIGNED-PAYLOAD', () => { + const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert.strictEqual(result.contentSHA256Stream, null); + }); + + it('should return null contentSHA256Stream for non-SigV4 (SigV2) auth even with a hex value', () => { + const request = makeRequest({ authorization: 'AWS AKID:sig', 'x-amz-content-sha256': bodyHex }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert.strictEqual(result.contentSHA256Stream, null); + }); + + it('should return null contentSHA256Stream when there is no authorization header', () => { + const request = makeRequest({ 'x-amz-content-sha256': bodyHex }); + const result = prepareStream(request, null, defaultChecksums, log, () => {}); + assert.strictEqual(result.contentSHA256Stream, null); + }); + + it('should accumulate the body sha256 so validateChecksum passes when the hex matches', done => { + const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': bodyHex }, bodyData); + const result = prepareStream(request, null, defaultChecksums, log, done); + result.stream.resume(); + result.stream.on('finish', () => { + assert.strictEqual(result.contentSHA256Stream.validateChecksum(), null); + done(); + }); + result.stream.on('error', done); + }); + + it('should pipe contentSHA256Stream upstream of a secondary checksum stream', done => { + const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': bodyHex }, bodyData); + const checksums = { + primary: { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }, + secondary: { algorithm: 'crc32', isTrailer: false, expected: undefined }, + }; + const result = prepareStream(request, null, checksums, log, done); + assert(result.contentSHA256Stream instanceof ContentSHA256Transform); + assert(result.secondaryChecksumStream instanceof ChecksumTransform); + result.stream.resume(); + result.stream.on('finish', () => { + // the content stream saw the full body even with a secondary present + assert.strictEqual(result.contentSHA256Stream.validateChecksum(), null); + done(); + }); + result.stream.on('error', done); + }); + + it('should invoke errCb only once when multiple streams error', () => { + const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': bodyHex }); + let count = 0; + const result = prepareStream(request, null, defaultChecksums, log, () => { count += 1; }); + result.contentSHA256Stream.emit('error', errors.InternalError); + result.stream.emit('error', errors.InternalError); + assert.strictEqual(count, 1); + }); + }); }); diff --git a/tests/unit/api/apiUtils/object/storeObject.js b/tests/unit/api/apiUtils/object/storeObject.js index 0ddd158a3c..ffef1fd717 100644 --- a/tests/unit/api/apiUtils/object/storeObject.js +++ b/tests/unit/api/apiUtils/object/storeObject.js @@ -1,4 +1,5 @@ const assert = require('assert'); +const crypto = require('crypto'); const sinon = require('sinon'); const { errors } = require('arsenal'); @@ -13,6 +14,13 @@ const defaultChecksums = { primary: defaultChecksumData, secondary: null }; const fakeDataRetrievalInfo = { key: 'test-key', dataStoreName: 'mem' }; +// A literal payload hash is only verified for SigV4 header-authenticated +// requests, so these tests carry an AWS4 Authorization header. +const sigV4Auth = 'AWS4-HMAC-SHA256 Credential=AK/20210101/us-east-1/s3/aws4_request, ' + + 'SignedHeaders=host, Signature=abc'; +const helloWorldHex = crypto.createHash('sha256').update(Buffer.from('hello world')).digest('hex'); +const wrongHex = 'a'.repeat(64); + function makeStream(headers = {}, body = '') { return new DummyRequest({ headers }, body ? Buffer.from(body) : undefined); } @@ -273,6 +281,62 @@ describe('dataStore', () => { }); }); + describe('x-amz-content-sha256 body validation', () => { + it('should call cb with XAmzContentSHA256Mismatch and delete stored data when the hash does not match', + done => { + batchDeleteSucceeds(); + putSucceeds(); + const request = makeStream( + { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, 'hello world'); + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { + assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); + assert(batchDeleteStub.calledOnce); + done(); + }); + }); + + it('should not delete stored data when the hash matches the body', done => { + putSucceeds(); + const request = makeStream( + { authorization: sigV4Auth, 'x-amz-content-sha256': helloWorldHex }, 'hello world'); + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { + assert.strictEqual(err, null); + assert(batchDeleteStub.notCalled); + done(); + }); + }); + + it('should validate x-amz-content-sha256 before the secondary checksum', done => { + batchDeleteSucceeds(); + putSucceeds(); + const request = makeStream( + { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, 'hello world'); + // The secondary checksum also mismatches, but the content-sha256 + // error is checked first and takes precedence. + const checksums = { + primary: { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }, + secondary: { algorithm: 'crc32', isTrailer: false, expected: 'AAAAAA==' }, + }; + dataStore({}, null, request, 0, null, {}, checksums, log, err => { + assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); + assert(batchDeleteStub.calledOnce); + done(); + }); + }); + + it('should call cb with XAmzContentSHA256Mismatch when the hash mismatches and batchDelete also fails', + done => { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); + putSucceeds(); + const request = makeStream( + { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, 'hello world'); + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { + assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); + done(); + }); + }); + }); + describe('dual-checksum behaviour', () => { it('should return client-facing checksum from secondary and storageChecksum from primary', done => { putSucceeds(); diff --git a/tests/unit/auth/ContentSHA256Transform.js b/tests/unit/auth/ContentSHA256Transform.js new file mode 100644 index 0000000000..a4f17f18e0 --- /dev/null +++ b/tests/unit/auth/ContentSHA256Transform.js @@ -0,0 +1,107 @@ +const assert = require('assert'); +const crypto = require('crypto'); + +const { ChecksumError } = require('../../../lib/api/apiUtils/integrity/validateChecksums'); +const ContentSHA256Transform = require('../../../lib/auth/streamingV4/ContentSHA256Transform'); +const { DummyRequestLogger } = require('../helpers'); + +const log = new DummyRequestLogger(); +const testData = Buffer.from('hello world'); +const testDigest = crypto.createHash('sha256').update(testData).digest('hex'); +const emptyDigest = crypto.createHash('sha256').update(Buffer.alloc(0)).digest('hex'); +const wrongDigest = 'a'.repeat(64); + +// Pipe chunks through the transform, collect output, resolve on end. +function runTransform(stream, chunks) { + return new Promise((resolve, reject) => { + const output = []; + stream.on('data', chunk => output.push(chunk)); + stream.on('end', () => resolve(Buffer.concat(output))); + stream.on('error', reject); + for (const chunk of chunks) { + stream.write(chunk); + } + stream.end(); + }); +} + +// Drain the transform without collecting output, resolve on finish. +function drainTransform(stream, chunks) { + return new Promise((resolve, reject) => { + stream.resume(); + stream.on('finish', resolve); + stream.on('error', reject); + for (const chunk of chunks) { + stream.write(chunk); + } + stream.end(); + }); +} + +describe('ContentSHA256Transform', () => { + describe('pass-through and digest', () => { + it('should pass data through unchanged', async () => { + const stream = new ContentSHA256Transform(testDigest, log); + const output = await runTransform(stream, [testData]); + assert.deepStrictEqual(output, testData); + }); + + it('should compute the sha256 hex digest after the stream ends', async () => { + const stream = new ContentSHA256Transform(testDigest, log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.digest, testDigest); + }); + + it('should compute the same digest for multi-chunk input', async () => { + const half = Math.floor(testData.length / 2); + const stream = new ContentSHA256Transform(testDigest, log); + await drainTransform(stream, [testData.subarray(0, half), testData.subarray(half)]); + assert.strictEqual(stream.digest, testDigest); + }); + + it('should handle Buffer and string chunks equally', async () => { + const streamBuf = new ContentSHA256Transform(testDigest, log); + const streamStr = new ContentSHA256Transform(testDigest, log); + await drainTransform(streamBuf, [testData]); + await drainTransform(streamStr, [testData.toString()]); + assert.strictEqual(streamBuf.digest, streamStr.digest); + }); + + it('should compute the sha256 of an empty body', async () => { + const stream = new ContentSHA256Transform(emptyDigest, log); + await drainTransform(stream, []); + assert.strictEqual(stream.digest, emptyDigest); + }); + }); + + describe('validateChecksum', () => { + it('should return null when the expected digest matches the body', async () => { + const stream = new ContentSHA256Transform(testDigest, log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.validateChecksum(), null); + }); + + it('should normalize an uppercase expected digest before comparing', async () => { + const stream = new ContentSHA256Transform(testDigest.toUpperCase(), log); + await drainTransform(stream, [testData]); + assert.strictEqual(stream.validateChecksum(), null); + }); + + it('should return ContentSHA256Mismatch with calculated/expected details on mismatch', async () => { + const stream = new ContentSHA256Transform(wrongDigest, log); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.ContentSHA256Mismatch); + assert.strictEqual(result.details.calculated, testDigest); + assert.strictEqual(result.details.expected, wrongDigest); + }); + + it('should return ContentSHA256Mismatch for an empty body when a non-empty digest is expected', async () => { + const stream = new ContentSHA256Transform(testDigest, log); + await drainTransform(stream, []); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.ContentSHA256Mismatch); + assert.strictEqual(result.details.calculated, emptyDigest); + }); + }); +}); From 2c5e09cdcfe8f2aabb837d023cc5eb97b04fbd83 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Fri, 26 Jun 2026 07:22:18 +0200 Subject: [PATCH 4/8] CLDSRV-932: validate x-amz-content-sha256 functional tests --- .../test/xAmzContentSha256Mismatch.js | 264 ++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 tests/functional/raw-node/test/xAmzContentSha256Mismatch.js diff --git a/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js b/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js new file mode 100644 index 0000000000..1c23df570d --- /dev/null +++ b/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js @@ -0,0 +1,264 @@ +const assert = require('assert'); +const crypto = require('crypto'); +const async = require('async'); + +const { makeS3Request } = require('../utils/makeRequest'); +const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); + +const config = require('../../config.json'); +const { checksumedMethods } = require('../../../../lib/api/apiUtils/integrity/validateChecksums'); + +// Regression test for S3C-10916: "[SigV4] x-amz-content-sha256 value not checked". +// +// CloudServer used to trust x-amz-content-sha256 without recomputing the body's +// SHA256, accepting a request signed with a wrong-but-well-formed hash. The fix +// verifies the header against the body on the buffered and streaming paths. These +// tests assert the AWS-correct 400 XAmzContentSHA256Mismatch, passing against real +// AWS (AWS_ON_AIR=true) and CloudServer with the fix. + +const bucket = 'contentsha256mismatchbucket'; +const objectKey = 'key'; + +const authCredentials = { + accessKey: config.accessKey, + secretKey: config.secretKey, +}; + +const host = process.env.AWS_ON_AIR ? 's3.amazonaws.com' : '127.0.0.1'; +const port = process.env.AWS_ON_AIR ? 80 : 8000; +const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; + +const objData = Buffer.from('the real request body content'); +const realSha256Hex = crypto.createHash('sha256').update(objData).digest('hex'); +const wrongSha256Hex = crypto.createHash('sha256') + .update('completely different content') + .digest('hex'); +const invalidSha256 = 'xxx'; + +// An arbitrary body that is never parsed: the x-amz-content-sha256 check rejects +// the request before the handler reads it, so its content is irrelevant. +const fakeBody = Buffer.from('not parsed before the content-sha256 check'); + +const bufferedEndpoints = { + multiObjectDelete: { method: 'POST', suffix: '?delete' }, + bucketPut: { method: 'PUT', suffix: '' }, // CreateBucket (no subresource) + bucketPutACL: { method: 'PUT', suffix: '?acl' }, + bucketPutCors: { method: 'PUT', suffix: '?cors' }, + bucketPutEncryption: { method: 'PUT', suffix: '?encryption' }, + bucketPutLifecycle: { method: 'PUT', suffix: '?lifecycle' }, + bucketPutLogging: { method: 'PUT', suffix: '?logging' }, + bucketPutNotification: { method: 'PUT', suffix: '?notification' }, + bucketPutPolicy: { method: 'PUT', suffix: '?policy' }, + bucketPutReplication: { method: 'PUT', suffix: '?replication' }, + bucketPutTagging: { method: 'PUT', suffix: '?tagging' }, + bucketPutVersioning: { method: 'PUT', suffix: '?versioning' }, + bucketPutWebsite: { method: 'PUT', suffix: '?website' }, + bucketPutObjectLock: { method: 'PUT', suffix: '?object-lock' }, + objectPutACL: { method: 'PUT', suffix: `/${objectKey}?acl` }, + objectPutLegalHold: { method: 'PUT', suffix: `/${objectKey}?legal-hold` }, + objectPutRetention: { method: 'PUT', suffix: `/${objectKey}?retention` }, + objectPutTagging: { method: 'PUT', suffix: `/${objectKey}?tagging` }, + objectRestore: { method: 'POST', suffix: `/${objectKey}?restore` }, + completeMultipartUpload: { method: 'POST', suffix: `/${objectKey}?uploadId=fakeUploadId` }, +}; + +const scalityExtensionEndpoints = { + bucketUpdateQuota: { method: 'PUT', suffix: '?quota' }, + bucketPutRateLimit: { method: 'PUT', suffix: '?rate-limit' }, +}; + +function doRequest(method, url, headers, body, callback) { + const req = new HttpRequestAuthV4( + url, + Object.assign({ method, headers }, authCredentials), + res => { + let data = ''; + res.on('data', chunk => { data += chunk; }); + res.on('end', () => callback(null, { + statusCode: res.statusCode, + body: data, + headers: res.headers, + })); + }, + ); + req.on('error', callback); + req.write(body); + req.end(); +} + +const doPutRequest = (url, headers, body, callback) => doRequest('PUT', url, headers, body, callback); + +function makeMismatchTests(urlFn, body = objData, correctHex = realSha256Hex) { + it('should reject a body whose x-amz-content-sha256 does not match with 400 XAmzContentSHA256Mismatch', + done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': wrongSha256Hex, + 'content-length': body.length, + }, body, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, + `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match(res.body, /XAmzContentSHA256Mismatch/, + `expected XAmzContentSHA256Mismatch in "${res.body}"`); + done(); + }); + }); + + it('should accept a body whose x-amz-content-sha256 matches', done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': correctHex, + 'content-length': body.length, + }, body, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, + `expected 200, got ${res.statusCode}: ${res.body}`); + done(); + }); + }); + + it('should reject an invalid x-amz-content-sha256 value with 400 InvalidArgument', done => { + doPutRequest(urlFn(), { + 'x-amz-content-sha256': invalidSha256, + 'content-length': body.length, + }, body, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, + `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match(res.body, /InvalidArgument/, + `expected InvalidArgument in "${res.body}"`); + done(); + }); + }); +} + +describe('SigV4 x-amz-content-sha256 body checksum validation (S3C-10916)', () => { + describe('PutObject', () => { + before(done => { + makeS3Request({ method: 'PUT', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => { + makeS3Request({ method: 'DELETE', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + }); + + makeMismatchTests(() => `http://${host}:${port}/${bucket}/${objectKey}`); + }); + + describe('UploadPart', () => { + let uploadId; + + before(done => { + async.series([ + next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), + next => makeS3Request({ + method: 'POST', + authCredentials, + bucket, + objectKey, + queryObj: { uploads: '' }, + }, (err, res) => { + if (err) { return next(err); } + const match = res.body.match(/([^<]+)<\/UploadId>/); + assert(match, `missing UploadId in response: ${res.body}`); + uploadId = match[1]; + return next(); + }), + ], err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + async.series([ + next => makeS3Request({ + method: 'DELETE', + authCredentials, + bucket, + objectKey, + queryObj: { uploadId }, + }, next), + // Delete the object key first (defensive: clears any state left by a previous run). + next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), + next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), + ], err => { + assert.ifError(err); + done(); + }); + }); + + makeMismatchTests(() => + `http://${host}:${port}/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId}`); + }); + + // Non-streaming (buffered) write path: validation happens in + // validateMethodChecksumNoChunking, before the handler reads the body. A + // mismatched hash must therefore be rejected on every concerned endpoint. + describe('buffered endpoints', () => { + before(done => { + makeS3Request({ method: 'PUT', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + + after(done => { + makeS3Request({ method: 'DELETE', authCredentials, bucket }, err => { + assert.ifError(err); + done(); + }); + }); + + // Regression sweep: a wrong-but-well-formed hash is rejected everywhere. + Object.entries(bufferedEndpoints).forEach(([apiMethod, ep]) => { + it(`should return 400 XAmzContentSHA256Mismatch for ${apiMethod}`, done => { + doRequest(ep.method, `http://${host}:${port}/${bucket}${ep.suffix}`, { + 'x-amz-content-sha256': wrongSha256Hex, + 'content-length': fakeBody.length, + }, fakeBody, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, + `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match(res.body, /XAmzContentSHA256Mismatch/, + `expected XAmzContentSHA256Mismatch in "${res.body}"`); + done(); + }); + }); + }); + + // Fails if a new checksumed/buffered method is added without test coverage. + it('should exercise every buffered checksumed method', () => { + const expected = new Set([...Object.keys(checksumedMethods), 'completeMultipartUpload']); + const covered = new Set(Object.keys(bufferedEndpoints)); + expected.forEach(method => + assert(covered.has(method), `missing buffered-endpoint coverage for ${method}`)); + }); + }); + + // Scality-only admin extensions: same choke point, but absent from AWS. + describe('buffered Scality extensions (skipped on AWS)', () => { + Object.entries(scalityExtensionEndpoints).forEach(([apiMethod, ep]) => { + itSkipIfAWS(`should return 400 XAmzContentSHA256Mismatch for ${apiMethod}`, done => { + doRequest(ep.method, `http://${host}:${port}/${bucket}${ep.suffix}`, { + 'x-amz-content-sha256': wrongSha256Hex, + 'content-length': fakeBody.length, + }, fakeBody, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, + `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match(res.body, /XAmzContentSHA256Mismatch/, + `expected XAmzContentSHA256Mismatch in "${res.body}"`); + done(); + }); + }); + }); + }); +}); From 42e0e74094783de10e11bda66f857e5dfcf2ba02 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Thu, 2 Jul 2026 16:32:50 +0200 Subject: [PATCH 5/8] CLDSRV-932: validate x-amz-content-sha256 for zero-byte objects --- .../apiUtils/object/createAndStoreObject.js | 10 +++++-- .../test/xAmzContentSha256Mismatch.js | 27 +++++++++++++++++++ .../apiUtils/integrity/validateChecksums.js | 14 ++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 48d1ec5f6f..48c035a8c0 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -19,6 +19,7 @@ const { defaultChecksumData, getChecksumDataFromHeaders, arsenalErrorFromChecksumError, + validateXAmzContentSHA256, } = require('../integrity/validateChecksums'); const { externalBackends, versioningNotImplBackends } = constants; @@ -28,8 +29,9 @@ const externalVersioningErrorMessage = 'We do not currently support putting ' + /** * Validate and compute the checksum for a zero-size object body. - * Parses the checksum headers, validates the client-supplied digest against - * the empty-body hash, sets metadataStoreParams.checksum on success, and + * Verifies the SigV4 payload hash (x-amz-content-sha256) against the empty + * body, then parses the checksum headers, validates the client-supplied digest + * against the empty-body hash, sets metadataStoreParams.checksum on success, and * calls back with an error on mismatch or invalid headers. * * @param {object} headers - request headers @@ -38,6 +40,10 @@ const externalVersioningErrorMessage = 'We do not currently support putting ' + * @return {undefined} */ function zeroSizeBodyChecksumCheck(headers, metadataStoreParams, callback) { + const contentSHA256Err = validateXAmzContentSHA256(headers, Buffer.alloc(0)); + if (contentSHA256Err) { + return callback(arsenalErrorFromChecksumError(contentSHA256Err)); + } const checksumData = getChecksumDataFromHeaders(headers) || defaultChecksumData; if (checksumData.error) { return callback(arsenalErrorFromChecksumError(checksumData)); diff --git a/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js b/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js index 1c23df570d..5eb4d003c8 100644 --- a/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js +++ b/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js @@ -30,6 +30,8 @@ const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; const objData = Buffer.from('the real request body content'); const realSha256Hex = crypto.createHash('sha256').update(objData).digest('hex'); +const emptyBody = Buffer.alloc(0); +const emptySha256Hex = crypto.createHash('sha256').update(emptyBody).digest('hex'); const wrongSha256Hex = crypto.createHash('sha256') .update('completely different content') .digest('hex'); @@ -150,6 +152,10 @@ describe('SigV4 x-amz-content-sha256 body checksum validation (S3C-10916)', () = }); makeMismatchTests(() => `http://${host}:${port}/${bucket}/${objectKey}`); + + describe('with an empty body (zero-byte path)', () => { + makeMismatchTests(() => `http://${host}:${port}/${bucket}/${objectKey}`, emptyBody, emptySha256Hex); + }); }); describe('UploadPart', () => { @@ -197,6 +203,12 @@ describe('SigV4 x-amz-content-sha256 body checksum validation (S3C-10916)', () = makeMismatchTests(() => `http://${host}:${port}/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId}`); + + describe('with an empty body (zero-byte path)', () => { + makeMismatchTests(() => + `http://${host}:${port}/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId}`, + emptyBody, emptySha256Hex); + }); }); // Non-streaming (buffered) write path: validation happens in @@ -234,6 +246,21 @@ describe('SigV4 x-amz-content-sha256 body checksum validation (S3C-10916)', () = }); }); + it('should return 400 XAmzContentSHA256Mismatch for a zero-byte buffered body', done => { + const ep = bufferedEndpoints.bucketPutCors; + doRequest(ep.method, `http://${host}:${port}/${bucket}${ep.suffix}`, { + 'x-amz-content-sha256': wrongSha256Hex, + 'content-length': emptyBody.length, + }, emptyBody, (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, + `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match(res.body, /XAmzContentSHA256Mismatch/, + `expected XAmzContentSHA256Mismatch in "${res.body}"`); + done(); + }); + }); + // Fails if a new checksumed/buffered method is added without test coverage. it('should exercise every buffered checksumed method', () => { const expected = new Set([...Object.keys(checksumedMethods), 'completeMultipartUpload']); diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index 0dd415f30f..d39ad4acb6 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -1409,6 +1409,20 @@ describe('validateXAmzContentSHA256', () => { { authorization: 'AWS AKID:sig', 'x-amz-content-sha256': wrongHex }, body)); }); + it('should return null when the hash matches an empty body', () => { + const emptyHex = crypto.createHash('sha256').update(Buffer.alloc(0)).digest('hex'); + assert.ifError(validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': emptyHex }, Buffer.alloc(0))); + }); + + it('should return ContentSHA256Mismatch for an empty body with a wrong hash', () => { + const result = validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, Buffer.alloc(0)); + assert.strictEqual(result.error, ChecksumError.ContentSHA256Mismatch); + assert.strictEqual(result.details.calculated, + crypto.createHash('sha256').update(Buffer.alloc(0)).digest('hex')); + }); + describe('mapped through arsenalErrorFromChecksumError', () => { it('should map mismatch to XAmzContentSHA256Mismatch (400)', () => { const result = validateXAmzContentSHA256( From d0095579d3c6e5087907b20e3828aba271d14f8a Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Fri, 3 Jul 2026 16:17:14 +0200 Subject: [PATCH 6/8] CLDSRV-932: prettier format --- .../apiUtils/object/createAndStoreObject.js | 397 ++++++++++-------- lib/api/apiUtils/object/prepareStream.js | 26 +- lib/api/apiUtils/object/storeObject.js | 32 +- lib/api/objectPut.js | 334 ++++++++------- .../test/xAmzContentSha256Mismatch.js | 285 +++++++------ .../functional/raw-node/utils/makeRequest.js | 45 +- .../apiUtils/integrity/validateChecksums.js | 110 +++-- .../unit/api/apiUtils/object/prepareStream.js | 41 +- tests/unit/api/apiUtils/object/storeObject.js | 214 +++++----- .../api/apiUtils/validatePayloadProtocol.js | 4 +- 10 files changed, 840 insertions(+), 648 deletions(-) diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 48c035a8c0..11b3d185bc 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -11,8 +11,7 @@ const { versioningPreprocessing, overwritingVersioning, decodeVID } = require('. const removeAWSChunked = require('./removeAWSChunked'); const getReplicationInfo = require('./getReplicationInfo'); const { config } = require('../../../Config'); -const validateWebsiteHeader = require('./websiteServing') - .validateWebsiteHeader; +const validateWebsiteHeader = require('./websiteServing').validateWebsiteHeader; const applyZenkoUserMD = require('./applyZenkoUserMD'); const { algorithms, @@ -24,8 +23,8 @@ const { const { externalBackends, versioningNotImplBackends } = constants; -const externalVersioningErrorMessage = 'We do not currently support putting ' + - 'a versioned object to a location-constraint of type Azure or GCP.'; +const externalVersioningErrorMessage = + 'We do not currently support putting ' + 'a versioned object to a location-constraint of type Azure or GCP.'; /** * Validate and compute the checksum for a zero-size object body. @@ -52,34 +51,43 @@ function zeroSizeBodyChecksumCheck(headers, metadataStoreParams, callback) { // never read (stream bypassed), so expected is always undefined here. // We still compute and store the empty-body hash for the announced algorithm. const { algorithm, expected } = checksumData; - return Promise.resolve(algorithms[algorithm].digest(Buffer.alloc(0))) - .then(value => { + return Promise.resolve(algorithms[algorithm].digest(Buffer.alloc(0))).then( + value => { if (expected !== undefined && expected !== value) { - return callback(errorInstances.BadDigest.customizeDescription( - `The ${algorithm.toUpperCase()} you specified did not match the calculated checksum.` - )); + return callback( + errorInstances.BadDigest.customizeDescription( + `The ${algorithm.toUpperCase()} you specified did not match the calculated checksum.`, + ), + ); } // eslint-disable-next-line no-param-reassign metadataStoreParams.checksum = { algorithm, value, type: 'FULL_OBJECT' }; return callback(null); - }, err => callback(err)); + }, + err => callback(err), + ); } -function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle, - metadataStoreParams, dataToDelete, log, requestMethod, callback) { - services.metadataStoreObject(bucketName, dataGetInfo, - cipherBundle, metadataStoreParams, (err, result) => { - if (err) { - return callback(err); - } - if (dataToDelete) { - const newDataStoreName = Array.isArray(dataGetInfo) ? - dataGetInfo[0].dataStoreName : null; - return data.batchDelete(dataToDelete, requestMethod, - newDataStoreName, log, err => callback(err, result)); - } - return callback(null, result); - }); +function _storeInMDandDeleteData( + bucketName, + dataGetInfo, + cipherBundle, + metadataStoreParams, + dataToDelete, + log, + requestMethod, + callback, +) { + services.metadataStoreObject(bucketName, dataGetInfo, cipherBundle, metadataStoreParams, (err, result) => { + if (err) { + return callback(err); + } + if (dataToDelete) { + const newDataStoreName = Array.isArray(dataGetInfo) ? dataGetInfo[0].dataStoreName : null; + return data.batchDelete(dataToDelete, requestMethod, newDataStoreName, log, err => callback(err, result)); + } + return callback(null, result); + }); } /** createAndStoreObject - store data, store metadata, and delete old data @@ -104,9 +112,22 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle, * result.contentMD5 - content md5 of new object or version * result.versionId - unencrypted versionId returned by metadata */ -function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, - canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params, - overheadField, log, originOp, callback) { +function createAndStoreObject( + bucketName, + bucketMD, + objectKey, + objMD, + authInfo, + canonicalID, + cipherBundle, + request, + isDeleteMarker, + streamingV4Params, + overheadField, + log, + originOp, + callback, +) { const putVersionId = request.headers['x-scal-s3-version-id']; const isPutVersion = putVersionId || putVersionId === ''; @@ -115,12 +136,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, // delete marker, for our purposes we consider this to be a 'PUT' // operation const requestMethod = 'PUT'; - const websiteRedirectHeader = - request.headers['x-amz-website-redirect-location']; + const websiteRedirectHeader = request.headers['x-amz-website-redirect-location']; if (!validateWebsiteHeader(websiteRedirectHeader)) { const err = errors.InvalidRedirectLocation; - log.debug('invalid x-amz-website-redirect-location' + - `value ${websiteRedirectHeader}`, { error: err }); + log.debug('invalid x-amz-website-redirect-location' + `value ${websiteRedirectHeader}`, { error: err }); return callback(err); } @@ -160,8 +179,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, size, headers, isDeleteMarker, - replicationInfo: getReplicationInfo(config, - objectKey, bucketMD, false, size, null, null, authInfo), + replicationInfo: getReplicationInfo(config, objectKey, bucketMD, false, size, null, null, authInfo), overheadField, log, }; @@ -186,17 +204,13 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, if (!isDeleteMarker) { metadataStoreParams.contentType = request.headers['content-type']; metadataStoreParams.cacheControl = request.headers['cache-control']; - metadataStoreParams.contentDisposition = - request.headers['content-disposition']; - metadataStoreParams.contentEncoding = - removeAWSChunked(request.headers['content-encoding']); + metadataStoreParams.contentDisposition = request.headers['content-disposition']; + metadataStoreParams.contentEncoding = removeAWSChunked(request.headers['content-encoding']); metadataStoreParams.expires = request.headers.expires; metadataStoreParams.tagging = request.headers['x-amz-tagging']; - const defaultObjectLockConfiguration - = bucketMD.getObjectLockConfiguration(); + const defaultObjectLockConfiguration = bucketMD.getObjectLockConfiguration(); if (defaultObjectLockConfiguration) { - metadataStoreParams.defaultRetention - = defaultObjectLockConfiguration; + metadataStoreParams.defaultRetention = defaultObjectLockConfiguration; } } @@ -204,12 +218,10 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, // the object's location constraint metaheader to determine backend info if (isDeleteMarker && objMD) { // eslint-disable-next-line no-param-reassign - request.headers[constants.objectLocationConstraintHeader] = - objMD[constants.objectLocationConstraintHeader]; + request.headers[constants.objectLocationConstraintHeader] = objMD[constants.objectLocationConstraintHeader]; } - const backendInfoObj = - locationConstraintCheck(request, null, bucketMD, log); + const backendInfoObj = locationConstraintCheck(request, null, bucketMD, log); if (backendInfoObj.err) { return process.nextTick(() => { callback(backendInfoObj.err); @@ -218,8 +230,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, const backendInfo = backendInfoObj.backendInfo; const location = backendInfo.getControllingLocationConstraint(); - const locationType = backendInfoObj.defaultedToDataBackend ? location : - config.getLocationConstraintType(location); + const locationType = backendInfoObj.defaultedToDataBackend ? location : config.getLocationConstraintType(location); metadataStoreParams.dataStoreName = location; if (versioningNotImplBackends[locationType]) { @@ -227,11 +238,9 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; if (isVersionedObj) { - log.debug(externalVersioningErrorMessage, - { method: 'createAndStoreObject', error: errors.NotImplemented }); + log.debug(externalVersioningErrorMessage, { method: 'createAndStoreObject', error: errors.NotImplemented }); return process.nextTick(() => { - callback(errorInstances.NotImplemented.customizeDescription( - externalVersioningErrorMessage)); + callback(errorInstances.NotImplemented.customizeDescription(externalVersioningErrorMessage)); }); } } @@ -257,145 +266,173 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, const mdOnlyHeader = request.headers['x-amz-meta-mdonly']; const mdOnlySize = request.headers['x-amz-meta-size']; - return async.waterfall([ - function storeData(next) { - if (size === 0) { - if (!dontSkipBackend[locationType]) { - metadataStoreParams.contentMD5 = constants.emptyFileMd5; - // Delete markers are zero-byte versioned tombstones with - // no body, ETag, or checksum — skip checksum handling. - if (isDeleteMarker) { - return next(null, null, null, null); + return async.waterfall( + [ + function storeData(next) { + if (size === 0) { + if (!dontSkipBackend[locationType]) { + metadataStoreParams.contentMD5 = constants.emptyFileMd5; + // Delete markers are zero-byte versioned tombstones with + // no body, ETag, or checksum — skip checksum handling. + if (isDeleteMarker) { + return next(null, null, null, null); + } + return zeroSizeBodyChecksumCheck(request.headers, metadataStoreParams, err => + next(err, null, null, null), + ); } - return zeroSizeBodyChecksumCheck(request.headers, metadataStoreParams, - err => next(err, null, null, null)); - } - // Handle mdOnlyHeader as a metadata only operation. If - // the object in question is actually 0 byte or has a body size - // then handle normally. - if (mdOnlyHeader === 'true' && mdOnlySize > 0) { - log.debug('metadata only operation x-amz-meta-mdonly'); - const md5 = request.headers['x-amz-meta-md5chksum'] - ? Buffer.from(request.headers['x-amz-meta-md5chksum'], - 'base64').toString('hex') : null; - const numParts = request.headers['x-amz-meta-md5numparts']; - let _md5; - if (numParts === undefined) { - _md5 = md5; - } else { - _md5 = `${md5}-${numParts}`; + // Handle mdOnlyHeader as a metadata only operation. If + // the object in question is actually 0 byte or has a body size + // then handle normally. + if (mdOnlyHeader === 'true' && mdOnlySize > 0) { + log.debug('metadata only operation x-amz-meta-mdonly'); + const md5 = request.headers['x-amz-meta-md5chksum'] + ? Buffer.from(request.headers['x-amz-meta-md5chksum'], 'base64').toString('hex') + : null; + const numParts = request.headers['x-amz-meta-md5numparts']; + let _md5; + if (numParts === undefined) { + _md5 = md5; + } else { + _md5 = `${md5}-${numParts}`; + } + const versionId = request.headers['x-amz-meta-version-id']; + const dataGetInfo = { + key: objectKey, + dataStoreName: location, + dataStoreType: locationType, + dataStoreVersionId: versionId, + dataStoreMD5: _md5, + }; + return next(null, dataGetInfo, _md5, null); } - const versionId = request.headers['x-amz-meta-version-id']; - const dataGetInfo = { - key: objectKey, - dataStoreName: location, - dataStoreType: locationType, - dataStoreVersionId: versionId, - dataStoreMD5: _md5, - }; - return next(null, dataGetInfo, _md5, null); } - } - const headerChecksum = getChecksumDataFromHeaders(request.headers); - if (headerChecksum && headerChecksum.error) { - return next(arsenalErrorFromChecksumError(headerChecksum)); - } - const checksums = { - primary: headerChecksum || defaultChecksumData, - secondary: null, - }; - return dataStore(objectKeyContext, cipherBundle, request, size, - streamingV4Params, backendInfo, checksums, log, next); - }, - function processDataResult(dataGetInfo, calculatedHash, checksum, next) { - if (dataGetInfo === null || dataGetInfo === undefined) { - return next(null, null); - } - // So that data retrieval information for MPU's and - // regular puts are stored in the same data structure, - // place the retrieval info here into a single element array - const { key, dataStoreName, dataStoreType, dataStoreETag, - dataStoreVersionId } = dataGetInfo; - const prefixedDataStoreETag = dataStoreETag - ? `1:${dataStoreETag}` - : `1:${calculatedHash}`; - const dataGetInfoArr = [{ key, size, start: 0, dataStoreName, - dataStoreType, dataStoreETag: prefixedDataStoreETag, - dataStoreVersionId - }]; - if (cipherBundle) { - dataGetInfoArr[0].cryptoScheme = cipherBundle.cryptoScheme; - dataGetInfoArr[0].cipheredDataKey = - cipherBundle.cipheredDataKey; - } - if (mdOnlyHeader === 'true') { - metadataStoreParams.size = mdOnlySize; - dataGetInfoArr[0].size = mdOnlySize; - } - metadataStoreParams.contentMD5 = calculatedHash; - if (checksum) { - // eslint-disable-next-line no-param-reassign - checksum.type = 'FULL_OBJECT'; - metadataStoreParams.checksum = checksum; - } - return next(null, dataGetInfoArr); - }, - function getVersioningInfo(infoArr, next) { - // if x-scal-s3-version-id header is specified, we overwrite the object/version metadata. - if (isPutVersion) { - const options = overwritingVersioning(objMD, metadataStoreParams); - return process.nextTick(() => next(null, options, infoArr)); - } + const headerChecksum = getChecksumDataFromHeaders(request.headers); + if (headerChecksum && headerChecksum.error) { + return next(arsenalErrorFromChecksumError(headerChecksum)); + } + const checksums = { + primary: headerChecksum || defaultChecksumData, + secondary: null, + }; + return dataStore( + objectKeyContext, + cipherBundle, + request, + size, + streamingV4Params, + backendInfo, + checksums, + log, + next, + ); + }, + function processDataResult(dataGetInfo, calculatedHash, checksum, next) { + if (dataGetInfo === null || dataGetInfo === undefined) { + return next(null, null); + } + // So that data retrieval information for MPU's and + // regular puts are stored in the same data structure, + // place the retrieval info here into a single element array + const { key, dataStoreName, dataStoreType, dataStoreETag, dataStoreVersionId } = dataGetInfo; + const prefixedDataStoreETag = dataStoreETag ? `1:${dataStoreETag}` : `1:${calculatedHash}`; + const dataGetInfoArr = [ + { + key, + size, + start: 0, + dataStoreName, + dataStoreType, + dataStoreETag: prefixedDataStoreETag, + dataStoreVersionId, + }, + ]; + if (cipherBundle) { + dataGetInfoArr[0].cryptoScheme = cipherBundle.cryptoScheme; + dataGetInfoArr[0].cipheredDataKey = cipherBundle.cipheredDataKey; + } + if (mdOnlyHeader === 'true') { + metadataStoreParams.size = mdOnlySize; + dataGetInfoArr[0].size = mdOnlySize; + } + metadataStoreParams.contentMD5 = calculatedHash; + if (checksum) { + // eslint-disable-next-line no-param-reassign + checksum.type = 'FULL_OBJECT'; + metadataStoreParams.checksum = checksum; + } + return next(null, dataGetInfoArr); + }, + function getVersioningInfo(infoArr, next) { + // if x-scal-s3-version-id header is specified, we overwrite the object/version metadata. + if (isPutVersion) { + const options = overwritingVersioning(objMD, metadataStoreParams); + return process.nextTick(() => next(null, options, infoArr)); + } - if (!bucketMD.isVersioningEnabled() && objMD?.archive?.archiveInfo) { - // Ensure we trigger a "delete" event in the oplog for the previously archived object - metadataStoreParams.needOplogUpdate = 's3:ReplaceArchivedObject'; - } + if (!bucketMD.isVersioningEnabled() && objMD?.archive?.archiveInfo) { + // Ensure we trigger a "delete" event in the oplog for the previously archived object + metadataStoreParams.needOplogUpdate = 's3:ReplaceArchivedObject'; + } - return versioningPreprocessing(bucketName, bucketMD, - metadataStoreParams.objectKey, objMD, log, (err, options) => { - if (err) { - // TODO: check AWS error when user requested a specific - // version before any versions have been put - const logLvl = err.is.BadRequest ? - 'debug' : 'error'; - log[logLvl]('error getting versioning info', { - error: err, - method: 'versioningPreprocessing', - }); - } + return versioningPreprocessing( + bucketName, + bucketMD, + metadataStoreParams.objectKey, + objMD, + log, + (err, options) => { + if (err) { + // TODO: check AWS error when user requested a specific + // version before any versions have been put + const logLvl = err.is.BadRequest ? 'debug' : 'error'; + log[logLvl]('error getting versioning info', { + error: err, + method: 'versioningPreprocessing', + }); + } - const location = infoArr?.[0]?.dataStoreName; - if (location === bucketMD.getLocationConstraint() && bucketMD.isIngestionBucket()) { - // If the object is being written to the "ingested" storage location, keep the same - // versionId for consistency and to avoid creating an extra version when it gets - // ingested - const backendVersionId = decodeVID(infoArr[0].dataStoreVersionId); - if (!(backendVersionId instanceof Error)) { - options.versionId = backendVersionId; // eslint-disable-line no-param-reassign + const location = infoArr?.[0]?.dataStoreName; + if (location === bucketMD.getLocationConstraint() && bucketMD.isIngestionBucket()) { + // If the object is being written to the "ingested" storage location, keep the same + // versionId for consistency and to avoid creating an extra version when it gets + // ingested + const backendVersionId = decodeVID(infoArr[0].dataStoreVersionId); + if (!(backendVersionId instanceof Error)) { + options.versionId = backendVersionId; // eslint-disable-line no-param-reassign + } } - } - return next(err, options, infoArr); - }); - }, - function storeMDAndDeleteData(options, infoArr, next) { - metadataStoreParams.versionId = options.versionId; - metadataStoreParams.versioning = options.versioning; - metadataStoreParams.isNull = options.isNull; - metadataStoreParams.deleteNullKey = options.deleteNullKey; - - if (options.extraMD) { - Object.assign(metadataStoreParams, options.extraMD); - } + return next(err, options, infoArr); + }, + ); + }, + function storeMDAndDeleteData(options, infoArr, next) { + metadataStoreParams.versionId = options.versionId; + metadataStoreParams.versioning = options.versioning; + metadataStoreParams.isNull = options.isNull; + metadataStoreParams.deleteNullKey = options.deleteNullKey; + + if (options.extraMD) { + Object.assign(metadataStoreParams, options.extraMD); + } - return _storeInMDandDeleteData(bucketName, infoArr, - cipherBundle, metadataStoreParams, - options.dataToDelete, log, requestMethod, next); - }, - ], callback); + return _storeInMDandDeleteData( + bucketName, + infoArr, + cipherBundle, + metadataStoreParams, + options.dataToDelete, + log, + requestMethod, + next, + ); + }, + ], + callback, + ); } module.exports = createAndStoreObject; diff --git a/lib/api/apiUtils/object/prepareStream.js b/lib/api/apiUtils/object/prepareStream.js index 66895fba6f..9ec1e5ed2a 100644 --- a/lib/api/apiUtils/object/prepareStream.js +++ b/lib/api/apiUtils/object/prepareStream.js @@ -56,14 +56,16 @@ function prepareStream(request, streamingV4Params, checksums, log, errCb) { let stream = v4Transform; if (secondary) { secondaryChecksumStream = new ChecksumTransform( - secondary.algorithm, secondary.expected, - secondary.isTrailer, log); + secondary.algorithm, + secondary.expected, + secondary.isTrailer, + log, + ); secondaryChecksumStream.on('error', onStreamError); stream = v4Transform.pipe(secondaryChecksumStream); } - const primaryStream = new ChecksumTransform( - primary.algorithm, primary.expected, primary.isTrailer, log); + const primaryStream = new ChecksumTransform(primary.algorithm, primary.expected, primary.isTrailer, log); primaryStream.on('error', onStreamError); return { error: null, stream: stream.pipe(primaryStream), secondaryChecksumStream }; } @@ -78,8 +80,11 @@ function prepareStream(request, streamingV4Params, checksums, log, errCb) { let stream = trailingChecksumTransform; if (secondary) { secondaryChecksumStream = new ChecksumTransform( - secondary.algorithm, secondary.expected, - secondary.isTrailer, log); + secondary.algorithm, + secondary.expected, + secondary.isTrailer, + log, + ); secondaryChecksumStream.on('error', onStreamError); stream = trailingChecksumTransform.pipe(secondaryChecksumStream); trailingChecksumTransform.on('trailer', (name, value) => { @@ -107,7 +112,7 @@ function prepareStream(request, streamingV4Params, checksums, log, errCb) { const parsedContentSHA256 = parseContentSHA256(request.headers); const shouldValidateContentSHA256 = parsedContentSHA256.type === ContentSHA256Type.HexSHA256; - const onStreamError = (secondary || shouldValidateContentSHA256) ? jsutil.once(errCb) : errCb; + const onStreamError = secondary || shouldValidateContentSHA256 ? jsutil.once(errCb) : errCb; let contentSHA256Stream = null; let secondaryChecksumStream = null; let stream = request; @@ -118,8 +123,11 @@ function prepareStream(request, streamingV4Params, checksums, log, errCb) { } if (secondary) { secondaryChecksumStream = new ChecksumTransform( - secondary.algorithm, secondary.expected, - secondary.isTrailer, log); + secondary.algorithm, + secondary.expected, + secondary.isTrailer, + log, + ); secondaryChecksumStream.on('error', onStreamError); stream = stream.pipe(secondaryChecksumStream); } diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js index 06dd669052..ee0baa03e8 100644 --- a/lib/api/apiUtils/object/storeObject.js +++ b/lib/api/apiUtils/object/storeObject.js @@ -75,7 +75,12 @@ function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, return process.nextTick(() => cbOnce(checksumedStream.error)); } return data.put( - cipherBundle, checksumedStream.stream, size, objectContext, backendInfo, log, + cipherBundle, + checksumedStream.stream, + size, + objectContext, + backendInfo, + log, (err, dataRetrievalInfo, hashedStream) => { if (err) { log.error('error in datastore', { error: err }); @@ -143,15 +148,26 @@ function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, }); } if (!secondaryChecksumStream) { - return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, - checksumedStream.stream, log, cbOnce); + return checkHashMatchMD5( + stream, + hashedStream, + dataRetrievalInfo, + checksumedStream.stream, + log, + cbOnce, + ); } // Dual-checksum: checkHashMatchMD5 returns the primary // (storage) checksum. Swap it to the client-facing one // from the secondary stream and attach the primary as // storageChecksum. - return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, - checksumedStream.stream, log, (err, dataInfo, hash, primaryChecksum) => { + return checkHashMatchMD5( + stream, + hashedStream, + dataRetrievalInfo, + checksumedStream.stream, + log, + (err, dataInfo, hash, primaryChecksum) => { if (err) { return cbOnce(err); } @@ -161,7 +177,8 @@ function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, storageChecksum: primaryChecksum, }; return cbOnce(null, dataInfo, hash, checksum); - }); + }, + ); }; // ChecksumTransform._flush computes the digest asynchronously for @@ -174,7 +191,8 @@ function dataStore(objectContext, cipherBundle, stream, size, streamingV4Params, } checksumedStream.stream.once('finish', doValidate); return undefined; - }); + }, + ); } module.exports = { diff --git a/lib/api/objectPut.js b/lib/api/objectPut.js index 8bdce061ed..4025e90a0d 100644 --- a/lib/api/objectPut.js +++ b/lib/api/objectPut.js @@ -28,7 +28,8 @@ const versionIdUtils = versioning.VersionID; const { updateEncryption } = require('./apiUtils/bucket/updateEncryption'); const invalidSSEError = errorInstances.InvalidArgument.customizeDescription( - 'The encryption method specified is not supported'); + 'The encryption method specified is not supported', +); /** * PUT Object in the requested bucket. Steps include: @@ -69,25 +70,15 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) { versionId = decodedVidResult; } - const { - bucketName, - headers, - method, - objectKey, - parsedContentLength, - query, - } = request; - if (headers['x-amz-storage-class'] && - !constants.validStorageClasses.includes(headers['x-amz-storage-class'])) { + const { bucketName, headers, method, objectKey, parsedContentLength, query } = request; + if (headers['x-amz-storage-class'] && !constants.validStorageClasses.includes(headers['x-amz-storage-class'])) { log.trace('invalid storage-class header'); - monitoring.promMetrics('PUT', request.bucketName, - errorInstances.InvalidStorageClass.code, 'putObject'); + monitoring.promMetrics('PUT', request.bucketName, errorInstances.InvalidStorageClass.code, 'putObject'); return callback(errors.InvalidStorageClass); } if (!aclUtils.checkGrantHeaderValidity(headers)) { log.trace('invalid acl header'); - monitoring.promMetrics('PUT', request.bucketName, 400, - 'putObject'); + monitoring.promMetrics('PUT', request.bucketName, 400, 'putObject'); return callback(errors.InvalidArgument); } const queryContainsVersionId = checkQueryVersionId(query); @@ -101,22 +92,19 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) { } const size = request.parsedContentLength; - if (Number.parseInt(size, 10) > constants.maximumAllowedUploadSize - && !config.bypassMaxPutObjectSize) { - log.debug('Upload size exceeds maximum allowed for a single PUT', - { size }); + if (Number.parseInt(size, 10) > constants.maximumAllowedUploadSize && !config.bypassMaxPutObjectSize) { + log.debug('Upload size exceeds maximum allowed for a single PUT', { size }); return callback(errors.EntityTooLarge); } const requestType = request.apiMethods || 'objectPut'; - const valParams = { authInfo, bucketName, objectKey, versionId, - requestType, request, withVersionId: isPutVersion }; + const valParams = { authInfo, bucketName, objectKey, versionId, requestType, request, withVersionId: isPutVersion }; const canonicalID = authInfo.getCanonicalID(); if (hasNonPrintables(objectKey)) { - return callback(errorInstances.InvalidInput.customizeDescription( - 'object keys cannot contain non-printable characters', - )); + return callback( + errorInstances.InvalidInput.customizeDescription('object keys cannot contain non-printable characters'), + ); } const payloadProtocolErr = validatePayloadProtocol(headers); @@ -126,158 +114,178 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) { log.trace('owner canonicalID to send to data', { canonicalID }); - return standardMetadataValidateBucketAndObj(valParams, request.actionImplicitDenies, log, - (err, bucket, objMD) => updateEncryption(err, bucket, objMD, objectKey, log, { skipObject: true }, - (err, bucket, objMD) => { - const responseHeaders = collectCorsHeaders(headers.origin, - method, bucket); - if (err) { - log.trace('error processing request', { - error: err, - method: 'metadataValidateBucketAndObj', - }); - monitoring.promMetrics('PUT', bucketName, err.code, 'putObject'); - return callback(err, responseHeaders); - } - if (bucket.hasDeletedFlag() && canonicalID !== bucket.getOwner()) { - log.trace('deleted flag on bucket and request ' + - 'from non-owner account'); - monitoring.promMetrics('PUT', bucketName, 404, 'putObject'); - return callback(errors.NoSuchBucket); - } - - if (isPutVersion) { - const error = validatePutVersionId(objMD, putVersionId, log); - if (error) { - return callback(error); + return standardMetadataValidateBucketAndObj(valParams, request.actionImplicitDenies, log, (err, bucket, objMD) => + updateEncryption(err, bucket, objMD, objectKey, log, { skipObject: true }, (err, bucket, objMD) => { + const responseHeaders = collectCorsHeaders(headers.origin, method, bucket); + if (err) { + log.trace('error processing request', { + error: err, + method: 'metadataValidateBucketAndObj', + }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putObject'); + return callback(err, responseHeaders); + } + if (bucket.hasDeletedFlag() && canonicalID !== bucket.getOwner()) { + log.trace('deleted flag on bucket and request ' + 'from non-owner account'); + monitoring.promMetrics('PUT', bucketName, 404, 'putObject'); + return callback(errors.NoSuchBucket); } - } - return async.waterfall([ - function handleTransientOrDeleteBuckets(next) { - if (bucket.hasTransientFlag() || bucket.hasDeletedFlag()) { - return cleanUpBucket(bucket, canonicalID, log, next); + if (isPutVersion) { + const error = validatePutVersionId(objMD, putVersionId, log); + if (error) { + return callback(error); } - return next(); - }, - function getSSEConfig(next) { - return getObjectSSEConfiguration(headers, bucket, log, - (err, sseConfig) => { - if (err) { - log.error('error getting server side encryption config', { err }); - return next(invalidSSEError); + } + + return async.waterfall( + [ + function handleTransientOrDeleteBuckets(next) { + if (bucket.hasTransientFlag() || bucket.hasDeletedFlag()) { + return cleanUpBucket(bucket, canonicalID, log, next); } - return next(null, sseConfig); - } - ); - }, - function createCipherBundle(serverSideEncryptionConfig, next) { - if (serverSideEncryptionConfig) { - return kms.createCipherBundle( - serverSideEncryptionConfig, log, (err, cipherBundle) => { + return next(); + }, + function getSSEConfig(next) { + return getObjectSSEConfiguration(headers, bucket, log, (err, sseConfig) => { if (err) { - return next(err); + log.error('error getting server side encryption config', { err }); + return next(invalidSSEError); } - setSSEHeaders(responseHeaders, - cipherBundle.algorithm, - cipherBundle.configuredMasterKeyId || cipherBundle.masterKeyId); - return next(null, cipherBundle); + return next(null, sseConfig); }); - } - return next(null, null); - }, - function objectCreateAndStore(cipherBundle, next) { - const objectLockValidationError - = validateHeaders(bucket, headers, log); - if (objectLockValidationError) { - return next(objectLockValidationError); - } - writeContinue(request, request._response); - return createAndStoreObject(bucketName, - bucket, objectKey, objMD, authInfo, canonicalID, cipherBundle, - request, false, streamingV4Params, overheadField, log, 's3:ObjectCreated:Put', next); - }, - ], (err, storingResult) => { - if (err) { - monitoring.promMetrics('PUT', bucketName, err.code, - 'putObject'); - return callback(err, responseHeaders); - } - // ingestSize assumes that these custom headers indicate - // an ingestion PUT which is a metadata only operation. - // Since these headers can be modified client side, they - // should be used with caution if needed for precise - // metrics. - const ingestSize = (request.headers['x-amz-meta-mdonly'] - && !Number.isNaN(request.headers['x-amz-meta-size'])) - ? Number.parseInt(request.headers['x-amz-meta-size'], 10) : null; - const newByteLength = parsedContentLength; + }, + function createCipherBundle(serverSideEncryptionConfig, next) { + if (serverSideEncryptionConfig) { + return kms.createCipherBundle(serverSideEncryptionConfig, log, (err, cipherBundle) => { + if (err) { + return next(err); + } + setSSEHeaders( + responseHeaders, + cipherBundle.algorithm, + cipherBundle.configuredMasterKeyId || cipherBundle.masterKeyId, + ); + return next(null, cipherBundle); + }); + } + return next(null, null); + }, + function objectCreateAndStore(cipherBundle, next) { + const objectLockValidationError = validateHeaders(bucket, headers, log); + if (objectLockValidationError) { + return next(objectLockValidationError); + } + writeContinue(request, request._response); + return createAndStoreObject( + bucketName, + bucket, + objectKey, + objMD, + authInfo, + canonicalID, + cipherBundle, + request, + false, + streamingV4Params, + overheadField, + log, + 's3:ObjectCreated:Put', + next, + ); + }, + ], + (err, storingResult) => { + if (err) { + monitoring.promMetrics('PUT', bucketName, err.code, 'putObject'); + return callback(err, responseHeaders); + } + // ingestSize assumes that these custom headers indicate + // an ingestion PUT which is a metadata only operation. + // Since these headers can be modified client side, they + // should be used with caution if needed for precise + // metrics. + const ingestSize = + request.headers['x-amz-meta-mdonly'] && !Number.isNaN(request.headers['x-amz-meta-size']) + ? Number.parseInt(request.headers['x-amz-meta-size'], 10) + : null; + const newByteLength = parsedContentLength; - setExpirationHeaders(responseHeaders, { - lifecycleConfig: bucket.getLifecycleConfiguration(), - objectParams: { - key: objectKey, - date: storingResult.lastModified, - tags: storingResult.tags, - }, - }); + setExpirationHeaders(responseHeaders, { + lifecycleConfig: bucket.getLifecycleConfiguration(), + objectParams: { + key: objectKey, + date: storingResult.lastModified, + tags: storingResult.tags, + }, + }); - // Utapi expects null or a number for oldByteLength: - // * null - new object - // * 0 or > 0 - existing object with content-length 0 or > 0 - // objMD here is the master version that we would - // have overwritten if there was an existing version or object - // - // TODO: Handle utapi metrics for null version overwrites. - const oldByteLength = objMD && objMD['content-length'] - !== undefined ? objMD['content-length'] : null; - if (storingResult) { - // ETag's hex should always be enclosed in quotes - responseHeaders.ETag = `"${storingResult.contentMD5}"`; - if (storingResult.checksum) { - const { checksumAlgorithm, checksumValue } = storingResult.checksum; - responseHeaders[`x-amz-checksum-${checksumAlgorithm}`] = checksumValue; - } - } - const vcfg = bucket.getVersioningConfiguration(); - const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; - if (isVersionedObj) { - if (storingResult && storingResult.versionId) { - responseHeaders['x-amz-version-id'] = - versionIdUtils.encode(storingResult.versionId); - } - } + // Utapi expects null or a number for oldByteLength: + // * null - new object + // * 0 or > 0 - existing object with content-length 0 or > 0 + // objMD here is the master version that we would + // have overwritten if there was an existing version or object + // + // TODO: Handle utapi metrics for null version overwrites. + const oldByteLength = + objMD && objMD['content-length'] !== undefined ? objMD['content-length'] : null; + if (storingResult) { + // ETag's hex should always be enclosed in quotes + responseHeaders.ETag = `"${storingResult.contentMD5}"`; + if (storingResult.checksum) { + const { checksumAlgorithm, checksumValue } = storingResult.checksum; + responseHeaders[`x-amz-checksum-${checksumAlgorithm}`] = checksumValue; + } + } + const vcfg = bucket.getVersioningConfiguration(); + const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; + if (isVersionedObj) { + if (storingResult && storingResult.versionId) { + responseHeaders['x-amz-version-id'] = versionIdUtils.encode(storingResult.versionId); + } + } - // Only pre-existing non-versioned objects get 0 all others use 1 - const numberOfObjects = !isVersionedObj && oldByteLength !== null ? 0 : 1; + // Only pre-existing non-versioned objects get 0 all others use 1 + const numberOfObjects = !isVersionedObj && oldByteLength !== null ? 0 : 1; - // only the bucket owner's metrics should be updated, regardless of - // who the requester is - pushMetric('putObject', log, { - authInfo, - canonicalID: bucket.getOwner(), - bucket: bucketName, - keys: [objectKey], - newByteLength, - oldByteLength: isVersionedObj ? null : oldByteLength, - versionId: isVersionedObj && storingResult ? storingResult.versionId : undefined, - location: bucket.getLocationConstraint(), - numberOfObjects, - }); - monitoring.promMetrics('PUT', bucketName, '200', - 'putObject', newByteLength, oldByteLength, isVersionedObj, - null, ingestSize); + // only the bucket owner's metrics should be updated, regardless of + // who the requester is + pushMetric('putObject', log, { + authInfo, + canonicalID: bucket.getOwner(), + bucket: bucketName, + keys: [objectKey], + newByteLength, + oldByteLength: isVersionedObj ? null : oldByteLength, + versionId: isVersionedObj && storingResult ? storingResult.versionId : undefined, + location: bucket.getLocationConstraint(), + numberOfObjects, + }); + monitoring.promMetrics( + 'PUT', + bucketName, + '200', + 'putObject', + newByteLength, + oldByteLength, + isVersionedObj, + null, + ingestSize, + ); - if (isPutVersion) { - const durationMs = Date.now() - new Date(objMD.archive.restoreRequestedAt); - monitoring.lifecycleDuration.observe( - { type: 'restore', location: objMD.dataStoreName }, - durationMs / 1000); - } + if (isPutVersion) { + const durationMs = Date.now() - new Date(objMD.archive.restoreRequestedAt); + monitoring.lifecycleDuration.observe( + { type: 'restore', location: objMD.dataStoreName }, + durationMs / 1000, + ); + } - return callback(null, responseHeaders); - }); - })); + return callback(null, responseHeaders); + }, + ); + }), + ); } module.exports = objectPut; diff --git a/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js b/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js index 5eb4d003c8..44a82c19e9 100644 --- a/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js +++ b/tests/functional/raw-node/test/xAmzContentSha256Mismatch.js @@ -32,9 +32,7 @@ const objData = Buffer.from('the real request body content'); const realSha256Hex = crypto.createHash('sha256').update(objData).digest('hex'); const emptyBody = Buffer.alloc(0); const emptySha256Hex = crypto.createHash('sha256').update(emptyBody).digest('hex'); -const wrongSha256Hex = crypto.createHash('sha256') - .update('completely different content') - .digest('hex'); +const wrongSha256Hex = crypto.createHash('sha256').update('completely different content').digest('hex'); const invalidSha256 = 'xxx'; // An arbitrary body that is never parsed: the x-amz-content-sha256 check rejects @@ -70,19 +68,19 @@ const scalityExtensionEndpoints = { }; function doRequest(method, url, headers, body, callback) { - const req = new HttpRequestAuthV4( - url, - Object.assign({ method, headers }, authCredentials), - res => { - let data = ''; - res.on('data', chunk => { data += chunk; }); - res.on('end', () => callback(null, { + const req = new HttpRequestAuthV4(url, Object.assign({ method, headers }, authCredentials), res => { + let data = ''; + res.on('data', chunk => { + data += chunk; + }); + res.on('end', () => + callback(null, { statusCode: res.statusCode, body: data, headers: res.headers, - })); - }, - ); + }), + ); + }); req.on('error', callback); req.write(body); req.end(); @@ -91,45 +89,58 @@ function doRequest(method, url, headers, body, callback) { const doPutRequest = (url, headers, body, callback) => doRequest('PUT', url, headers, body, callback); function makeMismatchTests(urlFn, body = objData, correctHex = realSha256Hex) { - it('should reject a body whose x-amz-content-sha256 does not match with 400 XAmzContentSHA256Mismatch', - done => { - doPutRequest(urlFn(), { + it('should reject a body whose x-amz-content-sha256 does not match with 400 XAmzContentSHA256Mismatch', done => { + doPutRequest( + urlFn(), + { 'x-amz-content-sha256': wrongSha256Hex, 'content-length': body.length, - }, body, (err, res) => { + }, + body, + (err, res) => { assert.ifError(err); - assert.strictEqual(res.statusCode, 400, - `expected 400, got ${res.statusCode}: ${res.body}`); - assert.match(res.body, /XAmzContentSHA256Mismatch/, - `expected XAmzContentSHA256Mismatch in "${res.body}"`); + assert.strictEqual(res.statusCode, 400, `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match( + res.body, + /XAmzContentSHA256Mismatch/, + `expected XAmzContentSHA256Mismatch in "${res.body}"`, + ); done(); - }); - }); + }, + ); + }); it('should accept a body whose x-amz-content-sha256 matches', done => { - doPutRequest(urlFn(), { - 'x-amz-content-sha256': correctHex, - 'content-length': body.length, - }, body, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200, - `expected 200, got ${res.statusCode}: ${res.body}`); - done(); - }); + doPutRequest( + urlFn(), + { + 'x-amz-content-sha256': correctHex, + 'content-length': body.length, + }, + body, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200, `expected 200, got ${res.statusCode}: ${res.body}`); + done(); + }, + ); }); it('should reject an invalid x-amz-content-sha256 value with 400 InvalidArgument', done => { - doPutRequest(urlFn(), { - 'x-amz-content-sha256': invalidSha256, - 'content-length': body.length, - }, body, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 400, - `expected 400, got ${res.statusCode}: ${res.body}`); - assert.match(res.body, /InvalidArgument/, - `expected InvalidArgument in "${res.body}"`); - done(); - }); + doPutRequest( + urlFn(), + { + 'x-amz-content-sha256': invalidSha256, + 'content-length': body.length, + }, + body, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match(res.body, /InvalidArgument/, `expected InvalidArgument in "${res.body}"`); + done(); + }, + ); }); } @@ -162,52 +173,69 @@ describe('SigV4 x-amz-content-sha256 body checksum validation (S3C-10916)', () = let uploadId; before(done => { - async.series([ - next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), - next => makeS3Request({ - method: 'POST', - authCredentials, - bucket, - objectKey, - queryObj: { uploads: '' }, - }, (err, res) => { - if (err) { return next(err); } - const match = res.body.match(/([^<]+)<\/UploadId>/); - assert(match, `missing UploadId in response: ${res.body}`); - uploadId = match[1]; - return next(); - }), - ], err => { - assert.ifError(err); - done(); - }); + async.series( + [ + next => makeS3Request({ method: 'PUT', authCredentials, bucket }, next), + next => + makeS3Request( + { + method: 'POST', + authCredentials, + bucket, + objectKey, + queryObj: { uploads: '' }, + }, + (err, res) => { + if (err) { + return next(err); + } + const match = res.body.match(/([^<]+)<\/UploadId>/); + assert(match, `missing UploadId in response: ${res.body}`); + uploadId = match[1]; + return next(); + }, + ), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - async.series([ - next => makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - objectKey, - queryObj: { uploadId }, - }, next), - // Delete the object key first (defensive: clears any state left by a previous run). - next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), - next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), - ], err => { - assert.ifError(err); - done(); - }); + async.series( + [ + next => + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + objectKey, + queryObj: { uploadId }, + }, + next, + ), + // Delete the object key first (defensive: clears any state left by a previous run). + next => makeS3Request({ method: 'DELETE', authCredentials, bucket, objectKey }, () => next()), + next => makeS3Request({ method: 'DELETE', authCredentials, bucket }, next), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); - makeMismatchTests(() => - `http://${host}:${port}/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId}`); + makeMismatchTests(() => `http://${host}:${port}/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId}`); describe('with an empty body (zero-byte path)', () => { - makeMismatchTests(() => - `http://${host}:${port}/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId}`, - emptyBody, emptySha256Hex); + makeMismatchTests( + () => `http://${host}:${port}/${bucket}/${objectKey}?partNumber=1&uploadId=${uploadId}`, + emptyBody, + emptySha256Hex, + ); }); }); @@ -232,41 +260,56 @@ describe('SigV4 x-amz-content-sha256 body checksum validation (S3C-10916)', () = // Regression sweep: a wrong-but-well-formed hash is rejected everywhere. Object.entries(bufferedEndpoints).forEach(([apiMethod, ep]) => { it(`should return 400 XAmzContentSHA256Mismatch for ${apiMethod}`, done => { - doRequest(ep.method, `http://${host}:${port}/${bucket}${ep.suffix}`, { - 'x-amz-content-sha256': wrongSha256Hex, - 'content-length': fakeBody.length, - }, fakeBody, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 400, - `expected 400, got ${res.statusCode}: ${res.body}`); - assert.match(res.body, /XAmzContentSHA256Mismatch/, - `expected XAmzContentSHA256Mismatch in "${res.body}"`); - done(); - }); + doRequest( + ep.method, + `http://${host}:${port}/${bucket}${ep.suffix}`, + { + 'x-amz-content-sha256': wrongSha256Hex, + 'content-length': fakeBody.length, + }, + fakeBody, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match( + res.body, + /XAmzContentSHA256Mismatch/, + `expected XAmzContentSHA256Mismatch in "${res.body}"`, + ); + done(); + }, + ); }); }); it('should return 400 XAmzContentSHA256Mismatch for a zero-byte buffered body', done => { const ep = bufferedEndpoints.bucketPutCors; - doRequest(ep.method, `http://${host}:${port}/${bucket}${ep.suffix}`, { - 'x-amz-content-sha256': wrongSha256Hex, - 'content-length': emptyBody.length, - }, emptyBody, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 400, - `expected 400, got ${res.statusCode}: ${res.body}`); - assert.match(res.body, /XAmzContentSHA256Mismatch/, - `expected XAmzContentSHA256Mismatch in "${res.body}"`); - done(); - }); + doRequest( + ep.method, + `http://${host}:${port}/${bucket}${ep.suffix}`, + { + 'x-amz-content-sha256': wrongSha256Hex, + 'content-length': emptyBody.length, + }, + emptyBody, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match( + res.body, + /XAmzContentSHA256Mismatch/, + `expected XAmzContentSHA256Mismatch in "${res.body}"`, + ); + done(); + }, + ); }); // Fails if a new checksumed/buffered method is added without test coverage. it('should exercise every buffered checksumed method', () => { const expected = new Set([...Object.keys(checksumedMethods), 'completeMultipartUpload']); const covered = new Set(Object.keys(bufferedEndpoints)); - expected.forEach(method => - assert(covered.has(method), `missing buffered-endpoint coverage for ${method}`)); + expected.forEach(method => assert(covered.has(method), `missing buffered-endpoint coverage for ${method}`)); }); }); @@ -274,17 +317,25 @@ describe('SigV4 x-amz-content-sha256 body checksum validation (S3C-10916)', () = describe('buffered Scality extensions (skipped on AWS)', () => { Object.entries(scalityExtensionEndpoints).forEach(([apiMethod, ep]) => { itSkipIfAWS(`should return 400 XAmzContentSHA256Mismatch for ${apiMethod}`, done => { - doRequest(ep.method, `http://${host}:${port}/${bucket}${ep.suffix}`, { - 'x-amz-content-sha256': wrongSha256Hex, - 'content-length': fakeBody.length, - }, fakeBody, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 400, - `expected 400, got ${res.statusCode}: ${res.body}`); - assert.match(res.body, /XAmzContentSHA256Mismatch/, - `expected XAmzContentSHA256Mismatch in "${res.body}"`); - done(); - }); + doRequest( + ep.method, + `http://${host}:${port}/${bucket}${ep.suffix}`, + { + 'x-amz-content-sha256': wrongSha256Hex, + 'content-length': fakeBody.length, + }, + fakeBody, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 400, `expected 400, got ${res.statusCode}: ${res.body}`); + assert.match( + res.body, + /XAmzContentSHA256Mismatch/, + `expected XAmzContentSHA256Mismatch in "${res.body}"`, + ); + done(); + }, + ); }); }); }); diff --git a/tests/functional/raw-node/utils/makeRequest.js b/tests/functional/raw-node/utils/makeRequest.js index bfad8e774b..cd9010064e 100644 --- a/tests/functional/raw-node/utils/makeRequest.js +++ b/tests/functional/raw-node/utils/makeRequest.js @@ -11,18 +11,18 @@ const constructStringToSignV2 = require('arsenal/build/lib/auth/v2/constructStri function signGcpRequest(request, credentials, date) { if (!credentials || !credentials.secretKey || !credentials.accessKey) { - throw new Error('Invalid GCP credentials: must have accessKey and secretKey properties. ' + - `Got: ${JSON.stringify(credentials)}`); - } + throw new Error( + 'Invalid GCP credentials: must have accessKey and secretKey properties. ' + + `Got: ${JSON.stringify(credentials)}`, + ); + } // eslint-disable-next-line no-param-reassign request.headers['x-goog-date'] = date.toUTCString(); const data = Object.assign({}, request.headers); const logger = { trace: () => {} }; const stringToSign = constructStringToSignV2(request, data, logger, 'GCP'); // Sign with HMAC-SHA1 - const signature = crypto.createHmac('sha1', credentials.secretKey) - .update(stringToSign) - .digest('base64'); + const signature = crypto.createHmac('sha1', credentials.secretKey).update(stringToSign).digest('base64'); // eslint-disable-next-line no-param-reassign request.headers['Authorization'] = `GOOG1 ${credentials.accessKey}:${signature}`; } @@ -73,9 +73,18 @@ function _decodeURI(uri) { * @return {undefined} - and call callback */ function makeRequest(params, callback) { - const { hostname, port, method, queryObj, headers, path, - authCredentials, requestBody, jsonResponse, - urlForSignature } = params; + const { + hostname, + port, + method, + queryObj, + headers, + path, + authCredentials, + requestBody, + jsonResponse, + urlForSignature, + } = params; const options = { hostname, port, @@ -141,8 +150,16 @@ function makeRequest(params, callback) { if (authCredentials && !params.GCP) { // Pass an explicit payload (never undefined) so generateV4Headers signs // the real body for POST instead of falling back to the querystring. - auth.client.generateV4Headers(req, queryObj || '', - authCredentials.accessKey, authCredentials.secretKey, 's3', undefined, undefined, requestBody || ''); + auth.client.generateV4Headers( + req, + queryObj || '', + authCredentials.accessKey, + authCredentials.secretKey, + 's3', + undefined, + undefined, + requestBody || '', + ); } // restore original URL-encoded path req.path = savedPath; @@ -169,8 +186,7 @@ function makeRequest(params, callback) { * @return {undefined} - and call callback */ function makeS3Request(params, callback) { - const { method, queryObj, headers, bucket, objectKey, authCredentials, requestBody } - = params; + const { method, queryObj, headers, bucket, objectKey, authCredentials, requestBody } = params; const options = { authCredentials, hostname: process.env.AWS_ON_AIR ? 's3.amazonaws.com' : ipAddress, @@ -204,8 +220,7 @@ function makeS3Request(params, callback) { * @return {undefined} - and call callback */ function makeBackbeatRequest(params, callback) { - const { method, headers, bucket, objectKey, resourceType, - authCredentials, requestBody, queryObj } = params; + const { method, headers, bucket, objectKey, resourceType, authCredentials, requestBody, queryObj } = params; const options = { authCredentials, hostname: ipAddress, diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index d39ad4acb6..a45f414a25 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -1302,17 +1302,18 @@ describe('getCopyObjectChecksumAlgorithm', () => { }); }); -const sigV4Auth = 'AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/s3/aws4_request, ' - + 'SignedHeaders=host, Signature=abc'; +const sigV4Auth = + 'AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/s3/aws4_request, ' + 'SignedHeaders=host, Signature=abc'; describe('parseContentSHA256', () => { // build SigV4 header-auth headers carrying the given x-amz-content-sha256 const v4 = value => ({ authorization: sigV4Auth, 'x-amz-content-sha256': value }); it('should return Skip for non-SigV4 header auth, capturing the value', () => { - assert.deepStrictEqual( - parseContentSHA256({ authorization: 'AWS AKID:sig', 'x-amz-content-sha256': 'abc' }), - { type: ContentSHA256Type.Skip, value: 'abc' }); + assert.deepStrictEqual(parseContentSHA256({ authorization: 'AWS AKID:sig', 'x-amz-content-sha256': 'abc' }), { + type: ContentSHA256Type.Skip, + value: 'abc', + }); }); it('should return Skip with null value when there is no auth header', () => { @@ -1320,42 +1321,49 @@ describe('parseContentSHA256', () => { }); it('should return Absent when SigV4 header auth but the header is missing', () => { - assert.deepStrictEqual(parseContentSHA256({ authorization: sigV4Auth }), - { type: ContentSHA256Type.Absent, value: null }); + assert.deepStrictEqual(parseContentSHA256({ authorization: sigV4Auth }), { + type: ContentSHA256Type.Absent, + value: null, + }); }); it('should return Unsigned for UNSIGNED-PAYLOAD', () => { - assert.deepStrictEqual(parseContentSHA256(v4('UNSIGNED-PAYLOAD')), - { type: ContentSHA256Type.Unsigned, value: 'UNSIGNED-PAYLOAD' }); + assert.deepStrictEqual(parseContentSHA256(v4('UNSIGNED-PAYLOAD')), { + type: ContentSHA256Type.Unsigned, + value: 'UNSIGNED-PAYLOAD', + }); }); it('should return Streaming (supported) for a supported streaming token', () => { const tok = 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'; - assert.deepStrictEqual(parseContentSHA256(v4(tok)), - { type: ContentSHA256Type.Streaming, value: tok, supported: true }); + assert.deepStrictEqual(parseContentSHA256(v4(tok)), { + type: ContentSHA256Type.Streaming, + value: tok, + supported: true, + }); }); it('should return Streaming (not supported) for an unsupported streaming token', () => { const tok = 'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD'; - assert.deepStrictEqual(parseContentSHA256(v4(tok)), - { type: ContentSHA256Type.Streaming, value: tok, supported: false }); + assert.deepStrictEqual(parseContentSHA256(v4(tok)), { + type: ContentSHA256Type.Streaming, + value: tok, + supported: false, + }); }); it('should return HexSHA256 for a hex sha256, preserving the raw value', () => { const hex = 'a'.repeat(64); - assert.deepStrictEqual(parseContentSHA256(v4(hex)), - { type: ContentSHA256Type.HexSHA256, value: hex }); + assert.deepStrictEqual(parseContentSHA256(v4(hex)), { type: ContentSHA256Type.HexSHA256, value: hex }); }); it('should return HexSHA256 for an uppercase hex sha256', () => { const hex = 'A'.repeat(64); - assert.deepStrictEqual(parseContentSHA256(v4(hex)), - { type: ContentSHA256Type.HexSHA256, value: hex }); + assert.deepStrictEqual(parseContentSHA256(v4(hex)), { type: ContentSHA256Type.HexSHA256, value: hex }); }); it('should return Invalid for a malformed value', () => { - assert.deepStrictEqual(parseContentSHA256(v4('xxx')), - { type: ContentSHA256Type.Invalid, value: 'xxx' }); + assert.deepStrictEqual(parseContentSHA256(v4('xxx')), { type: ContentSHA256Type.Invalid, value: 'xxx' }); }); }); @@ -1365,76 +1373,92 @@ describe('validateXAmzContentSHA256', () => { const wrongHex = crypto.createHash('sha256').update('other').digest('hex'); it('should return null when the hash matches the body', () => { - assert.ifError(validateXAmzContentSHA256( - { authorization: sigV4Auth, 'x-amz-content-sha256': correctHex }, body)); + assert.ifError( + validateXAmzContentSHA256({ authorization: sigV4Auth, 'x-amz-content-sha256': correctHex }, body), + ); }); it('should return null for an uppercase hash that matches (case-insensitive)', () => { - assert.ifError(validateXAmzContentSHA256( - { authorization: sigV4Auth, 'x-amz-content-sha256': correctHex.toUpperCase() }, body)); + assert.ifError( + validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': correctHex.toUpperCase() }, + body, + ), + ); }); it('should return ContentSHA256Mismatch with calculated/expected details on mismatch', () => { - const result = validateXAmzContentSHA256( - { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, body); + const result = validateXAmzContentSHA256({ authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, body); assert.strictEqual(result.error, ChecksumError.ContentSHA256Mismatch); assert.strictEqual(result.details.expected, wrongHex); assert.strictEqual(result.details.calculated, correctHex); }); it('should return ContentSHA256Missing when the header is absent', () => { - assert.deepStrictEqual(validateXAmzContentSHA256({ authorization: sigV4Auth }, body), - { error: ChecksumError.ContentSHA256Missing }); + assert.deepStrictEqual(validateXAmzContentSHA256({ authorization: sigV4Auth }, body), { + error: ChecksumError.ContentSHA256Missing, + }); }); it('should return ContentSHA256Invalid for a malformed value', () => { - const result = validateXAmzContentSHA256( - { authorization: sigV4Auth, 'x-amz-content-sha256': 'xxx' }, body); + const result = validateXAmzContentSHA256({ authorization: sigV4Auth, 'x-amz-content-sha256': 'xxx' }, body); assert.strictEqual(result.error, ChecksumError.ContentSHA256Invalid); assert.strictEqual(result.details.value, 'xxx'); }); it('should return null for UNSIGNED-PAYLOAD', () => { - assert.ifError(validateXAmzContentSHA256( - { authorization: sigV4Auth, 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }, body)); + assert.ifError( + validateXAmzContentSHA256({ authorization: sigV4Auth, 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }, body), + ); }); it('should return null for a streaming token', () => { - assert.ifError(validateXAmzContentSHA256( - { authorization: sigV4Auth, 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }, body)); + assert.ifError( + validateXAmzContentSHA256( + { authorization: sigV4Auth, 'x-amz-content-sha256': 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' }, + body, + ), + ); }); it('should return null (skip) for non-SigV4 auth even with a wrong hash', () => { - assert.ifError(validateXAmzContentSHA256( - { authorization: 'AWS AKID:sig', 'x-amz-content-sha256': wrongHex }, body)); + assert.ifError( + validateXAmzContentSHA256({ authorization: 'AWS AKID:sig', 'x-amz-content-sha256': wrongHex }, body), + ); }); it('should return null when the hash matches an empty body', () => { const emptyHex = crypto.createHash('sha256').update(Buffer.alloc(0)).digest('hex'); - assert.ifError(validateXAmzContentSHA256( - { authorization: sigV4Auth, 'x-amz-content-sha256': emptyHex }, Buffer.alloc(0))); + assert.ifError( + validateXAmzContentSHA256({ authorization: sigV4Auth, 'x-amz-content-sha256': emptyHex }, Buffer.alloc(0)), + ); }); it('should return ContentSHA256Mismatch for an empty body with a wrong hash', () => { const result = validateXAmzContentSHA256( - { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, Buffer.alloc(0)); + { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, + Buffer.alloc(0), + ); assert.strictEqual(result.error, ChecksumError.ContentSHA256Mismatch); - assert.strictEqual(result.details.calculated, - crypto.createHash('sha256').update(Buffer.alloc(0)).digest('hex')); + assert.strictEqual( + result.details.calculated, + crypto.createHash('sha256').update(Buffer.alloc(0)).digest('hex'), + ); }); describe('mapped through arsenalErrorFromChecksumError', () => { it('should map mismatch to XAmzContentSHA256Mismatch (400)', () => { const result = validateXAmzContentSHA256( - { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, body); + { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, + body, + ); const err = arsenalErrorFromChecksumError(result); assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); assert.strictEqual(err.code, 400); }); it('should map invalid to InvalidArgument (400)', () => { - const result = validateXAmzContentSHA256( - { authorization: sigV4Auth, 'x-amz-content-sha256': 'xxx' }, body); + const result = validateXAmzContentSHA256({ authorization: sigV4Auth, 'x-amz-content-sha256': 'xxx' }, body); const err = arsenalErrorFromChecksumError(result); assert.strictEqual(err.message, 'InvalidArgument'); assert.strictEqual(err.code, 400); diff --git a/tests/unit/api/apiUtils/object/prepareStream.js b/tests/unit/api/apiUtils/object/prepareStream.js index 05bcc628c4..f48a842999 100644 --- a/tests/unit/api/apiUtils/object/prepareStream.js +++ b/tests/unit/api/apiUtils/object/prepareStream.js @@ -14,8 +14,8 @@ const defaultChecksums = { primary: defaultChecksumData, secondary: null }; // A literal payload hash is only verified for SigV4 header-authenticated // requests, so these tests carry an AWS4 Authorization header. -const sigV4Auth = 'AWS4-HMAC-SHA256 Credential=AK/20210101/us-east-1/s3/aws4_request, ' - + 'SignedHeaders=host, Signature=abc'; +const sigV4Auth = + 'AWS4-HMAC-SHA256 Credential=AK/20210101/us-east-1/s3/aws4_request, ' + 'SignedHeaders=host, Signature=abc'; const bodyData = 'the streamed body'; const bodyHex = crypto.createHash('sha256').update(Buffer.from(bodyData)).digest('hex'); @@ -111,10 +111,13 @@ describe('prepareStream', () => { it('should call setExpectedChecksum on ChecksumTransform when trailer event fires', done => { const body = '0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; - const request = makeRequest({ - 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', - 'x-amz-trailer': 'x-amz-checksum-crc32', - }, body); + const request = makeRequest( + { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }, + body, + ); const checksums = makeChecksums('crc32', undefined, true); const result = prepareStream(request, null, checksums, log, done); result.stream.resume(); @@ -128,10 +131,13 @@ describe('prepareStream', () => { it('should call errCb when TrailingChecksumTransform emits an error', done => { // malformed chunked data triggers an error in TrailingChecksumTransform - const request = makeRequest({ - 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', - 'x-amz-trailer': 'x-amz-checksum-crc32', - }, 'zz\r\n'); // invalid hex chunk size + const request = makeRequest( + { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-crc32', + }, + 'zz\r\n', + ); // invalid hex chunk size const checksums = makeChecksums('crc32', undefined, true); prepareStream(request, null, checksums, log, err => { assert.strictEqual(err.message, 'InvalidArgument'); @@ -215,10 +221,13 @@ describe('prepareStream', () => { it('should wire trailer to secondaryChecksumStream for STREAMING-UNSIGNED-PAYLOAD-TRAILER', done => { const body = '0\r\nx-amz-checksum-sha256:test-value\r\n'; - const request = makeRequest({ - 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', - 'x-amz-trailer': 'x-amz-checksum-sha256', - }, body); + const request = makeRequest( + { + 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER', + 'x-amz-trailer': 'x-amz-checksum-sha256', + }, + body, + ); const checksums = { primary: { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }, secondary: { algorithm: 'sha256', isTrailer: true, expected: undefined }, @@ -325,7 +334,9 @@ describe('prepareStream', () => { it('should invoke errCb only once when multiple streams error', () => { const request = makeRequest({ authorization: sigV4Auth, 'x-amz-content-sha256': bodyHex }); let count = 0; - const result = prepareStream(request, null, defaultChecksums, log, () => { count += 1; }); + const result = prepareStream(request, null, defaultChecksums, log, () => { + count += 1; + }); result.contentSHA256Stream.emit('error', errors.InternalError); result.stream.emit('error', errors.InternalError); assert.strictEqual(count, 1); diff --git a/tests/unit/api/apiUtils/object/storeObject.js b/tests/unit/api/apiUtils/object/storeObject.js index ffef1fd717..5d37b56c25 100644 --- a/tests/unit/api/apiUtils/object/storeObject.js +++ b/tests/unit/api/apiUtils/object/storeObject.js @@ -16,8 +16,8 @@ const fakeDataRetrievalInfo = { key: 'test-key', dataStoreName: 'mem' }; // A literal payload hash is only verified for SigV4 header-authenticated // requests, so these tests carry an AWS4 Authorization header. -const sigV4Auth = 'AWS4-HMAC-SHA256 Credential=AK/20210101/us-east-1/s3/aws4_request, ' - + 'SignedHeaders=host, Signature=abc'; +const sigV4Auth = + 'AWS4-HMAC-SHA256 Credential=AK/20210101/us-east-1/s3/aws4_request, ' + 'SignedHeaders=host, Signature=abc'; const helloWorldHex = crypto.createHash('sha256').update(Buffer.from('hello world')).digest('hex'); const wrongHex = 'a'.repeat(64); @@ -170,9 +170,12 @@ describe('dataStore', () => { batchDeleteSucceeds(); putSucceeds(); // CRC32 of 'hello world' is not 0x00000000 (AAAAAA==) - const request = makeStream({ - 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - }, 'hello world'); + const request = makeStream( + { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + }, + 'hello world', + ); const badChecksums = { primary: { algorithm: 'crc32', isTrailer: false, expected: 'AAAAAA==' }, secondary: null, @@ -186,9 +189,12 @@ describe('dataStore', () => { it('should not delete stored data when checksum validation passes', done => { putSucceeds(); - const request = makeStream({ - 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - }, 'hello world'); + const request = makeStream( + { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + }, + 'hello world', + ); const goodChecksums = { primary: { algorithm: 'crc32', isTrailer: false, expected: 'DUoRhQ==' }, secondary: null, @@ -200,49 +206,52 @@ describe('dataStore', () => { }); }); - it('should wait for finish before validating when checksumedStream is not yet writableFinished after data.put', - done => { - let capturedStream; - putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { - capturedStream = stream; - stream.resume(); - // Call cb synchronously — _flush uses Promise.resolve().then() so - // writableFinished is false here, exercising the finish-wait path. - cb(null, fakeDataRetrievalInfo, { completedHash: null }); - }); - const request = makeStream({ + // eslint-disable-next-line max-len + it('should wait for finish before validating when checksumedStream is not yet writableFinished after data.put', done => { + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + stream.resume(); + // Call cb synchronously — _flush uses Promise.resolve().then() so + // writableFinished is false here, exercising the finish-wait path. + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream( + { 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - }, 'hello world'); - const goodChecksums = { - primary: { algorithm: 'crc32', isTrailer: false, expected: 'DUoRhQ==' }, - secondary: null, - }; - dataStore({}, null, request, 0, null, {}, goodChecksums, log, err => { - assert.strictEqual(err, null); - assert(capturedStream.writableFinished); - done(); - }); + }, + 'hello world', + ); + const goodChecksums = { + primary: { algorithm: 'crc32', isTrailer: false, expected: 'DUoRhQ==' }, + secondary: null, + }; + dataStore({}, null, request, 0, null, {}, goodChecksums, log, err => { + assert.strictEqual(err, null); + assert(capturedStream.writableFinished); + done(); }); + }); - it('should delete stored data and call cb with the error when checksumedStream emits error after data.put', - done => { - batchDeleteSucceeds(); - let capturedStream; - putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { - capturedStream = stream; - // Do not resume — keeps writableFinished false, so onError listener is registered. - cb(null, fakeDataRetrievalInfo, { completedHash: null }); - }); - const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { - assert.deepStrictEqual(err, errors.InternalError); - assert(batchDeleteStub.calledOnce); - done(); - }); - // process.nextTick fires before Promise microtasks, so the error arrives - // before _flush resolves, ensuring onError fires rather than onFinish. - process.nextTick(() => capturedStream.emit('error', errors.InternalError)); + // eslint-disable-next-line max-len + it('should delete stored data and call cb with the error when checksumedStream emits error after data.put', done => { + batchDeleteSucceeds(); + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + // Do not resume — keeps writableFinished false, so onError listener is registered. + cb(null, fakeDataRetrievalInfo, { completedHash: null }); }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + assert(batchDeleteStub.calledOnce); + done(); + }); + // process.nextTick fires before Promise microtasks, so the error arrives + // before _flush resolves, ensuring onError fires rather than onFinish. + process.nextTick(() => capturedStream.emit('error', errors.InternalError)); + }); it('should call cb exactly once when finish fires (no double callback)', done => { let cbCount = 0; @@ -282,23 +291,24 @@ describe('dataStore', () => { }); describe('x-amz-content-sha256 body validation', () => { - it('should call cb with XAmzContentSHA256Mismatch and delete stored data when the hash does not match', - done => { - batchDeleteSucceeds(); - putSucceeds(); - const request = makeStream( - { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, 'hello world'); - dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { - assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); - assert(batchDeleteStub.calledOnce); - done(); - }); + // eslint-disable-next-line max-len + it('should call cb with XAmzContentSHA256Mismatch and delete stored data when the hash does not match', done => { + batchDeleteSucceeds(); + putSucceeds(); + const request = makeStream({ authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, 'hello world'); + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { + assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); + assert(batchDeleteStub.calledOnce); + done(); }); + }); it('should not delete stored data when the hash matches the body', done => { putSucceeds(); const request = makeStream( - { authorization: sigV4Auth, 'x-amz-content-sha256': helloWorldHex }, 'hello world'); + { authorization: sigV4Auth, 'x-amz-content-sha256': helloWorldHex }, + 'hello world', + ); dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { assert.strictEqual(err, null); assert(batchDeleteStub.notCalled); @@ -309,8 +319,7 @@ describe('dataStore', () => { it('should validate x-amz-content-sha256 before the secondary checksum', done => { batchDeleteSucceeds(); putSucceeds(); - const request = makeStream( - { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, 'hello world'); + const request = makeStream({ authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, 'hello world'); // The secondary checksum also mismatches, but the content-sha256 // error is checked first and takes precedence. const checksums = { @@ -324,25 +333,27 @@ describe('dataStore', () => { }); }); - it('should call cb with XAmzContentSHA256Mismatch when the hash mismatches and batchDelete also fails', - done => { - batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); - putSucceeds(); - const request = makeStream( - { authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, 'hello world'); - dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { - assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); - done(); - }); + // eslint-disable-next-line max-len + it('should call cb with XAmzContentSHA256Mismatch when the hash mismatches and batchDelete also fails', done => { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); + putSucceeds(); + const request = makeStream({ authorization: sigV4Auth, 'x-amz-content-sha256': wrongHex }, 'hello world'); + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { + assert.strictEqual(err.message, 'XAmzContentSHA256Mismatch'); + done(); }); + }); }); describe('dual-checksum behaviour', () => { it('should return client-facing checksum from secondary and storageChecksum from primary', done => { putSucceeds(); - const request = makeStream({ - 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - }, 'hello world'); + const request = makeStream( + { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + }, + 'hello world', + ); // Primary: crc64nvme (storage), Secondary: crc32 (client-facing) const dualChecksums = { primary: { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }, @@ -362,9 +373,12 @@ describe('dataStore', () => { it('should fail with BadDigest when secondary checksum does not match', done => { batchDeleteSucceeds(); putSucceeds(); - const request = makeStream({ - 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - }, 'hello world'); + const request = makeStream( + { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + }, + 'hello world', + ); const dualChecksums = { primary: { algorithm: 'crc64nvme', isTrailer: false, expected: undefined }, secondary: { algorithm: 'crc32', isTrailer: false, expected: 'AAAAAA==' }, @@ -378,9 +392,12 @@ describe('dataStore', () => { it('should return no storageChecksum when secondary is null', done => { putSucceeds(); - const request = makeStream({ - 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - }, 'hello world'); + const request = makeStream( + { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + }, + 'hello world', + ); const singleChecksums = { primary: { algorithm: 'crc32', isTrailer: false, expected: 'DUoRhQ==' }, secondary: null, @@ -399,9 +416,12 @@ describe('dataStore', () => { it('should call cb with checksum error when validateChecksum fails and batchDelete also fails', done => { batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); putSucceeds(); - const request = makeStream({ - 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', - }, 'hello world'); + const request = makeStream( + { + 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD', + }, + 'hello world', + ); const badChecksums = { primary: { algorithm: 'crc32', isTrailer: false, expected: 'AAAAAA==' }, secondary: null, @@ -423,20 +443,20 @@ describe('dataStore', () => { }); }); - it('should call cb with stream error when checksumedStream errors after data.put and batchDelete also fails', - done => { - batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.BadRequest)); - let capturedStream; - putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { - capturedStream = stream; - cb(null, fakeDataRetrievalInfo, { completedHash: null }); - }); - const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); - dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { - assert.deepStrictEqual(err, errors.InternalError); - done(); - }); - process.nextTick(() => capturedStream.emit('error', errors.InternalError)); + // eslint-disable-next-line max-len + it('should call cb with stream error when checksumedStream errors after data.put and batchDelete also fails', done => { + batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.BadRequest)); + let capturedStream; + putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { + capturedStream = stream; + cb(null, fakeDataRetrievalInfo, { completedHash: null }); + }); + const request = makeStream({ 'x-amz-content-sha256': 'UNSIGNED-PAYLOAD' }); + dataStore({}, null, request, 0, null, {}, defaultChecksums, log, err => { + assert.deepStrictEqual(err, errors.InternalError); + done(); }); + process.nextTick(() => capturedStream.emit('error', errors.InternalError)); + }); }); }); diff --git a/tests/unit/api/apiUtils/validatePayloadProtocol.js b/tests/unit/api/apiUtils/validatePayloadProtocol.js index 7e72c88dcd..12ec570aac 100644 --- a/tests/unit/api/apiUtils/validatePayloadProtocol.js +++ b/tests/unit/api/apiUtils/validatePayloadProtocol.js @@ -5,8 +5,8 @@ const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require(' // validatePayloadProtocol only validates x-amz-content-sha256 for SigV4 // header-authenticated requests (Authorization: "AWS4-..."). -const sigV4 = 'AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/s3/aws4_request, ' - + 'SignedHeaders=host, Signature=abc'; +const sigV4 = + 'AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/s3/aws4_request, ' + 'SignedHeaders=host, Signature=abc'; const sigV2 = 'AWS AKID:signature'; const validSha256Hex = 'a'.repeat(64); From 4de634b325200caefddbda1b0a5a7b4d74f6da20 Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Fri, 3 Jul 2026 17:34:22 +0200 Subject: [PATCH 7/8] CLDSRV-932: remove useless concats --- lib/api/apiUtils/object/createAndStoreObject.js | 2 +- lib/api/objectPut.js | 2 +- tests/unit/api/apiUtils/integrity/validateChecksums.js | 2 +- tests/unit/api/apiUtils/object/prepareStream.js | 2 +- tests/unit/api/apiUtils/object/storeObject.js | 2 +- tests/unit/api/apiUtils/validatePayloadProtocol.js | 3 +-- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 11b3d185bc..2c69a55ccf 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -24,7 +24,7 @@ const { const { externalBackends, versioningNotImplBackends } = constants; const externalVersioningErrorMessage = - 'We do not currently support putting ' + 'a versioned object to a location-constraint of type Azure or GCP.'; + 'We do not currently support putting a versioned object to a location-constraint of type Azure or GCP.'; /** * Validate and compute the checksum for a zero-size object body. diff --git a/lib/api/objectPut.js b/lib/api/objectPut.js index 4025e90a0d..d226729ffd 100644 --- a/lib/api/objectPut.js +++ b/lib/api/objectPut.js @@ -126,7 +126,7 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) { return callback(err, responseHeaders); } if (bucket.hasDeletedFlag() && canonicalID !== bucket.getOwner()) { - log.trace('deleted flag on bucket and request ' + 'from non-owner account'); + log.trace('deleted flag on bucket and request from non-owner account'); monitoring.promMetrics('PUT', bucketName, 404, 'putObject'); return callback(errors.NoSuchBucket); } diff --git a/tests/unit/api/apiUtils/integrity/validateChecksums.js b/tests/unit/api/apiUtils/integrity/validateChecksums.js index a45f414a25..c56f44d4a8 100644 --- a/tests/unit/api/apiUtils/integrity/validateChecksums.js +++ b/tests/unit/api/apiUtils/integrity/validateChecksums.js @@ -1303,7 +1303,7 @@ describe('getCopyObjectChecksumAlgorithm', () => { }); const sigV4Auth = - 'AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/s3/aws4_request, ' + 'SignedHeaders=host, Signature=abc'; + 'AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=abc'; describe('parseContentSHA256', () => { // build SigV4 header-auth headers carrying the given x-amz-content-sha256 diff --git a/tests/unit/api/apiUtils/object/prepareStream.js b/tests/unit/api/apiUtils/object/prepareStream.js index f48a842999..37f9456bf9 100644 --- a/tests/unit/api/apiUtils/object/prepareStream.js +++ b/tests/unit/api/apiUtils/object/prepareStream.js @@ -15,7 +15,7 @@ const defaultChecksums = { primary: defaultChecksumData, secondary: null }; // A literal payload hash is only verified for SigV4 header-authenticated // requests, so these tests carry an AWS4 Authorization header. const sigV4Auth = - 'AWS4-HMAC-SHA256 Credential=AK/20210101/us-east-1/s3/aws4_request, ' + 'SignedHeaders=host, Signature=abc'; + 'AWS4-HMAC-SHA256 Credential=AK/20210101/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=abc'; const bodyData = 'the streamed body'; const bodyHex = crypto.createHash('sha256').update(Buffer.from(bodyData)).digest('hex'); diff --git a/tests/unit/api/apiUtils/object/storeObject.js b/tests/unit/api/apiUtils/object/storeObject.js index 5d37b56c25..07bbc0b671 100644 --- a/tests/unit/api/apiUtils/object/storeObject.js +++ b/tests/unit/api/apiUtils/object/storeObject.js @@ -17,7 +17,7 @@ const fakeDataRetrievalInfo = { key: 'test-key', dataStoreName: 'mem' }; // A literal payload hash is only verified for SigV4 header-authenticated // requests, so these tests carry an AWS4 Authorization header. const sigV4Auth = - 'AWS4-HMAC-SHA256 Credential=AK/20210101/us-east-1/s3/aws4_request, ' + 'SignedHeaders=host, Signature=abc'; + 'AWS4-HMAC-SHA256 Credential=AK/20210101/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=abc'; const helloWorldHex = crypto.createHash('sha256').update(Buffer.from('hello world')).digest('hex'); const wrongHex = 'a'.repeat(64); diff --git a/tests/unit/api/apiUtils/validatePayloadProtocol.js b/tests/unit/api/apiUtils/validatePayloadProtocol.js index 12ec570aac..0a5e336c38 100644 --- a/tests/unit/api/apiUtils/validatePayloadProtocol.js +++ b/tests/unit/api/apiUtils/validatePayloadProtocol.js @@ -5,8 +5,7 @@ const { unsupportedSignatureChecksums, supportedSignatureChecksums } = require(' // validatePayloadProtocol only validates x-amz-content-sha256 for SigV4 // header-authenticated requests (Authorization: "AWS4-..."). -const sigV4 = - 'AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/s3/aws4_request, ' + 'SignedHeaders=host, Signature=abc'; +const sigV4 = 'AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=abc'; const sigV2 = 'AWS AKID:signature'; const validSha256Hex = 'a'.repeat(64); From 37fa6dc288ad94e4d0398674591c791c24ed481d Mon Sep 17 00:00:00 2001 From: Leif Henriksen Date: Fri, 3 Jul 2026 22:40:51 +0200 Subject: [PATCH 8/8] CLDSRV-932: bump arsenal to 8.5.3 --- package.json | 2 +- yarn.lock | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index dbd48351cd..8f4b7bfc8d 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "@opentelemetry/instrumentation-ioredis": "~0.64.0", "@opentelemetry/instrumentation-mongodb": "~0.69.0", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/arsenal#8.5.2", + "arsenal": "git+https://github.com/scality/arsenal#8.5.3", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", diff --git a/yarn.lock b/yarn.lock index 4b38e435c0..0769c5f221 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6595,9 +6595,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/arsenal#8.5.2": - version "8.5.2" - resolved "git+https://github.com/scality/arsenal#cd9ddfca043b522c9199a0edbf23a73dbc7973fd" +"arsenal@git+https://github.com/scality/arsenal#8.5.3": + version "8.5.3" + resolved "git+https://github.com/scality/arsenal#3ea1e99ef7119da9c656935afbaefa2c9b7138eb" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0" @@ -12844,7 +12844,6 @@ webidl-conversions@^7.0.0: "werelogs@github:scality/werelogs#8.2.2", werelogs@scality/werelogs#8.2.2: version "8.2.2" - uid e53bef5145697bf8af940dcbe59408988d64854f resolved "https://codeload.github.com/scality/werelogs/tar.gz/e53bef5145697bf8af940dcbe59408988d64854f" dependencies: fast-safe-stringify "^2.1.1" @@ -12857,7 +12856,7 @@ werelogs@scality/werelogs#8.2.0: fast-safe-stringify "^2.1.1" safe-json-stringify "^1.2.0" -"werelogs@scality/werelogs#semver:^8.2.4": +werelogs@scality/werelogs#8.2.4, "werelogs@scality/werelogs#semver:^8.2.4": version "8.2.4" resolved "https://codeload.github.com/scality/werelogs/tar.gz/a7bbb5917a08b035d3763b24b070d517483d6982" dependencies: