From 130f9371c41efb4d95662e5ce0b1df0e5830808e Mon Sep 17 00:00:00 2001 From: Taylor McKinnon Date: Wed, 1 Apr 2026 12:33:10 -0700 Subject: [PATCH] impr(CLDSRV-854): Update precheck --- lib/api/api.js | 423 ++++++++++++++++++++++++------------------------- 1 file changed, 207 insertions(+), 216 deletions(-) diff --git a/lib/api/api.js b/lib/api/api.js index 195441c3fe..b5db1df696 100644 --- a/lib/api/api.js +++ b/lib/api/api.js @@ -83,9 +83,10 @@ const constants = require('../../constants'); const { config } = require('../Config.js'); const { validateMethodChecksumNoChunking } = require('./apiUtils/integrity/validateChecksums'); const { - getRateLimitFromCache, - checkRateLimitWithConfig, - rateLimitApiActions, + requestNeedsRateCheck, + getCachedRateLimitConfig, + buildRateChecksFromConfig, + checkRateLimitsForRequest, } = require('./apiUtils/rateLimit/helpers'); const monitoringMap = policies.actionMaps.actionMonitoringMapS3; @@ -154,6 +155,180 @@ function handleAuthorizationResults(request, authorizationResults, apiMethod, re return callback(null, { returnTagCount }); } +function callApiHandler(apiMethod, apiHandler, request, response, log, callback) { + let returnTagCount = true; + + const validationRes = validateQueryAndHeaders(request, log); + if (validationRes.error) { + log.debug('request query / header validation failed', { + error: validationRes.error, + method: 'api.callApiMethod', + }); + return process.nextTick(callback, validationRes.error); + } + + // no need to check auth on website or cors preflight requests + if (apiMethod === 'websiteGet' || apiMethod === 'websiteHead' || + apiMethod === 'corsPreflight') { + request.actionImplicitDenies = false; + return apiHandler(request, log, callback); + } + + const { sourceBucket, sourceObject, sourceVersionId, parsingError } = + parseCopySource(apiMethod, request.headers['x-amz-copy-source']); + if (parsingError) { + log.debug('error parsing copy source', { + error: parsingError, + }); + return process.nextTick(callback, parsingError); + } + + const { httpHeadersSizeError } = checkHttpHeadersSize(request.headers); + if (httpHeadersSizeError) { + log.debug('http header size limit exceeded', { + error: httpHeadersSizeError, + }); + return process.nextTick(callback, httpHeadersSizeError); + } + + const requestContexts = prepareRequestContexts(apiMethod, request, + sourceBucket, sourceObject, sourceVersionId); + + // Extract all the _apiMethods and store them in an array + const apiMethods = requestContexts ? requestContexts.map(context => context._apiMethod) : []; + // Attach the names to the current request + request.apiMethods = apiMethods; + + return async.waterfall([ + next => auth.server.doAuth( + request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => { + if (request.serverAccessLog) { + request.serverAccessLog.authInfo = userInfo; + } + if (err) { + // VaultClient returns standard errors, but the route requires + // Arsenal errors + const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError; + log.trace('authentication error', { error: err }); + return next(arsenalError); + } + return next(null, userInfo, authorizationResults, streamingV4Params, infos); + }, 's3', requestContexts), + (userInfo, authorizationResults, streamingV4Params, infos, next) => { + const authNames = { accountName: userInfo.getAccountDisplayName() }; + if (userInfo.isRequesterAnIAMUser()) { + authNames.userName = userInfo.getIAMdisplayName(); + } + if (isRequesterASessionUser(userInfo)) { + authNames.sessionName = userInfo.getShortid().split(':')[1]; + } + log.addDefaultFields(authNames); + if (request.serverAccessLog) { + request.serverAccessLog.analyticsAccountName = authNames.accountName; + request.serverAccessLog.analyticsUserName = authNames.userName; + } + if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') { + return next(null, userInfo, authorizationResults, streamingV4Params, infos); + } + // issue 100 Continue to the client + writeContinue(request, response); + + const defaultMaxBodyLength = request.method === 'POST' ? + constants.oneMegaBytes : constants.halfMegaBytes; + const MAX_BODY_LENGTH = config.apiBodySizeLimits[apiMethod] || defaultMaxBodyLength; + const post = []; + let bodyLength = 0; + request.on('data', chunk => { + bodyLength += chunk.length; + // Sanity check on post length + if (bodyLength <= MAX_BODY_LENGTH) { + post.push(chunk); + } + }); + + request.on('error', err => { + log.trace('error receiving request', { + error: err, + }); + return next(errors.InternalError); + }); + + request.on('end', () => { + if (request.serverAccessLog) { + request.serverAccessLog.startTurnAroundTime = process.hrtime.bigint(); + } + + if (bodyLength > MAX_BODY_LENGTH) { + log.error('body length is too long for request type', + { bodyLength }); + return next(errors.InvalidRequest); + } + + const buff = Buffer.concat(post, bodyLength); + + const err = validateMethodChecksumNoChunking(request, buff, log); + if (err) { + return next(err); + } + + // Convert array of post buffers into one string + request.post = buff.toString(); + return next(null, userInfo, authorizationResults, streamingV4Params, infos); + }); + return undefined; + }, + // Tag condition keys require information from CloudServer for evaluation + (userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth( + authorizationResults, + request, + requestContexts, + apiMethod, + log, + (err, authResultsWithTags) => { + if (err) { + log.trace('tag authentication error', { error: err }); + return next(err); + } + return next(null, userInfo, authResultsWithTags, streamingV4Params, infos); + }, + ), + (userInfo, authorizationResults, streamingV4Params, infos, next) => handleAuthorizationResults( + request, authorizationResults, apiMethod, returnTagCount, log, (err, res) => { + request.accountQuotas = infos?.accountQuota; + request.accountLimits = infos?.limits; + if (err) { + return next(err); + } + returnTagCount = res.returnTagCount; + return next(null, userInfo, authorizationResults, streamingV4Params); + }), + ], (err, userInfo, authorizationResults, streamingV4Params) => { + if (err) { + return callback(err); + } + const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5, + (hook, done) => hook(err, done), + () => callback(err, ...results)); + + if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') { + request._response = response; + return apiHandler(userInfo, request, streamingV4Params, + log, methodCallback, authorizationResults); + } + if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') { + return apiHandler(userInfo, request, sourceBucket, + sourceObject, sourceVersionId, log, methodCallback); + } + if (apiMethod === 'objectGet') { + // remove objectGetTagging/objectGetTaggingVersion from apiMethods, these were added by + // prepareRequestContexts to determine the value of returnTagCount. + request.apiMethods = request.apiMethods.filter(methodName => !methodName.includes('Tagging')); + return apiHandler(userInfo, request, returnTagCount, log, callback); + } + return apiHandler(userInfo, request, log, methodCallback); + }); +} + const api = { callApiMethod(apiMethod, request, response, log, callback) { // Attach the apiMethod method to the request, so it can used by monitoring in the server @@ -186,236 +361,52 @@ const api = { request.serverAccessLog.objectKey = request.objectKey; request.serverAccessLog.analyticsAction = actionLog; } - let returnTagCount = true; // Initialize rate limit tracker flag - request.rateLimitAlreadyChecked = false; + request.rateLimitBucketAlreadyChecked = false; + request.rateLimitAccountAlreadyChecked = false; + + const apiHandler = this[apiMethod]; // Process the request with validation, authentication, and execution - const processRequest = () => { - const validationRes = validateQueryAndHeaders(request, log); - if (validationRes.error) { - log.debug('request query / header validation failed', { - error: validationRes.error, - method: 'api.callApiMethod', - }); - return process.nextTick(callback, validationRes.error); - } - // no need to check auth on website or cors preflight requests - if (apiMethod === 'websiteGet' || apiMethod === 'websiteHead' || - apiMethod === 'corsPreflight') { - request.actionImplicitDenies = false; - return this[apiMethod](request, log, callback); + if (request.bucketName === undefined || !requestNeedsRateCheck(request)) { + return process.nextTick(callApiHandler, apiMethod, apiHandler, request, response, log, callback); } - const { sourceBucket, sourceObject, sourceVersionId, parsingError } = - parseCopySource(apiMethod, request.headers['x-amz-copy-source']); - if (parsingError) { - log.debug('error parsing copy source', { - error: parsingError, - }); - return process.nextTick(callback, parsingError); + const checks = []; + const rateLimitConfig = getCachedRateLimitConfig(request); + + if (rateLimitConfig.bucket !== undefined) { + request.rateLimitBucketAlreadyChecked = true; + checks.push(...buildRateChecksFromConfig('bucket', request.bucketName, rateLimitConfig.bucket)); } - const { httpHeadersSizeError } = checkHttpHeadersSize(request.headers); - if (httpHeadersSizeError) { - log.debug('http header size limit exceeded', { - error: httpHeadersSizeError, - }); - return process.nextTick(callback, httpHeadersSizeError); + if (rateLimitConfig.account !== undefined) { + request.rateLimitAccountAlreadyChecked = true; + checks.push(...buildRateChecksFromConfig('account', rateLimitConfig.bucketOwner, rateLimitConfig.account)); } - const requestContexts = prepareRequestContexts(apiMethod, request, - sourceBucket, sourceObject, sourceVersionId); - - // Extract all the _apiMethods and store them in an array - const apiMethods = requestContexts ? requestContexts.map(context => context._apiMethod) : []; - // Attach the names to the current request - request.apiMethods = apiMethods; - - return async.waterfall([ - next => auth.server.doAuth( - request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => { - if (request.serverAccessLog) { - request.serverAccessLog.authInfo = userInfo; - } - if (err) { - // VaultClient returns standard errors, but the route requires - // Arsenal errors - const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError; - log.trace('authentication error', { error: err }); - return next(arsenalError); - } - return next(null, userInfo, authorizationResults, streamingV4Params, infos); - }, 's3', requestContexts), - (userInfo, authorizationResults, streamingV4Params, infos, next) => { - const authNames = { accountName: userInfo.getAccountDisplayName() }; - if (userInfo.isRequesterAnIAMUser()) { - authNames.userName = userInfo.getIAMdisplayName(); - } - if (isRequesterASessionUser(userInfo)) { - authNames.sessionName = userInfo.getShortid().split(':')[1]; - } - log.addDefaultFields(authNames); + if (checks.length > 0) { + const { allowed, rateLimitSource } = checkRateLimitsForRequest(checks, log); + if (!allowed) { + log.addDefaultFields({ + rateLimited: true, + rateLimitSource, + }); + // Add to server access log if (request.serverAccessLog) { - request.serverAccessLog.analyticsAccountName = authNames.accountName; - request.serverAccessLog.analyticsUserName = authNames.userName; - } - if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') { - return next(null, userInfo, authorizationResults, streamingV4Params, infos); + /* eslint-disable no-param-reassign */ + request.serverAccessLog.rateLimited = true; + request.serverAccessLog.rateLimitSource = rateLimitSource; + /* eslint-enable no-param-reassign */ } - // issue 100 Continue to the client - writeContinue(request, response); - - const defaultMaxBodyLength = request.method === 'POST' ? - constants.oneMegaBytes : constants.halfMegaBytes; - const MAX_BODY_LENGTH = config.apiBodySizeLimits[apiMethod] || defaultMaxBodyLength; - const post = []; - let bodyLength = 0; - request.on('data', chunk => { - bodyLength += chunk.length; - // Sanity check on post length - if (bodyLength <= MAX_BODY_LENGTH) { - post.push(chunk); - } - }); - - request.on('error', err => { - log.trace('error receiving request', { - error: err, - }); - return next(errors.InternalError); - }); - request.on('end', () => { - if (request.serverAccessLog) { - request.serverAccessLog.startTurnAroundTime = process.hrtime.bigint(); - } - - if (bodyLength > MAX_BODY_LENGTH) { - log.error('body length is too long for request type', - { bodyLength }); - return next(errors.InvalidRequest); - } - - const buff = Buffer.concat(post, bodyLength); - - const err = validateMethodChecksumNoChunking(request, buff, log); - if (err) { - return next(err); - } - - // Convert array of post buffers into one string - request.post = buff.toString(); - return next(null, userInfo, authorizationResults, streamingV4Params, infos); - }); - return undefined; - }, - // Tag condition keys require information from CloudServer for evaluation - (userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth( - authorizationResults, - request, - requestContexts, - apiMethod, - log, - (err, authResultsWithTags) => { - if (err) { - log.trace('tag authentication error', { error: err }); - return next(err); - } - return next(null, userInfo, authResultsWithTags, streamingV4Params, infos); - }, - ), - (userInfo, authorizationResults, streamingV4Params, infos, next) => handleAuthorizationResults( - request, authorizationResults, apiMethod, returnTagCount, log, (err, res) => { - request.accountQuotas = infos?.accountQuota; - if (err) { - return next(err); - } - returnTagCount = res.returnTagCount; - return next(null, userInfo, authorizationResults, streamingV4Params); - }), - ], (err, userInfo, authorizationResults, streamingV4Params) => { - if (err) { - return callback(err); - } - const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5, - (hook, done) => hook(err, done), - () => callback(err, ...results)); - - if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') { - request._response = response; - return this[apiMethod](userInfo, request, streamingV4Params, - log, methodCallback, authorizationResults); - } - if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') { - return this[apiMethod](userInfo, request, sourceBucket, - sourceObject, sourceVersionId, log, methodCallback); - } - if (apiMethod === 'objectGet') { - // remove objectGetTagging/objectGetTaggingVersion from apiMethods, these were added by - // prepareRequestContexts to determine the value of returnTagCount. - request.apiMethods = request.apiMethods.filter(methodName => !methodName.includes('Tagging')); - return this[apiMethod](userInfo, request, returnTagCount, log, callback); - } - return this[apiMethod](userInfo, request, log, methodCallback); - }); - }; // End of processRequest helper function - - const applyRateLimit = request.bucketName - && config.rateLimiting?.enabled - && !rateLimitApiActions.includes(apiMethod) // Don't limit any rate limit admin actions - && !request.isInternalServiceRequest; // Don't limit any calls from internal services - - if (applyRateLimit) { - // Cache-only rate limit check (fast path, no metadata fetch) - // If config is cached, apply rate limiting now - // If not cached, metadata validation functions will handle it - const cachedConfig = getRateLimitFromCache(request.bucketName); - - if (cachedConfig !== undefined) { - // Cache hit - apply rate limiting NOW (fast path) - return checkRateLimitWithConfig( - request.bucketName, - cachedConfig, - log, - (rateLimitErr, rateLimited) => { - if (rateLimitErr) { - log.error('Rate limit check error in api.js', { - error: rateLimitErr, - }); - } - - if (rateLimited) { - log.addDefaultFields({ - rateLimited: true, - rateLimitSource: cachedConfig.source, - }); - // Add to server access log - if (request.serverAccessLog) { - /* eslint-disable no-param-reassign */ - request.serverAccessLog.rateLimited = true; - request.serverAccessLog.rateLimitSource = cachedConfig.source; - /* eslint-enable no-param-reassign */ - } - return callback(config.rateLimiting.error); - } - - // Set tracker - rate limiting already applied - // eslint-disable-next-line no-param-reassign - request.rateLimitAlreadyChecked = true; - - // Continue with normal flow - return processRequest(); - } - ); + return process.nextTick(callback, config.rateLimiting.error); } } - // Cache miss or no bucket name - continue to metadata validation - // Rate limiting will happen there after metadata fetch - return processRequest(); + return process.nextTick(callApiHandler, apiMethod, apiHandler, request, response, log, callback); }, bucketDelete, bucketDeleteCors,