diff --git a/lib/aws/s3-body-to-string.js b/lib/aws/s3-body-to-string.js new file mode 100644 index 000000000..2318a4f1e --- /dev/null +++ b/lib/aws/s3-body-to-string.js @@ -0,0 +1,98 @@ +'use strict'; + +const ServerlessError = require('../serverless-error'); + +function chunkToBuffer(chunk, encoding) { + if (typeof chunk === 'string') { + return Buffer.from(chunk, encoding); + } + + if (Buffer.isBuffer(chunk)) { + return chunk; + } + + if (chunk instanceof Uint8Array) { + return Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength); + } + + if (chunk instanceof ArrayBuffer) { + return Buffer.from(chunk); + } + + return Buffer.from(chunk); +} + +async function webReadableStreamToString(stream, encoding) { + const chunks = []; + const reader = stream.getReader(); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + + chunks.push(chunkToBuffer(value, encoding)); + } + } finally { + reader.releaseLock(); + } + + return Buffer.concat(chunks).toString(encoding); +} + +async function nodeReadableStreamToString(stream, encoding) { + const chunks = []; + + for await (const chunk of stream) { + chunks.push(chunkToBuffer(chunk, encoding)); + } + + return Buffer.concat(chunks).toString(encoding); +} + +async function s3BodyToString(body, { encoding = 'utf8' } = {}) { + if (body == null) { + return ''; + } + + if (typeof body === 'string') { + return body; + } + + if (typeof body.transformToString === 'function') { + return body.transformToString(encoding); + } + + if (Buffer.isBuffer(body)) { + return body.toString(encoding); + } + + if (body instanceof Uint8Array) { + return Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString(encoding); + } + + if (body instanceof ArrayBuffer) { + return Buffer.from(body).toString(encoding); + } + + if (typeof Blob === 'function' && body instanceof Blob) { + return Buffer.from(await body.arrayBuffer()).toString(encoding); + } + + if (typeof body.getReader === 'function') { + return webReadableStreamToString(body, encoding); + } + + if (typeof body[Symbol.asyncIterator] === 'function') { + return nodeReadableStreamToString(body, encoding); + } + + throw new ServerlessError( + `Unsupported S3 GetObject Body type: ${Object.prototype.toString.call(body)}`, + 'UNSUPPORTED_S3_GET_OBJECT_BODY' + ); +} + +module.exports = s3BodyToString; diff --git a/lib/configuration/variables/sources/instance-dependent/create-cached-aws-variable-source-command-sender.js b/lib/configuration/variables/sources/instance-dependent/create-cached-aws-variable-source-command-sender.js new file mode 100644 index 000000000..321e8380f --- /dev/null +++ b/lib/configuration/variables/sources/instance-dependent/create-cached-aws-variable-source-command-sender.js @@ -0,0 +1,105 @@ +'use strict'; + +const promiseLimit = require('ext/promise/limit').bind(Promise); +const deepSortObjectByKey = require('../../../../utils/deep-sort-object-by-key'); + +const maxConcurrentVariableSourceCommands = 2; + +function normalizeRegion(region) { + return region === undefined ? null : region; +} + +function getCommandName(Command) { + return Command.name || 'Command'; +} + +function createCacheKey({ commandName, region, input }) { + return JSON.stringify({ + command: commandName, + region: normalizeRegion(region), + input: input === undefined ? null : deepSortObjectByKey(input), + }); +} + +function createCachedAwsVariableSourceCommandSender({ + getProvider, + Client, + transformResult = ({ result }) => result, +}) { + const clients = new Map(); + const requests = new Map(); + const sendQueue = promiseLimit(maxConcurrentVariableSourceCommands, async (task) => task()); + let provider; + + function resolveProvider() { + if (!provider) provider = getProvider(); + return provider; + } + + function getEffectiveRegion(region) { + return region === undefined ? resolveProvider().getRegion() : region; + } + + async function getClient(region) { + const cacheKey = JSON.stringify({ region: normalizeRegion(region) }); + + if (!clients.has(cacheKey)) { + const clientPromise = (async () => { + const config = await resolveProvider().getAwsSdkV3Config({ region }); + return new Client(config); + })(); + + clients.set( + cacheKey, + clientPromise.catch((error) => { + clients.delete(cacheKey); + throw error; + }) + ); + } + + return clients.get(cacheKey); + } + + async function send(Command, input, options = {}) { + const { region } = options; + const effectiveRegion = getEffectiveRegion(region); + const commandName = getCommandName(Command); + const cacheKey = createCacheKey({ + commandName, + region: effectiveRegion, + input, + }); + + if (!requests.has(cacheKey)) { + const requestPromise = (async () => { + const client = await getClient(effectiveRegion); + + return sendQueue(async () => { + const result = await client.send(new Command(input)); + return transformResult({ + result, + commandName, + input, + region, + effectiveRegion, + }); + }); + })(); + + requests.set( + cacheKey, + requestPromise.catch((error) => { + requests.delete(cacheKey); + throw error; + }) + ); + } + + return requests.get(cacheKey); + } + + return { send }; +} + +module.exports = createCachedAwsVariableSourceCommandSender; diff --git a/lib/configuration/variables/sources/instance-dependent/get-aws.js b/lib/configuration/variables/sources/instance-dependent/get-aws.js index 45b7a021d..96c0eed17 100644 --- a/lib/configuration/variables/sources/instance-dependent/get-aws.js +++ b/lib/configuration/variables/sources/instance-dependent/get-aws.js @@ -1,9 +1,16 @@ 'use strict'; const ensureString = require('type/string/ensure'); +const { STSClient, GetCallerIdentityCommand } = require('@aws-sdk/client-sts'); const ServerlessError = require('../../../../serverless-error'); +const createCachedAwsVariableSourceCommandSender = require('./create-cached-aws-variable-source-command-sender'); module.exports = (serverlessInstance) => { + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider: () => serverlessInstance.getProvider('aws'), + Client: STSClient, + }); + return { resolve: async ({ address, options, resolveConfigurationProperty }) => { if (!address) { @@ -21,9 +28,7 @@ module.exports = (serverlessInstance) => { switch (address) { case 'accountId': { - const { Account } = await serverlessInstance - .getProvider('aws') - .request('STS', 'getCallerIdentity', {}, { useCache: true }); + const { Account } = await sender.send(GetCallerIdentityCommand, {}); return { value: Account }; } case 'region': { diff --git a/lib/configuration/variables/sources/instance-dependent/get-cf.js b/lib/configuration/variables/sources/instance-dependent/get-cf.js index ba0df3928..79da5d77d 100644 --- a/lib/configuration/variables/sources/instance-dependent/get-cf.js +++ b/lib/configuration/variables/sources/instance-dependent/get-cf.js @@ -1,9 +1,28 @@ 'use strict'; const ensureString = require('type/string/ensure'); +const { CloudFormationClient, DescribeStacksCommand } = require('@aws-sdk/client-cloudformation'); const ServerlessError = require('../../../../serverless-error'); +const createCachedAwsVariableSourceCommandSender = require('./create-cached-aws-variable-source-command-sender'); + +function isStackNotFoundError(error) { + const message = error && error.message ? error.message : ''; + + return ( + message.includes('does not exist') && + (error.name === 'ValidationError' || + error.Code === 'ValidationError' || + error.code === 'ValidationError' || + error.code === 'AWS_CLOUD_FORMATION_DESCRIBE_STACKS_VALIDATION_ERROR') + ); +} module.exports = (serverlessInstance) => { + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider: () => serverlessInstance.getProvider('aws'), + Client: CloudFormationClient, + }); + return { resolve: async ({ address, params }) => { // cf(region = null):stackName.outputLogicalId @@ -31,27 +50,19 @@ module.exports = (serverlessInstance) => { const result = await (async () => { try { - return await serverlessInstance - .getProvider('aws') - .request( - 'CloudFormation', - 'describeStacks', - { StackName: stackName }, - { useCache: true, region: params && params[0] } - ); + return await sender.send( + DescribeStacksCommand, + { StackName: stackName }, + { region: params && params[0] } + ); } catch (error) { - if ( - error.code === 'AWS_CLOUD_FORMATION_DESCRIBE_STACKS_VALIDATION_ERROR' && - error.message.includes('does not exist') - ) { - return null; - } + if (isStackNotFoundError(error)) return null; throw error; } })(); if (!result) return { value: null }; - const outputs = result.Stacks[0].Outputs; + const outputs = result.Stacks[0].Outputs || []; const output = outputs.find((x) => x.OutputKey === outputLogicalId); return { value: output ? output.OutputValue : null }; diff --git a/lib/configuration/variables/sources/instance-dependent/get-s3.js b/lib/configuration/variables/sources/instance-dependent/get-s3.js index 32a120446..109b32983 100644 --- a/lib/configuration/variables/sources/instance-dependent/get-s3.js +++ b/lib/configuration/variables/sources/instance-dependent/get-s3.js @@ -1,9 +1,28 @@ 'use strict'; const ensureString = require('type/string/ensure'); +const { S3Client, GetObjectCommand } = require('@aws-sdk/client-s3'); const ServerlessError = require('../../../../serverless-error'); +const s3BodyToString = require('../../../../aws/s3-body-to-string'); +const createCachedAwsVariableSourceCommandSender = require('./create-cached-aws-variable-source-command-sender'); + +function isNoSuchKeyError(error) { + return ( + error && + (error.name === 'NoSuchKey' || + error.Code === 'NoSuchKey' || + error.code === 'NoSuchKey' || + error.code === 'AWS_S3_GET_OBJECT_NO_SUCH_KEY') + ); +} module.exports = (serverlessInstance) => { + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider: () => serverlessInstance.getProvider('aws'), + Client: S3Client, + transformResult: async ({ result }) => s3BodyToString(result.Body), + }); + return { resolve: async ({ address }) => { // s3:bucketName/key @@ -31,19 +50,16 @@ module.exports = (serverlessInstance) => { const result = await (async () => { try { - return await serverlessInstance - .getProvider('aws') - .request('S3', 'getObject', { Bucket: bucketName, Key: key }, { useCache: true }); + return await sender.send(GetObjectCommand, { Bucket: bucketName, Key: key }); } catch (error) { - // Check for normalized error code instead of native one - if (error.code === 'AWS_S3_GET_OBJECT_NO_SUCH_KEY') return null; + if (isNoSuchKeyError(error)) return null; throw error; } })(); - if (!result) return { value: null }; + if (result == null) return { value: null }; - return { value: String(result.Body) }; + return { value: result }; }, }; }; diff --git a/lib/configuration/variables/sources/instance-dependent/get-ssm.js b/lib/configuration/variables/sources/instance-dependent/get-ssm.js index ee719e7cf..70f84d064 100644 --- a/lib/configuration/variables/sources/instance-dependent/get-ssm.js +++ b/lib/configuration/variables/sources/instance-dependent/get-ssm.js @@ -1,9 +1,26 @@ 'use strict'; const ensureString = require('type/string/ensure'); +const { SSMClient, GetParameterCommand } = require('@aws-sdk/client-ssm'); const ServerlessError = require('../../../../serverless-error'); +const createCachedAwsVariableSourceCommandSender = require('./create-cached-aws-variable-source-command-sender'); + +function isParameterNotFoundError(error) { + return ( + error && + (error.name === 'ParameterNotFound' || + error.Code === 'ParameterNotFound' || + error.code === 'ParameterNotFound' || + error.code === 'AWS_S_S_M_GET_PARAMETER_PARAMETER_NOT_FOUND') + ); +} module.exports = (serverlessInstance) => { + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider: () => serverlessInstance.getProvider('aws'), + Client: SSMClient, + }); + return { resolve: async ({ address, params }) => { // ssm(region = null):param-path @@ -27,23 +44,16 @@ module.exports = (serverlessInstance) => { const result = await (async () => { try { - return await serverlessInstance.getProvider('aws').request( - 'SSM', - 'getParameter', + return await sender.send( + GetParameterCommand, { Name: address, WithDecryption: !shouldSkipDecryption, }, - { useCache: true, region } + { region } ); } catch (error) { - // Check for normalized error code instead of native one - if ( - error.code === 'AWS_S_S_M_GET_PARAMETER_PARAMETER_NOT_FOUND' || - error.code === 'ParameterNotFound' - ) { - return null; - } + if (isParameterNotFoundError(error)) return null; throw error; } })(); diff --git a/package.json b/package.json index 7180e96fd..1f64e0d44 100644 --- a/package.json +++ b/package.json @@ -28,11 +28,14 @@ "dependencies": { "@apidevtools/json-schema-ref-parser": "^15.3.5", "@aws-sdk/client-api-gateway": "^3.975.0", + "@aws-sdk/client-cloudformation": "^3.975.0", "@aws-sdk/client-cognito-identity-provider": "^3.975.0", "@aws-sdk/client-eventbridge": "^3.975.0", "@aws-sdk/client-iam": "^3.975.0", "@aws-sdk/client-lambda": "^3.975.0", "@aws-sdk/client-s3": "^3.975.0", + "@aws-sdk/client-ssm": "^3.975.0", + "@aws-sdk/client-sts": "^3.975.0", "@aws-sdk/credential-providers": "^3.975.0", "@smithy/node-http-handler": "^4.6.1", "ajv": "^8.12.0", diff --git a/test/unit/lib/aws/s3-body-to-string.test.js b/test/unit/lib/aws/s3-body-to-string.test.js new file mode 100644 index 000000000..33a2a0489 --- /dev/null +++ b/test/unit/lib/aws/s3-body-to-string.test.js @@ -0,0 +1,130 @@ +'use strict'; + +const { Readable } = require('stream'); +const zlib = require('zlib'); +const { expect } = require('chai'); +const s3BodyToString = require('../../../../lib/aws/s3-body-to-string'); + +describe('test/unit/lib/aws/s3-body-to-string.test.js', () => { + it('returns string bodies unchanged', async () => { + await expect(s3BodyToString('value')).to.eventually.equal('value'); + }); + + it('converts Buffer bodies', async () => { + await expect(s3BodyToString(Buffer.from('value'))).to.eventually.equal('value'); + }); + + it('does not decompress gzip-encoded bodies', async () => { + const body = zlib.gzipSync('value'); + const result = await s3BodyToString(body); + + expect(result).to.equal(body.toString('utf8')); + expect(result).to.not.equal('value'); + }); + + it('converts Uint8Array bodies', async () => { + await expect(s3BodyToString(new Uint8Array(Buffer.from('value')))).to.eventually.equal('value'); + }); + + it('converts ArrayBuffer bodies', async () => { + const buffer = Buffer.from('value'); + const arrayBuffer = buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ); + + await expect(s3BodyToString(arrayBuffer)).to.eventually.equal('value'); + }); + + it('converts Node readable stream bodies', async () => { + await expect(s3BodyToString(Readable.from(['val', 'ue']))).to.eventually.equal('value'); + }); + + it('converts Web ReadableStream bodies when available', async () => { + if (typeof ReadableStream !== 'function') return; + + const body = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('val')); + controller.enqueue(new TextEncoder().encode('ue')); + controller.close(); + }, + }); + + await expect(s3BodyToString(body)).to.eventually.equal('value'); + }); + + it('converts Blob bodies when available', async () => { + if (typeof Blob !== 'function') return; + + await expect(s3BodyToString(new Blob(['value']))).to.eventually.equal('value'); + }); + + it('uses SDK transformToString when present', async () => { + const body = { + transformToString: async (encoding) => `transformed:${encoding}`, + [Symbol.asyncIterator]: async function* iterator() { + yield 'streamed'; + }, + }; + + await expect(s3BodyToString(body)).to.eventually.equal('transformed:utf8'); + }); + + it('passes custom encoding to SDK transformToString', async () => { + const body = { + transformToString: async (encoding) => encoding, + }; + + await expect(s3BodyToString(body, { encoding: 'latin1' })).to.eventually.equal('latin1'); + }); + + it('propagates transformToString errors', async () => { + const body = { + transformToString: async () => { + throw new Error('transform failed'); + }, + }; + + await expect(s3BodyToString(body)).to.be.rejectedWith('transform failed'); + }); + + it('propagates Node readable stream errors', async () => { + const body = new Readable({ + read() { + this.destroy(new Error('stream failed')); + }, + }); + + await expect(s3BodyToString(body)).to.be.rejectedWith('stream failed'); + }); + + it('returns an empty string for nullish bodies', async () => { + await expect(s3BodyToString(null)).to.eventually.equal(''); + await expect(s3BodyToString()).to.eventually.equal(''); + }); + + it('rejects unsupported body shapes without stringifying contents', async () => { + const body = { secret: 'do-not-log' }; + + try { + await s3BodyToString(body); + throw new Error('Expected s3BodyToString to reject'); + } catch (error) { + expect(error.code).to.equal('UNSUPPORTED_S3_GET_OBJECT_BODY'); + expect(error.message).to.include('[object Object]'); + expect(error.message).to.not.include('do-not-log'); + } + }); + + for (const body of [123, true, () => {}]) { + it(`rejects unsupported ${typeof body} bodies`, async () => { + try { + await s3BodyToString(body); + throw new Error('Expected s3BodyToString to reject'); + } catch (error) { + expect(error.code).to.equal('UNSUPPORTED_S3_GET_OBJECT_BODY'); + } + }); + } +}); diff --git a/test/unit/lib/configuration/variables/sources/instance-dependent/create-cached-aws-variable-source-command-sender.test.js b/test/unit/lib/configuration/variables/sources/instance-dependent/create-cached-aws-variable-source-command-sender.test.js new file mode 100644 index 000000000..9b448a01f --- /dev/null +++ b/test/unit/lib/configuration/variables/sources/instance-dependent/create-cached-aws-variable-source-command-sender.test.js @@ -0,0 +1,434 @@ +'use strict'; + +const { expect } = require('chai'); +const sinon = require('sinon'); +const createCachedAwsVariableSourceCommandSender = require('../../../../../../../lib/configuration/variables/sources/instance-dependent/create-cached-aws-variable-source-command-sender'); + +describe('test/unit/lib/configuration/variables/sources/instance-dependent/create-cached-aws-variable-source-command-sender.test.js', () => { + class TestCommand { + constructor(input) { + this.input = input; + } + } + + class OtherCommand { + constructor(input) { + this.input = input; + } + } + + function createDeferred() { + const deferred = {}; + deferred.promise = new Promise((resolve, reject) => { + deferred.resolve = resolve; + deferred.reject = reject; + }); + return deferred; + } + + function nextTick() { + return new Promise((resolve) => setImmediate(resolve)); + } + + function createClient({ send }) { + const instances = []; + + class TestClient { + constructor(config) { + this.config = config; + this.send = send; + instances.push(this); + } + } + + TestClient.instances = instances; + return TestClient; + } + + function createProvider(config = { region: 'us-east-1' }, region = 'us-east-1') { + return { + getRegion: sinon.stub().returns(region), + getAwsSdkV3Config: sinon.stub().resolves(config), + }; + } + + it('does not resolve the provider until a command is sent', () => { + const getProvider = sinon.stub().returns(createProvider()); + const TestClient = createClient({ send: sinon.stub() }); + + createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + }); + + expect(getProvider).to.not.have.been.called; + expect(TestClient.instances).to.have.length(0); + }); + + it('builds a client from provider SDK v3 config and sends commands', async () => { + const credentials = sinon.stub(); + const config = { region: 'eu-west-1', credentials }; + const provider = createProvider(config); + const getProvider = sinon.stub().returns(provider); + const send = sinon.stub().resolves({ ok: true }); + const TestClient = createClient({ send }); + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + }); + + await expect( + sender.send(TestCommand, { Name: 'parameter' }, { region: 'eu-west-1' }) + ).to.eventually.deep.equal({ ok: true }); + + expect(getProvider).to.have.been.calledOnce; + expect(provider.getAwsSdkV3Config).to.have.been.calledOnceWithExactly({ + region: 'eu-west-1', + }); + expect(TestClient.instances).to.have.length(1); + expect(TestClient.instances[0].config).to.equal(config); + expect(TestClient.instances[0].config.credentials).to.equal(credentials); + expect(send).to.have.been.calledOnce; + expect(send.firstCall.args[0]).to.be.instanceOf(TestCommand); + expect(send.firstCall.args[0].input).to.deep.equal({ Name: 'parameter' }); + }); + + it('reuses clients for the same effective region', async () => { + const provider = createProvider(); + const getProvider = sinon.stub().returns(provider); + const send = sinon.stub().resolves({ ok: true }); + const TestClient = createClient({ send }); + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + }); + + await sender.send(TestCommand, { Name: 'first' }, { region: 'eu-west-1' }); + await sender.send(TestCommand, { Name: 'second' }, { region: 'eu-west-1' }); + + expect(getProvider).to.have.been.calledOnce; + expect(TestClient.instances).to.have.length(1); + expect(provider.getAwsSdkV3Config).to.have.been.calledOnce; + expect(send).to.have.been.calledTwice; + }); + + it('uses different clients for different effective regions', async () => { + const provider = createProvider(); + const getProvider = sinon.stub().returns(provider); + const send = sinon.stub().resolves({ ok: true }); + const TestClient = createClient({ send }); + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + }); + + await sender.send(TestCommand, { Name: 'parameter' }, { region: 'eu-west-1' }); + await sender.send(TestCommand, { Name: 'parameter' }, { region: 'us-east-1' }); + + expect(TestClient.instances).to.have.length(2); + expect(provider.getAwsSdkV3Config).to.have.been.calledTwice; + }); + + it('shares cache entries between omitted region and explicit provider region', async () => { + const provider = createProvider({ region: 'us-east-1' }, 'us-east-1'); + const getProvider = sinon.stub().returns(provider); + const send = sinon.stub().resolves({ ok: true }); + const TestClient = createClient({ send }); + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + }); + + await sender.send(TestCommand, { Name: 'parameter' }); + await sender.send(TestCommand, { Name: 'parameter' }, { region: 'us-east-1' }); + + expect(provider.getAwsSdkV3Config).to.have.been.calledOnceWithExactly({ + region: 'us-east-1', + }); + expect(TestClient.instances).to.have.length(1); + expect(send).to.have.been.calledOnce; + }); + + it('caches identical transformed command results by default', async () => { + const provider = createProvider(); + const getProvider = sinon.stub().returns(provider); + const send = sinon.stub().resolves({ ok: true }); + const transformResult = sinon.stub().returns({ transformed: true }); + const TestClient = createClient({ send }); + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + transformResult, + }); + + await expect(sender.send(TestCommand, { Name: 'parameter' })).to.eventually.deep.equal({ + transformed: true, + }); + await expect(sender.send(TestCommand, { Name: 'parameter' })).to.eventually.deep.equal({ + transformed: true, + }); + + expect(send).to.have.been.calledOnce; + expect(transformResult).to.have.been.calledOnce; + }); + + it('passes command context to transformResult', async () => { + const rawResult = { Body: 'raw' }; + const input = { Name: 'parameter' }; + const provider = createProvider(); + const getProvider = sinon.stub().returns(provider); + const send = sinon.stub().resolves(rawResult); + const transformResult = sinon.stub().returns('mapped'); + const TestClient = createClient({ send }); + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + transformResult, + }); + + await expect(sender.send(TestCommand, input, { region: 'eu-west-1' })).to.eventually.equal( + 'mapped' + ); + + expect(transformResult).to.have.been.calledOnceWithExactly({ + result: rawResult, + commandName: 'TestCommand', + input, + region: 'eu-west-1', + effectiveRegion: 'eu-west-1', + }); + }); + + it('uses stable cache keys for deep-sorted command input', async () => { + const provider = createProvider(); + const getProvider = sinon.stub().returns(provider); + const send = sinon.stub().resolves({ ok: true }); + const TestClient = createClient({ send }); + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + }); + + await sender.send(TestCommand, { + Tags: [{ Value: 'one', Key: 'first' }], + Nested: { b: 2, a: 1 }, + }); + await sender.send(TestCommand, { + Nested: { a: 1, b: 2 }, + Tags: [{ Key: 'first', Value: 'one' }], + }); + + expect(send).to.have.been.calledOnce; + }); + + it('separates cached commands by command name, effective region, and input', async () => { + const provider = createProvider(); + const getProvider = sinon.stub().returns(provider); + const send = sinon.stub().resolves({ ok: true }); + const TestClient = createClient({ send }); + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + }); + + await sender.send(TestCommand, { Name: 'parameter' }, { region: 'eu-west-1' }); + await sender.send(TestCommand, { Name: 'parameter' }, { region: 'us-east-1' }); + await sender.send(TestCommand, { Name: 'other' }, { region: 'eu-west-1' }); + await sender.send(OtherCommand, { Name: 'parameter' }, { region: 'eu-west-1' }); + + expect(send).to.have.callCount(4); + }); + + it('evicts rejected cached command promises', async () => { + const provider = createProvider(); + const getProvider = sinon.stub().returns(provider); + const send = sinon + .stub() + .onFirstCall() + .rejects(new Error('temporary')) + .onSecondCall() + .resolves({ ok: true }); + const TestClient = createClient({ send }); + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + }); + + await expect(sender.send(TestCommand, { Name: 'parameter' })).to.be.rejectedWith('temporary'); + await expect(sender.send(TestCommand, { Name: 'parameter' })).to.eventually.deep.equal({ + ok: true, + }); + + expect(send).to.have.been.calledTwice; + }); + + it('evicts rejected transformResult promises', async () => { + const provider = createProvider(); + const getProvider = sinon.stub().returns(provider); + const send = sinon.stub().resolves({ ok: true }); + const transformResult = sinon + .stub() + .onFirstCall() + .rejects(new Error('transform failed')) + .onSecondCall() + .resolves({ transformed: true }); + const TestClient = createClient({ send }); + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + transformResult, + }); + + await expect(sender.send(TestCommand, { Name: 'parameter' })).to.be.rejectedWith( + 'transform failed' + ); + await expect(sender.send(TestCommand, { Name: 'parameter' })).to.eventually.deep.equal({ + transformed: true, + }); + + expect(send).to.have.been.calledTwice; + expect(transformResult).to.have.been.calledTwice; + }); + + it('evicts failed client construction promises', async () => { + const provider = { + getRegion: sinon.stub().returns('us-east-1'), + getAwsSdkV3Config: sinon + .stub() + .onFirstCall() + .rejects(new Error('config failed')) + .onSecondCall() + .resolves({ region: 'us-east-1' }), + }; + const getProvider = sinon.stub().returns(provider); + const send = sinon.stub().resolves({ ok: true }); + const TestClient = createClient({ send }); + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + }); + + await expect(sender.send(TestCommand, { Name: 'first' })).to.be.rejectedWith('config failed'); + await expect(sender.send(TestCommand, { Name: 'second' })).to.eventually.deep.equal({ + ok: true, + }); + + expect(provider.getAwsSdkV3Config).to.have.been.calledTwice; + expect(TestClient.instances).to.have.length(1); + }); + + it('limits command and transform concurrency per sender to 2', async () => { + const provider = createProvider(); + const getProvider = sinon.stub().returns(provider); + const send = sinon.stub().resolves({ ok: true }); + const transformResults = []; + const transformResult = sinon.stub().callsFake(() => { + const deferred = createDeferred(); + transformResults.push(deferred); + return deferred.promise; + }); + const TestClient = createClient({ send }); + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + transformResult, + }); + + const first = sender.send(TestCommand, { Name: 'first' }); + const second = sender.send(TestCommand, { Name: 'second' }); + const third = sender.send(TestCommand, { Name: 'third' }); + await nextTick(); + + expect(send).to.have.been.calledTwice; + expect(transformResult).to.have.been.calledTwice; + + transformResults[0].resolve('first'); + await nextTick(); + + expect(send).to.have.callCount(3); + expect(transformResult).to.have.callCount(3); + + transformResults[1].resolve('second'); + transformResults[2].resolve('third'); + + await expect(Promise.all([first, second, third])).to.eventually.deep.equal([ + 'first', + 'second', + 'third', + ]); + }); + + it('does not duplicate identical queued commands', async () => { + const provider = createProvider(); + const getProvider = sinon.stub().returns(provider); + const deferred = createDeferred(); + const send = sinon.stub().returns(deferred.promise); + const TestClient = createClient({ send }); + const sender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: TestClient, + }); + + const first = sender.send(TestCommand, { Name: 'parameter' }); + const second = sender.send(TestCommand, { Name: 'parameter' }); + const third = sender.send(TestCommand, { Name: 'parameter' }); + await nextTick(); + + expect(send).to.have.been.calledOnce; + + deferred.resolve({ ok: true }); + await expect(Promise.all([first, second, third])).to.eventually.deep.equal([ + { ok: true }, + { ok: true }, + { ok: true }, + ]); + }); + + it('scopes concurrency limits per sender instance', async () => { + const provider = createProvider(); + const getProvider = sinon.stub().returns(provider); + const transformResults = []; + const transformResult = sinon.stub().callsFake(() => { + const deferred = createDeferred(); + transformResults.push(deferred); + return deferred.promise; + }); + const send = sinon.stub().resolves({ ok: true }); + const FirstClient = createClient({ send }); + const SecondClient = createClient({ send }); + const firstSender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: FirstClient, + transformResult, + }); + const secondSender = createCachedAwsVariableSourceCommandSender({ + getProvider, + Client: SecondClient, + transformResult, + }); + + const promises = [ + firstSender.send(TestCommand, { Name: 'first' }), + firstSender.send(TestCommand, { Name: 'second' }), + firstSender.send(TestCommand, { Name: 'third' }), + secondSender.send(TestCommand, { Name: 'fourth' }), + secondSender.send(TestCommand, { Name: 'fifth' }), + secondSender.send(TestCommand, { Name: 'sixth' }), + ]; + await nextTick(); + + expect(send).to.have.callCount(4); + expect(transformResult).to.have.callCount(4); + + for (let index = 0; index < transformResults.length; index += 1) { + transformResults[index].resolve(index); + } + + await nextTick(); + for (let index = 4; index < transformResults.length; index += 1) { + transformResults[index].resolve(index); + } + + await expect(Promise.all(promises)).to.eventually.deep.equal([0, 1, 4, 2, 3, 5]); + }); +}); diff --git a/test/unit/lib/configuration/variables/sources/instance-dependent/get-aws.test.js b/test/unit/lib/configuration/variables/sources/instance-dependent/get-aws.test.js index 915e9eb5f..1327dfb45 100644 --- a/test/unit/lib/configuration/variables/sources/instance-dependent/get-aws.test.js +++ b/test/unit/lib/configuration/variables/sources/instance-dependent/get-aws.test.js @@ -1,39 +1,76 @@ 'use strict'; const { expect } = require('chai'); +const proxyquire = require('proxyquire'); +const sinon = require('sinon'); const resolveMeta = require('../../../../../../../lib/configuration/variables/resolve-meta'); const resolve = require('../../../../../../../lib/configuration/variables/resolve'); const selfSource = require('../../../../../../../lib/configuration/variables/sources/self'); -const getAwsSource = require('../../../../../../../lib/configuration/variables/sources/instance-dependent/get-aws'); const mergePlainObjects = require('../../../../../../../lib/utils/merge-plain-objects'); const Serverless = require('../../../../../../../lib/serverless'); describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-aws.test.js', () => { let configuration; let variablesMeta; - let serverlessInstance; + let sends; + let clientInstances; + let getAwsSdkV3Config; + let credentials; - const initializeServerless = async (configExt, options) => { + class GetCallerIdentityCommand { + constructor(input) { + this.input = input; + } + } + + function loadSource(handler) { + sends = []; + clientInstances = []; + credentials = sinon.stub(); + getAwsSdkV3Config = sinon.stub().callsFake(async ({ region }) => ({ + region, + credentials, + })); + + class STSClient { + constructor(config) { + this.config = config; + clientInstances.push(this); + } + + async send(command) { + sends.push({ config: this.config, input: command.input }); + return handler(command.input, this.config); + } + } + + return proxyquire + .noCallThru() + .load('../../../../../../../lib/configuration/variables/sources/instance-dependent/get-aws', { + '@aws-sdk/client-sts': { STSClient, GetCallerIdentityCommand }, + }); + } + + const initializeServerless = async ({ configExt, options, custom, handler } = {}) => { configuration = { service: 'foo', provider: { name: 'aws', }, - custom: { + custom: custom || { region: '${aws:region}', accountId: '${aws:accountId}', + accountIdAgain: '${aws:accountId}', missingAddress: '${aws:}', invalidAddress: '${aws:invalid}', nonStringAddress: '${aws:${self:custom.someObject}}', someObject: {}, }, }; - if (configExt) { - configuration = mergePlainObjects(configuration, configExt); - } + if (configExt) configuration = mergePlainObjects(configuration, configExt); variablesMeta = resolveMeta(configuration); - serverlessInstance = new Serverless({ + const serverlessInstance = new Serverless({ configuration, serviceDir: process.cwd(), configurationFilename: 'serverless.yml', @@ -45,12 +82,11 @@ describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-a constructor: { getProviderName: () => 'aws', }, - request: async () => { - return { - Account: '1234567890', - }; - }, + getRegion: () => 'us-east-1', + getAwsSdkV3Config, }); + const getAwsSource = loadSource(handler || (() => ({ Account: '1234567890' }))); + await resolve({ serviceDir: process.cwd(), configuration, @@ -61,48 +97,83 @@ describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-a }); }; - it('should resolve `accountId`', async () => { + it('should resolve `accountId` with STS', async () => { await initializeServerless(); + expect(configuration.custom.accountId).to.equal('1234567890'); + expect(configuration.custom.accountIdAgain).to.equal('1234567890'); + expect(getAwsSdkV3Config).to.have.been.calledOnceWithExactly({ + region: 'us-east-1', + }); + expect(clientInstances).to.have.length(1); + expect(clientInstances[0].config.credentials).to.equal(credentials); + expect(sends).to.have.length(1); + expect(sends[0].input).to.deep.equal({}); + }); + + it('should surface STS errors', async () => { + await initializeServerless({ + custom: { accountId: '${aws:accountId}' }, + handler: () => { + throw new Error('SSO session has expired'); + }, + }); + + expect(variablesMeta.get('custom\0accountId').error.code).to.equal('VARIABLE_RESOLUTION_ERROR'); + expect(variablesMeta.get('custom\0accountId').error.message).to.include( + 'SSO session has expired' + ); }); - it('should report with an error missing address', () => + it('should report with an error missing address', async () => { + await initializeServerless(); expect(variablesMeta.get('custom\0missingAddress').error.code).to.equal( 'VARIABLE_RESOLUTION_ERROR' - )); + ); + }); - it('should report with an error invalid address', () => + it('should report with an error invalid address', async () => { + await initializeServerless(); expect(variablesMeta.get('custom\0invalidAddress').error.code).to.equal( 'VARIABLE_RESOLUTION_ERROR' - )); + ); + }); - it('should report with an error a non-string address', () => + it('should report with an error a non-string address', async () => { + await initializeServerless(); expect(variablesMeta.get('custom\0nonStringAddress').error.code).to.equal( 'VARIABLE_RESOLUTION_ERROR' - )); + ); + }); it('should resolve ${aws:region}', async () => { - // us-east-1 by default - await initializeServerless(); + await initializeServerless({ custom: { region: '${aws:region}' } }); expect(configuration.custom.region).to.equal('us-east-1'); - // Resolves to provider.region if it exists + expect(clientInstances).to.have.length(0); + await initializeServerless({ - provider: { - region: 'eu-west-1', + custom: { region: '${aws:region}' }, + configExt: { + provider: { + region: 'eu-west-1', + }, }, }); expect(configuration.custom.region).to.equal('eu-west-1'); - // Resolves to `--region=` if the option is set - await initializeServerless( - { + expect(clientInstances).to.have.length(0); + + await initializeServerless({ + custom: { region: '${aws:region}' }, + configExt: { provider: { region: 'eu-west-1', }, }, - { + options: { region: 'eu-central-1', - } - ); + }, + }); expect(configuration.custom.region).to.equal('eu-central-1'); + expect(clientInstances).to.have.length(0); }); }); diff --git a/test/unit/lib/configuration/variables/sources/instance-dependent/get-cf.test.js b/test/unit/lib/configuration/variables/sources/instance-dependent/get-cf.test.js index a4372c73a..b1ae1b440 100644 --- a/test/unit/lib/configuration/variables/sources/instance-dependent/get-cf.test.js +++ b/test/unit/lib/configuration/variables/sources/instance-dependent/get-cf.test.js @@ -1,16 +1,61 @@ 'use strict'; const { expect } = require('chai'); +const proxyquire = require('proxyquire'); +const sinon = require('sinon'); const resolveMeta = require('../../../../../../../lib/configuration/variables/resolve-meta'); const resolve = require('../../../../../../../lib/configuration/variables/resolve'); const selfSource = require('../../../../../../../lib/configuration/variables/sources/self'); -const getCfSource = require('../../../../../../../lib/configuration/variables/sources/instance-dependent/get-cf'); describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-cf.test.js', () => { let configuration; let variablesMeta; - let serverlessInstance; + let sends; + let clientInstances; + let getAwsSdkV3Config; + let credentials; + + class DescribeStacksCommand { + constructor(input) { + this.input = input; + } + } + + function loadSource(handler) { + sends = []; + clientInstances = []; + credentials = sinon.stub(); + getAwsSdkV3Config = sinon.stub().callsFake(async ({ region }) => ({ + region, + credentials, + })); + + class CloudFormationClient { + constructor(config) { + this.config = config; + clientInstances.push(this); + } + + async send(command) { + sends.push({ config: this.config, input: command.input }); + return handler(command.input, this.config); + } + } + + const getCfSource = proxyquire + .noCallThru() + .load('../../../../../../../lib/configuration/variables/sources/instance-dependent/get-cf', { + '@aws-sdk/client-cloudformation': { + CloudFormationClient, + DescribeStacksCommand, + }, + }); + + return getCfSource({ + getProvider: () => ({ getRegion: () => 'us-east-1', getAwsSdkV3Config }), + }); + } before(async () => { configuration = { @@ -18,9 +63,11 @@ describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-c provider: { name: 'aws' }, custom: { existing: '${cf:existing.someOutput}', + existingAgain: '${cf:existing.someOutput}', existingInRegion: '${cf(eu-west-1):existing.someOutput}', noOutput: '${cf:existing.unrecognizedOutput, null}', noStack: '${cf:notExisting.someOutput, null}', + badValidation: '${cf:badValidation.someOutput}', missingAddress: '${cf:}', invalidAddress: '${cf:invalid}', nonStringAddress: '${cf:${self:custom.someObject}}', @@ -29,55 +76,94 @@ describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-c }; variablesMeta = resolveMeta(configuration); - serverlessInstance = { - getProvider: () => ({ - request: async (name, method, { StackName }, { region }) => { - if (StackName === 'existing') { - return { - Stacks: [ - { Outputs: [{ OutputKey: 'someOutput', OutputValue: region || 'someValue' }] }, + const source = loadSource(({ StackName }, { region }) => { + if (StackName === 'existing') { + return { + Stacks: [ + { + Outputs: [ + { + OutputKey: 'someOutput', + OutputValue: region === 'eu-west-1' ? region : 'someValue', + }, ], - }; - } - if (StackName === 'notExisting') { - throw Object.assign(new Error('Stack with id not-existing does not exist'), { - code: 'AWS_CLOUD_FORMATION_DESCRIBE_STACKS_VALIDATION_ERROR', - }); - } - throw new Error('Unexpected call'); - }, - }), - }; + }, + ], + }; + } + if (StackName === 'notExisting') { + throw Object.assign(new Error('Stack with id not-existing does not exist'), { + name: 'ValidationError', + }); + } + if (StackName === 'badValidation') { + throw Object.assign(new Error('Template validation failed'), { name: 'ValidationError' }); + } + throw new Error(`Unexpected CloudFormation call: ${StackName}`); + }); await resolve({ serviceDir: process.cwd(), configuration, variablesMeta, - sources: { self: selfSource, cf: getCfSource(serverlessInstance) }, + sources: { self: selfSource, cf: source }, options: {}, fulfilledSources: new Set(['cf', 'self']), }); }); + function getSendsByStack(stackName) { + return sends.filter(({ input }) => input.StackName === stackName); + } + it('should resolve existing output', () => { if (variablesMeta.get('custom\0existing')) throw variablesMeta.get('custom\0existing').error; expect(configuration.custom.existing).to.equal('someValue'); + expect(configuration.custom.existingAgain).to.equal('someValue'); + }); + + it('should use AWS SDK v3 config and preserve credential providers', () => { + expect(getAwsSdkV3Config).to.have.been.calledWith({ + region: 'us-east-1', + }); + expect(clientInstances[0].config.credentials).to.equal(credentials); }); + + it('should cache repeated stack lookups in the same region', () => { + expect( + getSendsByStack('existing').filter(({ config }) => config.region === 'us-east-1') + ).to.have.length(1); + }); + it('should resolve existing output in specific region', () => { if (variablesMeta.get('custom\0existingInRegion')) { throw variablesMeta.get('custom\0existingInRegion').error; } expect(configuration.custom.existingInRegion).to.equal('eu-west-1'); + expect(getAwsSdkV3Config).to.have.been.calledWith({ + region: 'eu-west-1', + }); }); + it('should resolve null on missing output', () => { if (variablesMeta.get('custom\0noOutput')) throw variablesMeta.get('custom\0noOutput').error; expect(configuration.custom.noOutput).to.equal(null); }); + it('should resolve null on missing stack', () => { if (variablesMeta.get('custom\0noStack')) throw variablesMeta.get('custom\0noStack').error; expect(configuration.custom.noStack).to.equal(null); }); + it('should surface non-missing ValidationError errors', () => { + expect(variablesMeta.get('custom\0badValidation').error.code).to.equal( + 'VARIABLE_RESOLUTION_ERROR' + ); + expect(variablesMeta.get('custom\0badValidation').error.message).to.include( + 'Template validation failed' + ); + }); + it('should report with an error missing address', () => expect(variablesMeta.get('custom\0missingAddress').error.code).to.equal( 'VARIABLE_RESOLUTION_ERROR' diff --git a/test/unit/lib/configuration/variables/sources/instance-dependent/get-s3.test.js b/test/unit/lib/configuration/variables/sources/instance-dependent/get-s3.test.js index f9140ecf0..816b5b84a 100644 --- a/test/unit/lib/configuration/variables/sources/instance-dependent/get-s3.test.js +++ b/test/unit/lib/configuration/variables/sources/instance-dependent/get-s3.test.js @@ -1,16 +1,59 @@ 'use strict'; +const { Readable } = require('stream'); const { expect } = require('chai'); +const proxyquire = require('proxyquire'); +const sinon = require('sinon'); const resolveMeta = require('../../../../../../../lib/configuration/variables/resolve-meta'); const resolve = require('../../../../../../../lib/configuration/variables/resolve'); const selfSource = require('../../../../../../../lib/configuration/variables/sources/self'); -const getS3Source = require('../../../../../../../lib/configuration/variables/sources/instance-dependent/get-s3'); describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-s3.test.js', () => { let configuration; let variablesMeta; - let serverlessInstance; + let sends; + let clientInstances; + let getAwsSdkV3Config; + let credentials; + + class GetObjectCommand { + constructor(input) { + this.input = input; + } + } + + function loadSource(handler) { + sends = []; + clientInstances = []; + credentials = sinon.stub(); + getAwsSdkV3Config = sinon.stub().callsFake(async ({ region }) => ({ + region, + credentials, + })); + + class S3Client { + constructor(config) { + this.config = config; + clientInstances.push(this); + } + + async send(command) { + sends.push({ config: this.config, input: command.input }); + return handler(command.input, this.config); + } + } + + const getS3Source = proxyquire + .noCallThru() + .load('../../../../../../../lib/configuration/variables/sources/instance-dependent/get-s3', { + '@aws-sdk/client-s3': { S3Client, GetObjectCommand }, + }); + + return getS3Source({ + getProvider: () => ({ getRegion: () => 'us-east-1', getAwsSdkV3Config }), + }); + } before(async () => { configuration = { @@ -18,8 +61,14 @@ describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-s provider: { name: 'aws' }, custom: { existing: '${s3:existing/someKey}', - noKey: '${s3:existing/unrecognizedKey, null}', + existingAgain: '${s3:existing/someKey}', + emptyBody: '${s3:existing/emptyKey}', + streamBody: '${s3:existing/streamKey}', + streamBodyAgain: '${s3:existing/streamKey}', + noKeyByName: '${s3:existing/noKeyByName, null}', + noKeyByCode: '${s3:existing/noKeyByCode, null}', noBucket: '${s3:notExisting/someKey, null}', + badBody: '${s3:existing/badBody}', missingAddress: '${s3:}', invalidAddress: '${s3:invalid}', nonStringAddress: '${s3:${self:custom.someObject}}', @@ -28,44 +77,101 @@ describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-s }; variablesMeta = resolveMeta(configuration); - serverlessInstance = { - getProvider: () => ({ - request: async (name, method, { Bucket, Key }) => { - if (Bucket === 'existing') { - if (Key === 'someKey') return { Body: 'foo' }; - throw Object.assign(new Error('The specified key does not exist.'), { - code: 'AWS_S3_GET_OBJECT_NO_SUCH_KEY', - }); - } - throw Object.assign(new Error('The specified bucket does not exist.'), { - code: 'NoSuchBucket', + const source = loadSource(({ Bucket, Key }) => { + if (Bucket === 'existing') { + if (Key === 'someKey') return { Body: 'foo' }; + if (Key === 'emptyKey') return { Body: '' }; + if (Key === 'streamKey') return { Body: Readable.from(['fo', 'o']) }; + if (Key === 'noKeyByName') { + throw Object.assign(new Error('The specified key does not exist.'), { + name: 'NoSuchKey', }); - }, - }), - }; + } + if (Key === 'noKeyByCode') { + throw Object.assign(new Error('The specified key does not exist.'), { + Code: 'NoSuchKey', + }); + } + if (Key === 'badBody') return { Body: { unsupported: true } }; + } + throw Object.assign(new Error('The specified bucket does not exist.'), { + name: 'NoSuchBucket', + }); + }); await resolve({ serviceDir: process.cwd(), configuration, variablesMeta, - sources: { self: selfSource, s3: getS3Source(serverlessInstance) }, + sources: { self: selfSource, s3: source }, options: {}, fulfilledSources: new Set(['s3', 'self']), }); }); + function getSendsByBucketAndKey(bucket, key) { + return sends.filter(({ input }) => input.Bucket === bucket && input.Key === key); + } + it('should resolve existing output', () => { if (variablesMeta.get('custom\0existing')) throw variablesMeta.get('custom\0existing').error; expect(configuration.custom.existing).to.equal('foo'); + expect(configuration.custom.existingAgain).to.equal('foo'); + }); + + it('should use AWS SDK v3 config and preserve credential providers', () => { + expect(getAwsSdkV3Config).to.have.been.calledWith({ region: 'us-east-1' }); + expect(clientInstances[0].config.credentials).to.equal(credentials); + }); + + it('should not request S3 acceleration config for variable reads', () => { + for (const call of getAwsSdkV3Config.getCalls()) { + expect(call.firstArg).to.not.have.property('service'); + expect(call.firstArg).to.not.have.property('useAccelerateEndpoint'); + } + for (const { config } of sends) { + expect(config).to.not.have.property('useAccelerateEndpoint'); + } + }); + + it('should cache repeated object lookups', () => { + expect(getSendsByBucketAndKey('existing', 'someKey')).to.have.length(1); + }); + + it('should resolve empty object bodies as empty strings', () => { + if (variablesMeta.get('custom\0emptyBody')) throw variablesMeta.get('custom\0emptyBody').error; + expect(configuration.custom.emptyBody).to.equal(''); + }); + + it('should convert stream bodies to strings', () => { + if (variablesMeta.get('custom\0streamBody')) + throw variablesMeta.get('custom\0streamBody').error; + if (variablesMeta.get('custom\0streamBodyAgain')) + throw variablesMeta.get('custom\0streamBodyAgain').error; + expect(configuration.custom.streamBody).to.equal('foo'); + expect(configuration.custom.streamBodyAgain).to.equal('foo'); + expect(getSendsByBucketAndKey('existing', 'streamKey')).to.have.length(1); }); it('should resolve null on missing key', () => { - if (variablesMeta.get('custom\0noKey')) throw variablesMeta.get('custom\0noKey').error; - expect(configuration.custom.noKey).to.equal(null); + if (variablesMeta.get('custom\0noKeyByName')) + throw variablesMeta.get('custom\0noKeyByName').error; + if (variablesMeta.get('custom\0noKeyByCode')) + throw variablesMeta.get('custom\0noKeyByCode').error; + expect(configuration.custom.noKeyByName).to.equal(null); + expect(configuration.custom.noKeyByCode).to.equal(null); }); + it('should report with an error missing bucket', () => expect(variablesMeta.get('custom\0noBucket').error.code).to.equal('VARIABLE_RESOLUTION_ERROR')); + it('should report with an error unsupported body', () => { + expect(variablesMeta.get('custom\0badBody').error.code).to.equal('VARIABLE_RESOLUTION_ERROR'); + expect(variablesMeta.get('custom\0badBody').error.message).to.include( + 'Unsupported S3 GetObject Body type' + ); + }); + it('should report with an error missing address', () => expect(variablesMeta.get('custom\0missingAddress').error.code).to.equal( 'VARIABLE_RESOLUTION_ERROR' diff --git a/test/unit/lib/configuration/variables/sources/instance-dependent/get-ssm.test.js b/test/unit/lib/configuration/variables/sources/instance-dependent/get-ssm.test.js index 19a9040e0..3cc17aaed 100644 --- a/test/unit/lib/configuration/variables/sources/instance-dependent/get-ssm.test.js +++ b/test/unit/lib/configuration/variables/sources/instance-dependent/get-ssm.test.js @@ -1,18 +1,61 @@ 'use strict'; const { expect } = require('chai'); +const proxyquire = require('proxyquire'); +const sinon = require('sinon'); const resolveMeta = require('../../../../../../../lib/configuration/variables/resolve-meta'); const resolve = require('../../../../../../../lib/configuration/variables/resolve'); const selfSource = require('../../../../../../../lib/configuration/variables/sources/self'); -const getSsmSource = require('../../../../../../../lib/configuration/variables/sources/instance-dependent/get-ssm'); - -const allowedRegionTypes = new Set(['undefined', 'string']); describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-ssm.test.js', () => { let configuration; let variablesMeta; - let serverlessInstance; + let sends; + let clientInstances; + let getAwsSdkV3Config; + let credentials; + + class GetParameterCommand { + constructor(input) { + this.input = input; + } + } + + function loadSource(handler) { + sends = []; + clientInstances = []; + credentials = sinon.stub().resolves({ + accessKeyId: 'accessKeyId', + secretAccessKey: 'secretAccessKey', + }); + getAwsSdkV3Config = sinon.stub().callsFake(async ({ region }) => ({ + region, + credentials, + })); + + class SSMClient { + constructor(config) { + this.config = config; + clientInstances.push(this); + } + + async send(command) { + sends.push({ config: this.config, input: command.input }); + return handler(command.input, this.config); + } + } + + const getSsmSource = proxyquire + .noCallThru() + .load('../../../../../../../lib/configuration/variables/sources/instance-dependent/get-ssm', { + '@aws-sdk/client-ssm': { SSMClient, GetParameterCommand }, + }); + + return getSsmSource({ + getProvider: () => ({ getRegion: () => 'us-east-1', getAwsSdkV3Config }), + }); + } before(async () => { configuration = { @@ -20,6 +63,7 @@ describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-s provider: { name: 'aws' }, custom: { existing: '${ssm:existing}', + existingAgain: '${ssm:existing}', existingInRegion: '${ssm(eu-west-1):existing}', existingList: '${ssm:existingList}', existingListRaw: '${ssm(raw):existingList}', @@ -29,7 +73,9 @@ describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-s encryptedWithSkipDecryptAndRegion: '${ssm(noDecrypt, eu-west-1):/secret/existing}', existingEncryptedDirect: '${ssm:/secret/direct}', existingEncryptedRaw: '${ssm(raw):/aws/reference/secretsmanager/existing}', - notExisting: '${ssm:notExisting, null}', + notExistingByName: '${ssm:notExistingByName, null}', + notExistingByCode: '${ssm:notExistingByCode, null}', + expiredSso: '${ssm:expiredSso}', missingAddress: '${ssm:}', nonStringAddress: '${ssm:${self:custom.someObject}}', someObject: {}, @@ -37,61 +83,71 @@ describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-s }; variablesMeta = resolveMeta(configuration); - serverlessInstance = { - getProvider: () => ({ - request: async (name, method, { Name, WithDecryption }, { region }) => { - if (!allowedRegionTypes.has(typeof region)) throw new Error('Invalid region value'); - if (Name === 'existing') { - return { Parameter: { Type: 'String', Value: region || 'value' } }; - } - if (Name === 'existingList') { - return { Parameter: { Type: 'StringList', Value: 'one,two,three' } }; - } - if (Name === '/secret/existing' || Name === '/aws/reference/secretsmanager/existing') { - return { - Parameter: { - Type: 'SecureString', - Value: WithDecryption ? '{"someSecret":"someValue"}' : 'ENCRYPTED', - }, - }; - } - if (Name === '/secret/direct') { - return { - Parameter: { - Type: 'SecureString', - Value: WithDecryption ? '12345678901234567890' : 'ENCRYPTED', - }, - }; - } - if (Name === 'notExisting') { - throw Object.assign( - new Error( - 'ParameterNotFound: An error occurred ' + - '(ParameterNotFound) when referencing Secrets Manager' - ), - { - code: 'AWS_S_S_M_GET_PARAMETER_PARAMETER_NOT_FOUND', - } - ); - } - throw new Error('Unexpected call'); - }, - }), - }; + const source = loadSource(({ Name, WithDecryption }, { region }) => { + if (Name === 'existing') { + return { Parameter: { Type: 'String', Value: region === 'eu-west-1' ? region : 'value' } }; + } + if (Name === 'existingList') { + return { Parameter: { Type: 'StringList', Value: 'one,two,three' } }; + } + if (Name === '/secret/existing' || Name === '/aws/reference/secretsmanager/existing') { + return { + Parameter: { + Type: 'SecureString', + Value: WithDecryption ? '{"someSecret":"someValue"}' : 'ENCRYPTED', + }, + }; + } + if (Name === '/secret/direct') { + return { + Parameter: { + Type: 'SecureString', + Value: WithDecryption ? '12345678901234567890' : 'ENCRYPTED', + }, + }; + } + if (Name === 'notExistingByName') { + throw Object.assign(new Error('ParameterNotFound'), { name: 'ParameterNotFound' }); + } + if (Name === 'notExistingByCode') { + throw Object.assign(new Error('ParameterNotFound'), { Code: 'ParameterNotFound' }); + } + if (Name === 'expiredSso') { + throw new Error('SSO session has expired'); + } + throw new Error(`Unexpected SSM call: ${Name}`); + }); await resolve({ serviceDir: process.cwd(), configuration, variablesMeta, - sources: { self: selfSource, ssm: getSsmSource(serverlessInstance) }, + sources: { self: selfSource, ssm: source }, options: {}, - fulfilledSources: new Set(['cf', 'self']), + fulfilledSources: new Set(['self', 'ssm']), }); }); + function getSendsByName(name) { + return sends.filter(({ input }) => input.Name === name); + } + it('should resolve existing string param', () => { if (variablesMeta.get('custom\0existing')) throw variablesMeta.get('custom\0existing').error; expect(configuration.custom.existing).to.equal('value'); + expect(configuration.custom.existingAgain).to.equal('value'); + }); + + it('should use AWS SDK v3 config and preserve credential providers', () => { + expect(getAwsSdkV3Config).to.have.been.calledWith({ region: 'us-east-1' }); + expect(clientInstances[0].config.credentials).to.equal(credentials); + }); + + it('should cache repeated lookups with the same name, region, and decryption setting', () => { + expect(getSendsByName('existing')).to.have.length(2); + expect( + getSendsByName('existing').filter(({ config }) => config.region === 'us-east-1') + ).to.have.length(1); }); it('should resolve existing string list param', () => { @@ -100,6 +156,7 @@ describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-s } expect(configuration.custom.existingList).to.deep.equal(['one', 'two', 'three']); }); + it('should support "raw" output for list param', () => { if (variablesMeta.get('custom\0existingListRaw')) { throw variablesMeta.get('custom\0existingListRaw').error; @@ -145,13 +202,37 @@ describe('test/unit/lib/configuration/variables/sources/instance-dependent/get-s throw variablesMeta.get('custom\0existingInRegion').error; } expect(configuration.custom.existingInRegion).to.equal('eu-west-1'); + expect(getAwsSdkV3Config).to.have.been.calledWith({ region: 'eu-west-1' }); }); - it('should resolve null on missing param', () => { - if (variablesMeta.get('custom\0notExisting')) { - throw variablesMeta.get('custom\0notExisting').error; + it('should pass decryption settings to GetParameterCommand', () => { + expect( + getSendsByName('/secret/existing').map(({ input }) => input.WithDecryption) + ).to.have.members([true, false, false]); + }); + + it('should separate cache entries by decryption setting and region', () => { + expect(getSendsByName('/secret/existing')).to.have.length(3); + }); + + it('should resolve null on missing params', () => { + if (variablesMeta.get('custom\0notExistingByName')) { + throw variablesMeta.get('custom\0notExistingByName').error; + } + if (variablesMeta.get('custom\0notExistingByCode')) { + throw variablesMeta.get('custom\0notExistingByCode').error; } - expect(configuration.custom.notExisting).to.equal(null); + expect(configuration.custom.notExistingByName).to.equal(null); + expect(configuration.custom.notExistingByCode).to.equal(null); + }); + + it('should not treat SSO errors as missing params', () => { + expect(variablesMeta.get('custom\0expiredSso').error.code).to.equal( + 'VARIABLE_RESOLUTION_ERROR' + ); + expect(variablesMeta.get('custom\0expiredSso').error.message).to.include( + 'SSO session has expired' + ); }); it('should report with an error missing address', () =>