[1/6] Migrate AWS variable sources to SDK v3

lib/aws/s3-body-to-string.js

@@ -0,0 +1,98 @@
+'use strict';
+
+const ServerlessError = require('../serverless-error');
+
+function chunkToBuffer(chunk, encoding) {
+  if (typeof chunk === 'string') {
+    return Buffer.from(chunk, encoding);
+  }
+
+  if (Buffer.isBuffer(chunk)) {
+    return chunk;
+  }
+
+  if (chunk instanceof Uint8Array) {
+    return Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength);
+  }
+
+  if (chunk instanceof ArrayBuffer) {
+    return Buffer.from(chunk);
+  }
+
+  return Buffer.from(chunk);
+}
+
+async function webReadableStreamToString(stream, encoding) {
+  const chunks = [];
+  const reader = stream.getReader();
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) {
+        break;
+      }
+
+      chunks.push(chunkToBuffer(value, encoding));
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  return Buffer.concat(chunks).toString(encoding);
+}
+
+async function nodeReadableStreamToString(stream, encoding) {
+  const chunks = [];
+
+  for await (const chunk of stream) {
+    chunks.push(chunkToBuffer(chunk, encoding));
+  }
+
+  return Buffer.concat(chunks).toString(encoding);
+}
+
+async function s3BodyToString(body, { encoding = 'utf8' } = {}) {
+  if (body == null) {
+    return '';
+  }
+
+  if (typeof body === 'string') {
+    return body;
+  }
+
+  if (typeof body.transformToString === 'function') {
+    return body.transformToString(encoding);
+  }
+
+  if (Buffer.isBuffer(body)) {
+    return body.toString(encoding);
+  }
+
+  if (body instanceof Uint8Array) {
+    return Buffer.from(body.buffer, body.byteOffset, body.byteLength).toString(encoding);
+  }
+
+  if (body instanceof ArrayBuffer) {
+    return Buffer.from(body).toString(encoding);
+  }
+
+  if (typeof Blob === 'function' && body instanceof Blob) {
+    return Buffer.from(await body.arrayBuffer()).toString(encoding);
+  }
+
+  if (typeof body.getReader === 'function') {
+    return webReadableStreamToString(body, encoding);
+  }
+
+  if (typeof body[Symbol.asyncIterator] === 'function') {
+    return nodeReadableStreamToString(body, encoding);
+  }
+
+  throw new ServerlessError(
+    `Unsupported S3 GetObject Body type: ${Object.prototype.toString.call(body)}`,
+    'UNSUPPORTED_S3_GET_OBJECT_BODY'
+  );
+}
+
+module.exports = s3BodyToString;

lib/configuration/variables/sources/instance-dependent/create-cached-aws-variable-source-command-sender.js

@@ -0,0 +1,105 @@
+'use strict';
+
+const promiseLimit = require('ext/promise/limit').bind(Promise);
+const deepSortObjectByKey = require('../../../../utils/deep-sort-object-by-key');
+
+const maxConcurrentVariableSourceCommands = 2;
+
+function normalizeRegion(region) {
+  return region === undefined ? null : region;
+}
+
+function getCommandName(Command) {
+  return Command.name || 'Command';
+}
+
+function createCacheKey({ commandName, region, input }) {
+  return JSON.stringify({
+    command: commandName,
+    region: normalizeRegion(region),
+    input: input === undefined ? null : deepSortObjectByKey(input),
+  });
+}
+
+function createCachedAwsVariableSourceCommandSender({
+  getProvider,
+  Client,
+  transformResult = ({ result }) => result,
+}) {
+  const clients = new Map();
+  const requests = new Map();
+  const sendQueue = promiseLimit(maxConcurrentVariableSourceCommands, async (task) => task());
+  let provider;
+
+  function resolveProvider() {
+    if (!provider) provider = getProvider();
+    return provider;
+  }
+
+  function getEffectiveRegion(region) {
+    return region === undefined ? resolveProvider().getRegion() : region;
+  }
+
+  async function getClient(region) {
+    const cacheKey = JSON.stringify({ region: normalizeRegion(region) });
+
+    if (!clients.has(cacheKey)) {
+      const clientPromise = (async () => {
+        const config = await resolveProvider().getAwsSdkV3Config({ region });
+        return new Client(config);
+      })();
+
+      clients.set(
+        cacheKey,
+        clientPromise.catch((error) => {
+          clients.delete(cacheKey);
+          throw error;
+        })
+      );
+    }
+
+    return clients.get(cacheKey);
+  }
+
+  async function send(Command, input, options = {}) {
+    const { region } = options;
+    const effectiveRegion = getEffectiveRegion(region);
+    const commandName = getCommandName(Command);
+    const cacheKey = createCacheKey({
+      commandName,
+      region: effectiveRegion,
+      input,
+    });
+
+    if (!requests.has(cacheKey)) {
+      const requestPromise = (async () => {
+        const client = await getClient(effectiveRegion);
+
+        return sendQueue(async () => {
+          const result = await client.send(new Command(input));
+          return transformResult({
+            result,
+            commandName,
+            input,
+            region,
+            effectiveRegion,
+          });
+        });
+      })();
+
+      requests.set(
+        cacheKey,
+        requestPromise.catch((error) => {
+          requests.delete(cacheKey);
+          throw error;
+        })
+      );
+    }
+
+    return requests.get(cacheKey);
+  }
+
+  return { send };
+}
+
+module.exports = createCachedAwsVariableSourceCommandSender;

lib/configuration/variables/sources/instance-dependent/get-aws.js

@@ -1,9 +1,16 @@
 'use strict';
 
 const ensureString = require('type/string/ensure');
+const { STSClient, GetCallerIdentityCommand } = require('@aws-sdk/client-sts');
 const ServerlessError = require('../../../../serverless-error');
+const createCachedAwsVariableSourceCommandSender = require('./create-cached-aws-variable-source-command-sender');
 
 module.exports = (serverlessInstance) => {
+  const sender = createCachedAwsVariableSourceCommandSender({
+    getProvider: () => serverlessInstance.getProvider('aws'),
+    Client: STSClient,
+  });
+
   return {
     resolve: async ({ address, options, resolveConfigurationProperty }) => {
       if (!address) {
@@ -21,9 +28,7 @@ module.exports = (serverlessInstance) => {
 
       switch (address) {
         case 'accountId': {
-          const { Account } = await serverlessInstance
-            .getProvider('aws')
-            .request('STS', 'getCallerIdentity', {}, { useCache: true });
+          const { Account } = await sender.send(GetCallerIdentityCommand, {});
           return { value: Account };
         }
         case 'region': {

lib/configuration/variables/sources/instance-dependent/get-cf.js

@@ -1,9 +1,28 @@
 'use strict';
 
 const ensureString = require('type/string/ensure');
+const { CloudFormationClient, DescribeStacksCommand } = require('@aws-sdk/client-cloudformation');
 const ServerlessError = require('../../../../serverless-error');
+const createCachedAwsVariableSourceCommandSender = require('./create-cached-aws-variable-source-command-sender');
+
+function isStackNotFoundError(error) {
+  const message = error && error.message ? error.message : '';
+
+  return (
+    message.includes('does not exist') &&
+    (error.name === 'ValidationError' ||
+      error.Code === 'ValidationError' ||
+      error.code === 'ValidationError' ||
+      error.code === 'AWS_CLOUD_FORMATION_DESCRIBE_STACKS_VALIDATION_ERROR')
+  );
+}
 
 module.exports = (serverlessInstance) => {
+  const sender = createCachedAwsVariableSourceCommandSender({
+    getProvider: () => serverlessInstance.getProvider('aws'),
+    Client: CloudFormationClient,
+  });
+
   return {
     resolve: async ({ address, params }) => {
       // cf(region = null):stackName.outputLogicalId
@@ -31,27 +50,19 @@ module.exports = (serverlessInstance) => {
 
       const result = await (async () => {
         try {
-          return await serverlessInstance
-            .getProvider('aws')
-            .request(
-              'CloudFormation',
-              'describeStacks',
-              { StackName: stackName },
-              { useCache: true, region: params && params[0] }
-            );
+          return await sender.send(
+            DescribeStacksCommand,
+            { StackName: stackName },
+            { region: params && params[0] }
+          );
         } catch (error) {
-          if (
-            error.code === 'AWS_CLOUD_FORMATION_DESCRIBE_STACKS_VALIDATION_ERROR' &&
-            error.message.includes('does not exist')
-          ) {
-            return null;
-          }
+          if (isStackNotFoundError(error)) return null;
           throw error;
         }
       })();
 
       if (!result) return { value: null };
-      const outputs = result.Stacks[0].Outputs;
+      const outputs = result.Stacks[0].Outputs || [];
       const output = outputs.find((x) => x.OutputKey === outputLogicalId);
 
       return { value: output ? output.OutputValue : null };

lib/configuration/variables/sources/instance-dependent/get-s3.js

@@ -1,9 +1,28 @@
 'use strict';
 
 const ensureString = require('type/string/ensure');
+const { S3Client, GetObjectCommand } = require('@aws-sdk/client-s3');
 const ServerlessError = require('../../../../serverless-error');
+const s3BodyToString = require('../../../../aws/s3-body-to-string');
+const createCachedAwsVariableSourceCommandSender = require('./create-cached-aws-variable-source-command-sender');
+
+function isNoSuchKeyError(error) {
+  return (
+    error &&
+    (error.name === 'NoSuchKey' ||
+      error.Code === 'NoSuchKey' ||
+      error.code === 'NoSuchKey' ||
+      error.code === 'AWS_S3_GET_OBJECT_NO_SUCH_KEY')
+  );
+}
 
 module.exports = (serverlessInstance) => {
+  const sender = createCachedAwsVariableSourceCommandSender({
+    getProvider: () => serverlessInstance.getProvider('aws'),
+    Client: S3Client,
+    transformResult: async ({ result }) => s3BodyToString(result.Body),
+  });
+
   return {
     resolve: async ({ address }) => {
       // s3:bucketName/key
@@ -31,19 +50,16 @@ module.exports = (serverlessInstance) => {
 
       const result = await (async () => {
         try {
-          return await serverlessInstance
-            .getProvider('aws')
-            .request('S3', 'getObject', { Bucket: bucketName, Key: key }, { useCache: true });
+          return await sender.send(GetObjectCommand, { Bucket: bucketName, Key: key });
         } catch (error) {
-          // Check for normalized error code instead of native one
-          if (error.code === 'AWS_S3_GET_OBJECT_NO_SUCH_KEY') return null;
+          if (isNoSuchKeyError(error)) return null;
           throw error;
         }
       })();
 
-      if (!result) return { value: null };
+      if (result == null) return { value: null };
 
-      return { value: String(result.Body) };
+      return { value: result };
     },
   };
 };