[2/6] Migrate AWS read paths and state discovery to SDK v3

lib/plugins/aws/deploy-list.js

@@ -1,10 +1,20 @@
 'use strict';
 
+const promiseLimit = require('ext/promise/limit').bind(Promise);
 const { log, writeText } = require('../../utils/serverless-utils/log');
 const validate = require('./lib/validate');
-const findAndGroupDeployments = require('./utils/find-and-group-deployments');
+const parseDeploymentObjectKey = require('./utils/parse-deployment-object-key');
 const setBucketName = require('./lib/set-bucket-name');
 const ServerlessError = require('../../serverless-error');
+const { S3Client, paginateListObjectsV2 } = require('@aws-sdk/client-s3');
+const {
+  LambdaClient,
+  GetFunctionCommand,
+  ListVersionsByFunctionCommand,
+} = require('@aws-sdk/client-lambda');
+const { isS3ListObjectsAccessDeniedError } = require('./utils/aws-sdk-v3-error');
+
+const limitLambdaRequests = promiseLimit(6, async (task) => task());
 
 class AwsDeployList {
   constructor(serverless, options) {
@@ -31,50 +41,71 @@ class AwsDeployList {
     const stage = this.provider.getStage();
     const prefix = this.provider.getDeploymentPrefix();
 
-    let response;
+    let foundDeployment = false;
+    let currentDirectory = null;
+    let currentDeployment = [];
+    const flushDeployment = () => {
+      if (!currentDeployment.length) return;
+      this.printDeployment(currentDeployment);
+      foundDeployment = true;
+      currentDeployment = [];
+    };
+
     try {
-      response = await this.provider.request('S3', 'listObjectsV2', {
-        Bucket: this.bucketName,
-        Prefix: `${prefix}/${service}/${stage}/`,
-      });
+      const s3 = new S3Client(await this.provider.getAwsSdkV3Config());
+      const paginator = paginateListObjectsV2(
+        { client: s3 },
+        {
+          Bucket: this.bucketName,
+          Prefix: `${prefix}/${service}/${stage}/`,
+        }
+      );
+      for await (const page of paginator) {
+        for (const object of page.Contents || []) {
+          const entry = parseDeploymentObjectKey(object.Key, prefix, service, stage);
+          if (!entry) continue;
+
+          if (currentDirectory && entry.directory !== currentDirectory) flushDeployment();
+          currentDirectory = entry.directory;
+          currentDeployment.push(entry);
+        }
+      }
+      flushDeployment();
     } catch (err) {
-      if (err.code === 'AWS_S3_LIST_OBJECTS_V2_ACCESS_DENIED') {
+      if (isS3ListObjectsAccessDeniedError(err)) {
         throw new ServerlessError(
           'Could not list objects in the deployment bucket. Make sure you have sufficient permissions to access it.',
-          err.code
+          'AWS_S3_LIST_OBJECTS_V2_ACCESS_DENIED'
         );
       }
       throw err;
     }
 
-    const directoryRegex = new RegExp('(.+)-(.+-.+-.+)');
-    const deployments = findAndGroupDeployments(response, prefix, service, stage);
-
-    if (deployments.length === 0) {
+    if (!foundDeployment) {
       log.notice();
       log.notice.skip(
         "No deployments found, if that's unexpected ensure that stage and region are correct"
       );
-      return;
     }
+  }
 
-    deployments.forEach((deployment) => {
-      const match = deployment[0].directory.match(directoryRegex);
-      const date = new Date(Date.parse(match[2]));
-      writeText(
-        `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, 0)}-${String(
-          date.getUTCDate()
-        ).padStart(2, 0)} ` +
-          `${String(date.getUTCHours()).padStart(2, 0)}:${String(date.getUTCMinutes()).padStart(
-            2,
-            0
-          )}:${String(date.getUTCSeconds()).padStart(2, 0)} UTC`,
-        `Timestamp: ${match[1]}`,
-        'Files:'
-      );
-      deployment.forEach((entry) => {
-        writeText(`  - ${entry.file}`);
-      });
+  printDeployment(deployment) {
+    const directoryRegex = new RegExp('(.+)-(.+-.+-.+)');
+    const match = deployment[0].directory.match(directoryRegex);
+    const date = new Date(Date.parse(match[2]));
+    writeText(
+      `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, 0)}-${String(
+        date.getUTCDate()
+      ).padStart(2, 0)} ` +
+        `${String(date.getUTCHours()).padStart(2, 0)}:${String(date.getUTCMinutes()).padStart(
+          2,
+          0
+        )}:${String(date.getUTCSeconds()).padStart(2, 0)} UTC`,
+      `Timestamp: ${match[1]}`,
+      'Files:'
+    );
+    deployment.forEach((entry) => {
+      writeText(`  - ${entry.file}`);
     });
   }
 
@@ -87,42 +118,48 @@ class AwsDeployList {
 
   async getFunctions() {
     const funcs = this.serverless.service.getAllFunctionsNames();
+    const lambda = new LambdaClient(await this.provider.getAwsSdkV3Config());
 
     const result = await Promise.all(
       funcs.map((funcName) => {
         const params = {
           FunctionName: funcName,
         };
 
-        return this.provider.request('Lambda', 'getFunction', params);
+        return limitLambdaRequests(() => lambda.send(new GetFunctionCommand(params)));
       })
     );
 
     return result.map((item) => item.Configuration);
   }
 
-  async getFunctionPaginatedVersions(params, totalVersions) {
-    const response = await this.provider.request('Lambda', 'listVersionsByFunction', params);
+  async getFunctionPaginatedVersions(params, totalVersions, lambda) {
+    if (!lambda) lambda = new LambdaClient(await this.provider.getAwsSdkV3Config());
+    const response = await lambda.send(new ListVersionsByFunctionCommand(params));
 
     const Versions = (totalVersions || []).concat(response.Versions);
     if (response.NextMarker) {
       return this.getFunctionPaginatedVersions(
         { ...params, Marker: response.NextMarker },
-        Versions
+        Versions,
+        lambda
       );
     }
 
     return { Versions };
   }
 
   async getFunctionVersions(funcs) {
+    const lambda = new LambdaClient(await this.provider.getAwsSdkV3Config());
     return Promise.all(
       funcs.map((func) => {
         const params = {
           FunctionName: func.FunctionName,
         };
 
-        return this.getFunctionPaginatedVersions(params);
+        return limitLambdaRequests(() =>
+          this.getFunctionPaginatedVersions(params, undefined, lambda)
+        );
       })
     );
   }

lib/plugins/aws/deploy/lib/check-for-changes.js

@@ -4,16 +4,35 @@ const fs = require('fs');
 const path = require('path');
 const crypto = require('crypto');
 const { isDeepStrictEqual } = require('node:util');
+const promiseLimit = require('ext/promise/limit').bind(Promise);
 const glob = require('../../../../utils/glob');
 const normalizeFiles = require('../../lib/normalize-files');
+const parseDeploymentObjectKey = require('../../utils/parse-deployment-object-key');
 const ServerlessError = require('../../../../serverless-error');
 const log = require('../../../../utils/serverless-utils/log').log.get('check-for-changes');
+const { S3Client, HeadObjectCommand, paginateListObjectsV2 } = require('@aws-sdk/client-s3');
+const { LambdaClient, GetFunctionCommand } = require('@aws-sdk/client-lambda');
+const {
+  CloudWatchLogsClient,
+  DescribeSubscriptionFiltersCommand,
+} = require('@aws-sdk/client-cloudwatch-logs');
+const {
+  CloudFormationClient,
+  DescribeStackResourceCommand,
+} = require('@aws-sdk/client-cloudformation');
+const {
+  isS3ListObjectsNoSuchBucketError,
+  isS3HeadObjectForbiddenError,
+  isLambdaAccessDeniedError,
+} = require('../../utils/aws-sdk-v3-error');
 
 const fsp = fs.promises;
 
-const isDeploymentDirToken = RegExp.prototype.test.bind(
-  /^[\d]+-\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/
-);
+const limitLambdaRequests = promiseLimit(6, async (task) => task());
+const limitS3Requests = promiseLimit(6, async (task) => task());
+const limitCloudWatchLogsRequests = promiseLimit(2, async (task) => task());
+const limitCloudFormationRequests = promiseLimit(6, async (task) => task());
+
 const isOtelExtensionName = RegExp.prototype.test.bind(/^sls-otel\.\d+\.\d+\.\d+\.zip$/);
 
 module.exports = {
@@ -38,61 +57,65 @@ module.exports = {
 
   async getMostRecentObjects() {
     const service = this.serverless.service.service;
+    const stage = this.provider.getStage();
+    const prefix = this.provider.getDeploymentPrefix();
 
     const params = {
       Bucket: this.bucketName,
-      Prefix: `${this.provider.getDeploymentPrefix()}/${service}/${this.provider.getStage()}/`,
+      Prefix: `${prefix}/${service}/${stage}/`,
     };
 
-    const result = await (async () => {
-      try {
-        return await this.provider.request('S3', 'listObjectsV2', params);
-      } catch (error) {
-        if (!error.message.includes('The specified bucket does not exist')) throw error;
-        const stackName = this.provider.naming.getStackName();
-        throw new ServerlessError(
-          [
-            `The serverless deployment bucket "${params.Bucket}" does not exist.`,
-            `Create it manually if you want to reuse the CloudFormation stack "${stackName}",`,
-            'or delete the stack if it is no longer required.',
-          ].join(' '),
-          'DEPLOYMENT_BUCKET_DOES_NOT_EXIST'
-        );
-      }
-    })();
-    const contents = result && result.Contents;
-    const objects = contents
-      ? contents.filter(({ Key: key }) => isDeploymentDirToken(key.split('/')[3]))
-      : [];
-    if (!objects.length) {
-      return [];
-    }
-
-    const ordered = [...objects].sort((objectA, objectB) => objectB.Key.localeCompare(objectA.Key));
-
-    const firstKey = ordered[0].Key;
-    const directory = firstKey.substring(0, firstKey.lastIndexOf('/'));
+    try {
+      const s3 = new S3Client(await this.provider.getAwsSdkV3Config());
+      let latestDirectory = null;
+      let latestObjects = [];
+
+      for await (const page of paginateListObjectsV2({ client: s3 }, params)) {
+        for (const object of page.Contents || []) {
+          const entry = parseDeploymentObjectKey(object.Key, prefix, service, stage);
+          if (!entry) continue;
+
+          if (!latestDirectory || entry.directory > latestDirectory) {
+            latestDirectory = entry.directory;
+            latestObjects = [object];
+            continue;
+          }
 
-    return ordered.filter((obj) => {
-      const objKey = obj.Key;
-      const objDirectory = objKey.substring(0, objKey.lastIndexOf('/'));
+          if (entry.directory === latestDirectory) latestObjects.push(object);
+        }
+      }
 
-      return directory === objDirectory;
-    });
+      return latestObjects.sort((objectA, objectB) => objectB.Key.localeCompare(objectA.Key));
+    } catch (error) {
+      if (!isS3ListObjectsNoSuchBucketError(error)) throw error;
+      const stackName = this.provider.naming.getStackName();
+      throw new ServerlessError(
+        [
+          `The serverless deployment bucket "${params.Bucket}" does not exist.`,
+          `Create it manually if you want to reuse the CloudFormation stack "${stackName}",`,
+          'or delete the stack if it is no longer required.',
+        ].join(' '),
+        'DEPLOYMENT_BUCKET_DOES_NOT_EXIST'
+      );
+    }
   },
 
   // Gives the least recent last modify date across all the functions in the service.
   async getFunctionsEarliestLastModifiedDate() {
     let couldNotAccessFunction = false;
+    const lambda = new LambdaClient(await this.provider.getAwsSdkV3Config());
     const getFunctionResults = this.serverless.service.getAllFunctions().map((funName) => {
       const functionObj = this.serverless.service.getFunction(funName);
-      return this.provider
-        .request('Lambda', 'getFunction', {
-          FunctionName: functionObj.name,
-        })
+      return limitLambdaRequests(() =>
+        lambda.send(
+          new GetFunctionCommand({
+            FunctionName: functionObj.name,
+          })
+        )
+      )
         .then((res) => new Date(res.Configuration.LastModified))
         .catch((err) => {
-          if (err.providerError && err.providerError.statusCode === 403) {
+          if (isLambdaAccessDeniedError(err)) {
             couldNotAccessFunction = true;
           }
           return new Date(0);
@@ -114,20 +137,25 @@ module.exports = {
   },
 
   async getObjectMetadata(objects) {
+    const s3 = new S3Client(await this.provider.getAwsSdkV3Config());
     return Promise.all(
       objects.map(async (obj) => {
         try {
-          const result = await this.provider.request('S3', 'headObject', {
-            Bucket: this.bucketName,
-            Key: obj.Key,
-          });
+          const result = await limitS3Requests(() =>
+            s3.send(
+              new HeadObjectCommand({
+                Bucket: this.bucketName,
+                Key: obj.Key,
+              })
+            )
+          );
           result.Key = obj.Key;
           return result;
         } catch (err) {
-          if (err.code === 'AWS_S3_HEAD_OBJECT_FORBIDDEN') {
+          if (isS3HeadObjectForbiddenError(err)) {
             throw new ServerlessError(
               'Could not access objects in the deployment bucket. Make sure you have sufficient permissions to access it.',
-              err.code
+              'AWS_S3_HEAD_OBJECT_FORBIDDEN'
             );
           }
           throw err;
@@ -300,11 +328,10 @@ module.exports = {
     const cloudwatchLogEvents = params.cloudwatchLogEvents;
     const CLOUDWATCHLOG_LOG_GROUP_EVENT_PER_FUNCTION_LIMIT = 2;
 
-    const response = await this.provider
-      .request('CloudWatchLogs', 'describeSubscriptionFilters', {
-        logGroupName,
-      })
-      .catch(() => ({ subscriptionFilters: [] }));
+    const cloudWatchLogs = new CloudWatchLogsClient(await this.provider.getAwsSdkV3Config());
+    const response = await limitCloudWatchLogsRequests(() =>
+      cloudWatchLogs.send(new DescribeSubscriptionFiltersCommand({ logGroupName }))
+    ).catch(() => ({ subscriptionFilters: [] }));
     if (response.subscriptionFilters.length === 0) {
       log.debug('no subscription filters detected');
       return false;
@@ -394,13 +421,14 @@ module.exports = {
 
   async isInternalSubscriptionFilter(stackName, logicalResourceId, physicalResourceId) {
     try {
-      const { StackResourceDetail } = await this.provider.request(
-        'CloudFormation',
-        'describeStackResource',
-        {
-          StackName: stackName,
-          LogicalResourceId: logicalResourceId,
-        }
+      const cloudFormation = new CloudFormationClient(await this.provider.getAwsSdkV3Config());
+      const { StackResourceDetail } = await limitCloudFormationRequests(() =>
+        cloudFormation.send(
+          new DescribeStackResourceCommand({
+            StackName: stackName,
+            LogicalResourceId: logicalResourceId,
+          })
+        )
       );
 
       return physicalResourceId === StackResourceDetail.PhysicalResourceId;