Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 16 additions & 17 deletions lib/plugins/aws/deploy/lib/cleanup-s3-bucket.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const isS3ListAccessDeniedError = require('../../utils/is-s3-list-access-denied-
const parseDeploymentObjectKey = require('../../utils/parse-deployment-object-key');
const ServerlessError = require('../../../../serverless-error');
const { log } = require('../../../../utils/serverless-utils/log');
const { S3Client, DeleteObjectsCommand, paginateListObjectsV2 } = require('@aws-sdk/client-s3');

const maxDeleteObjectsCount = 1000;

Expand All @@ -31,19 +32,12 @@ const createDeleteObjectsError = (bucketName, firstError) => {
);
};

async function listObjectsV2(provider, params) {
async function listObjectsV2(s3, params) {
const Contents = [];
let ContinuationToken;

do {
const response = await provider.request('S3', 'listObjectsV2', {
...params,
...(ContinuationToken ? { ContinuationToken } : {}),
});

Contents.push(...(response?.Contents || []));
ContinuationToken = response && response.NextContinuationToken;
} while (ContinuationToken);
for await (const response of paginateListObjectsV2({ client: s3 }, params)) {
Contents.push(...(response.Contents || []));
}

return { Contents };
}
Expand All @@ -58,7 +52,8 @@ module.exports = {
const prefix = this.provider.getDeploymentPrefix();

try {
const response = await listObjectsV2(this.provider, {
const s3 = new S3Client(await this.provider.getAwsSdkV3Config());
const response = await listObjectsV2(s3, {
Bucket: this.bucketName,
Prefix: `${prefix}/${service}/${stage}/`,
});
Expand All @@ -76,12 +71,15 @@ module.exports = {
async removeObjects(objectsToRemove) {
if (!objectsToRemove || !objectsToRemove.length) return;

const s3 = new S3Client(await this.provider.getAwsSdkV3Config());
for (let index = 0; index < objectsToRemove.length; index += maxDeleteObjectsCount) {
const batch = objectsToRemove.slice(index, index + maxDeleteObjectsCount);
const result = await this.provider.request('S3', 'deleteObjects', {
Bucket: this.bucketName,
Delete: { Objects: batch },
});
const result = await s3.send(
new DeleteObjectsCommand({
Bucket: this.bucketName,
Delete: { Objects: batch },
})
);

if (result && result.Errors && result.Errors.length) {
throw createDeleteObjectsError(this.bucketName, result.Errors[0]);
Expand Down Expand Up @@ -127,7 +125,8 @@ module.exports = {
'INVALID_EMPTY_CHANGE_SET_ARTIFACT_DIRECTORY'
);
}
response = await listObjectsV2(this.provider, {
const s3 = new S3Client(await this.provider.getAwsSdkV3Config());
response = await listObjectsV2(s3, {
Bucket: this.bucketName,
Prefix: `${artifactDirectoryName}/`,
});
Expand Down
5 changes: 3 additions & 2 deletions lib/plugins/aws/deploy/lib/upload-artifacts.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const crypto = require('crypto');
const limit = require('ext/promise/limit').bind(Promise);
const { filesize } = require('filesize');
const normalizeFiles = require('../../lib/normalize-files');
const uploadS3Object = require('../../lib/upload-s3-object');
const getLambdaLayerArtifactPath = require('../../utils/get-lambda-layer-artifact-path');
const ServerlessError = require('../../../../serverless-error');
const setS3UploadEncryptionOptions = require('../../../../aws/set-s3-upload-encryption-options');
Expand Down Expand Up @@ -71,7 +72,7 @@ module.exports = {
params = setS3UploadEncryptionOptions(params, deploymentBucketObject);
}

return this.provider.request('S3', 'upload', params);
return uploadS3Object(this.provider, params);
},
async uploadStateFile() {
log.info('Uploading State file to S3');
Expand Down Expand Up @@ -101,7 +102,7 @@ module.exports = {
params = setS3UploadEncryptionOptions(params, deploymentBucketObject);
}

return this.provider.request('S3', 'upload', params);
return uploadS3Object(this.provider, params);
},

async getFunctionArtifactFilePaths() {
Expand Down
49 changes: 49 additions & 0 deletions lib/plugins/aws/lib/upload-s3-object.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
'use strict';

const { S3Client } = require('@aws-sdk/client-s3');
const { Upload } = require('@aws-sdk/lib-storage');
const ServerlessError = require('../../../serverless-error');
const { getAwsErrorCode, getAwsErrorStatusCode } = require('../utils/aws-sdk-v3-error');

const uploadQueueSize = 6;
const uploadPartSize = 5 * 1024 * 1024;
const normalizerPattern = /(?<!^)([A-Z])/g;

const normalizeErrorCodePostfix = (code) => {
if (typeof code === 'number') return `HTTP_${code}_ERROR`;
return String(code).replace(normalizerPattern, '_$1').toUpperCase();
};

module.exports = async (provider, params) => {
const s3 = new S3Client(
await provider.getAwsSdkV3Config({
useAccelerateEndpoint: provider.isS3TransferAccelerationEnabled(),
})
);

try {
return await new Upload({
client: s3,
params,
queueSize: uploadQueueSize,
partSize: uploadPartSize,
leavePartsOnError: false,
}).done();
} catch (error) {
const providerErrorCode = getAwsErrorCode(error) || getAwsErrorStatusCode(error);
const providerErrorCodeExtension = providerErrorCode
? normalizeErrorCodePostfix(providerErrorCode)
: 'ERROR';

throw Object.assign(
new ServerlessError(
error && error.message ? error.message : String(providerErrorCode || 'Error'),
`AWS_S3_UPLOAD_${providerErrorCodeExtension}`
),
{
providerError: error,
providerErrorCodeExtension,
}
);
}
};
3 changes: 2 additions & 1 deletion lib/plugins/aws/lib/upload-zip-file.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ const fs = require('fs');
const crypto = require('crypto');
const log = require('../../../utils/serverless-utils/log').log.get('deploy:upload');
const setS3UploadEncryptionOptions = require('../../../aws/set-s3-upload-encryption-options');
const uploadS3Object = require('./upload-s3-object');

module.exports = {
async uploadZipFile({ filename, s3KeyDirname, basename }) {
Expand Down Expand Up @@ -38,7 +39,7 @@ module.exports = {
params = setS3UploadEncryptionOptions(params, deploymentBucketObject);
}

const response = await this.provider.request('S3', 'upload', params);
const response = await uploadS3Object(this.provider, params);
// Interestingly, if request handling was queued, and stream errored (before being consumed by
// AWS SDK) then SDK call succeeds without actually uploading a file to S3 bucket.
// Below line ensures that eventual stream error is communicated
Expand Down
37 changes: 14 additions & 23 deletions lib/plugins/aws/package/compile/functions.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
'use strict';

const AWS = require('../../../../aws/sdk-v2');
const crypto = require('crypto');
const fs = require('fs');
const path = require('path');
const { isDeepStrictEqual } = require('node:util');
const { pipeline } = require('node:stream/promises');
const { S3Client, GetObjectCommand } = require('@aws-sdk/client-s3');
const isObject = require('type/object/is');
const ServerlessError = require('../../../../serverless-error');
const deepSortObjectByKey = require('../../../../utils/deep-sort-object-by-key');
Expand Down Expand Up @@ -115,9 +116,6 @@ class AwsCompileFunctions {
}

async downloadPackageArtifact(functionName) {
const { region } = this.options;
const S3 = new AWS.S3({ region });

const functionObject = this.serverless.service.getFunction(functionName);
if (functionObject.image) return;

Expand All @@ -128,25 +126,18 @@ class AwsCompileFunctions {
const s3Object = parseS3URI(artifactFilePath);
if (!s3Object) return;
log.info(`Downloading ${s3Object.Key} from bucket ${s3Object.Bucket}`);
await new Promise((resolve, reject) => {
const tmpDir = this.serverless.utils.getTmpDirPath();
const filePath = path.join(tmpDir, path.basename(s3Object.Key));

const readStream = S3.getObject(s3Object).createReadStream();

const writeStream = fs.createWriteStream(filePath);
readStream
.pipe(writeStream)
.on('error', reject)
.on('close', () => {
if (functionObject.package.artifact) {
functionObject.package.artifact = filePath;
} else {
this.serverless.service.package.artifact = filePath;
}
return resolve(filePath);
});
});
const tmpDir = this.serverless.utils.getTmpDirPath();
const filePath = path.join(tmpDir, path.basename(s3Object.Key));
const { region } = this.options;
const s3 = new S3Client(await this.provider.getAwsSdkV3Config({ region }));
const response = await s3.send(new GetObjectCommand(s3Object));

await pipeline(response.Body, fs.createWriteStream(filePath));
if (functionObject.package && functionObject.package.artifact) {
functionObject.package.artifact = filePath;
} else {
this.serverless.service.package.artifact = filePath;
}
}

async addFileToHash(filePath, hash) {
Expand Down
54 changes: 34 additions & 20 deletions lib/plugins/aws/remove/lib/bucket.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@ const { log } = require('../../../../utils/serverless-utils/log');
const ServerlessError = require('../../../../serverless-error');
const isS3ListAccessDeniedError = require('../../utils/is-s3-list-access-denied-error');
const { isCloudFormationValidationError } = require('../../utils/aws-sdk-v3-error');
const {
S3Client,
DeleteObjectsCommand,
ListObjectsV2Command,
ListObjectVersionsCommand,
} = require('@aws-sdk/client-s3');

const maxDeleteObjectsCount = 1000;

Expand Down Expand Up @@ -35,17 +41,20 @@ module.exports = {
this.serverless.service.service
}/${this.provider.getStage()}/`;

const s3 = new S3Client(await this.provider.getAwsSdkV3Config());
let ContinuationToken;

do {
let result;

try {
result = await this.provider.request('S3', 'listObjectsV2', {
Bucket: this.bucketName,
Prefix: prefix,
...(ContinuationToken ? { ContinuationToken } : {}),
});
result = await s3.send(
new ListObjectsV2Command({
Bucket: this.bucketName,
Prefix: prefix,
...(ContinuationToken ? { ContinuationToken } : {}),
})
);
} catch (err) {
if (isS3ListAccessDeniedError(err)) throw createS3ListObjectsAccessDeniedError();
throw err;
Expand All @@ -56,7 +65,7 @@ module.exports = {
}));

const nextContinuationToken = result && result.NextContinuationToken;
await this.deleteObjectBatches(pageObjects);
await this.deleteObjectBatches(pageObjects, s3);
ContinuationToken = nextContinuationToken;
} while (ContinuationToken);
},
Expand All @@ -66,19 +75,22 @@ module.exports = {
this.serverless.service.service
}/${this.provider.getStage()}/`;

const s3 = new S3Client(await this.provider.getAwsSdkV3Config());
let KeyMarker;
let VersionIdMarker;

do {
let result;

try {
result = await this.provider.request('S3', 'listObjectVersions', {
Bucket: this.bucketName,
Prefix: prefix,
...(KeyMarker ? { KeyMarker } : {}),
...(VersionIdMarker ? { VersionIdMarker } : {}),
});
result = await s3.send(
new ListObjectVersionsCommand({
Bucket: this.bucketName,
Prefix: prefix,
...(KeyMarker ? { KeyMarker } : {}),
...(VersionIdMarker ? { VersionIdMarker } : {}),
})
);
} catch (err) {
if (isS3ListAccessDeniedError(err)) throw createS3ListObjectsAccessDeniedError();
throw err;
Expand All @@ -97,7 +109,7 @@ module.exports = {

const nextKeyMarker = result && result.NextKeyMarker;
const nextVersionIdMarker = result && result.NextVersionIdMarker;
await this.deleteObjectBatches(pageObjects);
await this.deleteObjectBatches(pageObjects, s3);
KeyMarker = nextKeyMarker;
VersionIdMarker = nextVersionIdMarker;
} while (KeyMarker || VersionIdMarker);
Expand All @@ -110,17 +122,19 @@ module.exports = {
: this.listObjectsV2();
},

async deleteObjectBatches(objects) {
async deleteObjectBatches(objects, s3) {
if (!objects.length) return;

for (let index = 0; index < objects.length; index += maxDeleteObjectsCount) {
const batch = objects.slice(index, index + maxDeleteObjectsCount);
const data = await this.provider.request('S3', 'deleteObjects', {
Bucket: this.bucketName,
Delete: {
Objects: batch,
},
});
const data = await s3.send(
new DeleteObjectsCommand({
Bucket: this.bucketName,
Delete: {
Objects: batch,
},
})
);
if (data && data.Errors && data.Errors.length) {
const firstErrorCode = data.Errors[0].Code;

Expand Down
Loading