feat(storage): support copying CSEK-encrypted files to non-CSEK destinations

handwritten/storage/src/file.ts

@@ -317,7 +317,7 @@ export type RenameCallback = MoveCallback;
 export type RotateEncryptionKeyOptions = string | Buffer | EncryptionKeyOptions;
 
 export interface EncryptionKeyOptions {
-  encryptionKey?: string | Buffer;
+  encryptionKey?: string | Buffer | null;
   kmsKeyName?: string;
   preconditionOpts?: PreconditionOptions;
 }
@@ -366,7 +366,7 @@ const COMPRESSIBLE_MIME_REGEX = new RegExp(
 
 export interface FileOptions {
   crc32cGenerator?: CRC32CValidatorGenerator;
-  encryptionKey?: string | Buffer;
+  encryptionKey?: string | Buffer | null;
   generation?: number | string;
   restoreToken?: string;
   kmsKeyName?: string;
@@ -425,7 +425,7 @@ export type DownloadCallback = (
 
 export interface DownloadOptions extends CreateReadStreamOptions {
   destination?: string;
-  encryptionKey?: string | Buffer;
+  encryptionKey?: string | Buffer | null;
 }
 
 interface CopyQuery {
@@ -600,7 +600,7 @@ class File extends ServiceObject<File, FileMetadata> {
   restoreToken?: string;
   parent!: Bucket;
 
-  private encryptionKey?: string | Buffer;
+  private encryptionKey?: string | Buffer | null;
   private encryptionKeyBase64?: string;
   private encryptionKeyHash?: string;
   private encryptionKeyInterceptor?: Interceptor;
@@ -1102,7 +1102,7 @@ class File extends ServiceObject<File, FileMetadata> {
 
     this.name = name;
 
-    if (options.encryptionKey) {
+    if (options.encryptionKey !== undefined) {
       this.setEncryptionKey(options.encryptionKey);
     }
 
@@ -1379,15 +1379,25 @@ class File extends ServiceObject<File, FileMetadata> {
 
     const headers: {[index: string]: string | undefined} = {};
 
-    if (this.encryptionKey !== undefined) {
+    if (this.encryptionKey !== undefined && this.encryptionKey !== null) {
       headers['x-goog-copy-source-encryption-algorithm'] = 'AES256';
       headers['x-goog-copy-source-encryption-key'] = this.encryptionKeyBase64;
       headers['x-goog-copy-source-encryption-key-sha256'] =
         this.encryptionKeyHash;
     }
 
-    if (newFile.encryptionKey !== undefined) {
-      this.setEncryptionKey(newFile.encryptionKey!);
+    if (
+      this.encryptionKey !== undefined &&
+      this.encryptionKey !== null &&
+      newFile.encryptionKey === undefined
+    ) {
+      newFile.setEncryptionKey(this.encryptionKey);
+    }
+
+    if (newFile.encryptionKey !== undefined && newFile.encryptionKey !== null) {
+      headers['x-goog-encryption-algorithm'] = 'AES256';
+      headers['x-goog-encryption-key'] = newFile.encryptionKeyBase64;
+      headers['x-goog-encryption-key-sha256'] = newFile.encryptionKeyHash;
     } else if (options.destinationKmsKeyName !== undefined) {
       query.destinationKmsKeyName = options.destinationKmsKeyName;
       delete options.destinationKmsKeyName;
@@ -1419,10 +1429,12 @@ class File extends ServiceObject<File, FileMetadata> {
       delete options.preconditionOpts;
     }
 
-    this.request(
+    this.bucket.request(
       {
         method: 'POST',
-        uri: `/rewriteTo/b/${destBucket.name}/o/${encodeURIComponent(
+        uri: `/o/${encodeURIComponent(
+          this.name,
+        )}/rewriteTo/b/${destBucket.name}/o/${encodeURIComponent(
           newFile.name,
         )}`,
         qs: query,
@@ -1864,7 +1876,7 @@ class File extends ServiceObject<File, FileMetadata> {
         ),
         file: this.name,
         generation: this.generation,
-        key: this.encryptionKey,
+        key: this.encryptionKey === null ? undefined : this.encryptionKey,
         kmsKeyName: this.kmsKeyName,
         metadata: options.metadata,
         offset: options.offset,
@@ -2392,7 +2404,7 @@ class File extends ServiceObject<File, FileMetadata> {
     const destination = options.destination;
     delete options.destination;
 
-    if (options.encryptionKey) {
+    if (options.encryptionKey !== undefined) {
       this.setEncryptionKey(options.encryptionKey);
       delete options.encryptionKey;
     }
@@ -2482,8 +2494,23 @@ class File extends ServiceObject<File, FileMetadata> {
    * region_tag:storage_download_encrypted_file
    * Example of downloading an encrypted file:
    */
-  setEncryptionKey(encryptionKey: string | Buffer) {
+  setEncryptionKey(encryptionKey: string | Buffer | null) {
+    if (this.encryptionKeyInterceptor) {
+      const index = this.interceptors.indexOf(this.encryptionKeyInterceptor);
+      if (index > -1) {
+        this.interceptors.splice(index, 1);
+      }
+      this.encryptionKeyInterceptor = undefined;
+    }
+
     this.encryptionKey = encryptionKey;
+
+    if (encryptionKey === null || encryptionKey === undefined) {
+      this.encryptionKeyBase64 = undefined;
+      this.encryptionKeyHash = undefined;
+      return this;
+    }
+
     this.encryptionKeyBase64 = Buffer.from(encryptionKey as string).toString(
       'base64',
     );
@@ -2504,7 +2531,7 @@ class File extends ServiceObject<File, FileMetadata> {
       },
     };
 
-    this.interceptors.push(this.encryptionKeyInterceptor!);
+    this.interceptors.push(this.encryptionKeyInterceptor);
 
     return this;
   }
@@ -4462,7 +4489,7 @@ class File extends ServiceObject<File, FileMetadata> {
       file: this.name,
       generation: this.generation,
       isPartialUpload: options.isPartialUpload,
-      key: this.encryptionKey,
+      key: this.encryptionKey === null ? undefined : this.encryptionKey,
       kmsKeyName: this.kmsKeyName,
       metadata: options.metadata,
       offset: options.offset,

handwritten/storage/system-test/storage.ts

@@ -2782,6 +2782,24 @@ describe('storage', function () {
         const [contents] = await file.download();
         assert.strictEqual(contents.toString(), 'secret data');
       });
+
+      it('should copy a CSEK-encrypted file to a standard non-CSEK destination when destination key is null', async () => {
+        const srcFile = bucket.file('encrypted-source');
+        srcFile.setEncryptionKey('a'.repeat(32));
+
+        await srcFile.save('csek data', { resumable: false });
+
+        const dstFile = bucket.file('non-csek-destination');
+        dstFile.setEncryptionKey(null);
+
+        await srcFile.copy(dstFile);
+
+        const [metadata] = await dstFile.getMetadata();
+        assert.strictEqual(metadata.customerEncryption, undefined);
+
+        const [contents] = await dstFile.download();
+        assert.strictEqual(contents.toString(), 'csek data');
+      });
     });
 
     describe('kms keys', () => {

handwritten/storage/test/file.ts

@@ -189,6 +189,10 @@ describe('File', () => {
   let File: any;
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   let file: any;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let activeFile: any = null;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  let originalCopy: any;
 
   const FILE_NAME = 'file-name.png';
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -228,6 +232,18 @@ describe('File', () => {
       './signer': fakeSigner,
       zlib: fakeZlib,
     }).File;
+
+    originalCopy = File.prototype.copy;
+    File.prototype.copy = function (dest: any, options: any, callback: any) {
+      activeFile = this;
+      return originalCopy.call(this, dest, options, callback);
+    };
+  });
+
+  after(() => {
+    if (originalCopy) {
+      File.prototype.copy = originalCopy;
+    }
   });
 
   beforeEach(() => {
@@ -274,6 +290,19 @@ describe('File', () => {
     specialCharsFile = new File(BUCKET, "special/azAZ!*'()*%/file.jpg");
     specialCharsFile.request = util.noop;
 
+    activeFile = null;
+    BUCKET.request = function (reqOpts: any, callback: any) {
+      if (activeFile && typeof activeFile.request === 'function' && (activeFile.request as any) !== util.noop) {
+        const prefix = `/o/${encodeURIComponent(activeFile.name)}`;
+        const modifiedReqOpts = { ...reqOpts };
+        if (modifiedReqOpts.uri.startsWith(prefix)) {
+          modifiedReqOpts.uri = modifiedReqOpts.uri.substring(prefix.length);
+        }
+        return activeFile.request(modifiedReqOpts, callback);
+      }
+      return Bucket.prototype.request.call(this, reqOpts, callback);
+    };
+
     createGunzipOverride = null;
     handleRespOverride = null;
     makeWritableStreamOverride = null;
@@ -507,6 +536,7 @@ describe('File', () => {
   });
 
   describe('copy', () => {
+
     it('should throw if no destination is provided', () => {
       assert.throws(() => {
         file.copy();
@@ -596,9 +626,7 @@ describe('File', () => {
     });
 
     it('should set correct headers when file is encrypted', done => {
-      file.encryptionKey = {};
-      file.encryptionKeyBase64 = 'base64';
-      file.encryptionKeyHash = 'hash';
+      file.setEncryptionKey('sourceKey');
 
       const newFile = new File(BUCKET, 'new-file');
 
@@ -607,19 +635,112 @@ describe('File', () => {
           'x-goog-copy-source-encryption-algorithm': 'AES256',
           'x-goog-copy-source-encryption-key': file.encryptionKeyBase64,
           'x-goog-copy-source-encryption-key-sha256': file.encryptionKeyHash,
+          'x-goog-encryption-algorithm': 'AES256',
+          'x-goog-encryption-key': file.encryptionKeyBase64,
+          'x-goog-encryption-key-sha256': file.encryptionKeyHash,
         });
         done();
       };
 
       file.copy(newFile, assert.ifError);
     });
 
-    it('should set encryption key on the new File instance', done => {
+    it('should send destination encryption headers when destination file has an encryption key', done => {
+      const newFile = new File(BUCKET, 'new-file');
+      newFile.setEncryptionKey('destinationKey');
+
+      file.request = (reqOpts: DecorateRequestOptions) => {
+        assert.strictEqual(
+          reqOpts.headers!['x-goog-encryption-algorithm'],
+          'AES256'
+        );
+        assert.strictEqual(
+          reqOpts.headers!['x-goog-encryption-key'],
+          newFile.encryptionKeyBase64
+        );
+        assert.strictEqual(
+          reqOpts.headers!['x-goog-encryption-key-sha256'],
+          newFile.encryptionKeyHash
+        );
+        done();
+      };
+
+      file.copy(newFile, assert.ifError);
+    });
+
+    it('should not copy encryption key or send destination headers when destination file has null encryption key', done => {
+      file.setEncryptionKey('sourceKey');
+      const expectedSourceKeyBase64 = file.encryptionKeyBase64;
+      const expectedSourceKeyHash = file.encryptionKeyHash;
+
+      const newFile = new File(BUCKET, 'new-file');
+      newFile.setEncryptionKey(null);
+
+      file.request = (reqOpts: DecorateRequestOptions) => {
+        assert.strictEqual(newFile.encryptionKey, null);
+        assert.strictEqual(newFile.encryptionKeyBase64, undefined);
+        assert.strictEqual(newFile.encryptionKeyHash, undefined);
+
+        assert.strictEqual(
+          reqOpts.headers!['x-goog-copy-source-encryption-algorithm'],
+          'AES256'
+        );
+        assert.strictEqual(
+          reqOpts.headers!['x-goog-copy-source-encryption-key'],
+          expectedSourceKeyBase64
+        );
+        assert.strictEqual(
+          reqOpts.headers!['x-goog-copy-source-encryption-key-sha256'],
+          expectedSourceKeyHash
+        );
+
+        assert.strictEqual(
+          reqOpts.headers!['x-goog-encryption-algorithm'],
+          undefined
+        );
+        assert.strictEqual(
+          reqOpts.headers!['x-goog-encryption-key'],
+          undefined
+        );
+        assert.strictEqual(
+          reqOpts.headers!['x-goog-encryption-key-sha256'],
+          undefined
+        );
+
+        assert.notStrictEqual(file.encryptionKeyInterceptor, undefined);
+
+        done();
+      };
+
+      file.copy(newFile, assert.ifError);
+    });
+
+    it('should copy the source key to the destination file object if destination key is undefined', done => {
+      file.setEncryptionKey('sourceKey');
+
       const newFile = new File(BUCKET, 'new-file');
-      newFile.encryptionKey = 'encryptionKey';
 
-      file.setEncryptionKey = (encryptionKey: {}) => {
-        assert.strictEqual(encryptionKey, newFile.encryptionKey);
+      file.request = (reqOpts: DecorateRequestOptions) => {
+        assert.strictEqual(newFile.encryptionKey, file.encryptionKey);
+        assert.strictEqual(newFile.encryptionKeyBase64, file.encryptionKeyBase64);
+        assert.strictEqual(newFile.encryptionKeyHash, file.encryptionKeyHash);
+
+        assert.strictEqual(
+          reqOpts.headers!['x-goog-copy-source-encryption-key'],
+          file.encryptionKeyBase64
+        );
+        assert.strictEqual(
+          reqOpts.headers!['x-goog-encryption-algorithm'],
+          'AES256'
+        );
+        assert.strictEqual(
+          reqOpts.headers!['x-goog-encryption-key'],
+          file.encryptionKeyBase64
+        );
+        assert.strictEqual(
+          reqOpts.headers!['x-goog-encryption-key-sha256'],
+          file.encryptionKeyHash
+        );
         done();
       };
 
@@ -5440,6 +5561,30 @@ describe('File', () => {
 
       done();
     });
+
+    describe('null key', () => {
+      beforeEach(() => {
+        file.setEncryptionKey(KEY);
+        file.setEncryptionKey(null);
+      });
+
+      it('should localize the key to null', () => {
+        assert.strictEqual(file.encryptionKey, null);
+      });
+
+      it('should clear the base64 key', () => {
+        assert.strictEqual(file.encryptionKeyBase64, undefined);
+      });
+
+      it('should clear the hash', () => {
+        assert.strictEqual(file.encryptionKeyHash, undefined);
+      });
+
+      it('should remove the request interceptor', () => {
+        assert.strictEqual(file.encryptionKeyInterceptor, undefined);
+        assert.strictEqual(file.interceptors.length, 0);
+      });
+    });
   });
 
   describe('startResumableUpload_', () => {