fix: Prevent schema migration from deleting internal indexes created at startup

spec/DefinedSchemas.spec.js

@@ -482,11 +482,22 @@ describe('DefinedSchemas', () => {
 
     it('should detect protected indexes for _User class', () => {
       const definedSchema = new DefinedSchemas({}, {});
-      const protectedUserIndexes = ['_id_', 'case_insensitive_email', 'username_1', 'email_1'];
+      const protectedUserIndexes = [
+        '_id_',
+        'case_insensitive_username',
+        'case_insensitive_email',
+        'username_1',
+        'email_1',
+        '_email_verify_token',
+        '_perishable_token',
+        '_auth_data_admin_id',
+      ];
       protectedUserIndexes.forEach(field => {
         expect(definedSchema.isProtectedIndex('_User', field)).toEqual(true);
       });
       expect(definedSchema.isProtectedIndex('_User', 'test')).toEqual(false);
+      expect(definedSchema.isProtectedIndex('_User', 'auth_data_admin_id')).toEqual(false);
+      expect(definedSchema.isProtectedIndex('_User', '_auth_data_admin')).toEqual(false);
     });
 
     it('should detect protected indexes for _Role class', () => {
@@ -498,23 +509,104 @@ describe('DefinedSchemas', () => {
     it('should detect protected indexes for _Idempotency class', () => {
       const definedSchema = new DefinedSchemas({}, {});
       expect(definedSchema.isProtectedIndex('_Idempotency', 'reqId_1')).toEqual(true);
+      expect(definedSchema.isProtectedIndex('_Idempotency', 'ttl')).toEqual(true);
       expect(definedSchema.isProtectedIndex('_Idempotency', 'test')).toEqual(false);
     });
 
     it('should not detect protected indexes on user defined class', () => {
       const definedSchema = new DefinedSchemas({}, {});
       const protectedIndexes = [
+        'case_insensitive_username',
         'case_insensitive_email',
         'username_1',
         'email_1',
+        '_email_verify_token',
+        '_perishable_token',
+        '_auth_data_admin_id',
         'reqId_1',
+        'ttl',
         'name_1',
       ];
       protectedIndexes.forEach(field => {
         expect(definedSchema.isProtectedIndex('ExampleClass', field)).toEqual(false);
       });
       expect(definedSchema.isProtectedIndex('ExampleClass', '_id_')).toEqual(true);
     });
+
+    it_only_db('mongo')('should not delete internal _User indexes absent from the local schema', async () => {
+      const server = await reconfigureServer();
+      const config = Config.get(server.config.appId);
+      const adapter = config.database.adapter;
+      await adapter.ensureAuthDataUniqueness('admin');
+      await adapter.updateSchemaWithIndexes();
+
+      const userSchema = new Parse.Schema('_User');
+      userSchema.addIndex('custom_unprotected', { username: 1 });
+      await userSchema.update();
+      config.schemaCache.clear();
+
+      const before = await new Parse.Schema('_User').get();
+      expect(before.indexes._email_verify_token).toEqual({ _email_verify_token: 1 });
+      expect(before.indexes._perishable_token).toEqual({ _perishable_token: 1 });
+      expect(before.indexes._auth_data_admin_id).toEqual({ '_auth_data_admin.id': 1 });
+      expect(before.indexes.custom_unprotected).toEqual({ username: 1 });
+
+      await new DefinedSchemas(
+        { definitions: [{ className: '_User', indexes: {} }] },
+        server.config
+      ).execute();
+
+      const after = await new Parse.Schema('_User').get();
+      expect(after.indexes._email_verify_token).toEqual({ _email_verify_token: 1 });
+      expect(after.indexes._perishable_token).toEqual({ _perishable_token: 1 });
+      expect(after.indexes._auth_data_admin_id).toEqual({ '_auth_data_admin.id': 1 });
+      expect(after.indexes.custom_unprotected).toBeUndefined();
+    });
+
+    it_only_db('mongo')('should not delete the internal ttl index on _Idempotency', async () => {
+      const server = await reconfigureServer();
+      const config = Config.get(server.config.appId);
+      const adapter = config.database.adapter;
+      const mongoIndexes = await adapter.getIndexes('_Idempotency');
+      expect(mongoIndexes.find(index => index.name === 'ttl')).toBeDefined();
+
+      // _Idempotency is volatile, so startup does not copy its indexes into _SCHEMA.
+      // Put them in the cloud schema that the migration reads.
+      await adapter.database.collection(`${adapter._collectionPrefix}_SCHEMA`).updateOne(
+        { _id: '_Idempotency' },
+        {
+          $set: {
+            objectId: 'string',
+            updatedAt: 'string',
+            createdAt: 'string',
+            reqId: 'string',
+            expire: 'date',
+            _metadata: {
+              indexes: {
+                _id_: { _id: 1 },
+                reqId_1: { reqId: 1 },
+                ttl: { expire: 1 },
+                custom_unprotected: { reqId: 1 },
+              },
+            },
+          },
+        },
+        { upsert: true }
+      );
+      config.schemaCache.clear();
+
+      await new DefinedSchemas(
+        { definitions: [{ className: '_Idempotency', indexes: {} }] },
+        server.config
+      ).execute();
+
+      const after = await adapter.getClass('_Idempotency');
+      expect(after.indexes.ttl).toEqual({ expire: 1 });
+      expect(after.indexes.reqId_1).toEqual({ reqId: 1 });
+      expect(after.indexes.custom_unprotected).toBeUndefined();
+      const afterMongo = await adapter.getIndexes('_Idempotency');
+      expect(afterMongo.find(index => index.name === 'ttl')).toBeDefined();
+    });
   });
 
   describe('ClassLevelPermissions', () => {

spec/MongoStorageAdapter.spec.js

@@ -912,6 +912,42 @@ describe_only_db('mongo')('MongoStorageAdapter', () => {
       expect(userIndexes.find(idx => idx.name === '_perishable_token' || idx.name === '_perishable_token_1')).toBeDefined();
       expect(roleIndexes.find(idx => idx.name === 'name_1')).toBeDefined();
     });
+
+    it('should not throw when concurrent updates drop the same index', async () => {
+      const adapter = new MongoStorageAdapter({ uri: databaseURI });
+      const className = 'ConcurrentIndexDrop';
+      await adapter.createClass(className, {
+        fields: {
+          name: { type: 'String' },
+        },
+        indexes: {
+          name_1: { name: 1 },
+        },
+      });
+      const stored = await adapter.getClass(className);
+      const deleteOp = { name_1: { __op: 'Delete' } };
+      await Promise.all([
+        adapter.setIndexesWithSchemaFormat(
+          className,
+          deleteOp,
+          { ...stored.indexes },
+          stored.fields
+        ),
+        adapter.setIndexesWithSchemaFormat(
+          className,
+          deleteOp,
+          { ...stored.indexes },
+          stored.fields
+        ),
+      ]);
+      const indexes = await adapter.database.collection(className).indexes();
+      expect(indexes.find(index => index.name === 'name_1')).toBeUndefined();
+    });
+
+    it('should still reject dropIndex errors other than IndexNotFound', async () => {
+      const adapter = new MongoStorageAdapter({ uri: databaseURI });
+      await expectAsync(adapter.dropIndex('MissingDropIndexClass', 'missing_index')).toBeRejected();
+    });
   });
 
   describe('logClientEvents', () => {

spec/PostgresStorageAdapter.spec.js

@@ -579,6 +579,22 @@ describe_only_db('postgres')('PostgresStorageAdapter', () => {
     await adapter.deleteIdempotencyFunction();
     await client.none(qs);
   });
+
+  it('does not throw when dropping an index that is already gone', async () => {
+    const className = 'ConcurrentIndexDrop';
+    await dropTable(adapter._client, className);
+    await adapter.createTable(className, {
+      fields: {
+        name: { type: 'String' },
+      },
+    });
+    await adapter.createIndexes(className, [{ name: 'name_idx', key: 'name' }]);
+    await adapter.dropIndexes(className, ['name_idx']);
+    await adapter.dropIndexes(className, ['name_idx']);
+    const indexes = await adapter.getIndexes(className);
+    expect(indexes.find(index => index.indexname === 'name_idx')).toBeUndefined();
+    await dropTable(adapter._client, className);
+  });
 });
 
 describe_only_db('postgres')('PostgresStorageAdapter shutdown', () => {

src/Adapters/Storage/Mongo/MongoStorageAdapter.js

@@ -1259,7 +1259,13 @@ export class MongoStorageAdapter implements StorageAdapter {
   dropIndex(className: string, index: any) {
     return this._adaptiveCollection(className)
       .then(collection => collection._mongoCollection.dropIndex(index))
-      .catch(err => this.handleError(err));
+      .catch(err => {
+        // IndexNotFound: another instance already dropped this index.
+        if (err && err.code === 27) {
+          return;
+        }
+        return this.handleError(err);
+      });
   }
 
   dropAllIndexes(className: string) {

src/Adapters/Storage/Postgres/PostgresStorageAdapter.js

@@ -2557,7 +2557,7 @@ export class PostgresStorageAdapter implements StorageAdapter {
 
   async dropIndexes(className: string, indexes: any, conn: any): Promise<void> {
     const queries = indexes.map(i => ({
-      query: 'DROP INDEX $1:name',
+      query: 'DROP INDEX IF EXISTS $1:name',
       values: i,
     }));
     await (conn || this._client).tx(t => t.none(this._pgp.helpers.concat(queries)));

src/SchemaMigrations/DefinedSchemas.js

@@ -411,15 +411,21 @@ export class DefinedSchemas {
           'case_insensitive_username',
           'case_insensitive_email',
           'username_1',
-          'email_1'
+          'email_1',
+          '_email_verify_token',
+          '_perishable_token'
         );
+        // Unique indexes created for each auth provider: _auth_data_<provider>_id
+        if (/^_auth_data_.+_id$/.test(indexName)) {
+          return true;
+        }
         break;
       case '_Role':
         indexes.push('name_1');
         break;
 
       case '_Idempotency':
-        indexes.push('reqId_1');
+        indexes.push('reqId_1', 'ttl');
         break;
     }