diff --git a/apps/files/lib/AppInfo/Application.php b/apps/files/lib/AppInfo/Application.php index 1de8e60ab5a58..34f56e24c8251 100644 --- a/apps/files/lib/AppInfo/Application.php +++ b/apps/files/lib/AppInfo/Application.php @@ -36,7 +36,7 @@ use OCP\AppFramework\Bootstrap\IRegistrationContext; use OCP\Collaboration\Reference\RenderReferenceEvent; use OCP\Collaboration\Resources\IProviderManager; -use OCP\Files\Cache\CacheEntryRemovedEvent; +use OCP\Files\Cache\CacheEntriesRemovedEvent; use OCP\Files\Events\Node\BeforeNodeCopiedEvent; use OCP\Files\Events\Node\BeforeNodeDeletedEvent; use OCP\Files\Events\Node\BeforeNodeRenamedEvent; @@ -114,7 +114,7 @@ public function register(IRegistrationContext $context): void { $context->registerEventListener(RenderReferenceEvent::class, RenderReferenceEventListener::class); $context->registerEventListener(BeforeNodeRenamedEvent::class, SyncLivePhotosListener::class); $context->registerEventListener(BeforeNodeDeletedEvent::class, SyncLivePhotosListener::class); - $context->registerEventListener(CacheEntryRemovedEvent::class, SyncLivePhotosListener::class, 1); // Ensure this happen before the metadata are deleted. + $context->registerEventListener(CacheEntriesRemovedEvent::class, SyncLivePhotosListener::class, 1); // Ensure this happen before the metadata are deleted. $context->registerEventListener(BeforeNodeCopiedEvent::class, SyncLivePhotosListener::class); $context->registerEventListener(NodeCopiedEvent::class, SyncLivePhotosListener::class); $context->registerEventListener(LoadSearchPlugins::class, LoadSearchPluginsListener::class); diff --git a/apps/files/lib/Listener/SyncLivePhotosListener.php b/apps/files/lib/Listener/SyncLivePhotosListener.php index b6773e8c45239..a353f8bf9fc92 100644 --- a/apps/files/lib/Listener/SyncLivePhotosListener.php +++ b/apps/files/lib/Listener/SyncLivePhotosListener.php @@ -17,7 +17,7 @@ use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventListener; use OCP\Exceptions\AbortedEventException; -use OCP\Files\Cache\CacheEntryRemovedEvent; +use OCP\Files\Cache\CacheEntriesRemovedEvent; use OCP\Files\Events\Node\BeforeNodeCopiedEvent; use OCP\Files\Events\Node\BeforeNodeDeletedEvent; use OCP\Files\Events\Node\BeforeNodeRenamedEvent; @@ -63,8 +63,8 @@ public function handle(Event $event): void { $peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getSource()->getId()); } elseif ($event instanceof BeforeNodeDeletedEvent) { $peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getNode()->getId()); - } elseif ($event instanceof CacheEntryRemovedEvent) { - $peerFileId = $this->livePhotosService->getLivePhotoPeerId($event->getFileId()); + } elseif ($event instanceof CacheEntriesRemovedEvent) { + $this->handleCacheEntriesRemovedEvent($event); } if ($peerFileId === null) { @@ -83,12 +83,30 @@ public function handle(Event $event): void { $this->handleMove($event->getSource(), $event->getTarget(), $peerFile); } elseif ($event instanceof BeforeNodeDeletedEvent) { $this->handleDeletion($event, $peerFile); - } elseif ($event instanceof CacheEntryRemovedEvent) { - $peerFile->delete(); } } } + public function handleCacheEntriesRemovedEvent(CacheEntriesRemovedEvent $cacheEntriesRemovedEvent): void { + $entries = $cacheEntriesRemovedEvent->getCacheEntryRemovedEvents(); + $fileIds = []; + foreach ($entries as $entry) { + $fileIds[] = $entry->getFileId(); + } + + $peerFileIds = $this->livePhotosService->getLivePhotoPeerIds($fileIds); + + foreach ($peerFileIds as $peerFileId) { + // Check the user's folder. + $peerFile = $this->userFolder->getFirstNodeById($peerFileId); + + if ($peerFile === null) { + return; // Peer file not found. + } + $peerFile->delete(); + } + } + private function runMoveOrCopyChecks(Node $sourceFile, Node $targetFile, Node $peerFile): void { $targetParent = $targetFile->getParent(); $sourceExtension = $sourceFile->getExtension(); diff --git a/apps/files/lib/Service/LivePhotosService.php b/apps/files/lib/Service/LivePhotosService.php index 3ac6601d5dce4..f85933796ea4a 100644 --- a/apps/files/lib/Service/LivePhotosService.php +++ b/apps/files/lib/Service/LivePhotosService.php @@ -33,4 +33,22 @@ public function getLivePhotoPeerId(int $fileId): ?int { return (int)$metadata->getString('files-live-photo'); } + + /** + * Get the associated live photo for multiple file ids + * @param int[] $fileIds + * @return int[] + */ + public function getLivePhotoPeerIds(array $fileIds): array { + $metadata = $this->filesMetadataManager->getMetadataForFiles($fileIds); + $peersIds = []; + foreach ($metadata as $item) { + if (!$item->hasKey('files-live-photo')) { + continue; + } + + $peersIds[] = (int)$item->getString('files-live-photo'); + } + return $peersIds; + } } diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index f591a982a390a..5b7fa20e61169 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -396,6 +396,7 @@ 'OCP\\Files\\AlreadyExistsException' => $baseDir . '/lib/public/Files/AlreadyExistsException.php', 'OCP\\Files\\AppData\\IAppDataFactory' => $baseDir . '/lib/public/Files/AppData/IAppDataFactory.php', 'OCP\\Files\\Cache\\AbstractCacheEvent' => $baseDir . '/lib/public/Files/Cache/AbstractCacheEvent.php', + 'OCP\\Files\\Cache\\CacheEntriesRemovedEvent' => $baseDir . '/lib/public/Files/Cache/CacheEntriesRemovedEvent.php', 'OCP\\Files\\Cache\\CacheEntryInsertedEvent' => $baseDir . '/lib/public/Files/Cache/CacheEntryInsertedEvent.php', 'OCP\\Files\\Cache\\CacheEntryRemovedEvent' => $baseDir . '/lib/public/Files/Cache/CacheEntryRemovedEvent.php', 'OCP\\Files\\Cache\\CacheEntryUpdatedEvent' => $baseDir . '/lib/public/Files/Cache/CacheEntryUpdatedEvent.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 649886b76cad2..599aa406eb8ab 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -437,6 +437,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OCP\\Files\\AlreadyExistsException' => __DIR__ . '/../../..' . '/lib/public/Files/AlreadyExistsException.php', 'OCP\\Files\\AppData\\IAppDataFactory' => __DIR__ . '/../../..' . '/lib/public/Files/AppData/IAppDataFactory.php', 'OCP\\Files\\Cache\\AbstractCacheEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/AbstractCacheEvent.php', + 'OCP\\Files\\Cache\\CacheEntriesRemovedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/CacheEntriesRemovedEvent.php', 'OCP\\Files\\Cache\\CacheEntryInsertedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/CacheEntryInsertedEvent.php', 'OCP\\Files\\Cache\\CacheEntryRemovedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/CacheEntryRemovedEvent.php', 'OCP\\Files\\Cache\\CacheEntryUpdatedEvent' => __DIR__ . '/../../..' . '/lib/public/Files/Cache/CacheEntryUpdatedEvent.php', diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php index b6b550150f102..55d5849a6fc5c 100644 --- a/lib/private/Files/Cache/Cache.php +++ b/lib/private/Files/Cache/Cache.php @@ -18,6 +18,7 @@ use OCP\DB\Exception; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Cache\CacheEntriesRemovedEvent; use OCP\Files\Cache\CacheEntryInsertedEvent; use OCP\Files\Cache\CacheEntryRemovedEvent; use OCP\Files\Cache\CacheEntryUpdatedEvent; @@ -620,13 +621,17 @@ private function removeChildren(ICacheEntry $entry) { $query->executeStatement(); } + $cacheEntryRemovedEvents = []; foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) { - $cacheEntryRemovedEvent = new CacheEntryRemovedEvent( + $cacheEntryRemovedEvents[] = new CacheEntryRemovedEvent( $this->storage, $filePath, $fileId, $this->getNumericStorageId() ); + } + $this->eventDispatcher->dispatchTyped(new CacheEntriesRemovedEvent($cacheEntryRemovedEvents)); + foreach ($cacheEntryRemovedEvents as $cacheEntryRemovedEvent) { $this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent); } } @@ -784,7 +789,10 @@ public function moveFromCache(ICache $sourceCache, $sourcePath, $targetPath) { $this->connection->commit(); if ($sourceCache->getNumericStorageId() !== $this->getNumericStorageId()) { - $this->eventDispatcher->dispatchTyped(new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId())); + $event = new CacheEntryRemovedEvent($this->storage, $sourcePath, $sourceId, $sourceCache->getNumericStorageId()); + $this->eventDispatcher->dispatchTyped($event); + $this->eventDispatcher->dispatchTyped(new CacheEntriesRemovedEvent([$event])); + $event = new CacheEntryInsertedEvent($this->storage, $targetPath, $sourceId, $this->getNumericStorageId()); $this->eventDispatcher->dispatch(CacheInsertEvent::class, $event); $this->eventDispatcher->dispatchTyped($event); diff --git a/lib/private/FilesMetadata/FilesMetadataManager.php b/lib/private/FilesMetadata/FilesMetadataManager.php index 4846799b3d46c..188cb09e53a15 100644 --- a/lib/private/FilesMetadata/FilesMetadataManager.php +++ b/lib/private/FilesMetadata/FilesMetadataManager.php @@ -20,7 +20,7 @@ use OCP\DB\Exception as DBException; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\EventDispatcher\IEventDispatcher; -use OCP\Files\Cache\CacheEntryRemovedEvent; +use OCP\Files\Cache\CacheEntriesRemovedEvent; use OCP\Files\Events\Node\NodeWrittenEvent; use OCP\Files\InvalidPathException; use OCP\Files\Node; @@ -214,6 +214,20 @@ public function deleteMetadata(int $fileId): void { } } + public function deleteMetadataForFiles(int $storage, array $fileIds): void { + try { + $this->metadataRequestService->dropMetadataForFiles($storage, $fileIds); + } catch (Exception $e) { + $this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileIds' => $fileIds]); + } + + try { + $this->indexRequestService->dropIndexForFiles($fileIds); + } catch (Exception $e) { + $this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileIds' => $fileIds]); + } + } + /** * @param IQueryBuilder $qb * @param string $fileTableAlias alias of the table that contains data about files @@ -301,6 +315,6 @@ public function initMetadata( */ public static function loadListeners(IEventDispatcher $eventDispatcher): void { $eventDispatcher->addServiceListener(NodeWrittenEvent::class, MetadataUpdate::class); - $eventDispatcher->addServiceListener(CacheEntryRemovedEvent::class, MetadataDelete::class); + $eventDispatcher->addServiceListener(CacheEntriesRemovedEvent::class, MetadataDelete::class); } } diff --git a/lib/private/FilesMetadata/Listener/MetadataDelete.php b/lib/private/FilesMetadata/Listener/MetadataDelete.php index 226bd3bdafa92..5a404614df049 100644 --- a/lib/private/FilesMetadata/Listener/MetadataDelete.php +++ b/lib/private/FilesMetadata/Listener/MetadataDelete.php @@ -11,14 +11,14 @@ use Exception; use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventListener; -use OCP\Files\Cache\CacheEntryRemovedEvent; +use OCP\Files\Cache\CacheEntriesRemovedEvent; use OCP\FilesMetadata\IFilesMetadataManager; use Psr\Log\LoggerInterface; /** * Handle file deletion event and remove stored metadata related to the deleted file * - * @template-implements IEventListener + * @template-implements IEventListener */ class MetadataDelete implements IEventListener { public function __construct( @@ -28,14 +28,25 @@ public function __construct( } public function handle(Event $event): void { - if (!($event instanceof CacheEntryRemovedEvent)) { + if (!($event instanceof CacheEntriesRemovedEvent)) { return; } + $entries = $event->getCacheEntryRemovedEvents(); + $storageToFileIds = []; + + foreach ($entries as $entry) { + try { + $storageToFileIds[$entry->getStorageId()] ??= []; + $storageToFileIds[$entry->getStorageId()][] = $entry->getFileId(); + } catch (Exception $e) { + $this->logger->warning('issue while running MetadataDelete', ['exception' => $e]); + } + } + try { - $nodeId = $event->getFileId(); - if ($nodeId > 0) { - $this->filesMetadataManager->deleteMetadata($nodeId); + foreach ($storageToFileIds as $storageId => $fileIds) { + $this->filesMetadataManager->deleteMetadataForFiles($storageId, $fileIds); } } catch (Exception $e) { $this->logger->warning('issue while running MetadataDelete', ['exception' => $e]); diff --git a/lib/private/FilesMetadata/Service/IndexRequestService.php b/lib/private/FilesMetadata/Service/IndexRequestService.php index 91bd9f0b11e3a..90c2845cad283 100644 --- a/lib/private/FilesMetadata/Service/IndexRequestService.php +++ b/lib/private/FilesMetadata/Service/IndexRequestService.php @@ -175,4 +175,30 @@ public function dropIndex(int $fileId, string $key = ''): void { $qb->executeStatement(); } + + /** + * Drop indexes related to multiple file ids + * if a key is specified, only drop entries related to it + * + * @param int[] $fileIds file ids + * @param string $key metadata key + * + * @throws DbException + */ + public function dropIndexForFiles(array $fileIds, string $key = ''): void { + $chunks = array_chunk($fileIds, 1000); + + foreach ($chunks as $chunk) { + $qb = $this->dbConnection->getQueryBuilder(); + $expr = $qb->expr(); + $qb->delete(self::TABLE_METADATA_INDEX) + ->where($expr->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + + if ($key !== '') { + $qb->andWhere($expr->eq('meta_key', $qb->createNamedParameter($key))); + } + + $qb->executeStatement(); + } + } } diff --git a/lib/private/FilesMetadata/Service/MetadataRequestService.php b/lib/private/FilesMetadata/Service/MetadataRequestService.php index c308ae1c9c88f..55c547ef401d8 100644 --- a/lib/private/FilesMetadata/Service/MetadataRequestService.php +++ b/lib/private/FilesMetadata/Service/MetadataRequestService.php @@ -108,8 +108,10 @@ public function getMetadataFromFileId(int $fileId): IFilesMetadata { */ public function getMetadataFromFileIds(array $fileIds): array { $qb = $this->dbConnection->getQueryBuilder(); - $qb->select('file_id', 'json', 'sync_token')->from(self::TABLE_METADATA); - $qb->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))); + $qb->select('file_id', 'json', 'sync_token') + ->from(self::TABLE_METADATA) + ->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))) + ->runAcrossAllShards(); $list = []; $result = $qb->executeQuery(); @@ -143,6 +145,22 @@ public function dropMetadata(int $fileId): void { $qb->executeStatement(); } + /** + * @param int[] $fileIds + * @throws Exception + */ + public function dropMetadataForFiles(int $storage, array $fileIds): void { + $chunks = array_chunk($fileIds, 1000); + + foreach ($chunks as $chunk) { + $qb = $this->dbConnection->getQueryBuilder(); + $qb->delete(self::TABLE_METADATA) + ->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY))) + ->hintShardKey('storage', $storage); + $qb->executeStatement(); + } + } + /** * update metadata in the database * diff --git a/lib/public/Files/Cache/CacheEntriesRemovedEvent.php b/lib/public/Files/Cache/CacheEntriesRemovedEvent.php new file mode 100644 index 0000000000000..fee6a5996e3a4 --- /dev/null +++ b/lib/public/Files/Cache/CacheEntriesRemovedEvent.php @@ -0,0 +1,35 @@ +cacheEntryRemovedEvents; + } +} diff --git a/lib/public/Files/Cache/CacheEntryRemovedEvent.php b/lib/public/Files/Cache/CacheEntryRemovedEvent.php index d0f8c47b4309c..d015f3ba94dcd 100644 --- a/lib/public/Files/Cache/CacheEntryRemovedEvent.php +++ b/lib/public/Files/Cache/CacheEntryRemovedEvent.php @@ -11,7 +11,11 @@ /** * Event for when an existing entry in the cache gets removed * + * Prefer using CacheEntriesRemovedEvent as it is more efficient when deleting + * multiple files at the same time. + * * @since 21.0.0 + * @see CacheEntriesRemovedEvent */ class CacheEntryRemovedEvent extends AbstractCacheEvent implements ICacheEvent { } diff --git a/lib/public/FilesMetadata/IFilesMetadataManager.php b/lib/public/FilesMetadata/IFilesMetadataManager.php index 4b1a6d32e8e76..df16b9e3fde5d 100644 --- a/lib/public/FilesMetadata/IFilesMetadataManager.php +++ b/lib/public/FilesMetadata/IFilesMetadataManager.php @@ -8,6 +8,7 @@ namespace OCP\FilesMetadata; +use OCP\AppFramework\Attribute\Consumable; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\Node; use OCP\FilesMetadata\Exceptions\FilesMetadataException; @@ -20,6 +21,7 @@ * * @since 28.0.0 */ +#[Consumable(since: '28.0.0')] interface IFilesMetadataManager { /** @since 28.0.0 */ public const PROCESS_LIVE = 1; @@ -98,6 +100,16 @@ public function saveMetadata(IFilesMetadata $filesMetadata): void; */ public function deleteMetadata(int $fileId): void; + /** + * Delete metadata and its indexes of multiple file ids + * + * @param int $storage The storage id coresponding to the $fileIds + * @param array $fileIds file ids + * @return void + * @since 32.0.0 + */ + public function deleteMetadataForFiles(int $storage, array $fileIds): void; + /** * generate and return a MetadataQuery to help building sql queries *