From 788ee3e247e9bedf2bded278b222e9b43f66e6fe Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 7 Oct 2025 13:53:15 +0530 Subject: [PATCH 01/28] added database type for migrating database type --- src/Migration/Destinations/Appwrite.php | 1 + src/Migration/Resources/Database/Database.php | 7 +++++++ src/Migration/Sources/Appwrite.php | 3 ++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 4e38457..ee6a63b 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -338,6 +338,7 @@ protected function createDatabase(Database $resource): bool '$updatedAt' => $resource->getUpdatedAt(), 'originalId' => empty($resource->getOriginalId()) ? null : $resource->getOriginalId(), 'type' => empty($resource->getType()) ? 'legacy' : $resource->getType(), + 'database' => $resource->getDatabase() ])); $resource->setSequence($database->getSequence()); diff --git a/src/Migration/Resources/Database/Database.php b/src/Migration/Resources/Database/Database.php index 06882b2..8acc88e 100644 --- a/src/Migration/Resources/Database/Database.php +++ b/src/Migration/Resources/Database/Database.php @@ -26,6 +26,7 @@ public function __construct( protected bool $enabled = true, protected string $originalId = '', protected string $type = '', + protected string $database = '' ) { $this->id = $id; } @@ -50,6 +51,7 @@ public static function fromArray(array $array): self enabled: $array['enabled'] ?? true, originalId: $array['originalId'] ?? '', type: $array['type'] ?? 'legacy', + database: $array['database'] ); } @@ -92,4 +94,9 @@ public function getType(): string { return $this->type; } + + public function getDatabase(): string + { + return $this->database; + } } diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 9e33307..f848d35 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -738,7 +738,8 @@ private function exportDatabases(int $batchSize): void $database['name'], $database['$createdAt'], $database['$updatedAt'], - type: $database['type'] ?? 'legacy' + type: $database['type'] ?? 'legacy', + database: $database['database'] ); $databases[] = $newDatabase; From bfdf3f3137ad90674efd47c310fd62af0ab71aac Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 7 Oct 2025 16:25:23 +0530 Subject: [PATCH 02/28] linting --- src/Migration/Resources/Database/Database.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Migration/Resources/Database/Database.php b/src/Migration/Resources/Database/Database.php index 8acc88e..9c90498 100644 --- a/src/Migration/Resources/Database/Database.php +++ b/src/Migration/Resources/Database/Database.php @@ -39,6 +39,7 @@ public function __construct( * updatedAt: string, * enabled: bool, * originalId: string|null, + * database: string * } $array */ public static function fromArray(array $array): self From d063f46a62e73bab5a31c6afd45d729fd119c7ca Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 7 Oct 2025 19:46:23 +0530 Subject: [PATCH 03/28] * Multi-type database support for appwrite destination separated database object into dbForProject to interact with project metadata and dbForDatabase to work with the targetted database --- src/Migration/Destinations/Appwrite.php | 92 +++++++++++++------------ 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index ee6a63b..8eccc84 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -66,14 +66,16 @@ class Appwrite extends Destination * @param string $project * @param string $endpoint * @param string $key - * @param UtopiaDatabase $database + * @param UtopiaDatabase $dbForProject + * @param UtopiaDatabase $dbForDatabase * @param array> $collectionStructure */ public function __construct( string $project, string $endpoint, string $key, - protected UtopiaDatabase $database, + protected UtopiaDatabase $dbForProject, + protected UtopiaDatabase $dbForDatabase, protected array $collectionStructure ) { $this->project = $project; @@ -228,7 +230,7 @@ protected function import(array $resources, callable $callback): void $isLast = $index === $total - 1; try { - $this->database->setPreserveDates(true); + $this->dbForDatabase->setPreserveDates(true); $responseResource = match ($resource->getGroup()) { Transfer::GROUP_DATABASES => $this->importDatabaseResource($resource, $isLast), @@ -255,7 +257,7 @@ protected function import(array $resources, callable $callback): void $responseResource = $resource; } finally { - $this->database->setPreserveDates(false); + $this->dbForDatabase->setPreserveDates(false); } $this->cache->update($responseResource); @@ -329,7 +331,7 @@ protected function createDatabase(Database $resource): bool ); } - $database = $this->database->createDocument('databases', new UtopiaDocument([ + $database = $this->dbForProject->createDocument('databases', new UtopiaDocument([ '$id' => $resource->getId(), 'name' => $resource->getDatabaseName(), 'enabled' => $resource->getEnabled(), @@ -353,7 +355,7 @@ protected function createDatabase(Database $resource): bool $this->collectionStructure['indexes'] ); - $this->database->createCollection( + $this->dbForProject->createCollection( 'database_' . $database->getSequence(), $columns, $indexes @@ -385,7 +387,7 @@ protected function createTable(Table $resource): bool ); } - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getDatabase()->getId() ); @@ -399,7 +401,7 @@ protected function createTable(Table $resource): bool ); } - $table = $this->database->createDocument('database_' . $database->getSequence(), new UtopiaDocument([ + $table = $this->dbForProject->createDocument('database_' . $database->getSequence(), new UtopiaDocument([ '$id' => $resource->getId(), 'databaseInternalId' => $database->getSequence(), 'databaseId' => $resource->getDatabase()->getId(), @@ -414,7 +416,7 @@ protected function createTable(Table $resource): bool $resource->setSequence($table->getSequence()); - $this->database->createCollection( + $this->dbForDatabase->createCollection( 'database_' . $database->getSequence() . '_collection_' . $resource->getSequence(), permissions: $resource->getPermissions(), documentSecurity: $resource->getRowSecurity() @@ -447,7 +449,7 @@ protected function createColumn(Column $resource): bool default => throw new \Exception('Invalid resource type '.$resource->getType()), }; - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getTable()->getDatabase()->getId(), ); @@ -461,7 +463,7 @@ protected function createColumn(Column $resource): bool ); } - $table = $this->database->getDocument( + $table = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getTable()->getId(), ); @@ -506,7 +508,7 @@ protected function createColumn(Column $resource): bool if ($type === UtopiaDatabase::VAR_RELATIONSHIP) { $resource->getOptions()['side'] = UtopiaDatabase::RELATION_SIDE_PARENT; - $relatedTable = $this->database->getDocument( + $relatedTable = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getOptions()['relatedCollection'] ); @@ -543,9 +545,9 @@ protected function createColumn(Column $resource): bool '$updatedAt' => $resource->getUpdatedAt(), ]); - $this->database->checkAttribute($table, $column); + $this->dbForProject->checkAttribute($table, $column); - $column = $this->database->createDocument('attributes', $column); + $column = $this->dbForProject->createDocument('attributes', $column); } catch (DuplicateException) { throw new Exception( resourceName: $resource->getName(), @@ -561,13 +563,13 @@ protected function createColumn(Column $resource): bool message: 'Attribute limit exceeded', ); } catch (\Throwable $e) { - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); + $this->dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); throw $e; } - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); + $this->dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); $options = $resource->getOptions(); $twoWayKey = null; @@ -601,9 +603,9 @@ protected function createColumn(Column $resource): bool '$updatedAt' => $resource->getUpdatedAt(), ]); - $this->database->createDocument('attributes', $twoWayAttribute); + $this->dbForProject->createDocument('attributes', $twoWayAttribute); } catch (DuplicateException) { - $this->database->deleteDocument('attributes', $column->getId()); + $this->dbForProject->deleteDocument('attributes', $column->getId()); throw new Exception( resourceName: $resource->getName(), @@ -612,7 +614,7 @@ protected function createColumn(Column $resource): bool message: 'Attribute already exists', ); } catch (LimitException) { - $this->database->deleteDocument('attributes', $column->getId()); + $this->dbForProject->deleteDocument('attributes', $column->getId()); throw new Exception( resourceName: $resource->getName(), @@ -621,8 +623,8 @@ protected function createColumn(Column $resource): bool message: 'Column limit exceeded', ); } catch (\Throwable $e) { - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); + $this->dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence()); throw $e; } } @@ -630,7 +632,7 @@ protected function createColumn(Column $resource): bool try { switch ($type) { case UtopiaDatabase::VAR_RELATIONSHIP: - if (!$this->database->createRelationship( + if (!$this->dbForDatabase->createRelationship( collection: 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), relatedCollection: 'database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence(), type: $options['relationType'], @@ -648,7 +650,7 @@ protected function createColumn(Column $resource): bool } break; default: - if (!$this->database->createAttribute( + if (!$this->dbForDatabase->createAttribute( 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), $resource->getKey(), $type, @@ -665,10 +667,10 @@ protected function createColumn(Column $resource): bool } } } catch (\Throwable) { - $this->database->deleteDocument('attributes', $column->getId()); + $this->dbForProject->deleteDocument('attributes', $column->getId()); if (isset($twoWayAttribute)) { - $this->database->deleteDocument('attributes', $twoWayAttribute->getId()); + $this->dbForProject->deleteDocument('attributes', $twoWayAttribute->getId()); } throw new Exception( @@ -680,11 +682,11 @@ protected function createColumn(Column $resource): bool } if ($type === UtopiaDatabase::VAR_RELATIONSHIP && $options['twoWay']) { - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); } - $this->database->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->database->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); + $this->dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); return true; } @@ -695,7 +697,7 @@ protected function createColumn(Column $resource): bool */ protected function createIndex(Index $resource): bool { - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getTable()->getDatabase()->getId(), ); @@ -708,7 +710,7 @@ protected function createIndex(Index $resource): bool ); } - $table = $this->database->getDocument( + $table = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getTable()->getId(), ); @@ -721,12 +723,12 @@ protected function createIndex(Index $resource): bool ); } - $count = $this->database->count('indexes', [ + $count = $this->dbForProject->count('indexes', [ Query::equal('collectionInternalId', [$table->getSequence()]), Query::equal('databaseInternalId', [$database->getSequence()]) - ], $this->database->getLimitForIndexes()); + ], $this->dbForDatabase->getLimitForIndexes()); - if ($count >= $this->database->getLimitForIndexes()) { + if ($count >= $this->dbForDatabase->getLimitForIndexes()) { throw new Exception( resourceName: $resource->getName(), resourceGroup: $resource->getGroup(), @@ -845,9 +847,9 @@ protected function createIndex(Index $resource): bool $validator = new IndexValidator( $tableColumns, - $this->database->getAdapter()->getMaxIndexLength(), - $this->database->getAdapter()->getInternalIndexesKeys(), - $this->database->getAdapter()->getSupportForIndexArray(), + $this->dbForDatabase->getAdapter()->getMaxIndexLength(), + $this->dbForDatabase->getAdapter()->getInternalIndexesKeys(), + $this->dbForDatabase->getAdapter()->getSupportForIndexArray(), ); if (!$validator->isValid($index)) { @@ -859,10 +861,10 @@ protected function createIndex(Index $resource): bool ); } - $index = $this->database->createDocument('indexes', $index); + $index = $this->dbForProject->createDocument('indexes', $index); try { - $result = $this->database->createIndex( + $result = $this->dbForDatabase->createIndex( 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), $resource->getKey(), $resource->getType(), @@ -880,7 +882,7 @@ protected function createIndex(Index $resource): bool ); } } catch (\Throwable $th) { - $this->database->deleteDocument('indexes', $index->getId()); + $this->dbForProject->deleteDocument('indexes', $index->getId()); throw new Exception( resourceName: $resource->getName(), @@ -890,7 +892,7 @@ protected function createIndex(Index $resource): bool ); } - $this->database->purgeCachedDocument( + $this->dbForProject->purgeCachedDocument( 'database_' . $database->getSequence(), $table->getId() ); @@ -942,12 +944,12 @@ protected function createRow(Row $resource, bool $isLast): bool if ($isLast) { try { - $database = $this->database->getDocument( + $database = $this->dbForProject->getDocument( 'databases', $resource->getTable()->getDatabase()->getId(), ); - $table = $this->database->getDocument( + $table = $this->dbForProject->getDocument( 'database_' . $database->getSequence(), $resource->getTable()->getId(), ); @@ -980,7 +982,7 @@ protected function createRow(Row $resource, bool $isLast): bool } } - $this->database->skipRelationshipsExistCheck(fn () => $this->database->createDocuments( + $this->dbForDatabase->skipRelationshipsExistCheck(fn () => $this->dbForDatabase->createDocuments( 'database_' . $databaseInternalId . '_collection_' . $tableInternalId, $this->rowBuffer )); From 262c1c2c72d7429e504cde53b2b1774da8e9998c Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 7 Oct 2025 20:22:08 +0530 Subject: [PATCH 04/28] replaced static database db with a callback to create with dsn --- src/Migration/Destinations/Appwrite.php | 50 +++++++++++++++---------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 8eccc84..e3efe9d 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -57,6 +57,11 @@ class Appwrite extends Destination private Teams $teams; private Users $users; + /** + * @var callable(string $databaseDSN): UtopiaDatabase + */ + protected mixed $getDatabaseDB; + /** * @var array */ @@ -67,7 +72,7 @@ class Appwrite extends Destination * @param string $endpoint * @param string $key * @param UtopiaDatabase $dbForProject - * @param UtopiaDatabase $dbForDatabase + * @param callable(string $databaseDSN):UtopiaDatabase $getDatabaseDB * @param array> $collectionStructure */ public function __construct( @@ -75,7 +80,7 @@ public function __construct( string $endpoint, string $key, protected UtopiaDatabase $dbForProject, - protected UtopiaDatabase $dbForDatabase, + callable $getDatabaseDB, protected array $collectionStructure ) { $this->project = $project; @@ -91,6 +96,8 @@ public function __construct( $this->storage = new Storage($this->client); $this->teams = new Teams($this->client); $this->users = new Users($this->client); + + $this->getDatabaseDB = $getDatabaseDB; } public static function getName(): string @@ -230,7 +237,7 @@ protected function import(array $resources, callable $callback): void $isLast = $index === $total - 1; try { - $this->dbForDatabase->setPreserveDates(true); + $this->dbForProject->setPreserveDates(true); $responseResource = match ($resource->getGroup()) { Transfer::GROUP_DATABASES => $this->importDatabaseResource($resource, $isLast), @@ -257,7 +264,7 @@ protected function import(array $resources, callable $callback): void $responseResource = $resource; } finally { - $this->dbForDatabase->setPreserveDates(false); + $this->dbForProject->setPreserveDates(false); } $this->cache->update($responseResource); @@ -401,6 +408,8 @@ protected function createTable(Table $resource): bool ); } + $dbForDatabase = call_user_func($this->getDatabaseDB, $database->getAttribute('database')); + $table = $this->dbForProject->createDocument('database_' . $database->getSequence(), new UtopiaDocument([ '$id' => $resource->getId(), 'databaseInternalId' => $database->getSequence(), @@ -416,7 +425,7 @@ protected function createTable(Table $resource): bool $resource->setSequence($table->getSequence()); - $this->dbForDatabase->createCollection( + $dbForDatabase->createCollection( 'database_' . $database->getSequence() . '_collection_' . $resource->getSequence(), permissions: $resource->getPermissions(), documentSecurity: $resource->getRowSecurity() @@ -521,7 +530,7 @@ protected function createColumn(Column $resource): bool ); } } - + $dbForDatabase = call_user_func($this->getDatabaseDB, $database->getAttribute('database')); try { $column = new UtopiaDocument([ '$id' => ID::custom($database->getSequence() . '_' . $table->getSequence() . '_' . $resource->getKey()), @@ -564,12 +573,12 @@ protected function createColumn(Column $resource): bool ); } catch (\Throwable $e) { $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); throw $e; } $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); $options = $resource->getOptions(); $twoWayKey = null; @@ -624,7 +633,7 @@ protected function createColumn(Column $resource): bool ); } catch (\Throwable $e) { $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); - $this->dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence()); + $dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence()); throw $e; } } @@ -632,7 +641,7 @@ protected function createColumn(Column $resource): bool try { switch ($type) { case UtopiaDatabase::VAR_RELATIONSHIP: - if (!$this->dbForDatabase->createRelationship( + if (!$dbForDatabase->createRelationship( collection: 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), relatedCollection: 'database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence(), type: $options['relationType'], @@ -650,7 +659,7 @@ protected function createColumn(Column $resource): bool } break; default: - if (!$this->dbForDatabase->createAttribute( + if (!$dbForDatabase->createAttribute( 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), $resource->getKey(), $type, @@ -686,7 +695,7 @@ protected function createColumn(Column $resource): bool } $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $this->dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); return true; } @@ -722,13 +731,14 @@ protected function createIndex(Index $resource): bool message: 'Table not found', ); } + $dbForDatabase = call_user_func($this->getDatabaseDB, $database->getAttribute('database')); $count = $this->dbForProject->count('indexes', [ Query::equal('collectionInternalId', [$table->getSequence()]), Query::equal('databaseInternalId', [$database->getSequence()]) - ], $this->dbForDatabase->getLimitForIndexes()); + ], $dbForDatabase->getLimitForIndexes()); - if ($count >= $this->dbForDatabase->getLimitForIndexes()) { + if ($count >= $dbForDatabase->getLimitForIndexes()) { throw new Exception( resourceName: $resource->getName(), resourceGroup: $resource->getGroup(), @@ -847,9 +857,9 @@ protected function createIndex(Index $resource): bool $validator = new IndexValidator( $tableColumns, - $this->dbForDatabase->getAdapter()->getMaxIndexLength(), - $this->dbForDatabase->getAdapter()->getInternalIndexesKeys(), - $this->dbForDatabase->getAdapter()->getSupportForIndexArray(), + $dbForDatabase->getAdapter()->getMaxIndexLength(), + $dbForDatabase->getAdapter()->getInternalIndexesKeys(), + $dbForDatabase->getAdapter()->getSupportForIndexArray(), ); if (!$validator->isValid($index)) { @@ -864,7 +874,7 @@ protected function createIndex(Index $resource): bool $index = $this->dbForProject->createDocument('indexes', $index); try { - $result = $this->dbForDatabase->createIndex( + $result = $dbForDatabase->createIndex( 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), $resource->getKey(), $resource->getType(), @@ -981,8 +991,8 @@ protected function createRow(Row $resource, bool $isLast): bool } } } - - $this->dbForDatabase->skipRelationshipsExistCheck(fn () => $this->dbForDatabase->createDocuments( + $dbForDatabase = call_user_func($this->getDatabaseDB, $database->getAttribute('database')); + $dbForDatabase->skipRelationshipsExistCheck(fn () => $dbForDatabase->createDocuments( 'database_' . $databaseInternalId . '_collection_' . $tableInternalId, $this->rowBuffer )); From f072649b1102b3657f6624fac4e9a0ababfceeee Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 10 Oct 2025 20:22:21 +0530 Subject: [PATCH 05/28] removed document as string --- src/Migration/Destinations/Appwrite.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index e3efe9d..303214b 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -58,7 +58,7 @@ class Appwrite extends Destination private Users $users; /** - * @var callable(string $databaseDSN): UtopiaDatabase + * @var callable(UtopiaDocument $database): UtopiaDatabase */ protected mixed $getDatabaseDB; @@ -72,7 +72,7 @@ class Appwrite extends Destination * @param string $endpoint * @param string $key * @param UtopiaDatabase $dbForProject - * @param callable(string $databaseDSN):UtopiaDatabase $getDatabaseDB + * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabaseDB * @param array> $collectionStructure */ public function __construct( @@ -408,7 +408,7 @@ protected function createTable(Table $resource): bool ); } - $dbForDatabase = call_user_func($this->getDatabaseDB, $database->getAttribute('database')); + $dbForDatabase = call_user_func($this->getDatabaseDB, $database); $table = $this->dbForProject->createDocument('database_' . $database->getSequence(), new UtopiaDocument([ '$id' => $resource->getId(), @@ -530,7 +530,7 @@ protected function createColumn(Column $resource): bool ); } } - $dbForDatabase = call_user_func($this->getDatabaseDB, $database->getAttribute('database')); + $dbForDatabase = call_user_func($this->getDatabaseDB, $database); try { $column = new UtopiaDocument([ '$id' => ID::custom($database->getSequence() . '_' . $table->getSequence() . '_' . $resource->getKey()), @@ -731,7 +731,7 @@ protected function createIndex(Index $resource): bool message: 'Table not found', ); } - $dbForDatabase = call_user_func($this->getDatabaseDB, $database->getAttribute('database')); + $dbForDatabase = call_user_func($this->getDatabaseDB, $database); $count = $this->dbForProject->count('indexes', [ Query::equal('collectionInternalId', [$table->getSequence()]), @@ -991,7 +991,7 @@ protected function createRow(Row $resource, bool $isLast): bool } } } - $dbForDatabase = call_user_func($this->getDatabaseDB, $database->getAttribute('database')); + $dbForDatabase = call_user_func($this->getDatabaseDB, $database); $dbForDatabase->skipRelationshipsExistCheck(fn () => $dbForDatabase->createDocuments( 'database_' . $databaseInternalId . '_collection_' . $tableInternalId, $this->rowBuffer From b79066f3583d4351b532e55649961d2f488815a5 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sat, 11 Oct 2025 19:29:15 +0530 Subject: [PATCH 06/28] updated code --- composer.json | 2 +- composer.lock | 422 +++++++++++---- src/Migration/Destinations/Appwrite.php | 6 + src/Migration/Resource.php | 3 + .../Resources/Database/Collection.php | 15 + src/Migration/Resources/Database/Document.php | 13 + .../Resources/Database/DocumentsDB.php | 13 + src/Migration/Sources/Appwrite.php | 512 ++++++++++++------ src/Migration/Sources/Appwrite/Reader.php | 2 + src/Migration/Sources/Appwrite/Reader/API.php | 5 + .../Sources/Appwrite/Reader/Database.php | 5 + 11 files changed, 738 insertions(+), 260 deletions(-) create mode 100644 src/Migration/Resources/Database/Collection.php create mode 100644 src/Migration/Resources/Database/Document.php create mode 100644 src/Migration/Resources/Database/DocumentsDB.php diff --git a/composer.json b/composer.json index 0e282b2..3c34bc6 100644 --- a/composer.json +++ b/composer.json @@ -26,7 +26,7 @@ "ext-curl": "*", "ext-openssl": "*", "appwrite/appwrite": "15.*", - "utopia-php/database": "1.*", + "utopia-php/database": "dev-feat-documentsdb as 1.4.5", "utopia-php/storage": "0.18.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*" diff --git a/composer.lock b/composer.lock index ed29b12..8cb4331 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "bad9f5d99ff97a6ee5c807532b0720b8", + "content-hash": "ee13b1e778c4fc84737951d6b68b6317", "packages": [ { "name": "appwrite/appwrite", @@ -187,24 +187,21 @@ }, { "name": "google/protobuf", - "version": "v4.32.0", + "version": "v4.32.1", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "9a9a92ecbe9c671dc1863f6d4a91ea3ea12c8646" + "reference": "c4ed1c1f9bbc1e91766e2cd6c0af749324fe87cb" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/9a9a92ecbe9c671dc1863f6d4a91ea3ea12c8646", - "reference": "9a9a92ecbe9c671dc1863f6d4a91ea3ea12c8646", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/c4ed1c1f9bbc1e91766e2cd6c0af749324fe87cb", + "reference": "c4ed1c1f9bbc1e91766e2cd6c0af749324fe87cb", "shasum": "" }, "require": { "php": ">=8.1.0" }, - "provide": { - "ext-protobuf": "*" - }, "require-dev": { "phpunit/phpunit": ">=5.0.0 <8.5.27" }, @@ -228,9 +225,86 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.32.0" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.32.1" + }, + "time": "2025-09-14T05:14:52+00:00" + }, + { + "name": "mongodb/mongodb", + "version": "2.1.1", + "source": { + "type": "git", + "url": "https://github.com/mongodb/mongo-php-library.git", + "reference": "f399d24905dd42f97dfe0af9706129743ef247ac" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/f399d24905dd42f97dfe0af9706129743ef247ac", + "reference": "f399d24905dd42f97dfe0af9706129743ef247ac", + "shasum": "" + }, + "require": { + "composer-runtime-api": "^2.0", + "ext-mongodb": "^2.1", + "php": "^8.1", + "psr/log": "^1.1.4|^2|^3", + "symfony/polyfill-php85": "^1.32" + }, + "replace": { + "mongodb/builder": "*" + }, + "require-dev": { + "doctrine/coding-standard": "^12.0", + "phpunit/phpunit": "^10.5.35", + "rector/rector": "^1.2", + "squizlabs/php_codesniffer": "^3.7", + "vimeo/psalm": "6.5.*" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.x-dev" + } + }, + "autoload": { + "files": [ + "src/functions.php" + ], + "psr-4": { + "MongoDB\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Andreas Braun", + "email": "andreas.braun@mongodb.com" + }, + { + "name": "Jeremy Mikola", + "email": "jmikola@gmail.com" + }, + { + "name": "Jérôme Tamarelle", + "email": "jerome.tamarelle@mongodb.com" + } + ], + "description": "MongoDB driver library", + "homepage": "https://jira.mongodb.org/browse/PHPLIB", + "keywords": [ + "database", + "driver", + "mongodb", + "persistence" + ], + "support": { + "issues": "https://github.com/mongodb/mongo-php-library/issues", + "source": "https://github.com/mongodb/mongo-php-library/tree/2.1.1" }, - "time": "2025-08-14T20:00:33+00:00" + "time": "2025-08-13T20:50:05+00:00" }, { "name": "nyholm/psr7", @@ -378,20 +452,20 @@ }, { "name": "open-telemetry/api", - "version": "1.5.0", + "version": "1.7.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", - "reference": "7692075f486c14d8cfd37fba98a08a5667f089e5" + "reference": "610b79ad9d6d97e8368bcb6c4d42394fbb87b522" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/7692075f486c14d8cfd37fba98a08a5667f089e5", - "reference": "7692075f486c14d8cfd37fba98a08a5667f089e5", + "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/610b79ad9d6d97e8368bcb6c4d42394fbb87b522", + "reference": "610b79ad9d6d97e8368bcb6c4d42394fbb87b522", "shasum": "" }, "require": { - "open-telemetry/context": "^1.0", + "open-telemetry/context": "^1.4", "php": "^8.1", "psr/log": "^1.1|^2.0|^3.0", "symfony/polyfill-php82": "^1.26" @@ -407,7 +481,7 @@ ] }, "branch-alias": { - "dev-main": "1.4.x-dev" + "dev-main": "1.7.x-dev" } }, "autoload": { @@ -444,20 +518,20 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-08-07T23:07:38+00:00" + "time": "2025-10-02T23:44:28+00:00" }, { "name": "open-telemetry/context", - "version": "1.3.1", + "version": "1.4.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/context.git", - "reference": "438f71812242db3f196fb4c717c6f92cbc819be6" + "reference": "d4c4470b541ce72000d18c339cfee633e4c8e0cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/context/zipball/438f71812242db3f196fb4c717c6f92cbc819be6", - "reference": "438f71812242db3f196fb4c717c6f92cbc819be6", + "url": "https://api.github.com/repos/opentelemetry-php/context/zipball/d4c4470b541ce72000d18c339cfee633e4c8e0cf", + "reference": "d4c4470b541ce72000d18c339cfee633e4c8e0cf", "shasum": "" }, "require": { @@ -503,7 +577,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-08-13T01:12:00+00:00" + "time": "2025-09-19T00:05:49+00:00" }, { "name": "open-telemetry/exporter-otlp", @@ -571,16 +645,16 @@ }, { "name": "open-telemetry/gen-otlp-protobuf", - "version": "1.5.0", + "version": "1.8.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/gen-otlp-protobuf.git", - "reference": "585bafddd4ae6565de154610b10a787a455c9ba0" + "reference": "673af5b06545b513466081884b47ef15a536edde" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/gen-otlp-protobuf/zipball/585bafddd4ae6565de154610b10a787a455c9ba0", - "reference": "585bafddd4ae6565de154610b10a787a455c9ba0", + "url": "https://api.github.com/repos/opentelemetry-php/gen-otlp-protobuf/zipball/673af5b06545b513466081884b47ef15a536edde", + "reference": "673af5b06545b513466081884b47ef15a536edde", "shasum": "" }, "require": { @@ -630,27 +704,27 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-15T23:07:07+00:00" + "time": "2025-09-17T23:10:12+00:00" }, { "name": "open-telemetry/sdk", - "version": "1.7.1", + "version": "1.9.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", - "reference": "52690d4b37ae4f091af773eef3c238ed2bc0aa06" + "reference": "8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/52690d4b37ae4f091af773eef3c238ed2bc0aa06", - "reference": "52690d4b37ae4f091af773eef3c238ed2bc0aa06", + "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e", + "reference": "8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e", "shasum": "" }, "require": { "ext-json": "*", "nyholm/psr7-server": "^1.1", - "open-telemetry/api": "^1.4", - "open-telemetry/context": "^1.0", + "open-telemetry/api": "^1.7", + "open-telemetry/context": "^1.4", "open-telemetry/sem-conv": "^1.0", "php": "^8.1", "php-http/discovery": "^1.14", @@ -684,7 +758,7 @@ ] }, "branch-alias": { - "dev-main": "1.0.x-dev" + "dev-main": "1.9.x-dev" } }, "autoload": { @@ -727,7 +801,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-09-05T07:17:06+00:00" + "time": "2025-10-02T23:44:28+00:00" }, { "name": "open-telemetry/sem-conv", @@ -1351,16 +1425,16 @@ }, { "name": "symfony/http-client", - "version": "v7.3.3", + "version": "v7.3.4", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019" + "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019", - "reference": "333b9bd7639cbdaecd25a3a48a9d2dcfaa86e019", + "url": "https://api.github.com/repos/symfony/http-client/zipball/4b62871a01c49457cf2a8e560af7ee8a94b87a62", + "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62", "shasum": "" }, "require": { @@ -1427,7 +1501,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.3.3" + "source": "https://github.com/symfony/http-client/tree/v7.3.4" }, "funding": [ { @@ -1447,7 +1521,7 @@ "type": "tidelift" } ], - "time": "2025-08-27T07:45:05+00:00" + "time": "2025-09-11T10:12:26+00:00" }, { "name": "symfony/http-client-contracts", @@ -1772,6 +1846,86 @@ ], "time": "2025-07-08T02:45:35+00:00" }, + { + "name": "symfony/polyfill-php85", + "version": "v1.33.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php85.git", + "reference": "d4e5fcd4ab3d998ab16c0db48e6cbb9a01993f91" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/d4e5fcd4ab3d998ab16c0db48e6cbb9a01993f91", + "reference": "d4e5fcd4ab3d998ab16c0db48e6cbb9a01993f91", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php85\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.5+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php85/tree/v1.33.0" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2025-06-23T16:12:55+00:00" + }, { "name": "symfony/service-contracts", "version": "v3.6.0", @@ -2007,29 +2161,31 @@ }, { "name": "utopia-php/database", - "version": "1.4.4", + "version": "dev-feat-documentsdb", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "16f96e5d9784dae87d4f6b864e87da8e3be15507" + "reference": "a9e4f07fa2a2f36b4ed69adb7dd4e590acc05d60" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/16f96e5d9784dae87d4f6b864e87da8e3be15507", - "reference": "16f96e5d9784dae87d4f6b864e87da8e3be15507", + "url": "https://api.github.com/repos/utopia-php/database/zipball/a9e4f07fa2a2f36b4ed69adb7dd4e590acc05d60", + "reference": "a9e4f07fa2a2f36b4ed69adb7dd4e590acc05d60", "shasum": "" }, "require": { "ext-mbstring": "*", + "ext-mongodb": "*", "ext-pdo": "*", "php": ">=8.1", "utopia-php/cache": "0.13.*", "utopia-php/framework": "0.33.*", + "utopia-php/mongo": "0.10.*", "utopia-php/pools": "0.8.*" }, "require-dev": { "fakerphp/faker": "1.23.*", - "laravel/pint": "1.*", + "laravel/pint": "*", "pcov/clobber": "2.*", "phpstan/phpstan": "1.*", "phpunit/phpunit": "9.*", @@ -2057,9 +2213,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/1.4.4" + "source": "https://github.com/utopia-php/database/tree/feat-documentsdb" }, - "time": "2025-09-10T00:50:05+00:00" + "time": "2025-10-07T05:15:47+00:00" }, { "name": "utopia-php/dsn", @@ -2110,16 +2266,16 @@ }, { "name": "utopia-php/framework", - "version": "0.33.27", + "version": "0.33.28", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "d9d10a895e85c8c7675220347cc6109db9d3bd37" + "reference": "5aaa94d406577b0059ad28c78022606890dc6de0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/d9d10a895e85c8c7675220347cc6109db9d3bd37", - "reference": "d9d10a895e85c8c7675220347cc6109db9d3bd37", + "url": "https://api.github.com/repos/utopia-php/http/zipball/5aaa94d406577b0059ad28c78022606890dc6de0", + "reference": "5aaa94d406577b0059ad28c78022606890dc6de0", "shasum": "" }, "require": { @@ -2151,9 +2307,70 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.27" + "source": "https://github.com/utopia-php/http/tree/0.33.28" }, - "time": "2025-09-07T18:40:53+00:00" + "time": "2025-09-25T10:44:24+00:00" + }, + { + "name": "utopia-php/mongo", + "version": "0.10.0", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/mongo.git", + "reference": "ecfad6aad2e2e3fe5899ac2ebf1009a21b4d6b18" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/ecfad6aad2e2e3fe5899ac2ebf1009a21b4d6b18", + "reference": "ecfad6aad2e2e3fe5899ac2ebf1009a21b4d6b18", + "shasum": "" + }, + "require": { + "ext-mongodb": "2.1.*", + "mongodb/mongodb": "2.1.*", + "php": ">=8.0", + "ramsey/uuid": "4.9.*" + }, + "require-dev": { + "fakerphp/faker": "1.*", + "laravel/pint": "*", + "phpstan/phpstan": "*", + "phpunit/phpunit": "9.*", + "swoole/ide-helper": "5.1.*" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Mongo\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Eldad Fux", + "email": "eldad@appwrite.io" + }, + { + "name": "Wess", + "email": "wess@appwrite.io" + } + ], + "description": "A simple library to manage Mongo database", + "keywords": [ + "database", + "mongo", + "php", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/mongo/issues", + "source": "https://github.com/utopia-php/mongo/tree/0.10.0" + }, + "time": "2025-10-02T04:50:07+00:00" }, { "name": "utopia-php/pools", @@ -2209,16 +2426,16 @@ }, { "name": "utopia-php/storage", - "version": "0.18.13", + "version": "0.18.14", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "3d8ce53ae042173bf230445e996056c5f65ded22" + "reference": "4f14ec952c6f4006dd0613e55bbf7631814fbc00" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/3d8ce53ae042173bf230445e996056c5f65ded22", - "reference": "3d8ce53ae042173bf230445e996056c5f65ded22", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/4f14ec952c6f4006dd0613e55bbf7631814fbc00", + "reference": "4f14ec952c6f4006dd0613e55bbf7631814fbc00", "shasum": "" }, "require": { @@ -2261,9 +2478,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.18.13" + "source": "https://github.com/utopia-php/storage/tree/0.18.14" }, - "time": "2025-05-26T13:10:35+00:00" + "time": "2025-10-07T10:21:47+00:00" }, { "name": "utopia-php/system", @@ -2437,16 +2654,16 @@ }, { "name": "laravel/pint", - "version": "v1.24.0", + "version": "v1.25.1", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "0345f3b05f136801af8c339f9d16ef29e6b4df8a" + "reference": "5016e263f95d97670d71b9a987bd8996ade6d8d9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/0345f3b05f136801af8c339f9d16ef29e6b4df8a", - "reference": "0345f3b05f136801af8c339f9d16ef29e6b4df8a", + "url": "https://api.github.com/repos/laravel/pint/zipball/5016e263f95d97670d71b9a987bd8996ade6d8d9", + "reference": "5016e263f95d97670d71b9a987bd8996ade6d8d9", "shasum": "" }, "require": { @@ -2457,9 +2674,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.82.2", - "illuminate/view": "^11.45.1", - "larastan/larastan": "^3.5.0", + "friendsofphp/php-cs-fixer": "^3.87.2", + "illuminate/view": "^11.46.0", + "larastan/larastan": "^3.7.1", "laravel-zero/framework": "^11.45.0", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3.1", @@ -2470,9 +2687,6 @@ ], "type": "project", "autoload": { - "files": [ - "overrides/Runner/Parallel/ProcessFactory.php" - ], "psr-4": { "App\\": "app/", "Database\\Seeders\\": "database/seeders/", @@ -2502,7 +2716,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-07-10T18:09:32+00:00" + "time": "2025-09-19T02:57:12+00:00" }, { "name": "myclabs/deep-copy", @@ -2817,16 +3031,11 @@ }, { "name": "phpstan/phpstan", - "version": "1.12.28", - "source": { - "type": "git", - "url": "https://github.com/phpstan/phpstan.git", - "reference": "fcf8b71aeab4e1a1131d1783cef97b23a51b87a9" - }, + "version": "1.12.32", "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/fcf8b71aeab4e1a1131d1783cef97b23a51b87a9", - "reference": "fcf8b71aeab4e1a1131d1783cef97b23a51b87a9", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/2770dcdf5078d0b0d53f94317e06affe88419aa8", + "reference": "2770dcdf5078d0b0d53f94317e06affe88419aa8", "shasum": "" }, "require": { @@ -2871,7 +3080,7 @@ "type": "github" } ], - "time": "2025-07-17T17:15:39+00:00" + "time": "2025-09-30T10:16:31+00:00" }, { "name": "phpunit/php-code-coverage", @@ -3210,16 +3419,16 @@ }, { "name": "phpunit/phpunit", - "version": "11.5.36", + "version": "11.5.42", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "264a87c7ef68b1ab9af7172357740dc266df5957" + "reference": "1c6cb5dfe412af3d0dfd414cfd110e3b9cfdbc3c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/264a87c7ef68b1ab9af7172357740dc266df5957", - "reference": "264a87c7ef68b1ab9af7172357740dc266df5957", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/1c6cb5dfe412af3d0dfd414cfd110e3b9cfdbc3c", + "reference": "1c6cb5dfe412af3d0dfd414cfd110e3b9cfdbc3c", "shasum": "" }, "require": { @@ -3243,7 +3452,7 @@ "sebastian/comparator": "^6.3.2", "sebastian/diff": "^6.0.2", "sebastian/environment": "^7.2.1", - "sebastian/exporter": "^6.3.0", + "sebastian/exporter": "^6.3.2", "sebastian/global-state": "^7.0.2", "sebastian/object-enumerator": "^6.0.1", "sebastian/type": "^5.1.3", @@ -3291,7 +3500,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.36" + "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.42" }, "funding": [ { @@ -3315,7 +3524,7 @@ "type": "tidelift" } ], - "time": "2025-09-03T06:24:17+00:00" + "time": "2025-09-28T12:09:13+00:00" }, { "name": "sebastian/cli-parser", @@ -3782,16 +3991,16 @@ }, { "name": "sebastian/exporter", - "version": "6.3.0", + "version": "6.3.2", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/exporter.git", - "reference": "3473f61172093b2da7de1fb5782e1f24cc036dc3" + "reference": "70a298763b40b213ec087c51c739efcaa90bcd74" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/3473f61172093b2da7de1fb5782e1f24cc036dc3", - "reference": "3473f61172093b2da7de1fb5782e1f24cc036dc3", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/70a298763b40b213ec087c51c739efcaa90bcd74", + "reference": "70a298763b40b213ec087c51c739efcaa90bcd74", "shasum": "" }, "require": { @@ -3805,7 +4014,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "6.1-dev" + "dev-main": "6.3-dev" } }, "autoload": { @@ -3848,15 +4057,27 @@ "support": { "issues": "https://github.com/sebastianbergmann/exporter/issues", "security": "https://github.com/sebastianbergmann/exporter/security/policy", - "source": "https://github.com/sebastianbergmann/exporter/tree/6.3.0" + "source": "https://github.com/sebastianbergmann/exporter/tree/6.3.2" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/sebastian/exporter", + "type": "tidelift" } ], - "time": "2024-12-05T09:17:50+00:00" + "time": "2025-09-24T06:12:51+00:00" }, { "name": "sebastian/global-state", @@ -4694,9 +4915,18 @@ "time": "2025-04-30T23:37:27+00:00" } ], - "aliases": [], + "aliases": [ + { + "package": "utopia-php/database", + "version": "dev-feat-documentsdb", + "alias": "1.4.5", + "alias_normalized": "1.4.5.0" + } + ], "minimum-stability": "stable", - "stability-flags": {}, + "stability-flags": { + "utopia-php/database": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 303214b..3f43ad6 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -441,6 +441,12 @@ protected function createTable(Table $resource): bool */ protected function createColumn(Column $resource): bool { + // Skip columns for documents DB (schemaless) + if ($resource->getTable()->getDatabase()->getType() === 'documents') { + $resource->setStatus(Resource::STATUS_SKIPPED, 'Columns not supported for documents database'); + return false; + } + $type = match ($resource->getType()) { Column::TYPE_DATETIME => UtopiaDatabase::VAR_DATETIME, Column::TYPE_BOOLEAN => UtopiaDatabase::VAR_BOOLEAN, diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index 9645cc6..20315d4 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -30,6 +30,8 @@ abstract class Resource implements \JsonSerializable public const TYPE_DATABASE = 'database'; + public const TYPE_DOCUMENTSDB_DATABASE = 'documentsdb'; + public const TYPE_ROW = 'row'; public const TYPE_FILE = 'file'; @@ -70,6 +72,7 @@ abstract class Resource implements \JsonSerializable self::TYPE_BUCKET, self::TYPE_TABLE, self::TYPE_DATABASE, + self::TYPE_DOCUMENTSDB_DATABASE, self::TYPE_ROW, self::TYPE_FILE, self::TYPE_FUNCTION, diff --git a/src/Migration/Resources/Database/Collection.php b/src/Migration/Resources/Database/Collection.php new file mode 100644 index 0000000..c038e93 --- /dev/null +++ b/src/Migration/Resources/Database/Collection.php @@ -0,0 +1,15 @@ +client = (new Client()) ->setEndpoint($endpoint) @@ -87,19 +95,17 @@ public function __construct( $this->headers['X-Appwrite-Project'] = $this->project; $this->headers['X-Appwrite-Key'] = $this->key; - switch ($this->source) { - case static::SOURCE_API: - $this->database = new APIReader(new Databases($this->client)); - break; - case static::SOURCE_DATABASE: - if (\is_null($dbForProject)) { - throw new \Exception('Database is required for database source'); - } - $this->database = new DatabaseReader($dbForProject); - break; - default: - throw new \Exception('Unknown source'); + // Validate required parameters based on source type + if ($this->source === static::SOURCE_DATABASE) { + if (\is_null($dbForProject)) { + throw new \Exception('Database is required for database source'); + } + if (\is_null($getDatabaseDB)) { + throw new \Exception('getDatabaseDB callable is required for database source'); + } } + + $this->getDatabaseDB = $getDatabaseDB; } public static function getName(): string @@ -107,6 +113,24 @@ public static function getName(): string return 'Appwrite'; } + /** + * Create a reader instance for a specific database + * + * @param Database|null $database + * @return Reader + * @throws \Exception + */ + private function createReaderForDatabase(Database|null $database): Reader + { + return match ($this->source) { + static::SOURCE_API => new APIReader(new Databases($this->client)), + static::SOURCE_DATABASE => new DatabaseReader( + call_user_func($this->getDatabaseDB, new UtopiaDocument(['database' => $database])) + ), + default => throw new \Exception('Unknown source'), + }; + } + /** * @return array */ @@ -269,7 +293,15 @@ private function reportAuth(array $resources, array &$report): void */ private function reportDatabases(array $resources, array &$report): void { - $this->database->report($resources, $report); + // For reporting, we use a temporary reader with dbForProject for SOURCE_DATABASE + // or the API reader for SOURCE_API + $reader = match ($this->source) { + static::SOURCE_API => new APIReader(new Databases($this->client)), + static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject), + default => throw new \Exception('Unknown source'), + }; + + $reader->report($resources, $report); } /** @@ -625,76 +657,8 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void return; } - try { - if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) { - $this->exportTables($batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_TABLE, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - - try { - if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) { - $this->exportColumns($batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_COLUMN, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - - try { - if (\in_array(Resource::TYPE_INDEX, $resources)) { - $this->exportIndexes($batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_INDEX, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - - try { - if (Resource::isSupported(Resource::TYPE_ROW, $resources)) { - $this->exportRows($batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_ROW, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; + if (count($this->cache->get(Database::getName()))) { + $this->exportTablesDB($batchSize, $resources); } } @@ -706,8 +670,15 @@ private function exportDatabases(int $batchSize): void { $lastDatabase = null; + // Create a reader for listing databases (not database-specific) + $reader = match ($this->source) { + static::SOURCE_API => new APIReader(new Databases($this->client)), + static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject), + default => throw new \Exception('Unknown source'), + }; + while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$reader->queryLimit($batchSize)]; if ($this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_DATABASE) { $targetDatabaseId = $this->rootResourceId; @@ -720,28 +691,27 @@ private function exportDatabases(int $batchSize): void } } - $queries[] = $this->database->queryEqual('$id', [$targetDatabaseId]); - $queries[] = $this->database->queryLimit(1); + $queries[] = $reader->queryEqual('$id', [$targetDatabaseId]); + $queries[] = $reader->queryLimit(1); } $databases = []; if ($lastDatabase) { - $queries[] = $this->database->queryCursorAfter($lastDatabase); + $queries[] = $reader->queryCursorAfter($lastDatabase); } - $response = $this->database->listDatabases($queries); + $response = $reader->listDatabases($queries); foreach ($response as $database) { - $newDatabase = new Database( - $database['$id'], - $database['name'], - $database['$createdAt'], - $database['$updatedAt'], - type: $database['type'] ?? 'legacy', - database: $database['database'] - ); - + $newDatabase = self::getDatabase($database['type'], [ + 'id' => $database['$id'], + 'name' => $database['name'], + 'createdAt' => $database['$createdAt'], + 'updatedAt' => $database['$updatedAt'], + 'type' => $database['type'], + 'database' => $database['database'] + ]); $databases[] = $newDatabase; } @@ -760,19 +730,26 @@ private function exportDatabases(int $batchSize): void } /** + * @param string $databaseName * @param int $batchSize + * @param array $databases * @throws Exception */ - private function exportTables(int $batchSize): void + private function exportEntities(string $databaseName, int $batchSize, array $databases): void { - $databases = $this->cache->get(Database::getName()); - foreach ($databases as $database) { /** @var Database $database */ $lastTable = null; + // colections in the metadata + $reader = match ($this->source) { + static::SOURCE_API => new APIReader(new Databases($this->client)), + static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject), + default => throw new \Exception('Unknown source'), + }; + while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$reader->queryLimit($batchSize)]; $tables = []; // Filter to specific table if rootResourceType is database with database:collection format @@ -784,32 +761,37 @@ private function exportTables(int $batchSize): void $parts = \explode(':', $this->rootResourceId, 2); if (\count($parts) === 2) { $targetTableId = $parts[1]; // table ID - $queries[] = $this->database->queryEqual('$id', [$targetTableId]); - $queries[] = $this->database->queryLimit(1); + $queries[] = $reader->queryEqual('$id', [$targetTableId]); + $queries[] = $reader->queryLimit(1); } } elseif ( $this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_TABLE ) { $targetTableId = $this->rootResourceId; - $queries[] = $this->database->queryEqual('$id', [$targetTableId]); - $queries[] = $this->database->queryLimit(1); + $queries[] = $reader->queryEqual('$id', [$targetTableId]); + $queries[] = $reader->queryLimit(1); } elseif ($lastTable) { - $queries[] = $this->database->queryCursorAfter($lastTable); + $queries[] = $reader->queryCursorAfter($lastTable); } - $response = $this->database->listTables($database, $queries); + $response = $reader->listTables($database, $queries); foreach ($response as $table) { - $newTable = new Table( - $database, - $table['name'], - $table['$id'], - $table['documentSecurity'], - $table['$permissions'], - $table['$createdAt'], - $table['$updatedAt'], - ); + $newTable = self::getEntity($databaseName, [ + 'id' => $table['$id'], + 'name' => $table['name'], + 'documentSecurity' => $table['documentSecurity'], + 'permissions' => $table['$permissions'], + 'createdAt' => $table['$createdAt'], + 'updatedAt' => $table['$updatedAt'], + 'database' => [ + 'id' => $database->getId(), + 'name' => $database->getDatabaseName(), + 'type' => $database->getType(), + 'database' => $database->getDatabase(), + ] + ]); $tables[] = $newTable; } @@ -833,7 +815,7 @@ private function exportTables(int $batchSize): void * @param int $batchSize * @throws Exception */ - private function exportColumns(int $batchSize): void + private function exportFields(int $batchSize): void { $tables = $this->cache->get(Table::getName()); @@ -841,15 +823,21 @@ private function exportColumns(int $batchSize): void foreach ($tables as $table) { $lastColumn = null; + $reader = match ($this->source) { + static::SOURCE_API => new APIReader(new Databases($this->client)), + static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject), + default => throw new \Exception('Unknown source'), + }; + while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$reader->queryLimit($batchSize)]; $columns = []; if ($lastColumn) { - $queries[] = $this->database->queryCursorAfter($lastColumn); + $queries[] = $reader->queryCursorAfter($lastColumn); } - $response = $this->database->listColumns($table, $queries); + $response = $reader->listColumns($table, $queries); foreach ($response as $column) { if ( @@ -1039,26 +1027,32 @@ private function exportColumns(int $batchSize): void /** * @param int $batchSize + * @param array $entites * @throws Exception */ - private function exportIndexes(int $batchSize): void + private function exportIndexes(int $batchSize, array $entites): void { - $tables = $this->cache->get(Resource::TYPE_TABLE); - // Transfer Indexes - foreach ($tables as $table) { + foreach ($entites as $table) { /** @var Table $table */ $lastIndex = null; + // Create reader for this specific database + $reader = match ($this->source) { + static::SOURCE_API => new APIReader(new Databases($this->client)), + static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject), + default => throw new \Exception('Unknown source'), + }; + while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$reader->queryLimit($batchSize)]; $indexes = []; if ($lastIndex) { - $queries[] = $this->database->queryCursorAfter($lastIndex); + $queries[] = $reader->queryCursorAfter($lastIndex); } - $response = $this->database->listIndexes($table, $queries); + $response = $reader->listIndexes($table, $queries); foreach ($response as $index) { $indexes[] = new Index( @@ -1090,65 +1084,78 @@ private function exportIndexes(int $batchSize): void } /** + * @param string $databaseName + * @param int $batchSize + * @param array $entities * @throws Exception */ - private function exportRows(int $batchSize): void + private function exportRecords(string $databaseName, int $batchSize, array $entities): void { - $tables = $this->cache->get(Table::getName()); - foreach ($tables as $table) { + foreach ($entities as $table) { /** @var Table $table */ $lastRow = null; + // Create reader - use getDatabaseDB for row data operations + $reader = match ($this->source) { + static::SOURCE_API => new APIReader(new Databases($this->client)), + static::SOURCE_DATABASE => new DatabaseReader( + call_user_func($this->getDatabaseDB, new UtopiaDocument(['database' => $table->getDatabase()->getDatabase()])) + ), + default => throw new \Exception('Unknown source'), + }; + while (true) { - $queries = [$this->database->queryLimit($batchSize)]; + $queries = [$reader->queryLimit($batchSize)]; $rows = []; if ($lastRow) { - $queries[] = $this->database->queryCursorAfter($lastRow); + $queries[] = $reader->queryCursorAfter($lastRow); } $selects = ['*', '$id', '$permissions', '$updatedAt', '$createdAt']; // We want relations flat! $manyToMany = []; - $attributes = $this->cache->get(Column::getName()); - foreach ($attributes as $attribute) { - /** @var Relationship $attribute */ - if ( - $attribute->getTable()->getId() === $table->getId() && - $attribute->getType() === Column::TYPE_RELATIONSHIP && - $attribute->getSide() === 'parent' && - $attribute->getRelationType() == 'manyToMany' - ) { - /** - * Blockers: - * we should use but Does not work properly: - * $selects[] = $attribute->getKey() . '.$id'; - * when selecting for a relation we get all relations not just the one we were asking. - * when selecting for a relation like select(*, relation.$id) , all relations get resolve - */ - $manyToMany[] = $attribute->getKey(); + if ($reader->getSupportForAttributes()) { + $attributes = $this->cache->get(Column::getName()); + foreach ($attributes as $attribute) { + /** @var Relationship $attribute */ + if ( + $attribute->getTable()->getId() === $table->getId() && + $attribute->getType() === Column::TYPE_RELATIONSHIP && + $attribute->getSide() === 'parent' && + $attribute->getRelationType() == 'manyToMany' + ) { + /** + * Blockers: + * we should use but Does not work properly: + * $selects[] = $attribute->getKey() . '.$id'; + * when selecting for a relation we get all relations not just the one we were asking. + * when selecting for a relation like select(*, relation.$id) , all relations get resolve + */ + $manyToMany[] = $attribute->getKey(); + } } + /** @var Column|Relationship $attribute */ } - /** @var Column|Relationship $attribute */ - $queries[] = $this->database->querySelect($selects); + $queries[] = $reader->querySelect($selects); - $response = $this->database->listRows($table, $queries); + $response = $reader->listRows($table, $queries); foreach ($response as $row) { - // HACK: Handle many to many + // HACK: Handle many to many (only for schema-based databases) if (!empty($manyToMany)) { $stack = ['$id']; // Adding $id because we can't select only relations foreach ($manyToMany as $relation) { $stack[] = $relation . '.$id'; } - $rowItem = $this->database->getRow( + $rowItem = $reader->getRow( $table, $row['$id'], - [$this->database->querySelect($stack)] + [$reader->querySelect($stack)] ); foreach ($manyToMany as $key) { @@ -1169,12 +1176,23 @@ private function exportRows(int $batchSize): void unset($row['$sequence']); unset($row['$collection']); - $row = new Row( - $id, - $table, - $row, - $permissions - ); + $row = self::getRecord($databaseName, [ + 'id' => $id, + 'table' => [ + 'id' => $table->getId(), + 'name' => $table->getTableName(), + 'rowSecurity' => $table->getRowSecurity(), + 'permissions' => $table->getPermissions(), + 'database' => [ + 'id' => $table->getDatabase()->getId(), + 'name' => $table->getDatabase()->getDatabaseName(), + 'type' => $table->getDatabase()->getType(), + 'database' => $table->getDatabase()->getDatabase(), + ] + ], + 'data' => $row, + 'permissions' => $permissions + ]); $rows[] = $row; $lastRow = $row; @@ -1604,4 +1622,172 @@ private function exportDeploymentData(Func $func, array $deployment): void } } } + + private function exportTablesDB(int $batchSize, array $resources) + { + try { + if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) { + $databases = $this->cache->get(Database::getName()); + $this->exportEntities(Table::getName(), $batchSize, $databases); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + Resource::TYPE_TABLE, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } + + try { + if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) { + $this->exportFields($batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + Resource::TYPE_COLUMN, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } + + try { + if (\in_array(Resource::TYPE_INDEX, $resources)) { + $tables = $this->cache->get(Table::getName()); + $this->exportIndexes($batchSize, $tables); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + Resource::TYPE_INDEX, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } + + try { + if (Resource::isSupported(Resource::TYPE_ROW, $resources)) { + $tables = $this->cache->get(Table::getName()); + $this->exportRecords(Table::getName(), $batchSize, $tables); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + Resource::TYPE_ROW, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } + } + + + /** + * @param string $databaseType + * @param array $database { + * id: string, + * name: string, + * createdAt: string, + * updatedAt: string, + * enabled: bool, + * originalId: string|null, + * database: string + * } + */ + private static function getDatabase(string $databaseType, array $database): Resource + { + switch ($databaseType) { + case Resource::TYPE_DOCUMENTSDB_DATABASE: + return DocumentsDB::fromArray($database); + default: + return Database::fromArray($database); + } + } + + /** + * eg., tables,collections + * @param string $databaseType + * @param array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity?: bool, + * rowSecurity?: bool, + * permissions: ?array, + * createdAt: string, + * updatedAt: string, + * enabled: bool + * } $entity + */ + private static function getEntity(string $databaseType, array $entity): Resource + { + switch ($databaseType) { + case Resource::TYPE_DOCUMENTSDB_DATABASE: + return Collection::fromArray($entity); + default: + return Table::fromArray($entity); + } + } + + /** + * eg.,documents/attributes + * @param string $databaseType + * @param array{ + * id: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * data: array, + * permissions: ?array + * } $record + */ + private static function getRecord(string $databaseType, array $record): Resource + { + switch ($databaseType) { + case Resource::TYPE_DOCUMENTSDB_DATABASE: + return Document::fromArray($record); + default: + return Row::fromArray($record); + } + } } diff --git a/src/Migration/Sources/Appwrite/Reader.php b/src/Migration/Sources/Appwrite/Reader.php index b45f2bd..04cc3bc 100644 --- a/src/Migration/Sources/Appwrite/Reader.php +++ b/src/Migration/Sources/Appwrite/Reader.php @@ -106,4 +106,6 @@ public function queryCursorAfter(Resource|string $resource): mixed; * @return QueryType|string */ public function queryLimit(int $limit): mixed; + public function getSupportForAttributes(): bool; + } diff --git a/src/Migration/Sources/Appwrite/Reader/API.php b/src/Migration/Sources/Appwrite/Reader/API.php index da1a502..5790a60 100644 --- a/src/Migration/Sources/Appwrite/Reader/API.php +++ b/src/Migration/Sources/Appwrite/Reader/API.php @@ -240,4 +240,9 @@ public function queryLimit(int $limit): string { return Query::limit($limit); } + + public function getSupportForAttributes(): bool + { + return true; + } } diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index c43e246..4992c74 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -413,6 +413,11 @@ public function queryLimit(int $limit): Query return Query::limit($limit); } + public function getSupportForAttributes(): bool + { + return $this->dbForProject->getAdapter()->getSupportForAttributes(); + } + /** * @param string $table * @param array $queries From 1a45ca8117328801f4a28dd6b3946756cba0e273 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 14 Oct 2025 11:12:12 +0530 Subject: [PATCH 07/28] * linting * formatting --- src/Migration/Destinations/Appwrite.php | 49 ++--- src/Migration/Resource.php | 18 ++ src/Migration/Resources/Database/Database.php | 2 +- src/Migration/Resources/Database/Row.php | 2 +- src/Migration/Resources/Database/Table.php | 2 +- src/Migration/Sources/Appwrite.php | 191 +++++++++++------- .../Sources/Appwrite/Reader/Database.php | 116 ++++++++--- src/Migration/Transfer.php | 3 + 8 files changed, 264 insertions(+), 119 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 3f43ad6..c463800 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -118,6 +118,7 @@ public static function getSupportedResources(): array // Database Resource::TYPE_DATABASE, + Resource::TYPE_DOCUMENTSDB_DATABASE, Resource::TYPE_TABLE, Resource::TYPE_COLUMN, Resource::TYPE_INDEX, @@ -282,18 +283,19 @@ public function importDatabaseResource(Resource $resource, bool $isLast): Resour { switch ($resource->getName()) { case Resource::TYPE_DATABASE: + case Resource::TYPE_DOCUMENTSDB_DATABASE: /** @var Database $resource */ $success = $this->createDatabase($resource); break; case Resource::TYPE_TABLE: case Resource::TYPE_COLLECTION: /** @var Table $resource */ - $success = $this->createTable($resource); + $success = $this->createEntity($resource); break; case Resource::TYPE_COLUMN: case Resource::TYPE_ATTRIBUTE: /** @var Column $resource */ - $success = $this->createColumn($resource); + $success = $this->createField($resource); break; case Resource::TYPE_INDEX: /** @var Index $resource */ @@ -302,7 +304,7 @@ public function importDatabaseResource(Resource $resource, bool $isLast): Resour case Resource::TYPE_ROW: case Resource::TYPE_DOCUMENT: /** @var Row $resource */ - $success = $this->createRow($resource, $isLast); + $success = $this->createRecord($resource, $isLast); break; default: $success = false; @@ -377,7 +379,7 @@ protected function createDatabase(Database $resource): bool * @throws StructureException * @throws Exception */ - protected function createTable(Table $resource): bool + protected function createEntity(Table $resource): bool { if ($resource->getId() == 'unique()') { $resource->setId(ID::unique()); @@ -439,7 +441,7 @@ protected function createTable(Table $resource): bool * @throws \Exception * @throws \Throwable */ - protected function createColumn(Column $resource): bool + protected function createField(Column $resource): bool { // Skip columns for documents DB (schemaless) if ($resource->getTable()->getDatabase()->getType() === 'documents') { @@ -922,7 +924,7 @@ protected function createIndex(Index $resource): bool * @throws StructureException * @throws Exception */ - protected function createRow(Row $resource, bool $isLast): bool + protected function createRecord(Row $resource, bool $isLast): bool { if ($resource->getId() == 'unique()') { $resource->setId(ID::unique()); @@ -942,7 +944,7 @@ protected function createRow(Row $resource, bool $isLast): bool // Check if document has already been created $exists = \array_key_exists( $resource->getId(), - $this->cache->get(Resource::TYPE_ROW) + $this->cache->get($resource->getName()) ); if ($exists) { @@ -972,32 +974,33 @@ protected function createRow(Row $resource, bool $isLast): bool $databaseInternalId = $database->getSequence(); $tableInternalId = $table->getSequence(); - + $dbForDatabase = call_user_func($this->getDatabaseDB, $database); /** * This is in case an attribute was deleted from Appwrite attributes collection but was not deleted from the table * When creating an archive we select * which will include orphan attribute from the schema */ - foreach ($this->rowBuffer as $row) { - foreach ($row as $key => $value) { - if (\str_starts_with($key, '$')) { - continue; - } + if ($dbForDatabase->getAdapter()->getSupportForAttributes()) { + foreach ($this->rowBuffer as $row) { + foreach ($row as $key => $value) { + if (\str_starts_with($key, '$')) { + continue; + } - /** @var \Utopia\Database\Document $attribute */ - $found = false; - foreach ($table->getAttribute('attributes', []) as $attribute) { - if ($attribute->getAttribute('key') == $key) { - $found = true; - break; + /** @var \Utopia\Database\Document $attribute */ + $found = false; + foreach ($table->getAttribute('attributes', []) as $attribute) { + if ($attribute->getAttribute('key') == $key) { + $found = true; + break; + } } - } - if (! $found) { - $row->removeAttribute($key); + if (! $found) { + $row->removeAttribute($key); + } } } } - $dbForDatabase = call_user_func($this->getDatabaseDB, $database); $dbForDatabase->skipRelationshipsExistCheck(fn () => $dbForDatabase->createDocuments( 'database_' . $databaseInternalId . '_collection_' . $tableInternalId, $this->rowBuffer diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index 20315d4..b430696 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -30,6 +30,10 @@ abstract class Resource implements \JsonSerializable public const TYPE_DATABASE = 'database'; + public const TYPE_DATABASE_LEGACY = 'legacy'; + + public const TYPE_DATABASE_TABLESDB = 'tablesdb'; + public const TYPE_DOCUMENTSDB_DATABASE = 'documentsdb'; public const TYPE_ROW = 'row'; @@ -90,6 +94,20 @@ abstract class Resource implements \JsonSerializable self::TYPE_COLLECTION, ]; + // index terminology is same for all + public const DATABASE_TYPE_RESOURCE_MAP = [ + self::TYPE_DATABASE => [ + 'entity' => self::TYPE_TABLE, + 'fields' => self::TYPE_COLUMN, + 'records' => self::TYPE_ROW, + ], + self::TYPE_DOCUMENTSDB_DATABASE => [ + 'entity' => self::TYPE_COLLECTION, + 'records' => self::TYPE_DOCUMENT, + 'fields' => self::TYPE_ATTRIBUTE, + ], + ]; + protected string $id = ''; protected string $originalId = ''; diff --git a/src/Migration/Resources/Database/Database.php b/src/Migration/Resources/Database/Database.php index 9c90498..50d7af6 100644 --- a/src/Migration/Resources/Database/Database.php +++ b/src/Migration/Resources/Database/Database.php @@ -44,7 +44,7 @@ public function __construct( */ public static function fromArray(array $array): self { - return new self( + return new static( $array['id'], $array['name'], createdAt: $array['createdAt'] ?? '', diff --git a/src/Migration/Resources/Database/Row.php b/src/Migration/Resources/Database/Row.php index 42d5bfc..87dfcad 100644 --- a/src/Migration/Resources/Database/Row.php +++ b/src/Migration/Resources/Database/Row.php @@ -52,7 +52,7 @@ public function __construct( */ public static function fromArray(array $array): self { - return new self( + return new static( $array['id'], Table::fromArray($array['table'] ?? $array['collection']), $array['data'], diff --git a/src/Migration/Resources/Database/Table.php b/src/Migration/Resources/Database/Table.php index 889d90f..6c76f8c 100644 --- a/src/Migration/Resources/Database/Table.php +++ b/src/Migration/Resources/Database/Table.php @@ -49,7 +49,7 @@ public function __construct( */ public static function fromArray(array $array): self { - return new self( + return new static( Database::fromArray($array['database']), name: $array['name'], id: $array['id'], diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 4278db2..00a53be 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -114,19 +114,16 @@ public static function getName(): string } /** - * Create a reader instance for a specific database + * Create a reader instance for general operations (not database-specific) * - * @param Database|null $database * @return Reader * @throws \Exception */ - private function createReaderForDatabase(Database|null $database): Reader + private function createReader(): Reader { return match ($this->source) { static::SOURCE_API => new APIReader(new Databases($this->client)), - static::SOURCE_DATABASE => new DatabaseReader( - call_user_func($this->getDatabaseDB, new UtopiaDocument(['database' => $database])) - ), + static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabaseDB), default => throw new \Exception('Unknown source'), }; } @@ -154,6 +151,9 @@ public static function getSupportedResources(): array Resource::TYPE_ATTRIBUTE, Resource::TYPE_COLLECTION, + // documentsdb + Resource::TYPE_DOCUMENTSDB_DATABASE, + // Storage Resource::TYPE_BUCKET, Resource::TYPE_FILE, @@ -293,14 +293,7 @@ private function reportAuth(array $resources, array &$report): void */ private function reportDatabases(array $resources, array &$report): void { - // For reporting, we use a temporary reader with dbForProject for SOURCE_DATABASE - // or the API reader for SOURCE_API - $reader = match ($this->source) { - static::SOURCE_API => new APIReader(new Databases($this->client)), - static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject), - default => throw new \Exception('Unknown source'), - }; - + $reader = $this->createReader(); $reader->report($resources, $report); } @@ -640,8 +633,8 @@ private function exportMemberships(int $batchSize): void protected function exportGroupDatabases(int $batchSize, array $resources): void { try { - if (\in_array(Resource::TYPE_DATABASE, $resources)) { - $this->exportDatabases($batchSize); + if (\in_array(Resource::TYPE_DATABASE, $resources) || \in_array(Resource::TYPE_DOCUMENTSDB_DATABASE, $resources)) { + $this->exportDatabases($batchSize, $resources); } } catch (\Throwable $e) { $this->addError( @@ -657,30 +650,31 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void return; } - if (count($this->cache->get(Database::getName()))) { + if (count($this->cache->get(Database::getName())) > 0) { $this->exportTablesDB($batchSize, $resources); } + + if (count($this->cache->get(DocumentsDB::getName())) > 0) { + $this->exportDocumentsDB($batchSize, $resources); + } } /** * @param int $batchSize + * @param array $resources * @throws Exception */ - private function exportDatabases(int $batchSize): void + private function exportDatabases(int $batchSize, array $resources = []): void { $lastDatabase = null; // Create a reader for listing databases (not database-specific) - $reader = match ($this->source) { - static::SOURCE_API => new APIReader(new Databases($this->client)), - static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject), - default => throw new \Exception('Unknown source'), - }; + $reader = $this->createReader(); while (true) { $queries = [$reader->queryLimit($batchSize)]; - if ($this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_DATABASE) { + if ($this->rootResourceId !== '' && ($this->rootResourceType === Resource::TYPE_DATABASE || $this->rootResourceType === Resource::TYPE_DOCUMENTSDB_DATABASE)) { $targetDatabaseId = $this->rootResourceId; // Handle database:collection format - extract database ID @@ -704,15 +698,23 @@ private function exportDatabases(int $batchSize): void $response = $reader->listDatabases($queries); foreach ($response as $database) { - $newDatabase = self::getDatabase($database['type'], [ - 'id' => $database['$id'], - 'name' => $database['name'], - 'createdAt' => $database['$createdAt'], - 'updatedAt' => $database['$updatedAt'], - 'type' => $database['type'], - 'database' => $database['database'] - ]); - $databases[] = $newDatabase; + $databaseType = $database['type']; + if ($databaseType === Resource::TYPE_DATABASE_LEGACY || $databaseType === Resource::TYPE_DATABASE_TABLESDB) { + $databaseType = Resource::TYPE_DATABASE; + } + + if (Resource::isSupported($databaseType, $resources)) { + $newDatabase = self::getDatabase($databaseType, [ + 'id' => $database['$id'], + 'name' => $database['name'], + 'createdAt' => $database['$createdAt'], + 'updatedAt' => $database['$updatedAt'], + 'type' => $databaseType, + 'database' => $database['database'] + ]); + $databases[] = $newDatabase; + + } } if (empty($databases)) { @@ -735,18 +737,15 @@ private function exportDatabases(int $batchSize): void * @param array $databases * @throws Exception */ - private function exportEntities(string $databaseName, int $batchSize, array $databases): void + private function exportEntities(string $databaseName, int $batchSize): void { + $databases = $this->cache->get($databaseName); foreach ($databases as $database) { /** @var Database $database */ $lastTable = null; - // colections in the metadata - $reader = match ($this->source) { - static::SOURCE_API => new APIReader(new Databases($this->client)), - static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject), - default => throw new \Exception('Unknown source'), - }; + // collections in the metadata + $reader = $this->createReader(); while (true) { $queries = [$reader->queryLimit($batchSize)]; @@ -776,7 +775,6 @@ private function exportEntities(string $databaseName, int $batchSize, array $dat } $response = $reader->listTables($database, $queries); - foreach ($response as $table) { $newTable = self::getEntity($databaseName, [ 'id' => $table['$id'], @@ -787,7 +785,7 @@ private function exportEntities(string $databaseName, int $batchSize, array $dat 'updatedAt' => $table['$updatedAt'], 'database' => [ 'id' => $database->getId(), - 'name' => $database->getDatabaseName(), + 'name' => $databaseName, 'type' => $database->getType(), 'database' => $database->getDatabase(), ] @@ -823,11 +821,7 @@ private function exportFields(int $batchSize): void foreach ($tables as $table) { $lastColumn = null; - $reader = match ($this->source) { - static::SOURCE_API => new APIReader(new Databases($this->client)), - static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject), - default => throw new \Exception('Unknown source'), - }; + $reader = $this->createReader(); while (true) { $queries = [$reader->queryLimit($batchSize)]; @@ -1026,23 +1020,20 @@ private function exportFields(int $batchSize): void } /** + * @param string $entityType * @param int $batchSize - * @param array $entites * @throws Exception */ - private function exportIndexes(int $batchSize, array $entites): void + private function exportIndexes(string $entityType, int $batchSize): void { + $entities = $this->cache->get($entityType); // Transfer Indexes - foreach ($entites as $table) { + foreach ($entities as $table) { /** @var Table $table */ $lastIndex = null; // Create reader for this specific database - $reader = match ($this->source) { - static::SOURCE_API => new APIReader(new Databases($this->client)), - static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject), - default => throw new \Exception('Unknown source'), - }; + $reader = $this->createReader(); while (true) { $queries = [$reader->queryLimit($batchSize)]; @@ -1084,14 +1075,13 @@ private function exportIndexes(int $batchSize, array $entites): void } /** - * @param string $databaseName + * @param string $entityName * @param int $batchSize - * @param array $entities * @throws Exception */ - private function exportRecords(string $databaseName, int $batchSize, array $entities): void + private function exportRecords(string $entityName, int $batchSize): void { - + $entities = $this->cache->get($entityName); foreach ($entities as $table) { /** @var Table $table */ $lastRow = null; @@ -1100,7 +1090,8 @@ private function exportRecords(string $databaseName, int $batchSize, array $enti $reader = match ($this->source) { static::SOURCE_API => new APIReader(new Databases($this->client)), static::SOURCE_DATABASE => new DatabaseReader( - call_user_func($this->getDatabaseDB, new UtopiaDocument(['database' => $table->getDatabase()->getDatabase()])) + $this->dbForProject, + $this->getDatabaseDB ), default => throw new \Exception('Unknown source'), }; @@ -1176,7 +1167,7 @@ private function exportRecords(string $databaseName, int $batchSize, array $enti unset($row['$sequence']); unset($row['$collection']); - $row = self::getRecord($databaseName, [ + $row = self::getRecord($table->getDatabase()->getDatabaseName(), [ 'id' => $id, 'table' => [ 'id' => $table->getId(), @@ -1626,9 +1617,8 @@ private function exportDeploymentData(Func $func, array $deployment): void private function exportTablesDB(int $batchSize, array $resources) { try { - if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) { - $databases = $this->cache->get(Database::getName()); - $this->exportEntities(Table::getName(), $batchSize, $databases); + if (\in_array(Resource::TYPE_TABLE, $resources)) { + $this->exportEntities(Database::getName(), $batchSize); } } catch (\Throwable $e) { $this->addError( @@ -1645,7 +1635,7 @@ private function exportTablesDB(int $batchSize, array $resources) } try { - if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) { + if (\in_array(Resource::TYPE_COLUMN, $resources)) { $this->exportFields($batchSize); } } catch (\Throwable $e) { @@ -1664,8 +1654,7 @@ private function exportTablesDB(int $batchSize, array $resources) try { if (\in_array(Resource::TYPE_INDEX, $resources)) { - $tables = $this->cache->get(Table::getName()); - $this->exportIndexes($batchSize, $tables); + $this->exportIndexes(Table::getName(), $batchSize); } } catch (\Throwable $e) { $this->addError( @@ -1682,9 +1671,8 @@ private function exportTablesDB(int $batchSize, array $resources) } try { - if (Resource::isSupported(Resource::TYPE_ROW, $resources)) { - $tables = $this->cache->get(Table::getName()); - $this->exportRecords(Table::getName(), $batchSize, $tables); + if (\in_array(Resource::TYPE_ROW, $resources)) { + $this->exportRecords(Table::getName(), $batchSize); } } catch (\Throwable $e) { $this->addError( @@ -1701,6 +1689,69 @@ private function exportTablesDB(int $batchSize, array $resources) } } + /** + * Export DocumentsDB databases (collections and documents) + * + * @param int $batchSize + * @param array $resources + * @param array $documentsDBDatabases + */ + private function exportDocumentsDB(int $batchSize, array $resources): void + { + try { + if (\in_array(Resource::TYPE_COLLECTION, $resources)) { + $this->exportEntities(DocumentsDB::getName(), $batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + Resource::TYPE_COLLECTION, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } + + try { + if (\in_array(Resource::TYPE_INDEX, $resources)) { + $this->exportIndexes(Collection::getName(), $batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + Resource::TYPE_INDEX, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } + + try { + if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { + $this->exportRecords(Collection::getName(), $batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + Resource::TYPE_DOCUMENT, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } + } /** * @param string $databaseType diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index 4992c74..c1fa355 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -8,8 +8,10 @@ use Utopia\Database\Query; use Utopia\Migration\Exception; use Utopia\Migration\Resource; +use Utopia\Migration\Resources\Database\Collection as CollectionResource; use Utopia\Migration\Resources\Database\Column as ColumnResource; use Utopia\Migration\Resources\Database\Database as DatabaseResource; +use Utopia\Migration\Resources\Database\Document as DocumentResource; use Utopia\Migration\Resources\Database\Index as IndexResource; use Utopia\Migration\Resources\Database\Row as RowResource; use Utopia\Migration\Resources\Database\Table as TableResource; @@ -20,8 +22,28 @@ */ class Database implements Reader { - public function __construct(private readonly UtopiaDatabase $dbForProject) + /** + * @var callable(UtopiaDocument|null): UtopiaDatabase + */ + private mixed $getDatabaseDB; + + public function __construct( + private readonly UtopiaDatabase $dbForProject, + ?callable $getDatabaseDB = null + ) { + $this->getDatabaseDB = $getDatabaseDB; + } + + /** + * Get the appropriate database instance for the given database DSN + */ + private function getDatabase(?string $databaseDSN = null): UtopiaDatabase { + if ($this->getDatabaseDB !== null && $databaseDSN !== null) { + return call_user_func($this->getDatabaseDB, new UtopiaDocument(['database' => $databaseDSN])); + } + + return $this->dbForProject; } public function report(array $resources, array &$report): mixed @@ -32,6 +54,12 @@ public function report(array $resources, array &$report): mixed Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX, + // Documentsdb + Resource::TYPE_DOCUMENTSDB_DATABASE, + // Legacy types + Resource::TYPE_COLLECTION, + Resource::TYPE_DOCUMENT, + Resource::TYPE_ATTRIBUTE, ]; if (!Resource::isSupported($relevantResources, $resources)) { @@ -43,31 +71,35 @@ public function report(array $resources, array &$report): mixed $report[$resourceType] = 0; } } + $databases = $this->listDatabases(); - if (in_array(Resource::TYPE_DATABASE, $resources)) { - $report[Resource::TYPE_DATABASE] = $this->countResources('databases'); + foreach ($databases as $database) { + $databaseType = $database->getAttribute('type'); + if ($databaseType === Resource::TYPE_DATABASE_LEGACY || $databaseType === Resource::TYPE_DATABASE_TABLESDB) { + $databaseType = Resource::TYPE_DATABASE; + } + if (Resource::isSupported($databaseType, $resources)) { + $report[$databaseType] += 1; + } } - if (count(array_intersect($resources, $relevantResources)) === 1 && - in_array(Resource::TYPE_DATABASE, $resources)) { + if ( + count(array_intersect($resources, $relevantResources)) === 1 && + Resource::isSupported(array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP), $resources) + ) { return null; } $dbResources = []; - $databases = $this->listDatabases(); - - // Process each database foreach ($databases as $database) { - $databaseSequence = $database->getSequence(); - $tableId = "database_{$databaseSequence}"; - - if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) { - $report[Resource::TYPE_TABLE] += $this->countResources($tableId); + $databaseType = $database->getAttribute('type'); + if ($databaseType === Resource::TYPE_DATABASE_LEGACY || $databaseType === Resource::TYPE_DATABASE_TABLESDB) { + $databaseType = Resource::TYPE_DATABASE; } - if (!Resource::isSupported([Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX], $resources)) { - continue; - } + $databaseSpecificResources = Resource::DATABASE_TYPE_RESOURCE_MAP[$databaseType]; + + $databaseSequence = $database->getSequence(); if (!isset($dbResources[$database->getId()])) { $dbResources[$database->getId()] = new DatabaseResource( @@ -75,19 +107,29 @@ public function report(array $resources, array &$report): mixed $database->getAttribute('name'), $database->getCreatedAt(), $database->getUpdatedAt(), + $database->getAttribute('enabled', true), + $database->getAttribute('originalId', ''), + $database->getAttribute('type', ''), + $database->getAttribute('database', '') ); } $dbResource = $dbResources[$database->getId()]; $tables = $this->listTables($dbResource); + $count = count($tables); + + if (Resource::isSupported($databaseSpecificResources['entity'], $resources)) { + $report[$databaseSpecificResources['entity']] += $count; + } foreach ($tables as $table) { $tableSequence = $table->getSequence(); - if (Resource::isSupported(Resource::TYPE_ROW, $resources)) { + if (Resource::isSupported($databaseSpecificResources['records'], $resources)) { $rowTableId = "database_{$databaseSequence}_collection_{$tableSequence}"; - $report[Resource::TYPE_ROW] += $this->countResources($rowTableId); + $count = $this->countResources($rowTableId, [], $dbResource); + $report[$databaseSpecificResources['records']] += $count; } $commonQueries = [ @@ -95,8 +137,9 @@ public function report(array $resources, array &$report): mixed Query::equal('collectionInternalId', [$tableSequence]), ]; - if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) { - $report[Resource::TYPE_COLUMN] += $this->countResources('attributes', $commonQueries); + if (Resource::isSupported($databaseSpecificResources['fields'], $resources)) { + $count = $this->countResources('attributes', $commonQueries); + $report[$databaseSpecificResources['fields']] += $count; } if (in_array(Resource::TYPE_INDEX, $resources)) { @@ -295,8 +338,11 @@ public function listRows(TableResource $resource, array $queries = []): array $tableId = "database_{$database->getSequence()}_collection_{$table->getSequence()}"; + // Use the appropriate database instance for this specific database + $dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase()); + try { - $rows = $this->dbForProject->find($tableId, $queries); + $rows = $dbInstance->find($tableId, $queries); } catch (DatabaseException $e) { throw new Exception( resourceName: $resource->getName(), @@ -345,7 +391,10 @@ public function getRow(TableResource $resource, string $rowId, array $queries = $tableId = "database_{$database->getSequence()}_collection_{$table->getSequence()}"; - return $this->dbForProject->getDocument( + // Use the appropriate database instance for this specific database + $dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase()); + + return $dbInstance->getDocument( $tableId, $rowId, $queries @@ -385,19 +434,32 @@ public function queryCursorAfter(mixed $resource): Query switch ($resource::class) { case DatabaseResource::class: + /** @var DatabaseResource $resource */ + // Databases are always in dbForProject metadata $document = $this->dbForProject->getDocument('databases', $resource->getId()); break; case TableResource::class: + case CollectionResource::class: + /** @var TableResource|CollectionResource $resource */ + // Tables/Collections metadata is in dbForProject $database = $this->dbForProject->getDocument('databases', $resource->getDatabase()->getId()); $document = $this->dbForProject->getDocument('database_' . $database->getSequence(), $resource->getId()); break; case ColumnResource::class: + /** @var ColumnResource $resource */ + // Columns (attributes) are in dbForProject metadata $document = $this->dbForProject->getDocument('attributes', $resource->getId()); break; case IndexResource::class: + /** @var IndexResource $resource */ + // Indexes are in dbForProject metadata $document = $this->dbForProject->getDocument('indexes', $resource->getId()); break; case RowResource::class: + case DocumentResource::class: + /** @var RowResource|DocumentResource $resource */ + // Rows/Documents are in the specific database instance + // getRow() already uses getDatabase() internally $document = $this->getRow($resource->getTable(), $resource->getId()); $document = new UtopiaDocument($document); break; @@ -421,11 +483,19 @@ public function getSupportForAttributes(): bool /** * @param string $table * @param array $queries + * @param DatabaseResource|null $databaseResource * @return int * @throws DatabaseException */ - private function countResources(string $table, array $queries = []): int + private function countResources(string $table, array $queries = [], ?DatabaseResource $databaseResource = null): int { + // Use the appropriate database instance for row data + if ($databaseResource !== null) { + $dbInstance = $this->getDatabase($databaseResource->getDatabase()); + return $dbInstance->count($table, $queries); + } + + // Use dbForProject for metadata tables return $this->dbForProject->count($table, $queries); } } diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 2a447eb..7f4df89 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -36,10 +36,13 @@ class Transfer public const GROUP_DATABASES_RESOURCES = [ Resource::TYPE_DATABASE, + Resource::TYPE_DOCUMENTSDB_DATABASE, Resource::TYPE_TABLE, Resource::TYPE_INDEX, Resource::TYPE_COLUMN, Resource::TYPE_ROW, + Resource::TYPE_DOCUMENT, + Resource::TYPE_COLLECTION, ]; public const GROUP_SETTINGS_RESOURCES = []; From e8faed3943b0f1dee8ecfc679a5dc85f9af727d5 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 14 Oct 2025 13:22:31 +0530 Subject: [PATCH 08/28] replaced getDatabaseDB to getDatabasesDB --- src/Migration/Destinations/Appwrite.php | 16 ++++++++-------- src/Migration/Sources/Appwrite.php | 16 ++++++++-------- .../Sources/Appwrite/Reader/Database.php | 10 +++++----- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index c463800..2e87da3 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -60,7 +60,7 @@ class Appwrite extends Destination /** * @var callable(UtopiaDocument $database): UtopiaDatabase */ - protected mixed $getDatabaseDB; + protected mixed $getDatabasesDB; /** * @var array @@ -72,7 +72,7 @@ class Appwrite extends Destination * @param string $endpoint * @param string $key * @param UtopiaDatabase $dbForProject - * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabaseDB + * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB * @param array> $collectionStructure */ public function __construct( @@ -80,7 +80,7 @@ public function __construct( string $endpoint, string $key, protected UtopiaDatabase $dbForProject, - callable $getDatabaseDB, + callable $getDatabasesDB, protected array $collectionStructure ) { $this->project = $project; @@ -97,7 +97,7 @@ public function __construct( $this->teams = new Teams($this->client); $this->users = new Users($this->client); - $this->getDatabaseDB = $getDatabaseDB; + $this->getDatabasesDB = $getDatabasesDB; } public static function getName(): string @@ -410,7 +410,7 @@ protected function createEntity(Table $resource): bool ); } - $dbForDatabase = call_user_func($this->getDatabaseDB, $database); + $dbForDatabase = call_user_func($this->getDatabasesDB, $database); $table = $this->dbForProject->createDocument('database_' . $database->getSequence(), new UtopiaDocument([ '$id' => $resource->getId(), @@ -538,7 +538,7 @@ protected function createField(Column $resource): bool ); } } - $dbForDatabase = call_user_func($this->getDatabaseDB, $database); + $dbForDatabase = call_user_func($this->getDatabasesDB, $database); try { $column = new UtopiaDocument([ '$id' => ID::custom($database->getSequence() . '_' . $table->getSequence() . '_' . $resource->getKey()), @@ -739,7 +739,7 @@ protected function createIndex(Index $resource): bool message: 'Table not found', ); } - $dbForDatabase = call_user_func($this->getDatabaseDB, $database); + $dbForDatabase = call_user_func($this->getDatabasesDB, $database); $count = $this->dbForProject->count('indexes', [ Query::equal('collectionInternalId', [$table->getSequence()]), @@ -974,7 +974,7 @@ protected function createRecord(Row $resource, bool $isLast): bool $databaseInternalId = $database->getSequence(); $tableInternalId = $table->getSequence(); - $dbForDatabase = call_user_func($this->getDatabaseDB, $database); + $dbForDatabase = call_user_func($this->getDatabasesDB, $database); /** * This is in case an attribute was deleted from Appwrite attributes collection but was not deleted from the table * When creating an archive we select * which will include orphan attribute from the schema diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 00a53be..83baf27 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -69,7 +69,7 @@ class Appwrite extends Source /** * @var callable(UtopiaDocument $database|null): UtopiaDatabase */ - protected mixed $getDatabaseDB; + protected mixed $getDatabasesDB; /** * @throws \Exception @@ -80,7 +80,7 @@ public function __construct( protected string $key, protected string $source = self::SOURCE_API, protected ?UtopiaDatabase $dbForProject = null, - callable $getDatabaseDB + callable $getDatabasesDB ) { $this->client = (new Client()) ->setEndpoint($endpoint) @@ -100,12 +100,12 @@ public function __construct( if (\is_null($dbForProject)) { throw new \Exception('Database is required for database source'); } - if (\is_null($getDatabaseDB)) { - throw new \Exception('getDatabaseDB callable is required for database source'); + if (\is_null($getDatabasesDB)) { + throw new \Exception('getDatabasesDB callable is required for database source'); } } - $this->getDatabaseDB = $getDatabaseDB; + $this->getDatabasesDB = $getDatabasesDB; } public static function getName(): string @@ -123,7 +123,7 @@ private function createReader(): Reader { return match ($this->source) { static::SOURCE_API => new APIReader(new Databases($this->client)), - static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabaseDB), + static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabasesDB), default => throw new \Exception('Unknown source'), }; } @@ -1086,12 +1086,12 @@ private function exportRecords(string $entityName, int $batchSize): void /** @var Table $table */ $lastRow = null; - // Create reader - use getDatabaseDB for row data operations + // Create reader - use getDatabasesDB for row data operations $reader = match ($this->source) { static::SOURCE_API => new APIReader(new Databases($this->client)), static::SOURCE_DATABASE => new DatabaseReader( $this->dbForProject, - $this->getDatabaseDB + $this->getDatabasesDB ), default => throw new \Exception('Unknown source'), }; diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index c1fa355..beb809d 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -25,13 +25,13 @@ class Database implements Reader /** * @var callable(UtopiaDocument|null): UtopiaDatabase */ - private mixed $getDatabaseDB; + private mixed $getDatabasesDB; public function __construct( private readonly UtopiaDatabase $dbForProject, - ?callable $getDatabaseDB = null + ?callable $getDatabasesDB = null ) { - $this->getDatabaseDB = $getDatabaseDB; + $this->getDatabasesDB = $getDatabasesDB; } /** @@ -39,8 +39,8 @@ public function __construct( */ private function getDatabase(?string $databaseDSN = null): UtopiaDatabase { - if ($this->getDatabaseDB !== null && $databaseDSN !== null) { - return call_user_func($this->getDatabaseDB, new UtopiaDocument(['database' => $databaseDSN])); + if ($this->getDatabasesDB !== null && $databaseDSN !== null) { + return call_user_func($this->getDatabasesDB, new UtopiaDocument(['database' => $databaseDSN])); } return $this->dbForProject; From b77e8c2ced7f6dc412dc2669b43a719fba19fcd6 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 14 Oct 2025 20:32:00 +0530 Subject: [PATCH 09/28] * linting --- src/Migration/Destinations/Appwrite.php | 5 +-- src/Migration/Resource.php | 8 ++-- .../Resources/Database/Collection.php | 32 +++++++++++++++- src/Migration/Resources/Database/Database.php | 2 +- src/Migration/Resources/Database/Document.php | 38 +++++++++++++++++++ .../Resources/Database/DocumentsDB.php | 25 ++++++++++++ src/Migration/Resources/Database/Row.php | 2 +- src/Migration/Resources/Database/Table.php | 2 +- src/Migration/Sources/Appwrite.php | 6 +-- .../Sources/Appwrite/Reader/Database.php | 8 ++-- 10 files changed, 107 insertions(+), 21 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 2e87da3..8c4f689 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -443,9 +443,8 @@ protected function createEntity(Table $resource): bool */ protected function createField(Column $resource): bool { - // Skip columns for documents DB (schemaless) - if ($resource->getTable()->getDatabase()->getType() === 'documents') { - $resource->setStatus(Resource::STATUS_SKIPPED, 'Columns not supported for documents database'); + if ($resource->getTable()->getDatabase()->getDatabase() === Resource::TYPE_DOCUMENTSDB_DATABASE) { + $resource->setStatus(Resource::STATUS_SKIPPED, 'Columns not supported for DocumentsDB'); return false; } diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index b430696..3de172d 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -98,13 +98,13 @@ abstract class Resource implements \JsonSerializable public const DATABASE_TYPE_RESOURCE_MAP = [ self::TYPE_DATABASE => [ 'entity' => self::TYPE_TABLE, - 'fields' => self::TYPE_COLUMN, - 'records' => self::TYPE_ROW, + 'field' => self::TYPE_COLUMN, + 'record' => self::TYPE_ROW, ], self::TYPE_DOCUMENTSDB_DATABASE => [ 'entity' => self::TYPE_COLLECTION, - 'records' => self::TYPE_DOCUMENT, - 'fields' => self::TYPE_ATTRIBUTE, + 'record' => self::TYPE_DOCUMENT, + 'field' => self::TYPE_ATTRIBUTE, ], ]; diff --git a/src/Migration/Resources/Database/Collection.php b/src/Migration/Resources/Database/Collection.php index c038e93..b2c53ec 100644 --- a/src/Migration/Resources/Database/Collection.php +++ b/src/Migration/Resources/Database/Collection.php @@ -2,14 +2,42 @@ namespace Utopia\Migration\Resources\Database; -use Override; use Utopia\Migration\Resource; class Collection extends Table { - #[Override] public static function getName(): string { return Resource::TYPE_COLLECTION; } + + /** + * @param array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity?: bool, + * rowSecurity?: bool, + * permissions: ?array, + * createdAt: string, + * updatedAt: string, + * enabled: bool + * } $array + */ + public static function fromArray(array $array): self + { + return new self( + DocumentsDB::fromArray($array['database']), + name: $array['name'], + id: $array['id'], + rowSecurity: $array['rowSecurity'] ?? $array['documentSecurity'], + permissions: $array['permissions'] ?? [], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + enabled: $array['enabled'] ?? true, + ); + } } diff --git a/src/Migration/Resources/Database/Database.php b/src/Migration/Resources/Database/Database.php index 50d7af6..9c90498 100644 --- a/src/Migration/Resources/Database/Database.php +++ b/src/Migration/Resources/Database/Database.php @@ -44,7 +44,7 @@ public function __construct( */ public static function fromArray(array $array): self { - return new static( + return new self( $array['id'], $array['name'], createdAt: $array['createdAt'] ?? '', diff --git a/src/Migration/Resources/Database/Document.php b/src/Migration/Resources/Database/Document.php index 41b8a6c..07e9a5d 100644 --- a/src/Migration/Resources/Database/Document.php +++ b/src/Migration/Resources/Database/Document.php @@ -10,4 +10,42 @@ public static function getName(): string { return Resource::TYPE_DOCUMENT; } + + /** + * @param array{ + * id: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * data: array, + * permissions: ?array + * } $array + */ + public static function fromArray(array $array): self + { + // keeping table and collection to have backward compat + return new self( + $array['id'], + Collection::fromArray($array['table'] ?? $array['collection']), + $array['data'], + $array['permissions'] ?? [] + ); + } } diff --git a/src/Migration/Resources/Database/DocumentsDB.php b/src/Migration/Resources/Database/DocumentsDB.php index 7963476..46e5e86 100644 --- a/src/Migration/Resources/Database/DocumentsDB.php +++ b/src/Migration/Resources/Database/DocumentsDB.php @@ -10,4 +10,29 @@ public static function getName(): string { return Resource::TYPE_DOCUMENTSDB_DATABASE; } + + /** + * @param array{ + * id: string, + * name: string, + * createdAt: string, + * updatedAt: string, + * enabled: bool, + * originalId: string|null, + * database: string + * } $array + */ + public static function fromArray(array $array): self + { + return new self( + $array['id'], + $array['name'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + enabled: $array['enabled'] ?? true, + originalId: $array['originalId'] ?? '', + type: $array['type'] ?? 'legacy', + database: $array['database'] ?? 'legacy' + ); + } } diff --git a/src/Migration/Resources/Database/Row.php b/src/Migration/Resources/Database/Row.php index 87dfcad..42d5bfc 100644 --- a/src/Migration/Resources/Database/Row.php +++ b/src/Migration/Resources/Database/Row.php @@ -52,7 +52,7 @@ public function __construct( */ public static function fromArray(array $array): self { - return new static( + return new self( $array['id'], Table::fromArray($array['table'] ?? $array['collection']), $array['data'], diff --git a/src/Migration/Resources/Database/Table.php b/src/Migration/Resources/Database/Table.php index 6c76f8c..889d90f 100644 --- a/src/Migration/Resources/Database/Table.php +++ b/src/Migration/Resources/Database/Table.php @@ -49,7 +49,7 @@ public function __construct( */ public static function fromArray(array $array): self { - return new static( + return new self( Database::fromArray($array['database']), name: $array['name'], id: $array['id'], diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 83baf27..bcdd3b8 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -12,7 +12,6 @@ use Appwrite\Services\Users; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\DateTime as UtopiaDateTime; -use Utopia\Database\Document as UtopiaDocument; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Auth\Hash; @@ -78,9 +77,9 @@ public function __construct( protected string $project, protected string $endpoint, protected string $key, + callable $getDatabasesDB, protected string $source = self::SOURCE_API, protected ?UtopiaDatabase $dbForProject = null, - callable $getDatabasesDB ) { $this->client = (new Client()) ->setEndpoint($endpoint) @@ -114,7 +113,6 @@ public static function getName(): string } /** - * Create a reader instance for general operations (not database-specific) * * @return Reader * @throws \Exception @@ -734,7 +732,6 @@ private function exportDatabases(int $batchSize, array $resources = []): void /** * @param string $databaseName * @param int $batchSize - * @param array $databases * @throws Exception */ private function exportEntities(string $databaseName, int $batchSize): void @@ -1694,7 +1691,6 @@ private function exportTablesDB(int $batchSize, array $resources) * * @param int $batchSize * @param array $resources - * @param array $documentsDBDatabases */ private function exportDocumentsDB(int $batchSize, array $resources): void { diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index beb809d..2641a0f 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -126,10 +126,10 @@ public function report(array $resources, array &$report): mixed foreach ($tables as $table) { $tableSequence = $table->getSequence(); - if (Resource::isSupported($databaseSpecificResources['records'], $resources)) { + if (Resource::isSupported($databaseSpecificResources['record'], $resources)) { $rowTableId = "database_{$databaseSequence}_collection_{$tableSequence}"; $count = $this->countResources($rowTableId, [], $dbResource); - $report[$databaseSpecificResources['records']] += $count; + $report[$databaseSpecificResources['record']] += $count; } $commonQueries = [ @@ -137,9 +137,9 @@ public function report(array $resources, array &$report): mixed Query::equal('collectionInternalId', [$tableSequence]), ]; - if (Resource::isSupported($databaseSpecificResources['fields'], $resources)) { + if (Resource::isSupported($databaseSpecificResources['field'], $resources)) { $count = $this->countResources('attributes', $commonQueries); - $report[$databaseSpecificResources['fields']] += $count; + $report[$databaseSpecificResources['field']] += $count; } if (in_array(Resource::TYPE_INDEX, $resources)) { From 0a3abfaad2e4a35280d4e401cd510147d6ddd719 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 14 Oct 2025 21:00:04 +0530 Subject: [PATCH 10/28] linting --- src/Migration/Destinations/Appwrite.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 8c4f689..b90a723 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -443,7 +443,7 @@ protected function createEntity(Table $resource): bool */ protected function createField(Column $resource): bool { - if ($resource->getTable()->getDatabase()->getDatabase() === Resource::TYPE_DOCUMENTSDB_DATABASE) { + if ($resource->getTable()->getDatabase()->getName() === Resource::TYPE_DOCUMENTSDB_DATABASE) { $resource->setStatus(Resource::STATUS_SKIPPED, 'Columns not supported for DocumentsDB'); return false; } From de06db2b8ef50f315f2b61132b67d49d57c336ee Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 15 Oct 2025 00:59:55 +0530 Subject: [PATCH 11/28] updated destination to accept a dsn resolver for the datbase for cross database migration --- src/Migration/Destinations/Appwrite.php | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index b90a723..16536f8 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -59,9 +59,14 @@ class Appwrite extends Destination /** * @var callable(UtopiaDocument $database): UtopiaDatabase - */ + */ protected mixed $getDatabasesDB; + /** + * @var callable(string $databaseType):string + */ + protected mixed $getDatabaseDSN; + /** * @var array */ @@ -73,6 +78,7 @@ class Appwrite extends Destination * @param string $key * @param UtopiaDatabase $dbForProject * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB + * @param callable(string $databaseType):string $getDatabaseDSN * @param array> $collectionStructure */ public function __construct( @@ -81,6 +87,7 @@ public function __construct( string $key, protected UtopiaDatabase $dbForProject, callable $getDatabasesDB, + callable $getDatabasesDSN, protected array $collectionStructure ) { $this->project = $project; @@ -98,6 +105,7 @@ public function __construct( $this->users = new Users($this->client); $this->getDatabasesDB = $getDatabasesDB; + $this->getDatabaseDSN = $getDatabasesDSN; } public static function getName(): string @@ -349,7 +357,8 @@ protected function createDatabase(Database $resource): bool '$updatedAt' => $resource->getUpdatedAt(), 'originalId' => empty($resource->getOriginalId()) ? null : $resource->getOriginalId(), 'type' => empty($resource->getType()) ? 'legacy' : $resource->getType(), - 'database' => $resource->getDatabase() + // source and destination can be in different location + 'database' => call_user_func($this->getDatabaseDSN, $resource->getType()) ])); $resource->setSequence($database->getSequence()); From 1d607701d375b5897dbcf1f410b38a75c012c929 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 15 Oct 2025 13:58:49 +0530 Subject: [PATCH 12/28] * comments follow ups --- src/Migration/Destinations/Appwrite.php | 72 +++++++----- src/Migration/Resource.php | 6 +- .../Resources/Database/DocumentsDB.php | 2 +- src/Migration/Sources/Appwrite.php | 108 +++++++----------- .../Sources/Appwrite/Reader/Database.php | 10 +- src/Migration/Transfer.php | 2 +- 6 files changed, 97 insertions(+), 103 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 16536f8..5a724b3 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -60,12 +60,12 @@ class Appwrite extends Destination /** * @var callable(UtopiaDocument $database): UtopiaDatabase */ - protected mixed $getDatabasesDB; + protected $getDatabasesDB; /** * @var callable(string $databaseType):string */ - protected mixed $getDatabaseDSN; + protected $getDatabaseDSN; /** * @var array @@ -78,7 +78,7 @@ class Appwrite extends Destination * @param string $key * @param UtopiaDatabase $dbForProject * @param callable(UtopiaDocument $database):UtopiaDatabase $getDatabasesDB - * @param callable(string $databaseType):string $getDatabaseDSN + * @param callable(string $databaseType):string $getDatabasesDSN * @param array> $collectionStructure */ public function __construct( @@ -126,7 +126,7 @@ public static function getSupportedResources(): array // Database Resource::TYPE_DATABASE, - Resource::TYPE_DOCUMENTSDB_DATABASE, + Resource::TYPE_DATABASE_DOCUMENTSDB, Resource::TYPE_TABLE, Resource::TYPE_COLUMN, Resource::TYPE_INDEX, @@ -291,7 +291,7 @@ public function importDatabaseResource(Resource $resource, bool $isLast): Resour { switch ($resource->getName()) { case Resource::TYPE_DATABASE: - case Resource::TYPE_DOCUMENTSDB_DATABASE: + case Resource::TYPE_DATABASE_DOCUMENTSDB: /** @var Database $resource */ $success = $this->createDatabase($resource); break; @@ -358,7 +358,7 @@ protected function createDatabase(Database $resource): bool 'originalId' => empty($resource->getOriginalId()) ? null : $resource->getOriginalId(), 'type' => empty($resource->getType()) ? 'legacy' : $resource->getType(), // source and destination can be in different location - 'database' => call_user_func($this->getDatabaseDSN, $resource->getType()) + 'database' => ($this->getDatabaseDSN)($resource->getType()) ])); $resource->setSequence($database->getSequence()); @@ -419,7 +419,7 @@ protected function createEntity(Table $resource): bool ); } - $dbForDatabase = call_user_func($this->getDatabasesDB, $database); + $dbForDatabases = ($this->getDatabasesDB)($database); $table = $this->dbForProject->createDocument('database_' . $database->getSequence(), new UtopiaDocument([ '$id' => $resource->getId(), @@ -436,7 +436,7 @@ protected function createEntity(Table $resource): bool $resource->setSequence($table->getSequence()); - $dbForDatabase->createCollection( + $dbForDatabases->createCollection( 'database_' . $database->getSequence() . '_collection_' . $resource->getSequence(), permissions: $resource->getPermissions(), documentSecurity: $resource->getRowSecurity() @@ -452,7 +452,7 @@ protected function createEntity(Table $resource): bool */ protected function createField(Column $resource): bool { - if ($resource->getTable()->getDatabase()->getName() === Resource::TYPE_DOCUMENTSDB_DATABASE) { + if ($resource->getTable()->getDatabase()->getType() === Resource::TYPE_DATABASE_DOCUMENTSDB) { $resource->setStatus(Resource::STATUS_SKIPPED, 'Columns not supported for DocumentsDB'); return false; } @@ -546,7 +546,7 @@ protected function createField(Column $resource): bool ); } } - $dbForDatabase = call_user_func($this->getDatabasesDB, $database); + $dbForDatabases = ($this->getDatabasesDB)($database); try { $column = new UtopiaDocument([ '$id' => ID::custom($database->getSequence() . '_' . $table->getSequence() . '_' . $resource->getKey()), @@ -589,12 +589,12 @@ protected function createField(Column $resource): bool ); } catch (\Throwable $e) { $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); throw $e; } $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); $options = $resource->getOptions(); $twoWayKey = null; @@ -649,7 +649,7 @@ protected function createField(Column $resource): bool ); } catch (\Throwable $e) { $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $relatedTable->getId()); - $dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence()); throw $e; } } @@ -657,7 +657,7 @@ protected function createField(Column $resource): bool try { switch ($type) { case UtopiaDatabase::VAR_RELATIONSHIP: - if (!$dbForDatabase->createRelationship( + if (!$dbForDatabases->createRelationship( collection: 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), relatedCollection: 'database_' . $database->getSequence() . '_collection_' . $relatedTable->getSequence(), type: $options['relationType'], @@ -675,7 +675,7 @@ protected function createField(Column $resource): bool } break; default: - if (!$dbForDatabase->createAttribute( + if (!$dbForDatabases->createAttribute( 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), $resource->getKey(), $type, @@ -711,7 +711,7 @@ protected function createField(Column $resource): bool } $this->dbForProject->purgeCachedDocument('database_' . $database->getSequence(), $table->getId()); - $dbForDatabase->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); + $dbForDatabases->purgeCachedCollection('database_' . $database->getSequence() . '_collection_' . $table->getSequence()); return true; } @@ -747,14 +747,14 @@ protected function createIndex(Index $resource): bool message: 'Table not found', ); } - $dbForDatabase = call_user_func($this->getDatabasesDB, $database); + $dbForDatabases = ($this->getDatabasesDB)($database); $count = $this->dbForProject->count('indexes', [ Query::equal('collectionInternalId', [$table->getSequence()]), Query::equal('databaseInternalId', [$database->getSequence()]) - ], $dbForDatabase->getLimitForIndexes()); + ], $dbForDatabases->getLimitForIndexes()); - if ($count >= $dbForDatabase->getLimitForIndexes()) { + if ($count >= $dbForDatabases->getLimitForIndexes()) { throw new Exception( resourceName: $resource->getName(), resourceGroup: $resource->getGroup(), @@ -871,13 +871,31 @@ protected function createIndex(Index $resource): bool '$updatedAt' => $resource->getUpdatedAt(), ]); + $maxIndexLength = $dbForDatabases->getAdapter()->getMaxIndexLength(); + $internalIndexesKeys = $dbForDatabases->getAdapter()->getInternalIndexesKeys(); + $supportForIndexArray = $dbForDatabases->getAdapter()->getSupportForIndexArray(); + $supportForSpatialAttributes = $dbForDatabases->getAdapter()->getSupportForSpatialAttributes(); + $supportForSpatialIndexNull = $dbForDatabases->getAdapter()->getSupportForSpatialIndexNull(); + $supportForSpatialIndexOrder = $dbForDatabases->getAdapter()->getSupportForSpatialIndexOrder(); + $supportForAttributes = $dbForDatabases->getAdapter()->getSupportForAttributes(); + $supportForMultipleFulltextIndexes = $dbForDatabases->getAdapter()->getSupportForMultipleFulltextIndexes(); + $supportForIdenticalIndexes = $dbForDatabases->getAdapter()->getSupportForIdenticalIndexes(); + $validator = new IndexValidator( - $tableColumns, - $dbForDatabase->getAdapter()->getMaxIndexLength(), - $dbForDatabase->getAdapter()->getInternalIndexesKeys(), - $dbForDatabase->getAdapter()->getSupportForIndexArray(), + $table->getAttribute('attributes'), + $table->getAttribute('indexes', []), + $maxIndexLength, + $internalIndexesKeys, + $supportForIndexArray, + $supportForSpatialAttributes, + $supportForSpatialIndexNull, + $supportForSpatialIndexOrder, + $supportForAttributes, + $supportForMultipleFulltextIndexes, + $supportForIdenticalIndexes ); + if (!$validator->isValid($index)) { throw new Exception( resourceName: $resource->getName(), @@ -890,7 +908,7 @@ protected function createIndex(Index $resource): bool $index = $this->dbForProject->createDocument('indexes', $index); try { - $result = $dbForDatabase->createIndex( + $result = $dbForDatabases->createIndex( 'database_' . $database->getSequence() . '_collection_' . $table->getSequence(), $resource->getKey(), $resource->getType(), @@ -982,12 +1000,12 @@ protected function createRecord(Row $resource, bool $isLast): bool $databaseInternalId = $database->getSequence(); $tableInternalId = $table->getSequence(); - $dbForDatabase = call_user_func($this->getDatabasesDB, $database); + $dbForDatabases = ($this->getDatabasesDB)($database); /** * This is in case an attribute was deleted from Appwrite attributes collection but was not deleted from the table * When creating an archive we select * which will include orphan attribute from the schema */ - if ($dbForDatabase->getAdapter()->getSupportForAttributes()) { + if ($dbForDatabases->getAdapter()->getSupportForAttributes()) { foreach ($this->rowBuffer as $row) { foreach ($row as $key => $value) { if (\str_starts_with($key, '$')) { @@ -1009,7 +1027,7 @@ protected function createRecord(Row $resource, bool $isLast): bool } } } - $dbForDatabase->skipRelationshipsExistCheck(fn () => $dbForDatabase->createDocuments( + $dbForDatabases->skipRelationshipsExistCheck(fn () => $dbForDatabases->createDocuments( 'database_' . $databaseInternalId . '_collection_' . $tableInternalId, $this->rowBuffer )); diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index 3de172d..2b4e322 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -34,7 +34,7 @@ abstract class Resource implements \JsonSerializable public const TYPE_DATABASE_TABLESDB = 'tablesdb'; - public const TYPE_DOCUMENTSDB_DATABASE = 'documentsdb'; + public const TYPE_DATABASE_DOCUMENTSDB = 'documentsdb'; public const TYPE_ROW = 'row'; @@ -76,7 +76,7 @@ abstract class Resource implements \JsonSerializable self::TYPE_BUCKET, self::TYPE_TABLE, self::TYPE_DATABASE, - self::TYPE_DOCUMENTSDB_DATABASE, + self::TYPE_DATABASE_DOCUMENTSDB, self::TYPE_ROW, self::TYPE_FILE, self::TYPE_FUNCTION, @@ -101,7 +101,7 @@ abstract class Resource implements \JsonSerializable 'field' => self::TYPE_COLUMN, 'record' => self::TYPE_ROW, ], - self::TYPE_DOCUMENTSDB_DATABASE => [ + self::TYPE_DATABASE_DOCUMENTSDB => [ 'entity' => self::TYPE_COLLECTION, 'record' => self::TYPE_DOCUMENT, 'field' => self::TYPE_ATTRIBUTE, diff --git a/src/Migration/Resources/Database/DocumentsDB.php b/src/Migration/Resources/Database/DocumentsDB.php index 46e5e86..5409138 100644 --- a/src/Migration/Resources/Database/DocumentsDB.php +++ b/src/Migration/Resources/Database/DocumentsDB.php @@ -8,7 +8,7 @@ class DocumentsDB extends Database { public static function getName(): string { - return Resource::TYPE_DOCUMENTSDB_DATABASE; + return Resource::TYPE_DATABASE_DOCUMENTSDB; } /** diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index bcdd3b8..c48363c 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -57,6 +57,8 @@ class Appwrite extends Source protected Client $client; + private Reader $reader; + private Users $users; private Teams $teams; @@ -104,6 +106,12 @@ public function __construct( } } + $this->reader = match ($this->source) { + static::SOURCE_API => new APIReader(new Databases($this->client)), + static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabasesDB), + default => throw new \Exception('Unknown source'), + }; + $this->getDatabasesDB = $getDatabasesDB; } @@ -112,20 +120,6 @@ public static function getName(): string return 'Appwrite'; } - /** - * - * @return Reader - * @throws \Exception - */ - private function createReader(): Reader - { - return match ($this->source) { - static::SOURCE_API => new APIReader(new Databases($this->client)), - static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabasesDB), - default => throw new \Exception('Unknown source'), - }; - } - /** * @return array */ @@ -150,7 +144,7 @@ public static function getSupportedResources(): array Resource::TYPE_COLLECTION, // documentsdb - Resource::TYPE_DOCUMENTSDB_DATABASE, + Resource::TYPE_DATABASE_DOCUMENTSDB, // Storage Resource::TYPE_BUCKET, @@ -291,8 +285,7 @@ private function reportAuth(array $resources, array &$report): void */ private function reportDatabases(array $resources, array &$report): void { - $reader = $this->createReader(); - $reader->report($resources, $report); + $this->reader->report($resources, $report); } /** @@ -631,7 +624,7 @@ private function exportMemberships(int $batchSize): void protected function exportGroupDatabases(int $batchSize, array $resources): void { try { - if (\in_array(Resource::TYPE_DATABASE, $resources) || \in_array(Resource::TYPE_DOCUMENTSDB_DATABASE, $resources)) { + if (Resource::isSupported(array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP), $resources)) { $this->exportDatabases($batchSize, $resources); } } catch (\Throwable $e) { @@ -667,12 +660,11 @@ private function exportDatabases(int $batchSize, array $resources = []): void $lastDatabase = null; // Create a reader for listing databases (not database-specific) - $reader = $this->createReader(); while (true) { - $queries = [$reader->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; - if ($this->rootResourceId !== '' && ($this->rootResourceType === Resource::TYPE_DATABASE || $this->rootResourceType === Resource::TYPE_DOCUMENTSDB_DATABASE)) { + if ($this->rootResourceId !== '' && ($this->rootResourceType === Resource::TYPE_DATABASE || $this->rootResourceType === Resource::TYPE_DATABASE_DOCUMENTSDB)) { $targetDatabaseId = $this->rootResourceId; // Handle database:collection format - extract database ID @@ -683,24 +675,23 @@ private function exportDatabases(int $batchSize, array $resources = []): void } } - $queries[] = $reader->queryEqual('$id', [$targetDatabaseId]); - $queries[] = $reader->queryLimit(1); + $queries[] = $this->reader->queryEqual('$id', [$targetDatabaseId]); + $queries[] = $this->reader->queryLimit(1); } $databases = []; if ($lastDatabase) { - $queries[] = $reader->queryCursorAfter($lastDatabase); + $queries[] = $this->reader->queryCursorAfter($lastDatabase); } - $response = $reader->listDatabases($queries); + $response = $this->reader->listDatabases($queries); foreach ($response as $database) { $databaseType = $database['type']; - if ($databaseType === Resource::TYPE_DATABASE_LEGACY || $databaseType === Resource::TYPE_DATABASE_TABLESDB) { + if (in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY,Resource::TYPE_DATABASE_TABLESDB])) { $databaseType = Resource::TYPE_DATABASE; } - if (Resource::isSupported($databaseType, $resources)) { $newDatabase = self::getDatabase($databaseType, [ 'id' => $database['$id'], @@ -741,11 +732,8 @@ private function exportEntities(string $databaseName, int $batchSize): void /** @var Database $database */ $lastTable = null; - // collections in the metadata - $reader = $this->createReader(); - while (true) { - $queries = [$reader->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; $tables = []; // Filter to specific table if rootResourceType is database with database:collection format @@ -757,21 +745,21 @@ private function exportEntities(string $databaseName, int $batchSize): void $parts = \explode(':', $this->rootResourceId, 2); if (\count($parts) === 2) { $targetTableId = $parts[1]; // table ID - $queries[] = $reader->queryEqual('$id', [$targetTableId]); - $queries[] = $reader->queryLimit(1); + $queries[] = $this->reader->queryEqual('$id', [$targetTableId]); + $queries[] = $this->reader->queryLimit(1); } } elseif ( $this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_TABLE ) { $targetTableId = $this->rootResourceId; - $queries[] = $reader->queryEqual('$id', [$targetTableId]); - $queries[] = $reader->queryLimit(1); + $queries[] = $this->reader->queryEqual('$id', [$targetTableId]); + $queries[] = $this->reader->queryLimit(1); } elseif ($lastTable) { - $queries[] = $reader->queryCursorAfter($lastTable); + $queries[] = $this->reader->queryCursorAfter($lastTable); } - $response = $reader->listTables($database, $queries); + $response = $this->reader->listTables($database, $queries); foreach ($response as $table) { $newTable = self::getEntity($databaseName, [ 'id' => $table['$id'], @@ -818,17 +806,15 @@ private function exportFields(int $batchSize): void foreach ($tables as $table) { $lastColumn = null; - $reader = $this->createReader(); - while (true) { - $queries = [$reader->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; $columns = []; if ($lastColumn) { - $queries[] = $reader->queryCursorAfter($lastColumn); + $queries[] = $this->reader->queryCursorAfter($lastColumn); } - $response = $reader->listColumns($table, $queries); + $response = $this->reader->listColumns($table, $queries); foreach ($response as $column) { if ( @@ -1029,18 +1015,15 @@ private function exportIndexes(string $entityType, int $batchSize): void /** @var Table $table */ $lastIndex = null; - // Create reader for this specific database - $reader = $this->createReader(); - while (true) { - $queries = [$reader->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; $indexes = []; if ($lastIndex) { - $queries[] = $reader->queryCursorAfter($lastIndex); + $queries[] = $this->reader->queryCursorAfter($lastIndex); } - $response = $reader->listIndexes($table, $queries); + $response = $this->reader->listIndexes($table, $queries); foreach ($response as $index) { $indexes[] = new Index( @@ -1084,7 +1067,7 @@ private function exportRecords(string $entityName, int $batchSize): void $lastRow = null; // Create reader - use getDatabasesDB for row data operations - $reader = match ($this->source) { + $this->reader = match ($this->source) { static::SOURCE_API => new APIReader(new Databases($this->client)), static::SOURCE_DATABASE => new DatabaseReader( $this->dbForProject, @@ -1094,18 +1077,18 @@ private function exportRecords(string $entityName, int $batchSize): void }; while (true) { - $queries = [$reader->queryLimit($batchSize)]; + $queries = [$this->reader->queryLimit($batchSize)]; $rows = []; if ($lastRow) { - $queries[] = $reader->queryCursorAfter($lastRow); + $queries[] = $this->reader->queryCursorAfter($lastRow); } $selects = ['*', '$id', '$permissions', '$updatedAt', '$createdAt']; // We want relations flat! $manyToMany = []; - if ($reader->getSupportForAttributes()) { + if ($this->reader->getSupportForAttributes()) { $attributes = $this->cache->get(Column::getName()); foreach ($attributes as $attribute) { /** @var Relationship $attribute */ @@ -1115,22 +1098,15 @@ private function exportRecords(string $entityName, int $batchSize): void $attribute->getSide() === 'parent' && $attribute->getRelationType() == 'manyToMany' ) { - /** - * Blockers: - * we should use but Does not work properly: - * $selects[] = $attribute->getKey() . '.$id'; - * when selecting for a relation we get all relations not just the one we were asking. - * when selecting for a relation like select(*, relation.$id) , all relations get resolve - */ $manyToMany[] = $attribute->getKey(); } } /** @var Column|Relationship $attribute */ } - $queries[] = $reader->querySelect($selects); + $queries[] = $this->reader->querySelect($selects); - $response = $reader->listRows($table, $queries); + $response = $this->reader->listRows($table, $queries); foreach ($response as $row) { // HACK: Handle many to many (only for schema-based databases) @@ -1140,10 +1116,10 @@ private function exportRecords(string $entityName, int $batchSize): void $stack[] = $relation . '.$id'; } - $rowItem = $reader->getRow( + $rowItem = $this->reader->getRow( $table, $row['$id'], - [$reader->querySelect($stack)] + [$this->reader->querySelect($stack)] ); foreach ($manyToMany as $key) { @@ -1764,7 +1740,7 @@ private function exportDocumentsDB(int $batchSize, array $resources): void private static function getDatabase(string $databaseType, array $database): Resource { switch ($databaseType) { - case Resource::TYPE_DOCUMENTSDB_DATABASE: + case Resource::TYPE_DATABASE_DOCUMENTSDB: return DocumentsDB::fromArray($database); default: return Database::fromArray($database); @@ -1792,7 +1768,7 @@ private static function getDatabase(string $databaseType, array $database): Reso private static function getEntity(string $databaseType, array $entity): Resource { switch ($databaseType) { - case Resource::TYPE_DOCUMENTSDB_DATABASE: + case Resource::TYPE_DATABASE_DOCUMENTSDB: return Collection::fromArray($entity); default: return Table::fromArray($entity); @@ -1831,7 +1807,7 @@ private static function getEntity(string $databaseType, array $entity): Resource private static function getRecord(string $databaseType, array $record): Resource { switch ($databaseType) { - case Resource::TYPE_DOCUMENTSDB_DATABASE: + case Resource::TYPE_DATABASE_DOCUMENTSDB: return Document::fromArray($record); default: return Row::fromArray($record); diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index 2641a0f..d437727 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -40,7 +40,7 @@ public function __construct( private function getDatabase(?string $databaseDSN = null): UtopiaDatabase { if ($this->getDatabasesDB !== null && $databaseDSN !== null) { - return call_user_func($this->getDatabasesDB, new UtopiaDocument(['database' => $databaseDSN])); + return ($this->getDatabasesDB)(new UtopiaDocument(['database' => $databaseDSN])); } return $this->dbForProject; @@ -49,14 +49,14 @@ private function getDatabase(?string $databaseDSN = null): UtopiaDatabase public function report(array $resources, array &$report): mixed { $relevantResources = [ + // tablesdb Resource::TYPE_DATABASE, Resource::TYPE_TABLE, Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX, // Documentsdb - Resource::TYPE_DOCUMENTSDB_DATABASE, - // Legacy types + Resource::TYPE_DATABASE_DOCUMENTSDB, Resource::TYPE_COLLECTION, Resource::TYPE_DOCUMENT, Resource::TYPE_ATTRIBUTE, @@ -75,7 +75,7 @@ public function report(array $resources, array &$report): mixed foreach ($databases as $database) { $databaseType = $database->getAttribute('type'); - if ($databaseType === Resource::TYPE_DATABASE_LEGACY || $databaseType === Resource::TYPE_DATABASE_TABLESDB) { + if (in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY,Resource::TYPE_DATABASE_TABLESDB])) { $databaseType = Resource::TYPE_DATABASE; } if (Resource::isSupported($databaseType, $resources)) { @@ -93,7 +93,7 @@ public function report(array $resources, array &$report): mixed $dbResources = []; foreach ($databases as $database) { $databaseType = $database->getAttribute('type'); - if ($databaseType === Resource::TYPE_DATABASE_LEGACY || $databaseType === Resource::TYPE_DATABASE_TABLESDB) { + if (in_array($databaseType, [Resource::TYPE_DATABASE_LEGACY,Resource::TYPE_DATABASE_TABLESDB])) { $databaseType = Resource::TYPE_DATABASE; } diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 7f4df89..a50ec2d 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -36,7 +36,7 @@ class Transfer public const GROUP_DATABASES_RESOURCES = [ Resource::TYPE_DATABASE, - Resource::TYPE_DOCUMENTSDB_DATABASE, + Resource::TYPE_DATABASE_DOCUMENTSDB, Resource::TYPE_TABLE, Resource::TYPE_INDEX, Resource::TYPE_COLUMN, From 7b4a004ccb96c7b1e741afc34292ebc084b72263 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 15 Oct 2025 14:15:17 +0530 Subject: [PATCH 13/28] updated constructor order in the source appwrite --- src/Migration/Sources/Appwrite.php | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index c48363c..584c55a 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -96,15 +96,7 @@ public function __construct( $this->headers['X-Appwrite-Project'] = $this->project; $this->headers['X-Appwrite-Key'] = $this->key; - // Validate required parameters based on source type - if ($this->source === static::SOURCE_DATABASE) { - if (\is_null($dbForProject)) { - throw new \Exception('Database is required for database source'); - } - if (\is_null($getDatabasesDB)) { - throw new \Exception('getDatabasesDB callable is required for database source'); - } - } + $this->getDatabasesDB = $getDatabasesDB; $this->reader = match ($this->source) { static::SOURCE_API => new APIReader(new Databases($this->client)), @@ -112,7 +104,6 @@ public function __construct( default => throw new \Exception('Unknown source'), }; - $this->getDatabasesDB = $getDatabasesDB; } public static function getName(): string From 2a9e4b412fea8f943d2fcddd9c410cc43e6f594e Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 16 Oct 2025 14:06:57 +0530 Subject: [PATCH 14/28] updated db composer --- composer.json | 2 +- composer.lock | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/composer.json b/composer.json index 3c34bc6..091207a 100644 --- a/composer.json +++ b/composer.json @@ -26,7 +26,7 @@ "ext-curl": "*", "ext-openssl": "*", "appwrite/appwrite": "15.*", - "utopia-php/database": "dev-feat-documentsdb as 1.4.5", + "utopia-php/database": "dev-test-docsdb as 1.4.5", "utopia-php/storage": "0.18.*", "utopia-php/dsn": "0.2.*", "utopia-php/framework": "0.33.*" diff --git a/composer.lock b/composer.lock index 8cb4331..d893a1d 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ee13b1e778c4fc84737951d6b68b6317", + "content-hash": "ee2498f1ac5d16e0ab89ac1de91eb7eb", "packages": [ { "name": "appwrite/appwrite", @@ -187,16 +187,16 @@ }, { "name": "google/protobuf", - "version": "v4.32.1", + "version": "v4.33.0", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "c4ed1c1f9bbc1e91766e2cd6c0af749324fe87cb" + "reference": "b50269e23204e5ae859a326ec3d90f09efe3047d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/c4ed1c1f9bbc1e91766e2cd6c0af749324fe87cb", - "reference": "c4ed1c1f9bbc1e91766e2cd6c0af749324fe87cb", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/b50269e23204e5ae859a326ec3d90f09efe3047d", + "reference": "b50269e23204e5ae859a326ec3d90f09efe3047d", "shasum": "" }, "require": { @@ -225,9 +225,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.32.1" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.0" }, - "time": "2025-09-14T05:14:52+00:00" + "time": "2025-10-15T20:10:28+00:00" }, { "name": "mongodb/mongodb", @@ -2161,16 +2161,16 @@ }, { "name": "utopia-php/database", - "version": "dev-feat-documentsdb", + "version": "dev-test-docsdb", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "a9e4f07fa2a2f36b4ed69adb7dd4e590acc05d60" + "reference": "b4e3d5501f7de6c849677d2cd1c923ea16562781" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/a9e4f07fa2a2f36b4ed69adb7dd4e590acc05d60", - "reference": "a9e4f07fa2a2f36b4ed69adb7dd4e590acc05d60", + "url": "https://api.github.com/repos/utopia-php/database/zipball/b4e3d5501f7de6c849677d2cd1c923ea16562781", + "reference": "b4e3d5501f7de6c849677d2cd1c923ea16562781", "shasum": "" }, "require": { @@ -2213,9 +2213,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/feat-documentsdb" + "source": "https://github.com/utopia-php/database/tree/feat-mongodb" }, - "time": "2025-10-07T05:15:47+00:00" + "time": "2025-10-14T08:05:04+00:00" }, { "name": "utopia-php/dsn", @@ -4918,7 +4918,7 @@ "aliases": [ { "package": "utopia-php/database", - "version": "dev-feat-documentsdb", + "version": "dev-test-docsdb", "alias": "1.4.5", "alias_normalized": "1.4.5.0" } From c4a6fe9a575dee566c80c954f7f133275032583d Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 27 Oct 2025 15:49:57 +0530 Subject: [PATCH 15/28] updated db, index validator --- composer.json | 2 +- composer.lock | 401 +++++------------------- src/Migration/Destinations/Appwrite.php | 7 +- 3 files changed, 85 insertions(+), 325 deletions(-) diff --git a/composer.json b/composer.json index 3b974e0..d1c8dc0 100644 --- a/composer.json +++ b/composer.json @@ -26,7 +26,7 @@ "ext-curl": "*", "ext-openssl": "*", "appwrite/appwrite": "15.*", - "utopia-php/database": "3.*", + "utopia-php/database": "dev-var_object as 1.4.5", "utopia-php/storage": "0.18.*", "utopia-php/dsn": "0.2.*", "utopia-php/console": "0.0.*" diff --git a/composer.lock b/composer.lock index a6d7bd3..cfcfd41 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "ee2498f1ac5d16e0ab89ac1de91eb7eb", + "content-hash": "06e20c24c39c070353947cabfb51d3d0", "packages": [ { "name": "appwrite/appwrite", @@ -188,19 +188,15 @@ { "name": "google/protobuf", "version": "v4.33.0", - "version": "v4.33.0", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", "reference": "b50269e23204e5ae859a326ec3d90f09efe3047d" - "reference": "b50269e23204e5ae859a326ec3d90f09efe3047d" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/b50269e23204e5ae859a326ec3d90f09efe3047d", "reference": "b50269e23204e5ae859a326ec3d90f09efe3047d", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/b50269e23204e5ae859a326ec3d90f09efe3047d", - "reference": "b50269e23204e5ae859a326ec3d90f09efe3047d", "shasum": "" }, "require": { @@ -230,88 +226,9 @@ ], "support": { "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.0" - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.0" }, "time": "2025-10-15T20:10:28+00:00" }, - { - "name": "mongodb/mongodb", - "version": "2.1.1", - "source": { - "type": "git", - "url": "https://github.com/mongodb/mongo-php-library.git", - "reference": "f399d24905dd42f97dfe0af9706129743ef247ac" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/mongodb/mongo-php-library/zipball/f399d24905dd42f97dfe0af9706129743ef247ac", - "reference": "f399d24905dd42f97dfe0af9706129743ef247ac", - "shasum": "" - }, - "require": { - "composer-runtime-api": "^2.0", - "ext-mongodb": "^2.1", - "php": "^8.1", - "psr/log": "^1.1.4|^2|^3", - "symfony/polyfill-php85": "^1.32" - }, - "replace": { - "mongodb/builder": "*" - }, - "require-dev": { - "doctrine/coding-standard": "^12.0", - "phpunit/phpunit": "^10.5.35", - "rector/rector": "^1.2", - "squizlabs/php_codesniffer": "^3.7", - "vimeo/psalm": "6.5.*" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-master": "1.x-dev" - } - }, - "autoload": { - "files": [ - "src/functions.php" - ], - "psr-4": { - "MongoDB\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "Apache-2.0" - ], - "authors": [ - { - "name": "Andreas Braun", - "email": "andreas.braun@mongodb.com" - }, - { - "name": "Jeremy Mikola", - "email": "jmikola@gmail.com" - }, - { - "name": "Jérôme Tamarelle", - "email": "jerome.tamarelle@mongodb.com" - } - ], - "description": "MongoDB driver library", - "homepage": "https://jira.mongodb.org/browse/PHPLIB", - "keywords": [ - "database", - "driver", - "mongodb", - "persistence" - ], - "support": { - "issues": "https://github.com/mongodb/mongo-php-library/issues", - "source": "https://github.com/mongodb/mongo-php-library/tree/2.1.1" - }, - "time": "2025-08-13T20:50:05+00:00" - "time": "2025-10-15T20:10:28+00:00" - }, { "name": "mongodb/mongodb", "version": "2.1.1", @@ -536,23 +453,18 @@ { "name": "open-telemetry/api", "version": "1.7.0", - "version": "1.7.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/api.git", "reference": "610b79ad9d6d97e8368bcb6c4d42394fbb87b522" - "reference": "610b79ad9d6d97e8368bcb6c4d42394fbb87b522" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/610b79ad9d6d97e8368bcb6c4d42394fbb87b522", "reference": "610b79ad9d6d97e8368bcb6c4d42394fbb87b522", - "url": "https://api.github.com/repos/opentelemetry-php/api/zipball/610b79ad9d6d97e8368bcb6c4d42394fbb87b522", - "reference": "610b79ad9d6d97e8368bcb6c4d42394fbb87b522", "shasum": "" }, "require": { - "open-telemetry/context": "^1.4", "open-telemetry/context": "^1.4", "php": "^8.1", "psr/log": "^1.1|^2.0|^3.0", @@ -570,7 +482,6 @@ }, "branch-alias": { "dev-main": "1.7.x-dev" - "dev-main": "1.7.x-dev" } }, "autoload": { @@ -608,24 +519,19 @@ "source": "https://github.com/open-telemetry/opentelemetry-php" }, "time": "2025-10-02T23:44:28+00:00" - "time": "2025-10-02T23:44:28+00:00" }, { "name": "open-telemetry/context", "version": "1.4.0", - "version": "1.4.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/context.git", "reference": "d4c4470b541ce72000d18c339cfee633e4c8e0cf" - "reference": "d4c4470b541ce72000d18c339cfee633e4c8e0cf" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/opentelemetry-php/context/zipball/d4c4470b541ce72000d18c339cfee633e4c8e0cf", "reference": "d4c4470b541ce72000d18c339cfee633e4c8e0cf", - "url": "https://api.github.com/repos/opentelemetry-php/context/zipball/d4c4470b541ce72000d18c339cfee633e4c8e0cf", - "reference": "d4c4470b541ce72000d18c339cfee633e4c8e0cf", "shasum": "" }, "require": { @@ -672,7 +578,6 @@ "source": "https://github.com/open-telemetry/opentelemetry-php" }, "time": "2025-09-19T00:05:49+00:00" - "time": "2025-09-19T00:05:49+00:00" }, { "name": "open-telemetry/exporter-otlp", @@ -741,19 +646,15 @@ { "name": "open-telemetry/gen-otlp-protobuf", "version": "1.8.0", - "version": "1.8.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/gen-otlp-protobuf.git", "reference": "673af5b06545b513466081884b47ef15a536edde" - "reference": "673af5b06545b513466081884b47ef15a536edde" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/opentelemetry-php/gen-otlp-protobuf/zipball/673af5b06545b513466081884b47ef15a536edde", "reference": "673af5b06545b513466081884b47ef15a536edde", - "url": "https://api.github.com/repos/opentelemetry-php/gen-otlp-protobuf/zipball/673af5b06545b513466081884b47ef15a536edde", - "reference": "673af5b06545b513466081884b47ef15a536edde", "shasum": "" }, "require": { @@ -804,24 +705,19 @@ "source": "https://github.com/open-telemetry/opentelemetry-php" }, "time": "2025-09-17T23:10:12+00:00" - "time": "2025-09-17T23:10:12+00:00" }, { "name": "open-telemetry/sdk", "version": "1.9.0", - "version": "1.9.0", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/sdk.git", "reference": "8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e" - "reference": "8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e", "reference": "8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e", - "url": "https://api.github.com/repos/opentelemetry-php/sdk/zipball/8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e", - "reference": "8986bcbcbea79cb1ba9e91c1d621541ad63d6b3e", "shasum": "" }, "require": { @@ -829,8 +725,6 @@ "nyholm/psr7-server": "^1.1", "open-telemetry/api": "^1.7", "open-telemetry/context": "^1.4", - "open-telemetry/api": "^1.7", - "open-telemetry/context": "^1.4", "open-telemetry/sem-conv": "^1.0", "php": "^8.1", "php-http/discovery": "^1.14", @@ -865,7 +759,6 @@ }, "branch-alias": { "dev-main": "1.9.x-dev" - "dev-main": "1.9.x-dev" } }, "autoload": { @@ -909,7 +802,6 @@ "source": "https://github.com/open-telemetry/opentelemetry-php" }, "time": "2025-10-02T23:44:28+00:00" - "time": "2025-10-02T23:44:28+00:00" }, { "name": "open-telemetry/sem-conv", @@ -1534,19 +1426,15 @@ { "name": "symfony/http-client", "version": "v7.3.4", - "version": "v7.3.4", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62" - "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/symfony/http-client/zipball/4b62871a01c49457cf2a8e560af7ee8a94b87a62", "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62", - "url": "https://api.github.com/repos/symfony/http-client/zipball/4b62871a01c49457cf2a8e560af7ee8a94b87a62", - "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62", "shasum": "" }, "require": { @@ -1614,7 +1502,6 @@ ], "support": { "source": "https://github.com/symfony/http-client/tree/v7.3.4" - "source": "https://github.com/symfony/http-client/tree/v7.3.4" }, "funding": [ { @@ -1635,7 +1522,6 @@ } ], "time": "2025-09-11T10:12:26+00:00" - "time": "2025-09-11T10:12:26+00:00" }, { "name": "symfony/http-client-contracts", @@ -2040,86 +1926,6 @@ ], "time": "2025-06-23T16:12:55+00:00" }, - { - "name": "symfony/polyfill-php85", - "version": "v1.33.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-php85.git", - "reference": "d4e5fcd4ab3d998ab16c0db48e6cbb9a01993f91" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/d4e5fcd4ab3d998ab16c0db48e6cbb9a01993f91", - "reference": "d4e5fcd4ab3d998ab16c0db48e6cbb9a01993f91", - "shasum": "" - }, - "require": { - "php": ">=7.2" - }, - "type": "library", - "extra": { - "thanks": { - "url": "https://github.com/symfony/polyfill", - "name": "symfony/polyfill" - } - }, - "autoload": { - "files": [ - "bootstrap.php" - ], - "psr-4": { - "Symfony\\Polyfill\\Php85\\": "" - }, - "classmap": [ - "Resources/stubs" - ] - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Nicolas Grekas", - "email": "p@tchwork.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill backporting some PHP 8.5+ features to lower PHP versions", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "polyfill", - "portable", - "shim" - ], - "support": { - "source": "https://github.com/symfony/polyfill-php85/tree/v1.33.0" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://github.com/nicolas-grekas", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2025-06-23T16:12:55+00:00" - }, { "name": "symfony/service-contracts", "version": "v3.6.0", @@ -2307,52 +2113,6 @@ }, "time": "2025-05-09T14:43:52+00:00" }, - { - "name": "utopia-php/compression", - "version": "0.1.3", - "source": { - "type": "git", - "url": "https://github.com/utopia-php/compression.git", - "reference": "66f093557ba66d98245e562036182016c7dcfe8a" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/compression/zipball/66f093557ba66d98245e562036182016c7dcfe8a", - "reference": "66f093557ba66d98245e562036182016c7dcfe8a", - "shasum": "" - }, - "require": { - "php": ">=8.0" - }, - "require-dev": { - "laravel/pint": "1.2.*", - "phpunit/phpunit": "^9.3", - "vimeo/psalm": "4.0.1" - }, - "type": "library", - "autoload": { - "psr-4": { - "Utopia\\Compression\\": "src/Compression" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "description": "A simple Compression library to handle file compression", - "keywords": [ - "compression", - "framework", - "php", - "upf", - "utopia" - ], - "support": { - "issues": "https://github.com/utopia-php/compression/issues", - "source": "https://github.com/utopia-php/compression/tree/0.1.3" - }, - "time": "2025-01-15T15:15:51+00:00" - }, { "name": "utopia-php/console", "version": "0.0.1", @@ -2403,33 +2163,31 @@ }, { "name": "utopia-php/database", - "version": "dev-test-docsdb", + "version": "dev-var_object", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "b4e3d5501f7de6c849677d2cd1c923ea16562781" + "reference": "e2768d9d35e0e31a12e4e63d48f6806651de0225" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/b4e3d5501f7de6c849677d2cd1c923ea16562781", - "reference": "b4e3d5501f7de6c849677d2cd1c923ea16562781", + "url": "https://api.github.com/repos/utopia-php/database/zipball/e2768d9d35e0e31a12e4e63d48f6806651de0225", + "reference": "e2768d9d35e0e31a12e4e63d48f6806651de0225", "shasum": "" }, "require": { "ext-mbstring": "*", "ext-mongodb": "*", - "ext-mongodb": "*", "ext-pdo": "*", "php": ">=8.1", "utopia-php/cache": "0.13.*", - "utopia-php/framework": "0.33.*", - "utopia-php/mongo": "0.10.*", - "utopia-php/pools": "0.8.*" + "utopia-php/mongo": "0.11.*", + "utopia-php/pools": "0.8.*", + "utopia-php/validators": "0.0.*" }, "require-dev": { "fakerphp/faker": "1.23.*", "laravel/pint": "*", - "laravel/pint": "*", "pcov/clobber": "2.*", "phpstan/phpstan": "1.*", "phpunit/phpunit": "9.*", @@ -2457,9 +2215,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/feat-mongodb" + "source": "https://github.com/utopia-php/database/tree/var_object" }, - "time": "2025-10-14T08:05:04+00:00" + "time": "2025-10-27T10:06:02+00:00" }, { "name": "utopia-php/dsn", @@ -2510,68 +2268,65 @@ }, { "name": "utopia-php/framework", - "version": "0.33.28", - "version": "0.33.28", + "version": "0.34.10", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "5aaa94d406577b0059ad28c78022606890dc6de0" - "reference": "5aaa94d406577b0059ad28c78022606890dc6de0" + "reference": "334b99d47b08f016a160b67dcbee0390cc2b8544" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/5aaa94d406577b0059ad28c78022606890dc6de0", - "reference": "5aaa94d406577b0059ad28c78022606890dc6de0", - "url": "https://api.github.com/repos/utopia-php/http/zipball/5aaa94d406577b0059ad28c78022606890dc6de0", - "reference": "5aaa94d406577b0059ad28c78022606890dc6de0", + "url": "https://api.github.com/repos/utopia-php/http/zipball/334b99d47b08f016a160b67dcbee0390cc2b8544", + "reference": "334b99d47b08f016a160b67dcbee0390cc2b8544", "shasum": "" }, "require": { - "php": ">=8.1", - "utopia-php/compression": "0.1.*", - "utopia-php/telemetry": "0.1.*" + "ext-swoole": "*", + "php": ">=8.0" }, "require-dev": { - "laravel/pint": "^1.2", + "laravel/pint": "1.*", "phpbench/phpbench": "^1.2", - "phpstan/phpstan": "^1.10", - "phpunit/phpunit": "^9.5.25" + "phpstan/phpstan": "1.*", + "phpunit/phpunit": "^9.5.25", + "swoole/ide-helper": "4.8.3" }, "type": "library", "autoload": { "psr-4": { - "Utopia\\": "src/" + "Utopia\\": "src/", + "Tests\\E2E\\": "tests/e2e" } }, "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], - "description": "A simple, light and advanced PHP framework", + "description": "A simple, light and advanced PHP HTTP framework", "keywords": [ "framework", + "http", "php", "upf" ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.28" - "source": "https://github.com/utopia-php/http/tree/0.33.28" + "source": "https://github.com/utopia-php/http/tree/0.34.10" }, - "time": "2025-09-25T10:44:24+00:00" + "time": "2025-09-03T17:59:15+00:00" }, { "name": "utopia-php/mongo", - "version": "0.10.0", + "version": "0.11.0", "source": { "type": "git", "url": "https://github.com/utopia-php/mongo.git", - "reference": "ecfad6aad2e2e3fe5899ac2ebf1009a21b4d6b18" + "reference": "34bc0cda8ea368cde68702a6fffe2c3ac625398e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/mongo/zipball/ecfad6aad2e2e3fe5899ac2ebf1009a21b4d6b18", - "reference": "ecfad6aad2e2e3fe5899ac2ebf1009a21b4d6b18", + "url": "https://api.github.com/repos/utopia-php/mongo/zipball/34bc0cda8ea368cde68702a6fffe2c3ac625398e", + "reference": "34bc0cda8ea368cde68702a6fffe2c3ac625398e", "shasum": "" }, "require": { @@ -2617,9 +2372,9 @@ ], "support": { "issues": "https://github.com/utopia-php/mongo/issues", - "source": "https://github.com/utopia-php/mongo/tree/0.10.0" + "source": "https://github.com/utopia-php/mongo/tree/0.11.0" }, - "time": "2025-10-02T04:50:07+00:00" + "time": "2025-10-20T11:11:23+00:00" }, { "name": "utopia-php/pools", @@ -2676,19 +2431,15 @@ { "name": "utopia-php/storage", "version": "0.18.14", - "version": "0.18.14", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", "reference": "4f14ec952c6f4006dd0613e55bbf7631814fbc00" - "reference": "4f14ec952c6f4006dd0613e55bbf7631814fbc00" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/utopia-php/storage/zipball/4f14ec952c6f4006dd0613e55bbf7631814fbc00", "reference": "4f14ec952c6f4006dd0613e55bbf7631814fbc00", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/4f14ec952c6f4006dd0613e55bbf7631814fbc00", - "reference": "4f14ec952c6f4006dd0613e55bbf7631814fbc00", "shasum": "" }, "require": { @@ -2732,10 +2483,8 @@ "support": { "issues": "https://github.com/utopia-php/storage/issues", "source": "https://github.com/utopia-php/storage/tree/0.18.14" - "source": "https://github.com/utopia-php/storage/tree/0.18.14" }, "time": "2025-10-07T10:21:47+00:00" - "time": "2025-10-07T10:21:47+00:00" }, { "name": "utopia-php/system", @@ -2842,6 +2591,52 @@ "source": "https://github.com/utopia-php/telemetry/tree/0.1.1" }, "time": "2025-03-17T11:57:52+00:00" + }, + { + "name": "utopia-php/validators", + "version": "0.0.2", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/validators.git", + "reference": "894210695c5d35fa248fb65f7fe7237b6ff4fb0b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/validators/zipball/894210695c5d35fa248fb65f7fe7237b6ff4fb0b", + "reference": "894210695c5d35fa248fb65f7fe7237b6ff4fb0b", + "shasum": "" + }, + "require": { + "php": ">=8.1" + }, + "require-dev": { + "ext-xdebug": "*", + "laravel/pint": "^1.2", + "phpstan/phpstan": "1.*", + "phpunit/phpunit": "^9.5.25" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A lightweight collection of reusable validators for Utopia projects", + "keywords": [ + "php", + "utopia", + "validation", + "validator" + ], + "support": { + "issues": "https://github.com/utopia-php/validators/issues", + "source": "https://github.com/utopia-php/validators/tree/0.0.2" + }, + "time": "2025-10-20T21:52:28+00:00" } ], "packages-dev": [ @@ -2910,19 +2705,15 @@ { "name": "laravel/pint", "version": "v1.25.1", - "version": "v1.25.1", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", "reference": "5016e263f95d97670d71b9a987bd8996ade6d8d9" - "reference": "5016e263f95d97670d71b9a987bd8996ade6d8d9" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/laravel/pint/zipball/5016e263f95d97670d71b9a987bd8996ade6d8d9", "reference": "5016e263f95d97670d71b9a987bd8996ade6d8d9", - "url": "https://api.github.com/repos/laravel/pint/zipball/5016e263f95d97670d71b9a987bd8996ade6d8d9", - "reference": "5016e263f95d97670d71b9a987bd8996ade6d8d9", "shasum": "" }, "require": { @@ -2933,9 +2724,6 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.87.2", - "illuminate/view": "^11.46.0", - "larastan/larastan": "^3.7.1", "friendsofphp/php-cs-fixer": "^3.87.2", "illuminate/view": "^11.46.0", "larastan/larastan": "^3.7.1", @@ -2979,7 +2767,6 @@ "source": "https://github.com/laravel/pint" }, "time": "2025-09-19T02:57:12+00:00" - "time": "2025-09-19T02:57:12+00:00" }, { "name": "myclabs/deep-copy", @@ -3295,13 +3082,10 @@ { "name": "phpstan/phpstan", "version": "1.12.32", - "version": "1.12.32", "dist": { "type": "zip", "url": "https://api.github.com/repos/phpstan/phpstan/zipball/2770dcdf5078d0b0d53f94317e06affe88419aa8", "reference": "2770dcdf5078d0b0d53f94317e06affe88419aa8", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/2770dcdf5078d0b0d53f94317e06affe88419aa8", - "reference": "2770dcdf5078d0b0d53f94317e06affe88419aa8", "shasum": "" }, "require": { @@ -3347,7 +3131,6 @@ } ], "time": "2025-09-30T10:16:31+00:00" - "time": "2025-09-30T10:16:31+00:00" }, { "name": "phpunit/php-code-coverage", @@ -3687,19 +3470,15 @@ { "name": "phpunit/phpunit", "version": "11.5.42", - "version": "11.5.42", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", "reference": "1c6cb5dfe412af3d0dfd414cfd110e3b9cfdbc3c" - "reference": "1c6cb5dfe412af3d0dfd414cfd110e3b9cfdbc3c" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/1c6cb5dfe412af3d0dfd414cfd110e3b9cfdbc3c", "reference": "1c6cb5dfe412af3d0dfd414cfd110e3b9cfdbc3c", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/1c6cb5dfe412af3d0dfd414cfd110e3b9cfdbc3c", - "reference": "1c6cb5dfe412af3d0dfd414cfd110e3b9cfdbc3c", "shasum": "" }, "require": { @@ -3724,7 +3503,6 @@ "sebastian/diff": "^6.0.2", "sebastian/environment": "^7.2.1", "sebastian/exporter": "^6.3.2", - "sebastian/exporter": "^6.3.2", "sebastian/global-state": "^7.0.2", "sebastian/object-enumerator": "^6.0.1", "sebastian/type": "^5.1.3", @@ -3773,7 +3551,6 @@ "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.42" - "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.42" }, "funding": [ { @@ -3798,7 +3575,6 @@ } ], "time": "2025-09-28T12:09:13+00:00" - "time": "2025-09-28T12:09:13+00:00" }, { "name": "sebastian/cli-parser", @@ -4266,19 +4042,15 @@ { "name": "sebastian/exporter", "version": "6.3.2", - "version": "6.3.2", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/exporter.git", "reference": "70a298763b40b213ec087c51c739efcaa90bcd74" - "reference": "70a298763b40b213ec087c51c739efcaa90bcd74" }, "dist": { "type": "zip", "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/70a298763b40b213ec087c51c739efcaa90bcd74", "reference": "70a298763b40b213ec087c51c739efcaa90bcd74", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/70a298763b40b213ec087c51c739efcaa90bcd74", - "reference": "70a298763b40b213ec087c51c739efcaa90bcd74", "shasum": "" }, "require": { @@ -4293,7 +4065,6 @@ "extra": { "branch-alias": { "dev-main": "6.3-dev" - "dev-main": "6.3-dev" } }, "autoload": { @@ -4337,7 +4108,6 @@ "issues": "https://github.com/sebastianbergmann/exporter/issues", "security": "https://github.com/sebastianbergmann/exporter/security/policy", "source": "https://github.com/sebastianbergmann/exporter/tree/6.3.2" - "source": "https://github.com/sebastianbergmann/exporter/tree/6.3.2" }, "funding": [ { @@ -4352,25 +4122,12 @@ "url": "https://thanks.dev/u/gh/sebastianbergmann", "type": "thanks_dev" }, - { - "url": "https://tidelift.com/funding/github/packagist/sebastian/exporter", - "type": "tidelift" - }, - { - "url": "https://liberapay.com/sebastianbergmann", - "type": "liberapay" - }, - { - "url": "https://thanks.dev/u/gh/sebastianbergmann", - "type": "thanks_dev" - }, { "url": "https://tidelift.com/funding/github/packagist/sebastian/exporter", "type": "tidelift" } ], "time": "2025-09-24T06:12:51+00:00" - "time": "2025-09-24T06:12:51+00:00" }, { "name": "sebastian/global-state", @@ -5162,7 +4919,7 @@ "aliases": [ { "package": "utopia-php/database", - "version": "dev-test-docsdb", + "version": "dev-var_object", "alias": "1.4.5", "alias_normalized": "1.4.5.0" } diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 6273a60..f9fbfc2 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -881,6 +881,8 @@ protected function createIndex(Index $resource): bool $supportForAttributes = $dbForDatabases->getAdapter()->getSupportForAttributes(); $supportForMultipleFulltextIndexes = $dbForDatabases->getAdapter()->getSupportForMultipleFulltextIndexes(); $supportForIdenticalIndexes = $dbForDatabases->getAdapter()->getSupportForIdenticalIndexes(); + $supportForVectorIndexes = $dbForDatabases->getAdapter()->getSupportForVectors(); + $supportForObjectIndexes = $dbForDatabases->getAdapter()->getSupportForObject(); $validator = new IndexValidator( $table->getAttribute('attributes'), @@ -888,12 +890,13 @@ protected function createIndex(Index $resource): bool $maxIndexLength, $internalIndexesKeys, $supportForIndexArray, - $supportForSpatialAttributes, $supportForSpatialIndexNull, $supportForSpatialIndexOrder, + $supportForVectorIndexes, $supportForAttributes, $supportForMultipleFulltextIndexes, - $supportForIdenticalIndexes + $supportForIdenticalIndexes, + $supportForObjectIndexes, ); From 573aae2840e0727844b6614e57fd0fffbfc51518 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 28 Oct 2025 20:36:08 +0530 Subject: [PATCH 16/28] updated db version --- composer.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/composer.lock b/composer.lock index cfcfd41..4bd42c8 100644 --- a/composer.lock +++ b/composer.lock @@ -2167,12 +2167,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "e2768d9d35e0e31a12e4e63d48f6806651de0225" + "reference": "f62ff5183e4baf61746007c3b23548b85f33c71b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/e2768d9d35e0e31a12e4e63d48f6806651de0225", - "reference": "e2768d9d35e0e31a12e4e63d48f6806651de0225", + "url": "https://api.github.com/repos/utopia-php/database/zipball/f62ff5183e4baf61746007c3b23548b85f33c71b", + "reference": "f62ff5183e4baf61746007c3b23548b85f33c71b", "shasum": "" }, "require": { From 6a17cdf7579bd1bcddb7e635f5095659a076b4df Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sun, 9 Nov 2025 11:33:25 +0530 Subject: [PATCH 17/28] added vector db , object type --- composer.lock | 146 +++++++++--------- src/Migration/Destinations/Appwrite.php | 3 + src/Migration/Resource.php | 7 + src/Migration/Resources/Database/Column.php | 2 + .../Resources/Database/Columns/ObjectType.php | 74 +++++++++ src/Migration/Resources/Database/VectorDB.php | 40 +++++ src/Migration/Sources/Appwrite.php | 88 +++++++++++ .../Sources/Appwrite/Reader/Database.php | 2 + src/Migration/Transfer.php | 1 + 9 files changed, 289 insertions(+), 74 deletions(-) create mode 100644 src/Migration/Resources/Database/Columns/ObjectType.php create mode 100644 src/Migration/Resources/Database/VectorDB.php diff --git a/composer.lock b/composer.lock index 4bd42c8..c6b5db3 100644 --- a/composer.lock +++ b/composer.lock @@ -2113,6 +2113,52 @@ }, "time": "2025-05-09T14:43:52+00:00" }, + { + "name": "utopia-php/compression", + "version": "0.1.3", + "source": { + "type": "git", + "url": "https://github.com/utopia-php/compression.git", + "reference": "66f093557ba66d98245e562036182016c7dcfe8a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/utopia-php/compression/zipball/66f093557ba66d98245e562036182016c7dcfe8a", + "reference": "66f093557ba66d98245e562036182016c7dcfe8a", + "shasum": "" + }, + "require": { + "php": ">=8.0" + }, + "require-dev": { + "laravel/pint": "1.2.*", + "phpunit/phpunit": "^9.3", + "vimeo/psalm": "4.0.1" + }, + "type": "library", + "autoload": { + "psr-4": { + "Utopia\\Compression\\": "src/Compression" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "description": "A simple Compression library to handle file compression", + "keywords": [ + "compression", + "framework", + "php", + "upf", + "utopia" + ], + "support": { + "issues": "https://github.com/utopia-php/compression/issues", + "source": "https://github.com/utopia-php/compression/tree/0.1.3" + }, + "time": "2025-01-15T15:15:51+00:00" + }, { "name": "utopia-php/console", "version": "0.0.1", @@ -2167,12 +2213,12 @@ "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "f62ff5183e4baf61746007c3b23548b85f33c71b" + "reference": "9a0cea65abbfb325506cb1d619db60dac7ee7e55" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/f62ff5183e4baf61746007c3b23548b85f33c71b", - "reference": "f62ff5183e4baf61746007c3b23548b85f33c71b", + "url": "https://api.github.com/repos/utopia-php/database/zipball/9a0cea65abbfb325506cb1d619db60dac7ee7e55", + "reference": "9a0cea65abbfb325506cb1d619db60dac7ee7e55", "shasum": "" }, "require": { @@ -2181,9 +2227,9 @@ "ext-pdo": "*", "php": ">=8.1", "utopia-php/cache": "0.13.*", + "utopia-php/framework": "0.33.*", "utopia-php/mongo": "0.11.*", - "utopia-php/pools": "0.8.*", - "utopia-php/validators": "0.0.*" + "utopia-php/pools": "0.8.*" }, "require-dev": { "fakerphp/faker": "1.23.*", @@ -2217,7 +2263,7 @@ "issues": "https://github.com/utopia-php/database/issues", "source": "https://github.com/utopia-php/database/tree/var_object" }, - "time": "2025-10-27T10:06:02+00:00" + "time": "2025-11-06T09:02:12+00:00" }, { "name": "utopia-php/dsn", @@ -2268,52 +2314,50 @@ }, { "name": "utopia-php/framework", - "version": "0.34.10", + "version": "0.33.28", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "334b99d47b08f016a160b67dcbee0390cc2b8544" + "reference": "5aaa94d406577b0059ad28c78022606890dc6de0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/334b99d47b08f016a160b67dcbee0390cc2b8544", - "reference": "334b99d47b08f016a160b67dcbee0390cc2b8544", + "url": "https://api.github.com/repos/utopia-php/http/zipball/5aaa94d406577b0059ad28c78022606890dc6de0", + "reference": "5aaa94d406577b0059ad28c78022606890dc6de0", "shasum": "" }, "require": { - "ext-swoole": "*", - "php": ">=8.0" + "php": ">=8.1", + "utopia-php/compression": "0.1.*", + "utopia-php/telemetry": "0.1.*" }, "require-dev": { - "laravel/pint": "1.*", + "laravel/pint": "^1.2", "phpbench/phpbench": "^1.2", - "phpstan/phpstan": "1.*", - "phpunit/phpunit": "^9.5.25", - "swoole/ide-helper": "4.8.3" + "phpstan/phpstan": "^1.10", + "phpunit/phpunit": "^9.5.25" }, "type": "library", "autoload": { "psr-4": { - "Utopia\\": "src/", - "Tests\\E2E\\": "tests/e2e" + "Utopia\\": "src/" } }, "notification-url": "https://packagist.org/downloads/", "license": [ "MIT" ], - "description": "A simple, light and advanced PHP HTTP framework", + "description": "A simple, light and advanced PHP framework", "keywords": [ "framework", - "http", "php", "upf" ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.34.10" + "source": "https://github.com/utopia-php/http/tree/0.33.28" }, - "time": "2025-09-03T17:59:15+00:00" + "time": "2025-09-25T10:44:24+00:00" }, { "name": "utopia-php/mongo", @@ -2591,52 +2635,6 @@ "source": "https://github.com/utopia-php/telemetry/tree/0.1.1" }, "time": "2025-03-17T11:57:52+00:00" - }, - { - "name": "utopia-php/validators", - "version": "0.0.2", - "source": { - "type": "git", - "url": "https://github.com/utopia-php/validators.git", - "reference": "894210695c5d35fa248fb65f7fe7237b6ff4fb0b" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/validators/zipball/894210695c5d35fa248fb65f7fe7237b6ff4fb0b", - "reference": "894210695c5d35fa248fb65f7fe7237b6ff4fb0b", - "shasum": "" - }, - "require": { - "php": ">=8.1" - }, - "require-dev": { - "ext-xdebug": "*", - "laravel/pint": "^1.2", - "phpstan/phpstan": "1.*", - "phpunit/phpunit": "^9.5.25" - }, - "type": "library", - "autoload": { - "psr-4": { - "Utopia\\": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "description": "A lightweight collection of reusable validators for Utopia projects", - "keywords": [ - "php", - "utopia", - "validation", - "validator" - ], - "support": { - "issues": "https://github.com/utopia-php/validators/issues", - "source": "https://github.com/utopia-php/validators/tree/0.0.2" - }, - "time": "2025-10-20T21:52:28+00:00" } ], "packages-dev": [ @@ -3469,16 +3467,16 @@ }, { "name": "phpunit/phpunit", - "version": "11.5.42", + "version": "11.5.43", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "1c6cb5dfe412af3d0dfd414cfd110e3b9cfdbc3c" + "reference": "c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/1c6cb5dfe412af3d0dfd414cfd110e3b9cfdbc3c", - "reference": "1c6cb5dfe412af3d0dfd414cfd110e3b9cfdbc3c", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924", + "reference": "c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924", "shasum": "" }, "require": { @@ -3550,7 +3548,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.42" + "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.43" }, "funding": [ { @@ -3574,7 +3572,7 @@ "type": "tidelift" } ], - "time": "2025-09-28T12:09:13+00:00" + "time": "2025-10-30T08:39:39+00:00" }, { "name": "sebastian/cli-parser", diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index f9fbfc2..63528d1 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -127,6 +127,7 @@ public static function getSupportedResources(): array // Database Resource::TYPE_DATABASE, Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_DATABASE_VECTORDB, Resource::TYPE_TABLE, Resource::TYPE_COLUMN, Resource::TYPE_INDEX, @@ -288,6 +289,7 @@ public function importDatabaseResource(Resource $resource, bool $isLast): Resour switch ($resource->getName()) { case Resource::TYPE_DATABASE: case Resource::TYPE_DATABASE_DOCUMENTSDB: + case Resource::TYPE_DATABASE_VECTORDB: /** @var Database $resource */ $success = $this->createDatabase($resource); break; @@ -467,6 +469,7 @@ protected function createField(Column $resource): bool Column::TYPE_POINT => UtopiaDatabase::VAR_POINT, Column::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING, Column::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON, + Column::TYPE_OBJECT => UtopiaDatabase::VAR_OBJECT, default => throw new \Exception('Invalid resource type '.$resource->getType()), }; diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index 2b4e322..ef184fb 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -35,6 +35,7 @@ abstract class Resource implements \JsonSerializable public const TYPE_DATABASE_TABLESDB = 'tablesdb'; public const TYPE_DATABASE_DOCUMENTSDB = 'documentsdb'; + public const TYPE_DATABASE_VECTORDB = 'vectordb'; public const TYPE_ROW = 'row'; @@ -76,6 +77,7 @@ abstract class Resource implements \JsonSerializable self::TYPE_BUCKET, self::TYPE_TABLE, self::TYPE_DATABASE, + self::TYPE_DATABASE_VECTORDB, self::TYPE_DATABASE_DOCUMENTSDB, self::TYPE_ROW, self::TYPE_FILE, @@ -106,6 +108,11 @@ abstract class Resource implements \JsonSerializable 'record' => self::TYPE_DOCUMENT, 'field' => self::TYPE_ATTRIBUTE, ], + self::TYPE_DATABASE_VECTORDB => [ + 'entity' => self::TYPE_COLLECTION, + 'record' => self::TYPE_DOCUMENT, + 'field' => self::TYPE_ATTRIBUTE, + ] ]; protected string $id = ''; diff --git a/src/Migration/Resources/Database/Column.php b/src/Migration/Resources/Database/Column.php index e064a40..24cf95a 100644 --- a/src/Migration/Resources/Database/Column.php +++ b/src/Migration/Resources/Database/Column.php @@ -22,6 +22,8 @@ abstract class Column extends Resource public const TYPE_LINE = 'linestring'; public const TYPE_POLYGON = 'polygon'; + public const TYPE_OBJECT = 'object'; + /** * @param string $key * @param Table $table diff --git a/src/Migration/Resources/Database/Columns/ObjectType.php b/src/Migration/Resources/Database/Columns/ObjectType.php new file mode 100644 index 0000000..06a36a3 --- /dev/null +++ b/src/Migration/Resources/Database/Columns/ObjectType.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Table::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Column::TYPE_POINT; + } +} diff --git a/src/Migration/Resources/Database/VectorDB.php b/src/Migration/Resources/Database/VectorDB.php new file mode 100644 index 0000000..bcd6a6a --- /dev/null +++ b/src/Migration/Resources/Database/VectorDB.php @@ -0,0 +1,40 @@ +cache->get(DocumentsDB::getName())) > 0) { $this->exportDocumentsDB($batchSize, $resources); } + + // Export VectorDB databases using same flow as DocumentsDB + if (count($this->cache->get(VectorDB::getName())) > 0) { + $this->exportVectorDB($batchSize, $resources); + } } /** @@ -965,6 +974,16 @@ private function exportFields(int $batchSize): void updatedAt: $column['$updatedAt'] ?? '', ); break; + case Column::TYPE_OBJECT: + $col = new ObjectType( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ); + break; } if (!isset($col)) { @@ -1720,6 +1739,69 @@ private function exportDocumentsDB(int $batchSize, array $resources): void } } + /** + * Export VectorDB databases (collections and documents) + * + * @param int $batchSize + * @param array $resources + */ + private function exportVectorDB(int $batchSize, array $resources): void + { + try { + if (\in_array(Resource::TYPE_COLLECTION, $resources)) { + $this->exportEntities(VectorDB::getName(), $batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + Resource::TYPE_COLLECTION, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } + + try { + if (\in_array(Resource::TYPE_INDEX, $resources)) { + $this->exportIndexes(Collection::getName(), $batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + Resource::TYPE_INDEX, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } + + try { + if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { + $this->exportRecords(Collection::getName(), $batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + Resource::TYPE_DOCUMENT, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } + } + /** * @param string $databaseType * @param array $database { @@ -1737,6 +1819,8 @@ private static function getDatabase(string $databaseType, array $database): Reso switch ($databaseType) { case Resource::TYPE_DATABASE_DOCUMENTSDB: return DocumentsDB::fromArray($database); + case Resource::TYPE_DATABASE_VECTORDB: + return VectorDB::fromArray($database); default: return Database::fromArray($database); } @@ -1765,6 +1849,8 @@ private static function getEntity(string $databaseType, array $entity): Resource switch ($databaseType) { case Resource::TYPE_DATABASE_DOCUMENTSDB: return Collection::fromArray($entity); + case Resource::TYPE_DATABASE_VECTORDB: + return Collection::fromArray($entity); default: return Table::fromArray($entity); } @@ -1804,6 +1890,8 @@ private static function getRecord(string $databaseType, array $record): Resource switch ($databaseType) { case Resource::TYPE_DATABASE_DOCUMENTSDB: return Document::fromArray($record); + case Resource::TYPE_DATABASE_VECTORDB: + return Document::fromArray($record); default: return Row::fromArray($record); } diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index d437727..45524fd 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -55,6 +55,8 @@ public function report(array $resources, array &$report): mixed Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX, + // vectordb + Resource::TYPE_DATABASE_VECTORDB, // Documentsdb Resource::TYPE_DATABASE_DOCUMENTSDB, Resource::TYPE_COLLECTION, diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index a50ec2d..62d02be 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -37,6 +37,7 @@ class Transfer public const GROUP_DATABASES_RESOURCES = [ Resource::TYPE_DATABASE, Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_DATABASE_VECTORDB, Resource::TYPE_TABLE, Resource::TYPE_INDEX, Resource::TYPE_COLUMN, From d234c12129966e5d6f323fa840e8017f949b8052 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sun, 9 Nov 2025 20:05:45 +0530 Subject: [PATCH 18/28] added vector, objectype, vectordb --- src/Migration/Destinations/Appwrite.php | 6 ++ .../Resources/Database/Collection.php | 9 ++- src/Migration/Resources/Database/Column.php | 1 + .../Resources/Database/Columns/ObjectType.php | 2 +- .../Resources/Database/Columns/Vector.php | 78 +++++++++++++++++++ src/Migration/Sources/Appwrite.php | 50 ++++++++---- src/Migration/Transfer.php | 1 + 7 files changed, 131 insertions(+), 16 deletions(-) create mode 100644 src/Migration/Resources/Database/Columns/Vector.php diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 63528d1..34a38a5 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -419,6 +419,11 @@ protected function createEntity(Table $resource): bool $dbForDatabases = ($this->getDatabasesDB)($database); + // passing null in creates only creates the metadata collection + if (!$dbForDatabases->exists(null, UtopiaDatabase::METADATA)) { + $dbForDatabases->create(); + } + $table = $this->dbForProject->createDocument('database_' . $database->getSequence(), new UtopiaDocument([ '$id' => $resource->getId(), 'databaseInternalId' => $database->getSequence(), @@ -470,6 +475,7 @@ protected function createField(Column $resource): bool Column::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING, Column::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON, Column::TYPE_OBJECT => UtopiaDatabase::VAR_OBJECT, + Column::TYPE_VECTOR => UtopiaDatabase::VAR_VECTOR, default => throw new \Exception('Invalid resource type '.$resource->getType()), }; diff --git a/src/Migration/Resources/Database/Collection.php b/src/Migration/Resources/Database/Collection.php index b2c53ec..796420c 100644 --- a/src/Migration/Resources/Database/Collection.php +++ b/src/Migration/Resources/Database/Collection.php @@ -2,6 +2,7 @@ namespace Utopia\Migration\Resources\Database; +use Appwrite\Services\Databases; use Utopia\Migration\Resource; class Collection extends Table @@ -29,8 +30,14 @@ public static function getName(): string */ public static function fromArray(array $array): self { + $database = match ($array['database']['type']) { + Resource::TYPE_DATABASE_DOCUMENTSDB => DocumentsDB::fromArray($array['database']), + Resource::TYPE_DATABASE_VECTORDB => VectorDB::fromArray($array['database']), + default => Database::fromArray($array['database']) + }; + return new self( - DocumentsDB::fromArray($array['database']), + $database, name: $array['name'], id: $array['id'], rowSecurity: $array['rowSecurity'] ?? $array['documentSecurity'], diff --git a/src/Migration/Resources/Database/Column.php b/src/Migration/Resources/Database/Column.php index 24cf95a..e80d172 100644 --- a/src/Migration/Resources/Database/Column.php +++ b/src/Migration/Resources/Database/Column.php @@ -23,6 +23,7 @@ abstract class Column extends Resource public const TYPE_POLYGON = 'polygon'; public const TYPE_OBJECT = 'object'; + public const TYPE_VECTOR = 'vector'; /** * @param string $key diff --git a/src/Migration/Resources/Database/Columns/ObjectType.php b/src/Migration/Resources/Database/Columns/ObjectType.php index 06a36a3..38477cd 100644 --- a/src/Migration/Resources/Database/Columns/ObjectType.php +++ b/src/Migration/Resources/Database/Columns/ObjectType.php @@ -69,6 +69,6 @@ public static function fromArray(array $array): self public function getType(): string { - return Column::TYPE_POINT; + return Column::TYPE_OBJECT; } } diff --git a/src/Migration/Resources/Database/Columns/Vector.php b/src/Migration/Resources/Database/Columns/Vector.php new file mode 100644 index 0000000..e7ae005 --- /dev/null +++ b/src/Migration/Resources/Database/Columns/Vector.php @@ -0,0 +1,78 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * size: int, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Table::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + size:$array['size'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Column::TYPE_VECTOR; + } +} diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 822cc40..1f8cc7d 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -34,6 +34,7 @@ use Utopia\Migration\Resources\Database\Columns\Relationship; use Utopia\Migration\Resources\Database\Columns\Text; use Utopia\Migration\Resources\Database\Columns\URL; +use Utopia\Migration\Resources\Database\Columns\Vector; use Utopia\Migration\Resources\Database\Database; use Utopia\Migration\Resources\Database\Document; use Utopia\Migration\Resources\Database\DocumentsDB; @@ -796,15 +797,17 @@ private function exportEntities(string $databaseName, int $batchSize): void } /** + * @param string $entityType * @param int $batchSize * @throws Exception */ - private function exportFields(int $batchSize): void + private function exportFields(string $entityType,int $batchSize): void { - $tables = $this->cache->get(Table::getName()); + $entities = $this->cache->get($entityType); + // Transfer Indexes /** @var array $tables */ - foreach ($tables as $table) { + foreach ($entities as $table) { $lastColumn = null; while (true) { @@ -984,6 +987,17 @@ private function exportFields(int $batchSize): void updatedAt: $column['$updatedAt'] ?? '', ); break; + case Column::TYPE_VECTOR: + $col = new Vector( + $column['key'], + $table, + size:$column['size'], + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ); + break; } if (!isset($col)) { @@ -1077,16 +1091,6 @@ private function exportRecords(string $entityName, int $batchSize): void /** @var Table $table */ $lastRow = null; - // Create reader - use getDatabasesDB for row data operations - $this->reader = match ($this->source) { - static::SOURCE_API => new APIReader(new Databases($this->client)), - static::SOURCE_DATABASE => new DatabaseReader( - $this->dbForProject, - $this->getDatabasesDB - ), - default => throw new \Exception('Unknown source'), - }; - while (true) { $queries = [ $this->reader->queryLimit($batchSize), @@ -1623,7 +1627,7 @@ private function exportTablesDB(int $batchSize, array $resources) try { if (\in_array(Resource::TYPE_COLUMN, $resources)) { - $this->exportFields($batchSize); + $this->exportFields(Table::getName(),$batchSize); } } catch (\Throwable $e) { $this->addError( @@ -1765,6 +1769,24 @@ private function exportVectorDB(int $batchSize, array $resources): void return; } + try { + if (\in_array(Resource::TYPE_ATTRIBUTE, $resources)) { + $this->exportFields(Collection::getName(),$batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + Resource::TYPE_ATTRIBUTE, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } + try { if (\in_array(Resource::TYPE_INDEX, $resources)) { $this->exportIndexes(Collection::getName(), $batchSize); diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 62d02be..0d0e228 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -44,6 +44,7 @@ class Transfer Resource::TYPE_ROW, Resource::TYPE_DOCUMENT, Resource::TYPE_COLLECTION, + Resource::TYPE_ATTRIBUTE ]; public const GROUP_SETTINGS_RESOURCES = []; From 23e41a7692b98b125d9a12e6d80fec9fb69aa1f4 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Sun, 9 Nov 2025 21:18:34 +0530 Subject: [PATCH 19/28] added attribute support along with column --- src/Migration/Destinations/Appwrite.php | 33 +- .../Resources/Database/Attribute.php | 158 +++++ .../Resources/Database/Attribute/Boolean.php | 78 +++ .../Resources/Database/Attribute/DateTime.php | 79 +++ .../Resources/Database/Attribute/Decimal.php | 105 ++++ .../Resources/Database/Attribute/Email.php | 37 ++ .../Resources/Database/Attribute/Enum.php | 102 ++++ .../Resources/Database/Attribute/IP.php | 37 ++ .../Resources/Database/Attribute/Integer.php | 107 ++++ .../Resources/Database/Attribute/Line.php | 74 +++ .../Database/Attribute/ObjectType.php | 74 +++ .../Resources/Database/Attribute/Point.php | 74 +++ .../Resources/Database/Attribute/Polygon.php | 74 +++ .../Database/Attribute/Relationship.php | 125 ++++ .../Resources/Database/Attribute/Text.php | 97 ++++ .../Resources/Database/Attribute/URL.php | 37 ++ .../Resources/Database/Attribute/Vector.php | 78 +++ .../Resources/Database/Collection.php | 3 +- src/Migration/Resources/Database/VectorDB.php | 2 - src/Migration/Sources/Appwrite.php | 545 ++++++++++++------ 20 files changed, 1718 insertions(+), 201 deletions(-) create mode 100644 src/Migration/Resources/Database/Attribute.php create mode 100644 src/Migration/Resources/Database/Attribute/Boolean.php create mode 100644 src/Migration/Resources/Database/Attribute/DateTime.php create mode 100644 src/Migration/Resources/Database/Attribute/Decimal.php create mode 100644 src/Migration/Resources/Database/Attribute/Email.php create mode 100644 src/Migration/Resources/Database/Attribute/Enum.php create mode 100644 src/Migration/Resources/Database/Attribute/IP.php create mode 100644 src/Migration/Resources/Database/Attribute/Integer.php create mode 100644 src/Migration/Resources/Database/Attribute/Line.php create mode 100644 src/Migration/Resources/Database/Attribute/ObjectType.php create mode 100644 src/Migration/Resources/Database/Attribute/Point.php create mode 100644 src/Migration/Resources/Database/Attribute/Polygon.php create mode 100644 src/Migration/Resources/Database/Attribute/Relationship.php create mode 100644 src/Migration/Resources/Database/Attribute/Text.php create mode 100644 src/Migration/Resources/Database/Attribute/URL.php create mode 100644 src/Migration/Resources/Database/Attribute/Vector.php diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 34a38a5..064d16f 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -33,6 +33,7 @@ use Utopia\Migration\Resources\Auth\Membership; use Utopia\Migration\Resources\Auth\Team; use Utopia\Migration\Resources\Auth\User; +use Utopia\Migration\Resources\Database\Attribute; use Utopia\Migration\Resources\Database\Column; use Utopia\Migration\Resources\Database\Database; use Utopia\Migration\Resources\Database\Index; @@ -453,7 +454,7 @@ protected function createEntity(Table $resource): bool * @throws \Exception * @throws \Throwable */ - protected function createField(Column $resource): bool + protected function createField(Column|Attribute $resource): bool { if ($resource->getTable()->getDatabase()->getType() === Resource::TYPE_DATABASE_DOCUMENTSDB) { $resource->setStatus(Resource::STATUS_SKIPPED, 'Columns not supported for DocumentsDB'); @@ -461,21 +462,21 @@ protected function createField(Column $resource): bool } $type = match ($resource->getType()) { - Column::TYPE_DATETIME => UtopiaDatabase::VAR_DATETIME, - Column::TYPE_BOOLEAN => UtopiaDatabase::VAR_BOOLEAN, - Column::TYPE_INTEGER => UtopiaDatabase::VAR_INTEGER, - Column::TYPE_FLOAT => UtopiaDatabase::VAR_FLOAT, - Column::TYPE_RELATIONSHIP => UtopiaDatabase::VAR_RELATIONSHIP, - Column::TYPE_STRING, - Column::TYPE_IP, - Column::TYPE_EMAIL, - Column::TYPE_URL, - Column::TYPE_ENUM => UtopiaDatabase::VAR_STRING, - Column::TYPE_POINT => UtopiaDatabase::VAR_POINT, - Column::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING, - Column::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON, - Column::TYPE_OBJECT => UtopiaDatabase::VAR_OBJECT, - Column::TYPE_VECTOR => UtopiaDatabase::VAR_VECTOR, + Column::TYPE_DATETIME, Attribute::TYPE_DATETIME => UtopiaDatabase::VAR_DATETIME, + Column::TYPE_BOOLEAN, Attribute::TYPE_BOOLEAN => UtopiaDatabase::VAR_BOOLEAN, + Column::TYPE_INTEGER, Attribute::TYPE_INTEGER => UtopiaDatabase::VAR_INTEGER, + Column::TYPE_FLOAT, Attribute::TYPE_FLOAT => UtopiaDatabase::VAR_FLOAT, + Column::TYPE_RELATIONSHIP, Attribute::TYPE_RELATIONSHIP => UtopiaDatabase::VAR_RELATIONSHIP, + Column::TYPE_STRING, Attribute::TYPE_STRING, + Column::TYPE_IP, Attribute::TYPE_IP, + Column::TYPE_EMAIL, Attribute::TYPE_EMAIL, + Column::TYPE_URL, Attribute::TYPE_URL, + Column::TYPE_ENUM, Attribute::TYPE_ENUM => UtopiaDatabase::VAR_STRING, + Column::TYPE_POINT, Attribute::TYPE_POINT => UtopiaDatabase::VAR_POINT, + Column::TYPE_LINE, Attribute::TYPE_LINE => UtopiaDatabase::VAR_LINESTRING, + Column::TYPE_POLYGON, Attribute::TYPE_POLYGON => UtopiaDatabase::VAR_POLYGON, + Column::TYPE_OBJECT, Attribute::TYPE_OBJECT => UtopiaDatabase::VAR_OBJECT, + Column::TYPE_VECTOR, Attribute::TYPE_VECTOR => UtopiaDatabase::VAR_VECTOR, default => throw new \Exception('Invalid resource type '.$resource->getType()), }; diff --git a/src/Migration/Resources/Database/Attribute.php b/src/Migration/Resources/Database/Attribute.php new file mode 100644 index 0000000..244f2ba --- /dev/null +++ b/src/Migration/Resources/Database/Attribute.php @@ -0,0 +1,158 @@ + $formatOptions + * @param array $filters + * @param array $options + * @param string $createdAt + * @param string $updatedAt + */ + public function __construct( + protected readonly string $key, + protected readonly Table $table, + protected readonly int $size = 0, + protected readonly bool $required = false, + protected readonly mixed $default = null, + protected readonly bool $array = false, + protected readonly bool $signed = false, + protected readonly string $format = '', + protected readonly array $formatOptions = [], + protected readonly array $filters = [], + protected array $options = [], + protected string $createdAt = '', + protected string $updatedAt = '', + ) { + } + + /** + * @return array + */ + public function jsonSerialize(): array + { + return [ + 'key' => $this->key, + 'table' => $this->table, + 'type' => $this->getType(), + 'size' => $this->size, + 'required' => $this->required, + 'default' => $this->default, + 'array' => $this->array, + 'signed' => $this->signed, + 'format' => $this->format, + 'formatOptions' => $this->formatOptions, + 'filters' => $this->filters, + 'options' => $this->options, + 'createdAt' => $this->createdAt, + 'updatedAt' => $this->updatedAt, + ]; + } + + public static function getName(): string + { + return Resource::TYPE_ATTRIBUTE; + } + + abstract public function getType(): string; + + public function getGroup(): string + { + return Transfer::GROUP_DATABASES; + } + + public function getKey(): string + { + return $this->key; + } + + public function getTable(): Table + { + return $this->table; + } + + public function getSize(): int + { + return $this->size; + } + + public function isRequired(): bool + { + return $this->required; + } + + public function getDefault(): mixed + { + return $this->default; + } + + public function isArray(): bool + { + return $this->array; + } + + public function isSigned(): bool + { + return $this->signed; + } + + public function getFormat(): string + { + return $this->format; + } + + /** + * @return array + */ + public function getFormatOptions(): array + { + return $this->formatOptions; + } + + /** + * @return array + */ + public function getFilters(): array + { + return $this->filters; + } + + /** + * @return array + */ + public function &getOptions(): array + { + return $this->options; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Boolean.php b/src/Migration/Resources/Database/Attribute/Boolean.php new file mode 100644 index 0000000..6bbe4b1 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Boolean.php @@ -0,0 +1,78 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?bool, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_BOOLEAN; + } +} diff --git a/src/Migration/Resources/Database/Attribute/DateTime.php b/src/Migration/Resources/Database/Attribute/DateTime.php new file mode 100644 index 0000000..f81a375 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/DateTime.php @@ -0,0 +1,79 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?string, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } +} diff --git a/src/Migration/Resources/Database/Attribute/Decimal.php b/src/Migration/Resources/Database/Attribute/Decimal.php new file mode 100644 index 0000000..acbcad4 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Decimal.php @@ -0,0 +1,105 @@ + $min, + 'max' => $max, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?float, + * formatOptions: array{ + * min: ?float, + * max: ?float + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + min: $array['formatOptions']['min'], + max: $array['formatOptions']['max'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_FLOAT; + } + + public function getMin(): ?float + { + return (float)$this->formatOptions['min']; + } + + public function getMax(): ?float + { + return (float)$this->formatOptions['max']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Email.php b/src/Migration/Resources/Database/Attribute/Email.php new file mode 100644 index 0000000..2c460f3 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Email.php @@ -0,0 +1,37 @@ + $elements + */ + public function __construct( + string $key, + Collection $collection, + array $elements, + bool $required = false, + ?string $default = null, + bool $array = false, + int $size = 256, + string $createdAt = '', + string $updatedAt = '' + ) { + parent::__construct( + $key, + $collection, + size: $size, + required: $required, + default: $default, + array: $array, + format: 'enum', + formatOptions: [ + 'elements' => $elements, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * size: int, + * required: bool, + * default: ?string, + * array: bool, + * formatOptions: array{ + * elements: array + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + elements: $array['formatOptions']['elements'], + required: $array['required'], + default: $array['default'], + array: $array['array'], + size: $array['size'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_ENUM; + } + + /** + * @return array + */ + public function getElements(): array + { + return (array)$this->formatOptions['elements']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/IP.php b/src/Migration/Resources/Database/Attribute/IP.php new file mode 100644 index 0000000..e2be9ad --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/IP.php @@ -0,0 +1,37 @@ + 2147483647 ? 8 : 4; + + parent::__construct( + $key, + $collection, + size: $size, + required: $required, + default: $default, + array: $array, + signed: $signed, + formatOptions: [ + 'min' => $min, + 'max' => $max, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * array: bool, + * default: ?int, + * formatOptions: array{ + * min: ?int, + * max: ?int + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + array: $array['array'], + min: $array['formatOptions']['min'] ?? null, + max: $array['formatOptions']['max'] ?? null, + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_INTEGER; + } + + public function getMin(): ?int + { + return (int)$this->formatOptions['min']; + } + + public function getMax(): ?int + { + return (int)$this->formatOptions['max']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Line.php b/src/Migration/Resources/Database/Attribute/Line.php new file mode 100644 index 0000000..dfa360a --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Line.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_LINE; + } +} diff --git a/src/Migration/Resources/Database/Attribute/ObjectType.php b/src/Migration/Resources/Database/Attribute/ObjectType.php new file mode 100644 index 0000000..e3ef99d --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/ObjectType.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_OBJECT; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Point.php b/src/Migration/Resources/Database/Attribute/Point.php new file mode 100644 index 0000000..a82d7d3 --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Point.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_POINT; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Polygon.php b/src/Migration/Resources/Database/Attribute/Polygon.php new file mode 100644 index 0000000..28d2adc --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Polygon.php @@ -0,0 +1,74 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_POLYGON; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Relationship.php b/src/Migration/Resources/Database/Attribute/Relationship.php new file mode 100644 index 0000000..aa252ab --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Relationship.php @@ -0,0 +1,125 @@ + $relatedTable, + 'relationType' => $relationType, + 'twoWay' => $twoWay, + 'twoWayKey' => $twoWayKey, + 'onDelete' => $onDelete, + 'side' => $side, + ], + createdAt: $createdAt, + updatedAt: $updatedAt + ); + } + + /** + * @param array{ + * key: string, + * collection?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * documentSecurity: bool, + * permissions: ?array + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * options: array{ + * relatedCollection: string, + * relationType: string, + * twoWay: bool, + * twoWayKey: ?string, + * onDelete: string, + * side: string, + * }, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + relatedTable: $array['options']['relatedTable'] ?? $array['options']['relatedCollection'], + relationType: $array['options']['relationType'], + twoWay: $array['options']['twoWay'], + twoWayKey: $array['options']['twoWayKey'], + onDelete: $array['options']['onDelete'], + side: $array['options']['side'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_RELATIONSHIP; + } + + public function getRelatedTable(): string + { + return $this->options['relatedTable'] ?? $this->options['relatedCollection']; + } + + public function getRelationType(): string + { + return $this->options['relationType']; + } + + public function getTwoWay(): bool + { + return $this->options['twoWay']; + } + + public function getTwoWayKey(): ?string + { + return $this->options['twoWayKey']; + } + + public function getOnDelete(): string + { + return $this->options['onDelete']; + } + + public function getSide(): string + { + return $this->options['side']; + } +} diff --git a/src/Migration/Resources/Database/Attribute/Text.php b/src/Migration/Resources/Database/Attribute/Text.php new file mode 100644 index 0000000..a0c60ec --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/Text.php @@ -0,0 +1,97 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * required: bool, + * default: ?string, + * array: bool, + * size: int, + * format: string, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + default: $array['default'] ?? null, + array: $array['array'], + size: $array['size'], + format: $array['format'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_STRING; + } + + public function getSize(): int + { + return $this->size; + } + + public function getFormat(): string + { + return $this->format; + } +} diff --git a/src/Migration/Resources/Database/Attribute/URL.php b/src/Migration/Resources/Database/Attribute/URL.php new file mode 100644 index 0000000..2e83a0f --- /dev/null +++ b/src/Migration/Resources/Database/Attribute/URL.php @@ -0,0 +1,37 @@ + + * }, + * table?: array{ + * database: array{ + * id: string, + * name: string, + * }, + * name: string, + * id: string, + * rowSecurity: bool, + * permissions: ?array + * }, + * size: int, + * required: bool, + * default: ?array, + * createdAt: string, + * updatedAt: string, + * } $array + * @return self + */ + public static function fromArray(array $array): self + { + return new self( + $array['key'], + Collection::fromArray($array['table'] ?? $array['collection']), + required: $array['required'], + size:$array['size'], + default: $array['default'], + createdAt: $array['createdAt'] ?? '', + updatedAt: $array['updatedAt'] ?? '', + ); + } + + public function getType(): string + { + return Attribute::TYPE_VECTOR; + } +} diff --git a/src/Migration/Resources/Database/Collection.php b/src/Migration/Resources/Database/Collection.php index 796420c..117b31b 100644 --- a/src/Migration/Resources/Database/Collection.php +++ b/src/Migration/Resources/Database/Collection.php @@ -2,7 +2,6 @@ namespace Utopia\Migration\Resources\Database; -use Appwrite\Services\Databases; use Utopia\Migration\Resource; class Collection extends Table @@ -35,7 +34,7 @@ public static function fromArray(array $array): self Resource::TYPE_DATABASE_VECTORDB => VectorDB::fromArray($array['database']), default => Database::fromArray($array['database']) }; - + return new self( $database, name: $array['name'], diff --git a/src/Migration/Resources/Database/VectorDB.php b/src/Migration/Resources/Database/VectorDB.php index bcd6a6a..2d61c74 100644 --- a/src/Migration/Resources/Database/VectorDB.php +++ b/src/Migration/Resources/Database/VectorDB.php @@ -36,5 +36,3 @@ public static function fromArray(array $array): self ); } } - - diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 1f8cc7d..b4bc143 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -18,6 +18,22 @@ use Utopia\Migration\Resources\Auth\Membership; use Utopia\Migration\Resources\Auth\Team; use Utopia\Migration\Resources\Auth\User; +use Utopia\Migration\Resources\Database\Attribute; +use Utopia\Migration\Resources\Database\Attribute\Boolean as AttributeBoolean; +use Utopia\Migration\Resources\Database\Attribute\DateTime as AttributeDateTime; +use Utopia\Migration\Resources\Database\Attribute\Decimal as AttributeDecimal; +use Utopia\Migration\Resources\Database\Attribute\Email as AttributeEmail; +use Utopia\Migration\Resources\Database\Attribute\Enum as AttributeEnum; +use Utopia\Migration\Resources\Database\Attribute\Integer as AttributeInteger; +use Utopia\Migration\Resources\Database\Attribute\IP as AttributeIP; +use Utopia\Migration\Resources\Database\Attribute\Line as AttributeLine; +use Utopia\Migration\Resources\Database\Attribute\ObjectType as AttributeObjectType; +use Utopia\Migration\Resources\Database\Attribute\Point as AttributePoint; +use Utopia\Migration\Resources\Database\Attribute\Polygon as AttributePolygon; +use Utopia\Migration\Resources\Database\Attribute\Relationship as AttributeRelationship; +use Utopia\Migration\Resources\Database\Attribute\Text as AttributeText; +use Utopia\Migration\Resources\Database\Attribute\URL as AttributeURL; +use Utopia\Migration\Resources\Database\Attribute\Vector as AttributeVector; use Utopia\Migration\Resources\Database\Collection; use Utopia\Migration\Resources\Database\Column; use Utopia\Migration\Resources\Database\Columns\Boolean; @@ -28,9 +44,9 @@ use Utopia\Migration\Resources\Database\Columns\Integer; use Utopia\Migration\Resources\Database\Columns\IP; use Utopia\Migration\Resources\Database\Columns\Line; +use Utopia\Migration\Resources\Database\Columns\ObjectType; use Utopia\Migration\Resources\Database\Columns\Point; use Utopia\Migration\Resources\Database\Columns\Polygon; -use Utopia\Migration\Resources\Database\Columns\ObjectType; use Utopia\Migration\Resources\Database\Columns\Relationship; use Utopia\Migration\Resources\Database\Columns\Text; use Utopia\Migration\Resources\Database\Columns\URL; @@ -801,7 +817,7 @@ private function exportEntities(string $databaseName, int $batchSize): void * @param int $batchSize * @throws Exception */ - private function exportFields(string $entityType,int $batchSize): void + private function exportFields(string $entityType, int $batchSize): void { $entities = $this->cache->get($entityType); // Transfer Indexes @@ -828,177 +844,11 @@ private function exportFields(string $entityType,int $batchSize): void continue; } - switch ($column['type']) { - case Column::TYPE_STRING: - $col = match ($column['format'] ?? '') { - Column::TYPE_EMAIL => new Email( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 254, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - Column::TYPE_ENUM => new Enum( - $column['key'], - $table, - elements: $column['elements'], - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? UtopiaDatabase::LENGTH_KEY, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - Column::TYPE_URL => new URL( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 2000, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - Column::TYPE_IP => new IP( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 39, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - default => new Text( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - size: $column['size'] ?? 0, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ), - }; - - break; - case Column::TYPE_BOOLEAN: - $col = new Boolean( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_INTEGER: - $col = new Integer( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - min: $column['min'] ?? null, - max: $column['max'] ?? null, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_FLOAT: - $col = new Decimal( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - min: $column['min'] ?? null, - max: $column['max'] ?? null, - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_RELATIONSHIP: - $col = new Relationship( - $column['key'], - $table, - relatedTable: $column['relatedTable'] ?? $column['relatedCollection'], - relationType: $column['relationType'], - twoWay: $column['twoWay'], - twoWayKey: $column['twoWayKey'], - onDelete: $column['onDelete'], - side: $column['side'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_DATETIME: - $col = new DateTime( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - array: $column['array'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_POINT: - $col = new Point( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_LINE: - $col = new Line( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_POLYGON: - $col = new Polygon( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_OBJECT: - $col = new ObjectType( - $column['key'], - $table, - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - case Column::TYPE_VECTOR: - $col = new Vector( - $column['key'], - $table, - size:$column['size'], - required: $column['required'], - default: $column['default'], - createdAt: $column['$createdAt'] ?? '', - updatedAt: $column['$updatedAt'] ?? '', - ); - break; - } + /** @var Table $table */ + $col = match($table->getDatabase()->getType()) { + Resource::TYPE_DATABASE_VECTORDB => self::getAttribute($table, $column), + default => self::getColumn($table, $column), + }; if (!isset($col)) { throw new Exception( @@ -1081,10 +931,11 @@ private function exportIndexes(string $entityType, int $batchSize): void /** * @param string $entityName + * @param string $fieldName * @param int $batchSize * @throws Exception */ - private function exportRecords(string $entityName, int $batchSize): void + private function exportRecords(string $entityName, string $fieldName, int $batchSize): void { $entities = $this->cache->get($entityName); foreach ($entities as $table) { @@ -1107,12 +958,16 @@ private function exportRecords(string $entityName, int $batchSize): void $manyToMany = []; if ($this->reader->getSupportForAttributes()) { - $attributes = $this->cache->get(Column::getName()); + $attributes = $this->cache->get($fieldName); + $relationship = match($fieldName) { + Attribute::getName() => AttributeRelationship::getName(), + default => Column::getName() + }; foreach ($attributes as $attribute) { /** @var Relationship $attribute */ if ( $attribute->getTable()->getId() === $table->getId() && - $attribute->getType() === Column::TYPE_RELATIONSHIP && + $attribute->getType() === $relationship && $attribute->getSide() === 'parent' && $attribute->getRelationType() == 'manyToMany' ) { @@ -1627,7 +1482,7 @@ private function exportTablesDB(int $batchSize, array $resources) try { if (\in_array(Resource::TYPE_COLUMN, $resources)) { - $this->exportFields(Table::getName(),$batchSize); + $this->exportFields(Table::getName(), $batchSize); } } catch (\Throwable $e) { $this->addError( @@ -1663,7 +1518,7 @@ private function exportTablesDB(int $batchSize, array $resources) try { if (\in_array(Resource::TYPE_ROW, $resources)) { - $this->exportRecords(Table::getName(), $batchSize); + $this->exportRecords(Table::getName(), Column::getName(), $batchSize); } } catch (\Throwable $e) { $this->addError( @@ -1726,7 +1581,7 @@ private function exportDocumentsDB(int $batchSize, array $resources): void try { if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { - $this->exportRecords(Collection::getName(), $batchSize); + $this->exportRecords(Collection::getName(), Attribute::getName(), $batchSize); } } catch (\Throwable $e) { $this->addError( @@ -1771,7 +1626,7 @@ private function exportVectorDB(int $batchSize, array $resources): void try { if (\in_array(Resource::TYPE_ATTRIBUTE, $resources)) { - $this->exportFields(Collection::getName(),$batchSize); + $this->exportFields(Collection::getName(), $batchSize); } } catch (\Throwable $e) { $this->addError( @@ -1807,7 +1662,7 @@ private function exportVectorDB(int $batchSize, array $resources): void try { if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { - $this->exportRecords(Collection::getName(), $batchSize); + $this->exportRecords(Collection::getName(), '', $batchSize); } } catch (\Throwable $e) { $this->addError( @@ -1918,4 +1773,332 @@ private static function getRecord(string $databaseType, array $record): Resource return Row::fromArray($record); } } + private static function getColumn(Table $table, mixed $column): Column + { + return match ($column['type']) { + Column::TYPE_STRING => match ($column['format'] ?? '') { + Column::TYPE_EMAIL => new Email( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 254, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + Column::TYPE_ENUM => new Enum( + $column['key'], + $table, + elements: $column['elements'], + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? UtopiaDatabase::LENGTH_KEY, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + Column::TYPE_URL => new URL( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 2000, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + Column::TYPE_IP => new IP( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 39, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + default => new Text( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + size: $column['size'] ?? 0, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + }, + + Column::TYPE_BOOLEAN => new Boolean( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_INTEGER => new Integer( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + min: $column['min'] ?? null, + max: $column['max'] ?? null, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_FLOAT => new Decimal( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + min: $column['min'] ?? null, + max: $column['max'] ?? null, + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_RELATIONSHIP => new Relationship( + $column['key'], + $table, + relatedTable: $column['relatedTable'] ?? $column['relatedCollection'], + relationType: $column['relationType'], + twoWay: $column['twoWay'], + twoWayKey: $column['twoWayKey'], + onDelete: $column['onDelete'], + side: $column['side'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_DATETIME => new DateTime( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + array: $column['array'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_POINT => new Point( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_LINE => new Line( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_POLYGON => new Polygon( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_OBJECT => new ObjectType( + $column['key'], + $table, + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + Column::TYPE_VECTOR => new Vector( + $column['key'], + $table, + size: $column['size'], + required: $column['required'], + default: $column['default'], + createdAt: $column['$createdAt'] ?? '', + updatedAt: $column['$updatedAt'] ?? '', + ), + + default => throw new \InvalidArgumentException("Unsupported column type: {$column['type']}"), + }; + + } + + private static function getAttribute(Collection $collection, mixed $attribute): Attribute + { + return match ($attribute['type']) { + Attribute::TYPE_STRING => match ($attribute['format'] ?? '') { + Attribute::TYPE_EMAIL => new AttributeEmail( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 254, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_ENUM => new AttributeEnum( + $attribute['key'], + $collection, + elements: $attribute['elements'], + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? UtopiaDatabase::LENGTH_KEY, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_URL => new AttributeURL( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 2000, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_IP => new AttributeIP( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 39, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + default => new AttributeText( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 0, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + }, + + Attribute::TYPE_BOOLEAN => new AttributeBoolean( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_INTEGER => new AttributeInteger( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + min: $attribute['min'] ?? null, + max: $attribute['max'] ?? null, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_FLOAT => new AttributeDecimal( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + min: $attribute['min'] ?? null, + max: $attribute['max'] ?? null, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_RELATIONSHIP => new AttributeRelationship( + $attribute['key'], + $collection, + relatedTable: $attribute['relatedTable'] ?? $attribute['relatedCollection'], + relationType: $attribute['relationType'], + twoWay: $attribute['twoWay'], + twoWayKey: $attribute['twoWayKey'], + onDelete: $attribute['onDelete'], + side: $attribute['side'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_DATETIME => new AttributeDateTime( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_POINT => new AttributePoint( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_LINE => new AttributeLine( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_POLYGON => new AttributePolygon( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_OBJECT => new AttributeObjectType( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + Attribute::TYPE_VECTOR => new AttributeVector( + $attribute['key'], + $collection, + size: $attribute['size'], + required: $attribute['required'], + default: $attribute['default'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + + default => throw new \InvalidArgumentException("Unsupported attribute type: {$attribute['type']}"), + }; + } } From c9e1fdc7c76e45f24d69b3b7be14b7267c281f52 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 12 Nov 2025 19:14:09 +0530 Subject: [PATCH 20/28] * updated cache key resolution for collections * updated resource exporting way and removed redundancy in code --- src/Migration/Cache.php | 1 + src/Migration/Resource.php | 16 +- src/Migration/Sources/Appwrite.php | 300 +++++------------- .../Sources/Appwrite/Reader/Database.php | 5 +- 4 files changed, 93 insertions(+), 229 deletions(-) diff --git a/src/Migration/Cache.php b/src/Migration/Cache.php index a88d1c1..0e3d1e0 100644 --- a/src/Migration/Cache.php +++ b/src/Migration/Cache.php @@ -45,6 +45,7 @@ public function resolveResourceCacheKey(Resource $resource): string case Resource::TYPE_TABLE: case Resource::TYPE_COLLECTION: /** @var Table $resource */ + $keys[] = $resource->getDatabase()->getType(); $keys[] = $resource->getDatabase()->getSequence(); break; diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index ef184fb..5a6bc74 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -105,16 +105,28 @@ abstract class Resource implements \JsonSerializable ], self::TYPE_DATABASE_DOCUMENTSDB => [ 'entity' => self::TYPE_COLLECTION, - 'record' => self::TYPE_DOCUMENT, + // HACK: not required in documentsdb but adding it for consistency in the db reader(not gonna impact) 'field' => self::TYPE_ATTRIBUTE, + 'record' => self::TYPE_DOCUMENT, ], self::TYPE_DATABASE_VECTORDB => [ 'entity' => self::TYPE_COLLECTION, - 'record' => self::TYPE_DOCUMENT, 'field' => self::TYPE_ATTRIBUTE, + 'record' => self::TYPE_DOCUMENT, ] ]; + public const ENTITY_TYPE_RESOURCE_MAP = [ + self::TYPE_TABLE => [ + 'field' => self::TYPE_COLUMN, + 'record' => self::TYPE_ROW, + ], + self::TYPE_COLLECTION => [ + 'field' => self::TYPE_ATTRIBUTE, + 'record' => self::TYPE_DOCUMENT, + ], + ]; + protected string $id = ''; protected string $originalId = ''; diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index b4bc143..d4d0026 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -654,17 +654,84 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void return; } - if (count($this->cache->get(Database::getName())) > 0) { - $this->exportTablesDB($batchSize, $resources); + foreach (Resource::DATABASE_TYPE_RESOURCE_MAP as $databaseKey => $databaseResource) { + try { + if (\in_array($databaseResource['entity'], $resources)) { + $this->exportEntities($databaseKey, $batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + $databaseResource['entity'], + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } } - if (count($this->cache->get(DocumentsDB::getName())) > 0) { - $this->exportDocumentsDB($batchSize, $resources); + foreach (Resource::ENTITY_TYPE_RESOURCE_MAP as $entityKey => $entityResource) { + try { + if (\in_array($entityResource['field'], $resources)) { + $this->exportFields($entityKey, $batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + $entityResource['field'], + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } + } + + foreach (Resource::ENTITY_TYPE_RESOURCE_MAP as $entityKey => $entityResource) { + try { + if (\in_array($entityResource['field'], $resources)) { + $this->exportIndexes($entityKey, $batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + Resource::TYPE_INDEX, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } } - // Export VectorDB databases using same flow as DocumentsDB - if (count($this->cache->get(VectorDB::getName())) > 0) { - $this->exportVectorDB($batchSize, $resources); + foreach (Resource::ENTITY_TYPE_RESOURCE_MAP as $entityKey => $entityResource) { + try { + if (\in_array($entityResource['record'], $resources)) { + $this->exportRecords($entityKey, $entityResource['field'], $batchSize); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + $entityResource['record'], + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return; + } } } @@ -1460,225 +1527,6 @@ private function exportDeploymentData(Func $func, array $deployment): void } } - private function exportTablesDB(int $batchSize, array $resources) - { - try { - if (\in_array(Resource::TYPE_TABLE, $resources)) { - $this->exportEntities(Database::getName(), $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_TABLE, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - - try { - if (\in_array(Resource::TYPE_COLUMN, $resources)) { - $this->exportFields(Table::getName(), $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_COLUMN, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - - try { - if (\in_array(Resource::TYPE_INDEX, $resources)) { - $this->exportIndexes(Table::getName(), $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_INDEX, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - - try { - if (\in_array(Resource::TYPE_ROW, $resources)) { - $this->exportRecords(Table::getName(), Column::getName(), $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_ROW, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - } - - /** - * Export DocumentsDB databases (collections and documents) - * - * @param int $batchSize - * @param array $resources - */ - private function exportDocumentsDB(int $batchSize, array $resources): void - { - try { - if (\in_array(Resource::TYPE_COLLECTION, $resources)) { - $this->exportEntities(DocumentsDB::getName(), $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_COLLECTION, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - - try { - if (\in_array(Resource::TYPE_INDEX, $resources)) { - $this->exportIndexes(Collection::getName(), $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_INDEX, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - - try { - if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { - $this->exportRecords(Collection::getName(), Attribute::getName(), $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_DOCUMENT, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - } - - /** - * Export VectorDB databases (collections and documents) - * - * @param int $batchSize - * @param array $resources - */ - private function exportVectorDB(int $batchSize, array $resources): void - { - try { - if (\in_array(Resource::TYPE_COLLECTION, $resources)) { - $this->exportEntities(VectorDB::getName(), $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_COLLECTION, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - - try { - if (\in_array(Resource::TYPE_ATTRIBUTE, $resources)) { - $this->exportFields(Collection::getName(), $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_ATTRIBUTE, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - - try { - if (\in_array(Resource::TYPE_INDEX, $resources)) { - $this->exportIndexes(Collection::getName(), $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_INDEX, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - - try { - if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { - $this->exportRecords(Collection::getName(), '', $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_DOCUMENT, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } - } - /** * @param string $databaseType * @param array $database { diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index 45524fd..edc0315 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -139,7 +139,10 @@ public function report(array $resources, array &$report): mixed Query::equal('collectionInternalId', [$tableSequence]), ]; - if (Resource::isSupported($databaseSpecificResources['field'], $resources)) { + if ( + isset($databaseSpecificResources['field']) && + Resource::isSupported($databaseSpecificResources['field'], $resources) + ) { $count = $this->countResources('attributes', $commonQueries); $report[$databaseSpecificResources['field']] += $count; } From 07a988bc7af79978279748dbad5cc9d65e08ca63 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 13 Nov 2025 16:40:13 +0530 Subject: [PATCH 21/28] * updated index creation * removed redundancy in exporting of entity driven resources --- src/Migration/Destinations/Appwrite.php | 187 ++++++++++++------------ src/Migration/Resource.php | 2 + src/Migration/Sources/Appwrite.php | 86 ++++------- 3 files changed, 129 insertions(+), 146 deletions(-) diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index 064d16f..1796741 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -12,6 +12,7 @@ use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; +use Dom\Document; use Override; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Document as UtopiaDocument; @@ -769,101 +770,11 @@ protected function createIndex(Index $resource): bool ); } - /** - * @var array $tableColumns - */ - $tableColumns = $table->getAttribute('attributes', []); - - /** - * @var array $tableIndexes - */ - $tableIndexes = $table->getAttribute('indexes', []); - - $oldColumns = \array_map( - fn ($attr) => $attr->getArrayCopy(), - $tableColumns - ); - - $oldColumns[] = [ - 'key' => '$id', - 'type' => UtopiaDatabase::VAR_STRING, - 'status' => 'available', - 'required' => true, - 'array' => false, - 'default' => null, - 'size' => UtopiaDatabase::LENGTH_KEY - ]; - - $oldColumns[] = [ - 'key' => '$createdAt', - 'type' => UtopiaDatabase::VAR_DATETIME, - 'status' => 'available', - 'signed' => false, - 'required' => false, - 'array' => false, - 'default' => null, - 'size' => 0 - ]; - - $oldColumns[] = [ - 'key' => '$updatedAt', - 'type' => UtopiaDatabase::VAR_DATETIME, - 'status' => 'available', - 'signed' => false, - 'required' => false, - 'array' => false, - 'default' => null, - 'size' => 0 - ]; - // Lengths hidden by default $lengths = []; - foreach ($resource->getColumns() as $i => $column) { - // find attribute metadata in collection document - $columnIndex = \array_search( - $column, - \array_column($oldColumns, 'key') - ); - - if ($columnIndex === false) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Column not found in table: ' . $column, - ); - } - - $columnStatus = $oldColumns[$columnIndex]['status']; - $columnType = $oldColumns[$columnIndex]['type']; - $columnSize = $oldColumns[$columnIndex]['size']; - $columnArray = $oldColumns[$columnIndex]['array'] ?? false; - - if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Relationship columns are not supported in indexes', - ); - } - - // Ensure attribute is available - if ($columnStatus !== 'available') { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Column not available: ' . $column, - ); - } - - $lengths[$i] = null; - - if ($columnArray === true) { - $lengths[$i] = UtopiaDatabase::MAX_ARRAY_INDEX_LENGTH; - } + if ($dbForDatabases->getAdapter()->getSupportForAttributes()) { + $this->validateFieldsForIndexes($resource, $table, $lengths); } $index = new UtopiaDocument([ @@ -1488,4 +1399,96 @@ private function importDeployment(Deployment $deployment): Resource return $deployment; } + + private function validateFieldsForIndexes(Index $resource, UtopiaDocument $table, array &$lengths) + { + /** + * @var array $tableColumns + */ + $tableColumns = $table->getAttribute('attributes', []); + + $oldColumns = \array_map( + fn ($attr) => $attr->getArrayCopy(), + $tableColumns + ); + + $oldColumns[] = [ + 'key' => '$id', + 'type' => UtopiaDatabase::VAR_STRING, + 'status' => 'available', + 'required' => true, + 'array' => false, + 'default' => null, + 'size' => UtopiaDatabase::LENGTH_KEY + ]; + + $oldColumns[] = [ + 'key' => '$createdAt', + 'type' => UtopiaDatabase::VAR_DATETIME, + 'status' => 'available', + 'signed' => false, + 'required' => false, + 'array' => false, + 'default' => null, + 'size' => 0 + ]; + + $oldColumns[] = [ + 'key' => '$updatedAt', + 'type' => UtopiaDatabase::VAR_DATETIME, + 'status' => 'available', + 'signed' => false, + 'required' => false, + 'array' => false, + 'default' => null, + 'size' => 0 + ]; + + foreach ($resource->getColumns() as $i => $column) { + // find attribute metadata in collection document + $columnIndex = \array_search( + $column, + \array_column($oldColumns, 'key') + ); + + if ($columnIndex === false) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Column not found in table: ' . $column, + ); + } + + $columnStatus = $oldColumns[$columnIndex]['status']; + $columnType = $oldColumns[$columnIndex]['type']; + $columnSize = $oldColumns[$columnIndex]['size']; + $columnArray = $oldColumns[$columnIndex]['array'] ?? false; + + if ($columnType === UtopiaDatabase::VAR_RELATIONSHIP) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Relationship columns are not supported in indexes', + ); + } + + // Ensure attribute is available + if ($columnStatus !== 'available') { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Column not available: ' . $column, + ); + } + + $lengths[$i] = null; + + if ($columnArray === true) { + $lengths[$i] = UtopiaDatabase::MAX_ARRAY_INDEX_LENGTH; + } + } + } } diff --git a/src/Migration/Resource.php b/src/Migration/Resource.php index 5a6bc74..c511854 100644 --- a/src/Migration/Resource.php +++ b/src/Migration/Resource.php @@ -120,10 +120,12 @@ abstract class Resource implements \JsonSerializable self::TYPE_TABLE => [ 'field' => self::TYPE_COLUMN, 'record' => self::TYPE_ROW, + 'index' => self::TYPE_INDEX ], self::TYPE_COLLECTION => [ 'field' => self::TYPE_ATTRIBUTE, 'record' => self::TYPE_DOCUMENT, + 'index' => self::TYPE_INDEX ], ]; diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index d4d0026..5b2ca68 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -636,6 +636,29 @@ private function exportMemberships(int $batchSize): void protected function exportGroupDatabases(int $batchSize, array $resources): void { + $handleExportEntityScopedResources = function (string $resourceKey, callable $callback) use ($resources) { + foreach (Resource::ENTITY_TYPE_RESOURCE_MAP as $entityKey => $entityResource) { + try { + if (\in_array($entityResource[$resourceKey], $resources)) { + $callback($entityKey, $entityResource); + } + } catch (\Throwable $e) { + $this->addError( + new Exception( + $resourceKey, + Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ) + ); + + return false; + } + } + return true; + }; + try { if (Resource::isSupported(array_keys(Resource::DATABASE_TYPE_RESOURCE_MAP), $resources)) { $this->exportDatabases($batchSize, $resources); @@ -674,64 +697,19 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void } } - foreach (Resource::ENTITY_TYPE_RESOURCE_MAP as $entityKey => $entityResource) { - try { - if (\in_array($entityResource['field'], $resources)) { - $this->exportFields($entityKey, $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - $entityResource['field'], - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } + // field + if (!$handleExportEntityScopedResources('field', fn ($entityKey) => $this->exportFields($entityKey, $batchSize))) { + return; } - foreach (Resource::ENTITY_TYPE_RESOURCE_MAP as $entityKey => $entityResource) { - try { - if (\in_array($entityResource['field'], $resources)) { - $this->exportIndexes($entityKey, $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - Resource::TYPE_INDEX, - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } + // index + if (!$handleExportEntityScopedResources('index', fn ($entityKey) => $this->exportIndexes($entityKey, $batchSize))) { + return; } - foreach (Resource::ENTITY_TYPE_RESOURCE_MAP as $entityKey => $entityResource) { - try { - if (\in_array($entityResource['record'], $resources)) { - $this->exportRecords($entityKey, $entityResource['field'], $batchSize); - } - } catch (\Throwable $e) { - $this->addError( - new Exception( - $entityResource['record'], - Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ) - ); - - return; - } + // record + if (!$handleExportEntityScopedResources('record', fn ($entityKey, $entityResource) => $this->exportRecords($entityKey, $entityResource['field'], $batchSize))) { + return; } } From ea34cf073517732cc9037201b9ddec3ef8ca12dd Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 13 Nov 2025 16:40:20 +0530 Subject: [PATCH 22/28] linting --- src/Migration/Resources/Database/Attribute/Vector.php | 2 +- src/Migration/Resources/Database/Collection.php | 1 + src/Migration/Resources/Database/Columns/Vector.php | 2 +- src/Migration/Sources/Appwrite.php | 11 +---------- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/Migration/Resources/Database/Attribute/Vector.php b/src/Migration/Resources/Database/Attribute/Vector.php index c485684..f4dc166 100644 --- a/src/Migration/Resources/Database/Attribute/Vector.php +++ b/src/Migration/Resources/Database/Attribute/Vector.php @@ -11,7 +11,7 @@ public function __construct( string $key, Collection $collection, bool $required = false, - int $size, + int $size = 0, ?array $default = null, string $createdAt = '', string $updatedAt = '' diff --git a/src/Migration/Resources/Database/Collection.php b/src/Migration/Resources/Database/Collection.php index 117b31b..7969b93 100644 --- a/src/Migration/Resources/Database/Collection.php +++ b/src/Migration/Resources/Database/Collection.php @@ -16,6 +16,7 @@ public static function getName(): string * database: array{ * id: string, * name: string, + * type: string * }, * name: string, * id: string, diff --git a/src/Migration/Resources/Database/Columns/Vector.php b/src/Migration/Resources/Database/Columns/Vector.php index e7ae005..4cc898a 100644 --- a/src/Migration/Resources/Database/Columns/Vector.php +++ b/src/Migration/Resources/Database/Columns/Vector.php @@ -11,7 +11,7 @@ public function __construct( string $key, Table $table, bool $required = false, - int $size, + int $size = 0, ?array $default = null, string $createdAt = '', string $updatedAt = '' diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 5b2ca68..3fac4a1 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -867,7 +867,7 @@ private function exportFields(string $entityType, int $batchSize): void $entities = $this->cache->get($entityType); // Transfer Indexes - /** @var array
$tables */ + /** @var array $table */ foreach ($entities as $table) { $lastColumn = null; @@ -895,15 +895,6 @@ private function exportFields(string $entityType, int $batchSize): void default => self::getColumn($table, $column), }; - if (!isset($col)) { - throw new Exception( - resourceName: Resource::TYPE_COLUMN, - resourceGroup: Transfer::GROUP_DATABASES, - resourceId: $column['$id'], - message: 'Unknown column type: ' . $column['type'] - ); - } - $columns[] = $col; } From f5e175fa8da4bee6211dbeb4fe693a3eedb87672 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 13 Nov 2025 16:41:30 +0530 Subject: [PATCH 23/28] updated composer --- composer.json | 2 +- composer.lock | 75 +++++++++++++++++++++++++-------------------------- 2 files changed, 37 insertions(+), 40 deletions(-) diff --git a/composer.json b/composer.json index fb941b7..3b974e0 100644 --- a/composer.json +++ b/composer.json @@ -26,7 +26,7 @@ "ext-curl": "*", "ext-openssl": "*", "appwrite/appwrite": "15.*", - "utopia-php/database": "4.*", + "utopia-php/database": "3.*", "utopia-php/storage": "0.18.*", "utopia-php/dsn": "0.2.*", "utopia-php/console": "0.0.*" diff --git a/composer.lock b/composer.lock index 656fae2..dd58985 100644 --- a/composer.lock +++ b/composer.lock @@ -187,16 +187,16 @@ }, { "name": "google/protobuf", - "version": "v4.33.0", + "version": "v4.33.1", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "b50269e23204e5ae859a326ec3d90f09efe3047d" + "reference": "0cd73ccf0cd26c3e72299cce1ea6144091a57e12" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/b50269e23204e5ae859a326ec3d90f09efe3047d", - "reference": "b50269e23204e5ae859a326ec3d90f09efe3047d", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/0cd73ccf0cd26c3e72299cce1ea6144091a57e12", + "reference": "0cd73ccf0cd26c3e72299cce1ea6144091a57e12", "shasum": "" }, "require": { @@ -225,9 +225,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.0" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.33.1" }, - "time": "2025-10-15T20:10:28+00:00" + "time": "2025-11-12T21:58:05+00:00" }, { "name": "mongodb/mongodb", @@ -1425,16 +1425,16 @@ }, { "name": "symfony/http-client", - "version": "v7.3.4", + "version": "v7.3.6", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62" + "reference": "3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/4b62871a01c49457cf2a8e560af7ee8a94b87a62", - "reference": "4b62871a01c49457cf2a8e560af7ee8a94b87a62", + "url": "https://api.github.com/repos/symfony/http-client/zipball/3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de", + "reference": "3c0a55a2c8e21e30a37022801c11c7ab5a6cb2de", "shasum": "" }, "require": { @@ -1501,7 +1501,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.3.4" + "source": "https://github.com/symfony/http-client/tree/v7.3.6" }, "funding": [ { @@ -1521,7 +1521,7 @@ "type": "tidelift" } ], - "time": "2025-09-11T10:12:26+00:00" + "time": "2025-11-05T17:41:46+00:00" }, { "name": "symfony/http-client-contracts", @@ -1928,16 +1928,16 @@ }, { "name": "symfony/service-contracts", - "version": "v3.6.0", + "version": "v3.6.1", "source": { "type": "git", "url": "https://github.com/symfony/service-contracts.git", - "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4" + "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/service-contracts/zipball/f021b05a130d35510bd6b25fe9053c2a8a15d5d4", - "reference": "f021b05a130d35510bd6b25fe9053c2a8a15d5d4", + "url": "https://api.github.com/repos/symfony/service-contracts/zipball/45112560a3ba2d715666a509a0bc9521d10b6c43", + "reference": "45112560a3ba2d715666a509a0bc9521d10b6c43", "shasum": "" }, "require": { @@ -1991,7 +1991,7 @@ "standards" ], "support": { - "source": "https://github.com/symfony/service-contracts/tree/v3.6.0" + "source": "https://github.com/symfony/service-contracts/tree/v3.6.1" }, "funding": [ { @@ -2002,12 +2002,16 @@ "url": "https://github.com/fabpot", "type": "github" }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, { "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", "type": "tidelift" } ], - "time": "2025-04-25T09:37:31+00:00" + "time": "2025-07-15T11:30:57+00:00" }, { "name": "tbachert/spi", @@ -2209,16 +2213,16 @@ }, { "name": "utopia-php/database", - "version": "3.0.2", + "version": "3.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "b92554e2e7b3b00f0f0acb2b53c6a11e1349b81e" + "reference": "e10b4faa4f3a3ef30a5f6d76acdb605469924aec" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/b92554e2e7b3b00f0f0acb2b53c6a11e1349b81e", - "reference": "b92554e2e7b3b00f0f0acb2b53c6a11e1349b81e", + "url": "https://api.github.com/repos/utopia-php/database/zipball/e10b4faa4f3a3ef30a5f6d76acdb605469924aec", + "reference": "e10b4faa4f3a3ef30a5f6d76acdb605469924aec", "shasum": "" }, "require": { @@ -2261,9 +2265,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/3.0.2" + "source": "https://github.com/utopia-php/database/tree/3.4.0" }, - "time": "2025-10-20T23:58:56+00:00" + "time": "2025-11-13T06:34:20+00:00" }, { "name": "utopia-php/dsn", @@ -3467,16 +3471,16 @@ }, { "name": "phpunit/phpunit", - "version": "11.5.43", + "version": "11.5.44", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924" + "reference": "c346885c95423eda3f65d85a194aaa24873cda82" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924", - "reference": "c6b89b6cf4324a8b4cb86e1f5dfdd6c9e0371924", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/c346885c95423eda3f65d85a194aaa24873cda82", + "reference": "c346885c95423eda3f65d85a194aaa24873cda82", "shasum": "" }, "require": { @@ -3548,7 +3552,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.43" + "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.44" }, "funding": [ { @@ -3572,7 +3576,7 @@ "type": "tidelift" } ], - "time": "2025-10-30T08:39:39+00:00" + "time": "2025-11-13T07:17:35+00:00" }, { "name": "sebastian/cli-parser", @@ -4914,14 +4918,7 @@ "time": "2025-04-30T23:37:27+00:00" } ], - "aliases": [ - { - "package": "utopia-php/database", - "version": "dev-var_object", - "alias": "1.4.5", - "alias_normalized": "1.4.5.0" - } - ], + "aliases": [], "minimum-stability": "stable", "stability-flags": {}, "prefer-stable": false, @@ -4934,5 +4931,5 @@ "platform-dev": { "ext-pdo": "*" }, - "plugin-api-version": "2.2.0" + "plugin-api-version": "2.6.0" } From 105b48dcf2bc8a1a969497813f3d7d06231770e6 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 19 Nov 2025 18:17:12 +0530 Subject: [PATCH 24/28] * Added new transfer group for easier extraction of services * fixed many to many relationship bug --- src/Migration/Sources/Appwrite.php | 16 ++++++------- src/Migration/Transfer.php | 37 +++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 3fac4a1..943d018 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -995,22 +995,18 @@ private function exportRecords(string $entityName, string $fieldName, int $batch if ($this->reader->getSupportForAttributes()) { $attributes = $this->cache->get($fieldName); - $relationship = match($fieldName) { - Attribute::getName() => AttributeRelationship::getName(), - default => Column::getName() - }; + foreach ($attributes as $attribute) { /** @var Relationship $attribute */ if ( $attribute->getTable()->getId() === $table->getId() && - $attribute->getType() === $relationship && + $attribute->getType() === Column::TYPE_RELATIONSHIP && $attribute->getSide() === 'parent' && $attribute->getRelationType() == 'manyToMany' ) { $manyToMany[] = $attribute->getKey(); } } - /** @var Column|Relationship $attribute */ } $queries[] = $this->reader->querySelect($selects); @@ -1033,8 +1029,12 @@ private function exportRecords(string $entityName, string $fieldName, int $batch foreach ($manyToMany as $key) { $row[$key] = []; - foreach ($rowItem[$key] as $relatedRowItem) { - $row[$key][] = $relatedRowItem['$id']; + if (isset($rowItem[$key]) && is_array($rowItem[$key])) { + foreach ($rowItem[$key] as $relatedRowItem) { + if (is_array($relatedRowItem) && isset($relatedRowItem['$id'])) { + $row[$key][] = $relatedRowItem['$id']; + } + } } } } diff --git a/src/Migration/Transfer.php b/src/Migration/Transfer.php index 0d0e228..2bcceef 100644 --- a/src/Migration/Transfer.php +++ b/src/Migration/Transfer.php @@ -12,7 +12,13 @@ class Transfer public const GROUP_FUNCTIONS = 'functions'; + // separating databases and tablesdb out for easier separation in extract services + // migration can use group_databases for mentioning all resources but when mentioning specific resources go with specific type databases public const GROUP_DATABASES = 'databases'; + public const GROUP_DATABASES_TABLES_DB = 'tablesdb'; + public const GROUP_DATABASES_DOCUMENTS_DB = 'documentsdb'; + + public const GROUP_DATABASES_VECTOR_DB = 'vectordb'; public const GROUP_SETTINGS = 'settings'; @@ -34,6 +40,29 @@ class Transfer Resource::TYPE_DEPLOYMENT ]; + public const GROUP_TABLESDB_RESOURCES = [ + Resource::TYPE_DATABASE, + Resource::TYPE_TABLE, + Resource::TYPE_INDEX, + Resource::TYPE_COLUMN, + Resource::TYPE_ROW, + ]; + + public const GROUP_DOCUMENTSDB_RESOURCES = [ + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_INDEX, + Resource::TYPE_DOCUMENT + ]; + + public const GROUP_VECTORDB_RESOURCES = [ + Resource::TYPE_DATABASE_VECTORDB, + Resource::TYPE_COLLECTION, + Resource::TYPE_ATTRIBUTE, + Resource::TYPE_INDEX, + Resource::TYPE_DOCUMENT + ]; + public const GROUP_DATABASES_RESOURCES = [ Resource::TYPE_DATABASE, Resource::TYPE_DATABASE_DOCUMENTSDB, @@ -73,6 +102,8 @@ class Transfer public const ROOT_RESOURCES = [ Resource::TYPE_BUCKET, Resource::TYPE_DATABASE, + Resource::TYPE_DATABASE_DOCUMENTSDB, + Resource::TYPE_DATABASE_VECTORDB, Resource::TYPE_FUNCTION, Resource::TYPE_USER, Resource::TYPE_TEAM, @@ -232,7 +263,7 @@ public function run( } if (!in_array($rootResourceType, self::ROOT_RESOURCES)) { - throw new \Exception('Resource type must be one of ' . implode(', ', self::ROOT_RESOURCES)); + throw new \Exception('Got '.$rootResourceType.' Resource type must be one of ' . implode(', ', self::ROOT_RESOURCES)); } $rootResources = \array_intersect($computedResources, self::ROOT_RESOURCES); @@ -326,6 +357,7 @@ public function getReport(string $statusLevel = ''): array public static function extractServices(array $services): array { $resources = []; + $groupDatabasesIndex = array_search(Transfer::GROUP_DATABASES, $services); foreach ($services as $service) { $resources = match ($service) { self::GROUP_FUNCTIONS => array_merge($resources, self::GROUP_FUNCTIONS_RESOURCES), @@ -333,6 +365,9 @@ public static function extractServices(array $services): array self::GROUP_GENERAL => array_merge($resources, []), self::GROUP_AUTH => array_merge($resources, self::GROUP_AUTH_RESOURCES), self::GROUP_DATABASES => array_merge($resources, self::GROUP_DATABASES_RESOURCES), + self::GROUP_DATABASES_TABLES_DB => array_merge($resources, self::GROUP_TABLESDB_RESOURCES), + self::GROUP_DATABASES_DOCUMENTS_DB => array_merge($resources, self::GROUP_DOCUMENTSDB_RESOURCES), + self::GROUP_DATABASES_VECTOR_DB => array_merge($resources, self::GROUP_VECTORDB_RESOURCES), self::GROUP_SETTINGS => array_merge($resources, self::GROUP_SETTINGS_RESOURCES), default => throw new \Exception('No service group found'), }; From bcd7bcb634dea60485c16648e68ed7a255315560 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Thu, 20 Nov 2025 18:34:52 +0530 Subject: [PATCH 25/28] updated csv source structure for importing and exporting vectordb and connecting via databases dsn --- src/Migration/Sources/Appwrite.php | 10 ++-- src/Migration/Sources/CSV.php | 73 +++++++++++++++++++++++------- 2 files changed, 62 insertions(+), 21 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 943d018..26b079c 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -1508,7 +1508,7 @@ private function exportDeploymentData(Func $func, array $deployment): void * database: string * } */ - private static function getDatabase(string $databaseType, array $database): Resource + public static function getDatabase(string $databaseType, array $database): Resource { switch ($databaseType) { case Resource::TYPE_DATABASE_DOCUMENTSDB: @@ -1538,7 +1538,7 @@ private static function getDatabase(string $databaseType, array $database): Reso * enabled: bool * } $entity */ - private static function getEntity(string $databaseType, array $entity): Resource + public static function getEntity(string $databaseType, array $entity): Resource { switch ($databaseType) { case Resource::TYPE_DATABASE_DOCUMENTSDB: @@ -1579,7 +1579,7 @@ private static function getEntity(string $databaseType, array $entity): Resource * permissions: ?array * } $record */ - private static function getRecord(string $databaseType, array $record): Resource + public static function getRecord(string $databaseType, array $record): Resource { switch ($databaseType) { case Resource::TYPE_DATABASE_DOCUMENTSDB: @@ -1590,7 +1590,7 @@ private static function getRecord(string $databaseType, array $record): Resource return Row::fromArray($record); } } - private static function getColumn(Table $table, mixed $column): Column + public static function getColumn(Table $table, mixed $column): Column { return match ($column['type']) { Column::TYPE_STRING => match ($column['format'] ?? '') { @@ -1755,7 +1755,7 @@ private static function getColumn(Table $table, mixed $column): Column } - private static function getAttribute(Collection $collection, mixed $attribute): Attribute + public static function getAttribute(Collection $collection, mixed $attribute): Attribute { return match ($attribute['type']) { Attribute::TYPE_STRING => match ($attribute['format'] ?? '') { diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index a6ca0c2..c86234b 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -5,11 +5,9 @@ use Utopia\Console; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Migration\Exception; +use Utopia\Migration\Resource; use Utopia\Migration\Resource as UtopiaResource; use Utopia\Migration\Resources\Database\Column; -use Utopia\Migration\Resources\Database\Database; -use Utopia\Migration\Resources\Database\Row; -use Utopia\Migration\Resources\Database\Table; use Utopia\Migration\Resources\Storage\File; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite\Reader; @@ -44,12 +42,13 @@ public function __construct( string $resourceId, string $filePath, Device $device, - ?UtopiaDatabase $dbForProject + ?UtopiaDatabase $dbForProject, + ?callable $getDatabasesDB = null, ) { $this->device = $device; $this->filePath = $filePath; $this->resourceId = $resourceId; - $this->database = new DatabaseReader($dbForProject); + $this->database = new DatabaseReader($dbForProject, $getDatabasesDB); } public static function getName(): string @@ -61,6 +60,7 @@ public static function getSupportedResources(): array { return [ UtopiaResource::TYPE_ROW, + UtopiaResource::TYPE_DOCUMENT, ]; } @@ -104,7 +104,7 @@ protected function exportGroupAuth(int $batchSize, array $resources): void protected function exportGroupDatabases(int $batchSize, array $resources): void { try { - if (UtopiaResource::isSupported(UtopiaResource::TYPE_ROW, $resources)) { + if (UtopiaResource::isSupported($this->getSupportedResources(), $resources)) { $this->exportRows($batchSize); } } catch (\Throwable $e) { @@ -132,8 +132,47 @@ private function exportRows(int $batchSize): void $lastColumn = null; [$databaseId, $tableId] = \explode(':', $this->resourceId); - $database = new Database($databaseId, ''); - $table = new Table($database, '', $tableId); + + $databases = $this->database->listDatabases([ + $this->database->queryEqual('$id', [$databaseId]), + $this->database->queryLimit(1), + ]); + + if (empty($databases)) { + throw new \Exception('Database not found'); + } + + $databaseDocument = $databases[0]; + $databaseType = $databaseDocument->getAttribute('type', UtopiaResource::TYPE_DATABASE); + if (\in_array($databaseType, [UtopiaResource::TYPE_DATABASE_LEGACY, UtopiaResource::TYPE_DATABASE_TABLESDB], true)) { + $databaseType = UtopiaResource::TYPE_DATABASE; + } + + $databasePayload = [ + 'id' => $databaseDocument->getId(), + 'name' => $databaseDocument->getAttribute('name', $databaseDocument->getId()), + 'originalId' => $databaseDocument->getAttribute('originalId', ''), + 'type' => $databaseType, + 'database' => $databaseDocument->getAttribute('database', ''), + ]; + + $tablePayload = [ + 'id' => $tableId, + 'name' => $tableId, + 'documentSecurity' => false, + 'rowSecurity' => false, + 'permissions' => [], + 'createdAt' => '', + 'updatedAt' => '', + 'database' => [ + 'id' => $databasePayload['id'], + 'name' => $databasePayload['name'], + 'type' => $databasePayload['type'], + 'database' => $databasePayload['database'], + ], + ]; + + $table = Appwrite::getEntity($databaseType, $tablePayload); while (true) { $queries = [$this->database->queryLimit($batchSize)]; @@ -195,7 +234,7 @@ private function exportRows(int $batchSize): void } } - $this->withCsvStream(function ($stream, $delimiter) use ($columnTypes, $requiredColumns, $manyToManyKeys, $arrayKeys, $table, $batchSize) { + $this->withCsvStream(function ($stream, $delimiter) use ($columnTypes, $databaseType, $requiredColumns, $manyToManyKeys, $arrayKeys, $tablePayload, $batchSize) { $headers = \fgetcsv($stream, 0, $delimiter, '"', '"'); if (!\is_array($headers) || \count($headers) === 0) { @@ -300,7 +339,9 @@ private function exportRows(int $batchSize): void ), Column::TYPE_POINT, Column::TYPE_LINE, - Column::TYPE_POLYGON => \is_string($parsedValue) ? json_decode($parsedValue) : null, + Column::TYPE_POLYGON, + Column::TYPE_VECTOR, + Column::TYPE_OBJECT => \is_string($parsedValue) ? json_decode($parsedValue, true) : null, default => $parsedValue, }, }; @@ -311,12 +352,12 @@ private function exportRows(int $batchSize): void unset($parsedData['$id'], $parsedData['$permissions']); - $row = new Row( - $rowId, - $table, - $parsedData, - $permissions, - ); + $row = Appwrite::getRecord($databaseType, [ + 'id' => $rowId, + 'table' => $tablePayload, + 'data' => $parsedData, + 'permissions' => $permissions + ]); $buffer[] = $row; From c8d18d10cb6a7369a66fd2e43f64b67761f3ed77 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Nov 2025 17:26:02 +1300 Subject: [PATCH 26/28] Update src/Migration/Sources/Appwrite.php --- src/Migration/Sources/Appwrite.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 26b079c..15cd3a1 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -722,8 +722,6 @@ private function exportDatabases(int $batchSize, array $resources = []): void { $lastDatabase = null; - // Create a reader for listing databases (not database-specific) - while (true) { $queries = [$this->reader->queryLimit($batchSize)]; From 0afcc54ac5c34248ddd6647a7954544c306452b1 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Nov 2025 17:26:10 +1300 Subject: [PATCH 27/28] Update src/Migration/Sources/Appwrite.php --- src/Migration/Sources/Appwrite.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 15cd3a1..33ec45f 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -1588,6 +1588,7 @@ public static function getRecord(string $databaseType, array $record): Resource return Row::fromArray($record); } } + public static function getColumn(Table $table, mixed $column): Column { return match ($column['type']) { From 2321c28bba304df81555f1ab26ae70248ec44e35 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Fri, 21 Nov 2025 17:44:42 +1300 Subject: [PATCH 28/28] Lint --- src/Migration/Sources/Appwrite.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 33ec45f..f467de2 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -1588,7 +1588,7 @@ public static function getRecord(string $databaseType, array $record): Resource return Row::fromArray($record); } } - + public static function getColumn(Table $table, mixed $column): Column { return match ($column['type']) {