From ac67bdd69275a46fe236962165993ce87c5a0771 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 13 Mar 2025 12:49:20 +1300 Subject: [PATCH 01/38] Add source reads from DB for Appwrite --- bin/MigrationCLI.php | 32 +- composer.lock | 90 +-- src/Migration/Destinations/Appwrite.php | 2 + src/Migration/Sources/Appwrite.php | 700 +++++++++++-------- tests/Migration/Unit/Adapters/MockSource.php | 4 +- 5 files changed, 465 insertions(+), 363 deletions(-) diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index 04c80a5b..1b21d1d5 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -213,7 +213,8 @@ public function getSource(): Source return new Appwrite( $_ENV['SOURCE_APPWRITE_TEST_PROJECT'], $_ENV['SOURCE_APPWRITE_TEST_ENDPOINT'], - $_ENV['SOURCE_APPWRITE_TEST_KEY'] + $_ENV['SOURCE_APPWRITE_TEST_KEY'], + $this->getDatabase(), ); case 'supabase': return new Supabase( @@ -310,7 +311,7 @@ function (mixed $value) { }, function (mixed $value) { if (is_null($value)) { - return; + return null; } return json_decode($value, true)['value']; @@ -361,21 +362,18 @@ function (mixed $value, Document $attribute) { } ); - $database = new Database( - new MariaDB(new PDO( - $_ENV['DESTINATION_APPWRITE_TEST_DSN'], - $_ENV['DESTINATION_APPWRITE_TEST_USER'], - $_ENV['DESTINATION_APPWRITE_TEST_PASSWORD'], - [ - PDO::ATTR_TIMEOUT => 3, - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true - ], - )), - new Cache(new None()) - ); + $database = new Database(new MariaDB(new PDO( + $_ENV['DESTINATION_APPWRITE_TEST_DSN'], + $_ENV['DESTINATION_APPWRITE_TEST_USER'], + $_ENV['DESTINATION_APPWRITE_TEST_PASSWORD'], + [ + PDO::ATTR_TIMEOUT => 3, + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + ], + )), new Cache(new None())); $database ->setDatabase('appwrite') diff --git a/composer.lock b/composer.lock index a7ae9a3f..b172e0c7 100644 --- a/composer.lock +++ b/composer.lock @@ -636,16 +636,16 @@ }, { "name": "open-telemetry/exporter-otlp", - "version": "1.2.0", + "version": "1.2.1", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/exporter-otlp.git", - "reference": "243d9657c44a06f740cf384f486afe954c2b725f" + "reference": "b7580440b7481a98da97aceabeb46e1b276c8747" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/243d9657c44a06f740cf384f486afe954c2b725f", - "reference": "243d9657c44a06f740cf384f486afe954c2b725f", + "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/b7580440b7481a98da97aceabeb46e1b276c8747", + "reference": "b7580440b7481a98da97aceabeb46e1b276c8747", "shasum": "" }, "require": { @@ -696,7 +696,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-08T23:50:03+00:00" + "time": "2025-03-06T23:21:56+00:00" }, { "name": "open-telemetry/gen-otlp-protobuf", @@ -2231,16 +2231,16 @@ }, { "name": "utopia-php/framework", - "version": "0.33.17", + "version": "0.33.19", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644" + "reference": "64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/73fac6fbce9f56282dba4e52a58cf836ec434644", - "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644", + "url": "https://api.github.com/repos/utopia-php/http/zipball/64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0", + "reference": "64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0", "shasum": "" }, "require": { @@ -2272,9 +2272,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.17" + "source": "https://github.com/utopia-php/http/tree/0.33.19" }, - "time": "2025-02-24T17:35:48+00:00" + "time": "2025-03-06T11:37:49+00:00" }, { "name": "utopia-php/mongo", @@ -2338,22 +2338,24 @@ }, { "name": "utopia-php/storage", - "version": "0.18.9", + "version": "0.18.10", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c" + "reference": "76f31158f4251abb207f7a9b16f7cb0bfdb3b39e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/1cf455404e8700b3093fd73d74a38d41cdced90c", - "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/76f31158f4251abb207f7a9b16f7cb0bfdb3b39e", + "reference": "76f31158f4251abb207f7a9b16f7cb0bfdb3b39e", "shasum": "" }, "require": { "ext-brotli": "*", + "ext-curl": "*", "ext-fileinfo": "*", "ext-lz4": "*", + "ext-simplexml": "*", "ext-snappy": "*", "ext-xz": "*", "ext-zlib": "*", @@ -2387,9 +2389,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.18.9" + "source": "https://github.com/utopia-php/storage/tree/0.18.10" }, - "time": "2025-02-11T13:10:40+00:00" + "time": "2025-03-03T10:47:54+00:00" }, { "name": "utopia-php/system", @@ -2563,16 +2565,16 @@ }, { "name": "laravel/pint", - "version": "v1.21.0", + "version": "v1.21.1", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "531fa0871fbde719c51b12afa3a443b8f4e4b425" + "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/531fa0871fbde719c51b12afa3a443b8f4e4b425", - "reference": "531fa0871fbde719c51b12afa3a443b8f4e4b425", + "url": "https://api.github.com/repos/laravel/pint/zipball/c44bffbb2334e90fba560933c45948fa4a3f3e86", + "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86", "shasum": "" }, "require": { @@ -2583,9 +2585,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.68.5", - "illuminate/view": "^11.42.0", - "larastan/larastan": "^3.0.4", + "friendsofphp/php-cs-fixer": "^3.70.2", + "illuminate/view": "^11.44.1", + "larastan/larastan": "^3.1.0", "laravel-zero/framework": "^11.36.1", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3", @@ -2625,7 +2627,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-02-18T03:18:57+00:00" + "time": "2025-03-11T03:22:21+00:00" }, { "name": "myclabs/deep-copy", @@ -2940,16 +2942,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.12.20", + "version": "1.12.21", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "3240b1972042c7f73cf1045e879ea5bd5f761bb7" + "reference": "14276fdef70575106a3392a4ed553c06a984df28" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/3240b1972042c7f73cf1045e879ea5bd5f761bb7", - "reference": "3240b1972042c7f73cf1045e879ea5bd5f761bb7", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/14276fdef70575106a3392a4ed553c06a984df28", + "reference": "14276fdef70575106a3392a4ed553c06a984df28", "shasum": "" }, "require": { @@ -2994,7 +2996,7 @@ "type": "github" } ], - "time": "2025-03-05T13:37:43+00:00" + "time": "2025-03-09T09:24:50+00:00" }, { "name": "phpunit/php-code-coverage", @@ -3321,16 +3323,16 @@ }, { "name": "phpunit/phpunit", - "version": "11.5.11", + "version": "11.5.12", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "3946ac38410be7440186c6e74584f31b15107fc7" + "reference": "d42785840519401ed2113292263795eb4c0f95da" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/3946ac38410be7440186c6e74584f31b15107fc7", - "reference": "3946ac38410be7440186c6e74584f31b15107fc7", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/d42785840519401ed2113292263795eb4c0f95da", + "reference": "d42785840519401ed2113292263795eb4c0f95da", "shasum": "" }, "require": { @@ -3351,7 +3353,7 @@ "phpunit/php-timer": "^7.0.1", "sebastian/cli-parser": "^3.0.2", "sebastian/code-unit": "^3.0.2", - "sebastian/comparator": "^6.3.0", + "sebastian/comparator": "^6.3.1", "sebastian/diff": "^6.0.2", "sebastian/environment": "^7.2.0", "sebastian/exporter": "^6.3.0", @@ -3402,7 +3404,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.11" + "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.12" }, "funding": [ { @@ -3418,7 +3420,7 @@ "type": "tidelift" } ], - "time": "2025-03-05T07:36:02+00:00" + "time": "2025-03-07T07:31:03+00:00" }, { "name": "sebastian/cli-parser", @@ -3592,16 +3594,16 @@ }, { "name": "sebastian/comparator", - "version": "6.3.0", + "version": "6.3.1", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "d4e47a769525c4dd38cea90e5dcd435ddbbc7115" + "reference": "24b8fbc2c8e201bb1308e7b05148d6ab393b6959" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/d4e47a769525c4dd38cea90e5dcd435ddbbc7115", - "reference": "d4e47a769525c4dd38cea90e5dcd435ddbbc7115", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/24b8fbc2c8e201bb1308e7b05148d6ab393b6959", + "reference": "24b8fbc2c8e201bb1308e7b05148d6ab393b6959", "shasum": "" }, "require": { @@ -3620,7 +3622,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "6.2-dev" + "dev-main": "6.3-dev" } }, "autoload": { @@ -3660,7 +3662,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/comparator/issues", "security": "https://github.com/sebastianbergmann/comparator/security/policy", - "source": "https://github.com/sebastianbergmann/comparator/tree/6.3.0" + "source": "https://github.com/sebastianbergmann/comparator/tree/6.3.1" }, "funding": [ { @@ -3668,7 +3670,7 @@ "type": "github" } ], - "time": "2025-01-06T10:28:19+00:00" + "time": "2025-03-07T06:57:01+00:00" }, { "name": "sebastian/complexity", diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index e1b9eac2..1fd182e5 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -438,6 +438,7 @@ protected function createAttribute(Attribute $resource): bool 'databases', $resource->getCollection()->getDatabase()->getId(), ); + if ($database->isEmpty()) { throw new Exception( resourceName: $resource->getName(), @@ -451,6 +452,7 @@ protected function createAttribute(Attribute $resource): bool 'database_' . $database->getInternalId(), $resource->getCollection()->getId(), ); + if ($collection->isEmpty()) { throw new Exception( resourceName: $resource->getName(), diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index bd8f3f78..73160b08 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -10,7 +10,11 @@ use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; +use Appwrite\Utopia\Response; use Utopia\Database\Database as UtopiaDatabase; +use Utopia\Database\Exception\Timeout; +use Utopia\Database\Validator\Authorization; +use Utopia\Database\Validator\Query\Cursor; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Auth\Hash; @@ -48,16 +52,15 @@ class Appwrite extends Source private Teams $teams; - private Databases $database; - private Storage $storage; private Functions $functions; public function __construct( protected string $project, - string $endpoint, - protected string $key + protected string $endpoint, + protected string $key, + protected UtopiaDatabase $db ) { $this->client = (new Client()) ->setEndpoint($endpoint) @@ -66,12 +69,9 @@ public function __construct( $this->users = new Users($this->client); $this->teams = new Teams($this->client); - $this->database = new Databases($this->client); $this->storage = new Storage($this->client); $this->functions = new Functions($this->client); - $this->endpoint = $endpoint; - $this->headers['X-Appwrite-Project'] = $this->project; $this->headers['X-Appwrite-Key'] = $this->key; } @@ -152,7 +152,7 @@ public function report(array $resources = []): array // Databases $scope = 'databases.read'; if (\in_array(Resource::TYPE_DATABASE, $resources)) { - $report[Resource::TYPE_DATABASE] = $this->database->list()['total']; + $report[Resource::TYPE_DATABASE] = \count($this->listDatabases()); } $scope = 'collections.read'; @@ -160,6 +160,8 @@ public function report(array $resources = []): array $report[Resource::TYPE_COLLECTION] = 0; $databases = $this->database->list()['databases']; foreach ($databases as $database) { + $report[Resource::TYPE_COLLECTION] += $this->listCollections($database); + $report[Resource::TYPE_COLLECTION] += $this->database->listCollections( $database['$id'], [Query::limit(1)] @@ -618,283 +620,12 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void } } - /** - * @throws AppwriteException - */ - private function exportDocuments(int $batchSize): void - { - $collections = $this->cache->get(Collection::getName()); - - foreach ($collections as $collection) { - /** @var Collection $collection */ - $lastDocument = null; - - while (true) { - $queries = [Query::limit($batchSize)]; - - $documents = []; - - if ($lastDocument) { - $queries[] = Query::cursorAfter($lastDocument); - } - - $selects = ['*', '$id', '$permissions', '$updatedAt', '$createdAt']; // We want relations flat! - $manyToMany = []; - - $attributes = $this->cache->get(Attribute::getName()); - foreach ($attributes as $attribute) { - /** @var Relationship $attribute */ - if ( - $attribute->getCollection()->getId() === $collection->getId() && - $attribute->getType() === Attribute::TYPE_RELATIONSHIP && - $attribute->getSide() === 'parent' && - $attribute->getRelationType() == 'manyToMany' - ) { - /** - * Blockers: - * we should use but Does not work properly: - * $selects[] = $attribute->getKey() . '.$id'; - * when selecting for a relation we get all relations not just the one we were asking. - * when selecting for a relation like select(*, relation.$id) , all relations get resolve - */ - $manyToMany[] = $attribute->getKey(); - } - } - /** @var Attribute|Relationship $attribute */ - - $queries[] = Query::select($selects); - - $response = $this->database->listDocuments( - $collection->getDatabase()->getId(), - $collection->getId(), - $queries - ); - - foreach ($response['documents'] as $document) { - // HACK: Handle many to many - if (! empty($manyToMany)) { - $stack = ['$id']; // Adding $id because we can't select only relations - foreach ($manyToMany as $relation) { - $stack[] = $relation.'.$id'; - } - - $doc = $this->database->getDocument( - $collection->getDatabase()->getId(), - $collection->getId(), - $document['$id'], - [Query::select($stack)] - ); - - foreach ($manyToMany as $key) { - $document[$key] = []; - foreach ($doc[$key] as $relationDocument) { - $document[$key][] = $relationDocument['$id']; - } - } - } - - $id = $document['$id']; - $permissions = $document['$permissions']; - - unset($document['$id']); - unset($document['$permissions']); - unset($document['$collectionId']); - unset($document['$databaseId']); - - // Certain Appwrite versions allowed for data to be required but null - // This isn't allowed in modern versions so we need to remove it by comparing their attributes and replacing it with default value. - $attributes = $this->cache->get(Attribute::getName()); - foreach ($attributes as $attribute) { - /** @var Attribute $attribute */ - if ($attribute->getCollection()->getId() !== $collection->getId()) { - continue; - } - - if ($attribute->isRequired() && ! isset($document[$attribute->getKey()])) { - switch ($attribute->getType()) { - case Attribute::TYPE_BOOLEAN: - $document[$attribute->getKey()] = false; - break; - case Attribute::TYPE_STRING: - $document[$attribute->getKey()] = ''; - break; - case Attribute::TYPE_INTEGER: - $document[$attribute->getKey()] = 0; - break; - case Attribute::TYPE_FLOAT: - $document[$attribute->getKey()] = 0.0; - break; - case Attribute::TYPE_DATETIME: - $document[$attribute->getKey()] = '1970-01-01 00:00:00.000'; - break; - case Attribute::TYPE_URL: - $document[$attribute->getKey()] = 'http://null'; - break; - } - } - } - - $documents[] = new Document( - $id, - $collection, - $document, - $permissions - ); - - $lastDocument = $id; - } - - $this->callback($documents); - - if (count($response['documents']) < $batchSize) { - break; - } - } - } - } - - /** - * @throws \Exception - */ - private function convertAttribute(array $value, Collection $collection): Attribute - { - switch ($value['type']) { - case 'string': - if (! isset($value['format'])) { - return new Text( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - size: $value['size'] ?? 0, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ); - } - - return match ($value['format']) { - 'email' => new Email( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - size: $value['size'] ?? 254, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ), - 'enum' => new Enum( - $value['key'], - $collection, - elements: $value['elements'], - required: $value['required'], - default: $value['default'], - array: $value['array'], - size: $value['size'] ?? UtopiaDatabase::LENGTH_KEY, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ), - 'url' => new URL( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - size: $value['size'] ?? 2000, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ), - 'ip' => new IP( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - size: $value['size'] ?? 39, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ), - default => new Text( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - size: $value['size'] ?? 0, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ), - }; - case 'boolean': - return new Boolean( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ); - case 'integer': - return new Integer( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - min: $value['min'] ?? null, - max: $value['max'] ?? null, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ); - case 'double': - return new Decimal( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - min: $value['min'] ?? null, - max: $value['max'] ?? null, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ); - case 'relationship': - return new Relationship( - $value['key'], - $collection, - relatedCollection: $value['relatedCollection'], - relationType: $value['relationType'], - twoWay: $value['twoWay'], - twoWayKey: $value['twoWayKey'], - onDelete: $value['onDelete'], - side: $value['side'], - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ); - case 'datetime': - return new DateTime( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ); - } - - throw new \Exception('Unknown attribute type: '.$value['type']); - } /** * @throws AppwriteException */ private function exportDatabases(int $batchSize): void { - $this->database = new Databases($this->client); - $lastDatabase = null; while (true) { @@ -911,9 +642,9 @@ private function exportDatabases(int $batchSize): void $queries[] = Query::cursorAfter($lastDatabase); } - $response = $this->database->list($queries); + $response = $this->listDatabases($queries); - foreach ($response['databases'] as $database) { + foreach ($response as $database) { $newDatabase = new Database( $database['$id'], $database['name'], @@ -957,12 +688,9 @@ private function exportCollections(int $batchSize): void $queries[] = Query::cursorAfter($lastCollection); } - $response = $this->database->listCollections( - $database->getId(), - $queries - ); + $response = $this->listCollections($database, $queries); - foreach ($response['collections'] as $collection) { + foreach ($response as $collection) { $newCollection = new Collection( $database, $collection['name'], @@ -991,6 +719,7 @@ private function exportCollections(int $batchSize): void } } + /** * @throws AppwriteException * @throws \Exception @@ -1010,19 +739,140 @@ private function exportAttributes(int $batchSize): void $queries[] = Query::cursorAfter($lastAttribute); } - $response = $this->database->listAttributes( - $collection->getDatabase()->getId(), - $collection->getId(), - $queries - ); + $response = $this->listAttributes($collection, $queries); - foreach ($response['attributes'] as $attribute) { - /** @var array $attribute */ - if ($attribute['type'] === 'relationship' && $attribute['side'] === 'child') { + foreach ($response as $attribute) { + if ( + $attribute['type'] === UtopiaDatabase::VAR_RELATIONSHIP + && $attribute['side'] === UtopiaDatabase::RELATION_SIDE_CHILD + ) { continue; } - $attr = $this->convertAttribute($attribute, $collection); + switch ($attribute['type']) { + case UtopiaDatabase::VAR_STRING: + $attr = match ($attribute['format']) { + 'email' => new Email( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 254, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + 'enum' => new Enum( + $attribute['key'], + $collection, + elements: $attribute['elements'], + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? UtopiaDatabase::LENGTH_KEY, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + 'url' => new URL( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 2000, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + 'ip' => new IP( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 39, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + default => new Text( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 0, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + }; + + break; + case UtopiaDatabase::VAR_BOOLEAN: + $attr = new Boolean( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ); + break; + case UtopiaDatabase::VAR_INTEGER: + $attr = new Integer( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + min: $attribute['min'] ?? null, + max: $attribute['max'] ?? null, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ); + break; + case UtopiaDatabase::VAR_FLOAT: + $attr = new Decimal( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + min: $attribute['min'] ?? null, + max: $attribute['max'] ?? null, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ); + break; + case UtopiaDatabase::VAR_RELATIONSHIP: + $attr = new Relationship( + $attribute['key'], + $collection, + relatedCollection: $attribute['relatedCollection'], + relationType: $attribute['relationType'], + twoWay: $attribute['twoWay'], + twoWayKey: $attribute['twoWayKey'], + onDelete: $attribute['onDelete'], + side: $attribute['side'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ); + break; + case 'datetime': + $attr = new DateTime( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ); + break; + } + + if (!isset($attr)) { + throw new \Exception('Unknown attribute type: ' . $attribute['type']); + } $attributes[] = $attr; } @@ -1062,11 +912,7 @@ private function exportIndexes(int $batchSize): void $queries[] = Query::cursorAfter($lastIndex); } - $response = $this->database->listIndexes( - $collection->getDatabase()->getId(), - $collection->getId(), - $queries - ); + $response = $this->listIndexes($collection, $queries); foreach ($response['indexes'] as $index) { $indexes[] = new Index( @@ -1097,6 +943,260 @@ private function exportIndexes(int $batchSize): void } } + /** + * @throws AppwriteException + */ + private function exportDocuments(int $batchSize): void + { + $collections = $this->cache->get(Collection::getName()); + + foreach ($collections as $collection) { + /** @var Collection $collection */ + $lastDocument = null; + + while (true) { + $queries = [Query::limit($batchSize)]; + + $documents = []; + + if ($lastDocument) { + $queries[] = Query::cursorAfter($lastDocument); + } + + $selects = ['*', '$id', '$permissions', '$updatedAt', '$createdAt']; // We want relations flat! + $manyToMany = []; + + $attributes = $this->cache->get(Attribute::getName()); + foreach ($attributes as $attribute) { + /** @var Relationship $attribute */ + if ( + $attribute->getCollection()->getId() === $collection->getId() && + $attribute->getType() === Attribute::TYPE_RELATIONSHIP && + $attribute->getSide() === 'parent' && + $attribute->getRelationType() == 'manyToMany' + ) { + /** + * Blockers: + * we should use but Does not work properly: + * $selects[] = $attribute->getKey() . '.$id'; + * when selecting for a relation we get all relations not just the one we were asking. + * when selecting for a relation like select(*, relation.$id) , all relations get resolve + */ + $manyToMany[] = $attribute->getKey(); + } + } + /** @var Attribute|Relationship $attribute */ + + $queries[] = Query::select($selects); + + $response = $this->database->listDocuments( + $collection->getDatabase()->getId(), + $collection->getId(), + $queries + ); + + foreach ($response['documents'] as $document) { + // HACK: Handle many to many + if (! empty($manyToMany)) { + $stack = ['$id']; // Adding $id because we can't select only relations + foreach ($manyToMany as $relation) { + $stack[] = $relation.'.$id'; + } + + $doc = $this->database->getDocument( + $collection->getDatabase()->getId(), + $collection->getId(), + $document['$id'], + [Query::select($stack)] + ); + + foreach ($manyToMany as $key) { + $document[$key] = []; + foreach ($doc[$key] as $relationDocument) { + $document[$key][] = $relationDocument['$id']; + } + } + } + + $id = $document['$id']; + $permissions = $document['$permissions']; + + unset($document['$id']); + unset($document['$permissions']); + unset($document['$collectionId']); + unset($document['$databaseId']); + + // Certain Appwrite versions allowed for data to be required but null + // This isn't allowed in modern versions so we need to remove it by comparing their attributes and replacing it with default value. + $attributes = $this->cache->get(Attribute::getName()); + foreach ($attributes as $attribute) { + /** @var Attribute $attribute */ + if ($attribute->getCollection()->getId() !== $collection->getId()) { + continue; + } + + if ($attribute->isRequired() && ! isset($document[$attribute->getKey()])) { + switch ($attribute->getType()) { + case Attribute::TYPE_BOOLEAN: + $document[$attribute->getKey()] = false; + break; + case Attribute::TYPE_STRING: + $document[$attribute->getKey()] = ''; + break; + case Attribute::TYPE_INTEGER: + $document[$attribute->getKey()] = 0; + break; + case Attribute::TYPE_FLOAT: + $document[$attribute->getKey()] = 0.0; + break; + case Attribute::TYPE_DATETIME: + $document[$attribute->getKey()] = '1970-01-01 00:00:00.000'; + break; + case Attribute::TYPE_URL: + $document[$attribute->getKey()] = 'http://null'; + break; + } + } + } + + $documents[] = new Document( + $id, + $collection, + $document, + $permissions + ); + + $lastDocument = $id; + } + + $this->callback($documents); + + if (count($response['documents']) < $batchSize) { + break; + } + } + } + } + + /** + * @throws Timeout + * @throws \Utopia\Database\Exception + * @throws \Utopia\Database\Exception\Query + */ + private function listDatabases(array $queries = []): array + { + return $this->db->find('databases', $queries); + } + + /** + * @throws Exception + * @throws \Utopia\Database\Exception + * @throws Timeout + * @throws \Utopia\Database\Exception\Query + */ + private function listCollections(Database $resource, array $queries = []): array + { + $database = $this->db->getDocument('databases', $resource->getId()); + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + return $this->db->find( + 'database_' . $database->getInternalId(), + $queries + ); + } + + /** + * @throws Exception + * @throws \Utopia\Database\Exception + * @throws Timeout + * @throws \Utopia\Database\Exception\Query + */ + private function listAttributes(Collection $resource, array $queries = []): array + { + $database = $this->db->getDocument( + 'databases', + $resource->getDatabase()->getId(), + ); + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + $collection = $this->db->getDocument( + 'database_' . $database->getInternalId(), + $resource->getId(), + ); + + if ($collection->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Collection not found', + ); + } + + $queries[] = Query::equal('databaseInternalId', [$database->getInternalId()]); + $queries[] = Query::equal('collectionInternalId', [$collection->getInternalId()]); + + return $this->db->find('attributes', $queries); + } + + /** + * @throws Exception + * @throws \Utopia\Database\Exception + * @throws Timeout + * @throws \Utopia\Database\Exception\Query + */ + private function listIndexes(Collection $resource, array $queries = []): array + { + $database = $this->db->getDocument( + 'databases', + $resource->getDatabase()->getId(), + ); + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + $collection = $this->db->getDocument( + 'database_' . $database->getInternalId(), + $resource->getId(), + ); + + if ($collection->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Collection not found', + ); + } + + $queries[] = Query::equal('databaseInternalId', [$database->getInternalId()]); + $queries[] = Query::equal('collectionInternalId', [$collection->getInternalId()]); + + return $this->db->find('indexes', $queries); + } + protected function exportGroupStorage(int $batchSize, array $resources): void { try { diff --git a/tests/Migration/Unit/Adapters/MockSource.php b/tests/Migration/Unit/Adapters/MockSource.php index 66f22aca..e257a60c 100644 --- a/tests/Migration/Unit/Adapters/MockSource.php +++ b/tests/Migration/Unit/Adapters/MockSource.php @@ -12,11 +12,11 @@ class MockSource extends Source public function pushMockResource(Resource $resource): void { - if (!key_exists($resource->getGroup(), $this->mockResources)) { + if (!array_key_exists($resource->getGroup(), $this->mockResources)) { $this->mockResources[$resource->getGroup()] = []; } - if (!key_exists($resource->getName(), $this->mockResources[$resource->getGroup()])) { + if (!array_key_exists($resource->getName(), $this->mockResources[$resource->getGroup()])) { $this->mockResources[$resource->getGroup()][$resource->getName()] = []; } From 50bf788123622d6b01741efbbe287ab7721230bf Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 13 Mar 2025 19:25:28 +1300 Subject: [PATCH 02/38] Fix attribute types --- src/Migration/Resources/Database/Attribute.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Migration/Resources/Database/Attribute.php b/src/Migration/Resources/Database/Attribute.php index 826c2a9d..dab1d9a9 100644 --- a/src/Migration/Resources/Database/Attribute.php +++ b/src/Migration/Resources/Database/Attribute.php @@ -8,14 +8,14 @@ abstract class Attribute extends Resource { public const TYPE_STRING = 'string'; - public const TYPE_INTEGER = 'int'; - public const TYPE_FLOAT = 'float'; - public const TYPE_BOOLEAN = 'bool'; - public const TYPE_DATETIME = 'dateTime'; + public const TYPE_INTEGER = 'integer'; + public const TYPE_FLOAT = 'double'; + public const TYPE_BOOLEAN = 'boolean'; + public const TYPE_DATETIME = 'datetime'; public const TYPE_EMAIL = 'email'; public const TYPE_ENUM = 'enum'; - public const TYPE_IP = 'IP'; - public const TYPE_URL = 'URL'; + public const TYPE_IP = 'ip'; + public const TYPE_URL = 'url'; public const TYPE_RELATIONSHIP = 'relationship'; /** From 0545eda52af9c35f1e42df143cc8da48b8f14593 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 13 Mar 2025 19:26:00 +1300 Subject: [PATCH 03/38] Update attribute type checks --- src/Migration/Sources/Appwrite.php | 37 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 73160b08..d491bfc1 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -5,16 +5,12 @@ use Appwrite\AppwriteException; use Appwrite\Client; use Appwrite\Query; -use Appwrite\Services\Databases; use Appwrite\Services\Functions; use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; -use Appwrite\Utopia\Response; use Utopia\Database\Database as UtopiaDatabase; -use Utopia\Database\Exception\Timeout; -use Utopia\Database\Validator\Authorization; -use Utopia\Database\Validator\Query\Cursor; +use Utopia\Database\Exception as DatabaseException; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Auth\Hash; @@ -721,8 +717,8 @@ private function exportCollections(int $batchSize): void /** - * @throws AppwriteException - * @throws \Exception + * @param int $batchSize + * @throws Exception */ private function exportAttributes(int $batchSize): void { @@ -750,9 +746,9 @@ private function exportAttributes(int $batchSize): void } switch ($attribute['type']) { - case UtopiaDatabase::VAR_STRING: + case Attribute::TYPE_STRING: $attr = match ($attribute['format']) { - 'email' => new Email( + Attribute::TYPE_EMAIL => new Email( $attribute['key'], $collection, required: $attribute['required'], @@ -762,7 +758,7 @@ private function exportAttributes(int $batchSize): void createdAt: $attribute['$createdAt'] ?? '', updatedAt: $attribute['$updatedAt'] ?? '', ), - 'enum' => new Enum( + Attribute::TYPE_ENUM => new Enum( $attribute['key'], $collection, elements: $attribute['elements'], @@ -773,7 +769,7 @@ private function exportAttributes(int $batchSize): void createdAt: $attribute['$createdAt'] ?? '', updatedAt: $attribute['$updatedAt'] ?? '', ), - 'url' => new URL( + Attribute::TYPE_URL => new URL( $attribute['key'], $collection, required: $attribute['required'], @@ -783,7 +779,7 @@ private function exportAttributes(int $batchSize): void createdAt: $attribute['$createdAt'] ?? '', updatedAt: $attribute['$updatedAt'] ?? '', ), - 'ip' => new IP( + Attribute::TYPE_IP => new IP( $attribute['key'], $collection, required: $attribute['required'], @@ -806,7 +802,7 @@ private function exportAttributes(int $batchSize): void }; break; - case UtopiaDatabase::VAR_BOOLEAN: + case Attribute::TYPE_BOOLEAN: $attr = new Boolean( $attribute['key'], $collection, @@ -817,7 +813,7 @@ private function exportAttributes(int $batchSize): void updatedAt: $attribute['$updatedAt'] ?? '', ); break; - case UtopiaDatabase::VAR_INTEGER: + case Attribute::TYPE_INTEGER: $attr = new Integer( $attribute['key'], $collection, @@ -830,7 +826,7 @@ private function exportAttributes(int $batchSize): void updatedAt: $attribute['$updatedAt'] ?? '', ); break; - case UtopiaDatabase::VAR_FLOAT: + case Attribute::TYPE_FLOAT: $attr = new Decimal( $attribute['key'], $collection, @@ -843,7 +839,7 @@ private function exportAttributes(int $batchSize): void updatedAt: $attribute['$updatedAt'] ?? '', ); break; - case UtopiaDatabase::VAR_RELATIONSHIP: + case Attribute::TYPE_RELATIONSHIP: $attr = new Relationship( $attribute['key'], $collection, @@ -857,7 +853,7 @@ private function exportAttributes(int $batchSize): void updatedAt: $attribute['$updatedAt'] ?? '', ); break; - case 'datetime': + case Attribute::TYPE_DATETIME: $attr = new DateTime( $attribute['key'], $collection, @@ -871,7 +867,12 @@ private function exportAttributes(int $batchSize): void } if (!isset($attr)) { - throw new \Exception('Unknown attribute type: ' . $attribute['type']); + throw new Exception( + resourceName: Resource::TYPE_ATTRIBUTE, + resourceGroup: Transfer::GROUP_DATABASES, + resourceId: $attribute['$id'], + message: 'Unknown attribute type: ' . $attribute['type'] + ); } $attributes[] = $attr; From 1fa9d5563d40c1d54a5a56a890319edcc9cef099 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 13 Mar 2025 19:27:01 +1300 Subject: [PATCH 04/38] Update resource counting --- src/Migration/Sources/Appwrite.php | 90 ++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 28 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index d491bfc1..62cf1c36 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -109,7 +109,7 @@ public static function getSupportedResources(): array } /** - * @param array $resources + * @param array $resources * @return array * * @throws \Exception @@ -148,35 +148,38 @@ public function report(array $resources = []): array // Databases $scope = 'databases.read'; if (\in_array(Resource::TYPE_DATABASE, $resources)) { - $report[Resource::TYPE_DATABASE] = \count($this->listDatabases()); + $report[Resource::TYPE_DATABASE] = $this->countResources('databases'); } $scope = 'collections.read'; if (\in_array(Resource::TYPE_COLLECTION, $resources)) { $report[Resource::TYPE_COLLECTION] = 0; - $databases = $this->database->list()['databases']; + $databases = $this->listDatabases(); foreach ($databases as $database) { - $report[Resource::TYPE_COLLECTION] += $this->listCollections($database); + $collectionId = "database_{$database->getInternalId()}"; - $report[Resource::TYPE_COLLECTION] += $this->database->listCollections( - $database['$id'], - [Query::limit(1)] - )['total']; + $report[Resource::TYPE_COLLECTION] += $this->countResources($collectionId); } } $scope = 'documents.read'; if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { $report[Resource::TYPE_DOCUMENT] = 0; - $databases = $this->database->list()['databases']; + $databases = $this->listDatabases(); foreach ($databases as $database) { - $collections = $this->database->listCollections($database['$id'])['collections']; + $dbResource = new Database( + $database->getId(), + $database->getAttribute('name'), + $database->getCreatedAt(), + $database->getUpdatedAt(), + ); + + $collections = $this->listCollections($dbResource); + foreach ($collections as $collection) { - $report[Resource::TYPE_DOCUMENT] += $this->database->listDocuments( - $database['$id'], - $collection['$id'], - [Query::limit(1)] - )['total']; + $collectionId = "database_{$database->getInternal()}_collection_{$collection->getInternalId()}"; + + $report[Resource::TYPE_DOCUMENT] += $this->countResources($collectionId); } } } @@ -184,14 +187,22 @@ public function report(array $resources = []): array $scope = 'attributes.read'; if (\in_array(Resource::TYPE_ATTRIBUTE, $resources)) { $report[Resource::TYPE_ATTRIBUTE] = 0; - $databases = $this->database->list()['databases']; + $databases = $this->listDatabases(); foreach ($databases as $database) { - $collections = $this->database->listCollections($database['$id'])['collections']; + $dbResource = new Database( + $database->getId(), + $database->getAttribute('name'), + $database->getCreatedAt(), + $database->getUpdatedAt(), + ); + + $collections = $this->listCollections($dbResource); + foreach ($collections as $collection) { - $report[Resource::TYPE_ATTRIBUTE] += $this->database->listAttributes( - $database['$id'], - $collection['$id'] - )['total']; + $report[Resource::TYPE_ATTRIBUTE] += $this->countResources('attributes', [ + Query::equal('databaseInternalId', [$database->getInternalId()]), + Query::equal('collectionInternalId', [$collection->getInternalId()]), + ]); } } } @@ -199,14 +210,22 @@ public function report(array $resources = []): array $scope = 'indexes.read'; if (\in_array(Resource::TYPE_INDEX, $resources)) { $report[Resource::TYPE_INDEX] = 0; - $databases = $this->database->list()['databases']; + $databases = $this->listDatabases(); foreach ($databases as $database) { - $collections = $this->database->listCollections($database['$id'])['collections']; + $dbResource = new Database( + $database->getId(), + $database->getAttribute('name'), + $database->getCreatedAt(), + $database->getUpdatedAt(), + ); + + $collections = $this->listCollections($dbResource); + foreach ($collections as $collection) { - $report[Resource::TYPE_INDEX] += $this->database->listIndexes( - $database['$id'], - $collection['$id'] - )['total']; + $report[Resource::TYPE_INDEX] += $this->countResources('indexes', [ + Query::equal('databaseInternalId', [$database->getInternalId()]), + Query::equal('collectionInternalId', [$collection->getInternalId()]), + ]); } } } @@ -280,7 +299,7 @@ public function report(array $resources = []): array $report[Resource::TYPE_DEPLOYMENT] = 0; $functions = $this->functions->list()['functions']; foreach ($functions as $function) { - if (! empty($function['deployment'])) { + if (!empty($function['deployment'])) { $report[Resource::TYPE_DEPLOYMENT] += 1; } } @@ -1198,6 +1217,21 @@ private function listIndexes(Collection $resource, array $queries = []): array return $this->db->find('indexes', $queries); } + /** + * @param string $collection + * @param array $queries + * @return int + * @throws Exception + */ + private function countResources(string $collection, array $queries = []): int + { + try { + return $this->db->count($collection, $queries); + } catch (DatabaseException $e) { + throw new Exception($e->getMessage(), $e->getCode(), $e); + } + } + protected function exportGroupStorage(int $batchSize, array $resources): void { try { From 00da9d4712ce43cd8cb5b88dad350c05274bb84e Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 13 Mar 2025 19:27:42 +1300 Subject: [PATCH 05/38] Add document export --- src/Migration/Sources/Appwrite.php | 69 ++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 62cf1c36..3b288b83 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -637,7 +637,8 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void /** - * @throws AppwriteException + * @param int $batchSize + * @throws Exception */ private function exportDatabases(int $batchSize): void { @@ -685,7 +686,8 @@ private function exportDatabases(int $batchSize): void } /** - * @throws AppwriteException + * @param int $batchSize + * @throws Exception */ private function exportCollections(int $batchSize): void { @@ -913,7 +915,8 @@ private function exportAttributes(int $batchSize): void } /** - * @throws AppwriteException + * @param int $batchSize + * @throws Exception */ private function exportIndexes(int $batchSize): void { @@ -964,7 +967,7 @@ private function exportIndexes(int $batchSize): void } /** - * @throws AppwriteException + * @throws Exception */ private function exportDocuments(int $batchSize): void { @@ -1009,13 +1012,9 @@ private function exportDocuments(int $batchSize): void $queries[] = Query::select($selects); - $response = $this->database->listDocuments( - $collection->getDatabase()->getId(), - $collection->getId(), - $queries - ); + $response = $this->listDocuments($collection, $queries); - foreach ($response['documents'] as $document) { + foreach ($response as $document) { // HACK: Handle many to many if (! empty($manyToMany)) { $stack = ['$id']; // Adding $id because we can't select only relations @@ -1023,8 +1022,7 @@ private function exportDocuments(int $batchSize): void $stack[] = $relation.'.$id'; } - $doc = $this->database->getDocument( - $collection->getDatabase()->getId(), + $doc = $this->db->getDocument( $collection->getId(), $document['$id'], [Query::select($stack)] @@ -1091,7 +1089,7 @@ private function exportDocuments(int $batchSize): void $this->callback($documents); - if (count($response['documents']) < $batchSize) { + if (count($response) < $batchSize) { break; } } @@ -1215,6 +1213,51 @@ private function listIndexes(Collection $resource, array $queries = []): array $queries[] = Query::equal('collectionInternalId', [$collection->getInternalId()]); return $this->db->find('indexes', $queries); + + private function listDocuments(Collection $resource, array $queries = []): array + { + $database = $this->db->getDocument( + 'databases', + $resource->getDatabase()->getId(), + ); + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + $collection = $this->db->getDocument( + 'database_' . $database->getInternalId(), + $resource->getId(), + ); + + if ($collection->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Collection not found', + ); + } + + $collectionId = "database_{$database->getInternalId()}_collection_{$collection->getInternalId()}"; + + try { + return $this->db->find($collectionId, $queries); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } } /** From cf4b0fca3dfc247102598658c4eba1d62c365736 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 13 Mar 2025 19:27:51 +1300 Subject: [PATCH 06/38] Remove redundant logic --- src/Migration/Sources/Appwrite.php | 33 ------------------------------ 1 file changed, 33 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 3b288b83..241f06d7 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -1044,39 +1044,6 @@ private function exportDocuments(int $batchSize): void unset($document['$collectionId']); unset($document['$databaseId']); - // Certain Appwrite versions allowed for data to be required but null - // This isn't allowed in modern versions so we need to remove it by comparing their attributes and replacing it with default value. - $attributes = $this->cache->get(Attribute::getName()); - foreach ($attributes as $attribute) { - /** @var Attribute $attribute */ - if ($attribute->getCollection()->getId() !== $collection->getId()) { - continue; - } - - if ($attribute->isRequired() && ! isset($document[$attribute->getKey()])) { - switch ($attribute->getType()) { - case Attribute::TYPE_BOOLEAN: - $document[$attribute->getKey()] = false; - break; - case Attribute::TYPE_STRING: - $document[$attribute->getKey()] = ''; - break; - case Attribute::TYPE_INTEGER: - $document[$attribute->getKey()] = 0; - break; - case Attribute::TYPE_FLOAT: - $document[$attribute->getKey()] = 0.0; - break; - case Attribute::TYPE_DATETIME: - $document[$attribute->getKey()] = '1970-01-01 00:00:00.000'; - break; - case Attribute::TYPE_URL: - $document[$attribute->getKey()] = 'http://null'; - break; - } - } - } - $documents[] = new Document( $id, $collection, From 93887b19f7420462e2e500168322d6ff7c630228 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 13 Mar 2025 19:28:12 +1300 Subject: [PATCH 07/38] Handle exceptions --- src/Migration/Sources/Appwrite.php | 96 +++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 20 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 241f06d7..d728e90e 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -1064,24 +1064,44 @@ private function exportDocuments(int $batchSize): void } /** - * @throws Timeout - * @throws \Utopia\Database\Exception - * @throws \Utopia\Database\Exception\Query + * @throws Exception + * + * @returns array */ private function listDatabases(array $queries = []): array { - return $this->db->find('databases', $queries); + try { + return $this->db->find('databases', $queries); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: 'databases', + resourceGroup: Transfer::GROUP_DATABASES, + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } } /** * @throws Exception - * @throws \Utopia\Database\Exception - * @throws Timeout - * @throws \Utopia\Database\Exception\Query + * + * @returns array */ private function listCollections(Database $resource, array $queries = []): array { - $database = $this->db->getDocument('databases', $resource->getId()); + try { + $database = $this->db->getDocument('databases', $resource->getId()); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } if ($database->isEmpty()) { throw new Exception( @@ -1092,17 +1112,29 @@ private function listCollections(Database $resource, array $queries = []): array ); } - return $this->db->find( - 'database_' . $database->getInternalId(), - $queries - ); + try { + return $this->db->find( + 'database_' . $database->getInternalId(), + $queries + ); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } } /** + * @param Collection $resource + * @param array $queries + * @return array + * @throws DatabaseException * @throws Exception - * @throws \Utopia\Database\Exception - * @throws Timeout - * @throws \Utopia\Database\Exception\Query */ private function listAttributes(Collection $resource, array $queries = []): array { @@ -1137,14 +1169,26 @@ private function listAttributes(Collection $resource, array $queries = []): arra $queries[] = Query::equal('databaseInternalId', [$database->getInternalId()]); $queries[] = Query::equal('collectionInternalId', [$collection->getInternalId()]); - return $this->db->find('attributes', $queries); + try { + return $this->db->find('attributes', $queries); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } } /** + * @param Collection $resource + * @param array $queries + * @return array + * @throws DatabaseException * @throws Exception - * @throws \Utopia\Database\Exception - * @throws Timeout - * @throws \Utopia\Database\Exception\Query */ private function listIndexes(Collection $resource, array $queries = []): array { @@ -1179,7 +1223,19 @@ private function listIndexes(Collection $resource, array $queries = []): array $queries[] = Query::equal('databaseInternalId', [$database->getInternalId()]); $queries[] = Query::equal('collectionInternalId', [$collection->getInternalId()]); - return $this->db->find('indexes', $queries); + try { + return $this->db->find('indexes', $queries); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } + } private function listDocuments(Collection $resource, array $queries = []): array { From 3689c8f58e01f8deac60e46d93e8d23a883ea5ee Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Thu, 13 Mar 2025 19:28:37 +1300 Subject: [PATCH 08/38] Allow overriding source batch size per resource type --- src/Migration/Source.php | 79 ++++++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/src/Migration/Source.php b/src/Migration/Source.php index 46333b61..fb4a146a 100644 --- a/src/Migration/Source.php +++ b/src/Migration/Source.php @@ -4,6 +4,8 @@ abstract class Source extends Target { + protected static int $defaultBatchSize = 100; + /** * @var callable(array): void $transferCallback */ @@ -14,6 +16,26 @@ abstract class Source extends Target */ public array $previousReport = []; + public function getAuthBatchSize(): int + { + return static::$defaultBatchSize; + } + + public function getDatabasesBatchSize(): int + { + return static::$defaultBatchSize; + } + + public function getStorageBatchSize(): int + { + return static::$defaultBatchSize; + } + + public function getFunctionsBatchSize(): int + { + return static::$defaultBatchSize; + } + /** * @param array $resources * @return void @@ -39,7 +61,7 @@ public function run(array $resources, callable $callback, string $rootResourceId $prunedResources = []; foreach ($returnedResources as $resource) { /** @var Resource $resource */ - if (! in_array($resource->getName(), $resources)) { + if (!in_array($resource->getName(), $resources)) { $resource->setStatus(Resource::STATUS_SKIPPED); } else { $prunedResources[] = $resource; @@ -56,24 +78,24 @@ public function run(array $resources, callable $callback, string $rootResourceId /** * Export Resources * - * @param array $resources Resources to export + * @param array $resources Resources to export */ public function exportResources(array $resources): void { - // Convert Resources back into their relevant groups - - $batchSize = $this->getBatchSize(); - $groups = []; foreach ($resources as $resource) { - if (\in_array($resource, Transfer::GROUP_AUTH_RESOURCES)) { - $groups[Transfer::GROUP_AUTH][] = $resource; - } elseif (\in_array($resource, Transfer::GROUP_DATABASES_RESOURCES)) { - $groups[Transfer::GROUP_DATABASES][] = $resource; - } elseif (\in_array($resource, Transfer::GROUP_STORAGE_RESOURCES)) { - $groups[Transfer::GROUP_STORAGE][] = $resource; - } elseif (\in_array($resource, Transfer::GROUP_FUNCTIONS_RESOURCES)) { - $groups[Transfer::GROUP_FUNCTIONS][] = $resource; + $mapping = [ + Transfer::GROUP_AUTH => Transfer::GROUP_AUTH_RESOURCES, + Transfer::GROUP_DATABASES => Transfer::GROUP_DATABASES_RESOURCES, + Transfer::GROUP_STORAGE => Transfer::GROUP_STORAGE_RESOURCES, + Transfer::GROUP_FUNCTIONS => Transfer::GROUP_FUNCTIONS_RESOURCES, + ]; + + foreach ($mapping as $group => $resources) { + if (\in_array($resource, $resources, true)) { + $groups[$group][] = $resource; + break; + } } } @@ -81,58 +103,53 @@ public function exportResources(array $resources): void return; } - // Send each group to the relevant export function foreach ($groups as $group => $resources) { switch ($group) { case Transfer::GROUP_AUTH: - $this->exportGroupAuth($batchSize, $resources); + $this->exportGroupAuth($this->getAuthBatchSize(), $resources); break; case Transfer::GROUP_DATABASES: - $this->exportGroupDatabases($batchSize, $resources); + $this->exportGroupDatabases($this->getDatabasesBatchSize(), $resources); break; case Transfer::GROUP_STORAGE: - $this->exportGroupStorage($batchSize, $resources); + $this->exportGroupStorage($this->getStorageBatchSize(), $resources); break; case Transfer::GROUP_FUNCTIONS: - $this->exportGroupFunctions($batchSize, $resources); + $this->exportGroupFunctions($this->getFunctionsBatchSize(), $resources); break; } } } - public function getBatchSize(): int - { - return 100; - } /** * Export Auth Group * - * @param int $batchSize Max 100 - * @param array $resources Resources to export + * @param int $batchSize + * @param array $resources Resources to export */ abstract protected function exportGroupAuth(int $batchSize, array $resources): void; /** * Export Databases Group * - * @param int $batchSize Max 100 - * @param array $resources Resources to export + * @param int $batchSize + * @param array $resources Resources to export */ abstract protected function exportGroupDatabases(int $batchSize, array $resources): void; /** * Export Storage Group * - * @param int $batchSize Max 5 - * @param array $resources Resources to export + * @param int $batchSize Max 5 + * @param array $resources Resources to export */ abstract protected function exportGroupStorage(int $batchSize, array $resources): void; /** * Export Functions Group * - * @param int $batchSize Max 100 - * @param array $resources Resources to export + * @param int $batchSize + * @param array $resources Resources to export */ abstract protected function exportGroupFunctions(int $batchSize, array $resources): void; } From a6f6febad59c02dc9c16d2618e2ff365d3ba499d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sat, 15 Mar 2025 21:38:31 +1300 Subject: [PATCH 09/38] Add data source --- src/Migration/Sources/Appwrite.php | 6 ++++-- src/Migration/Sources/DataSource.php | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 src/Migration/Sources/DataSource.php diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 73c88e28..0b73c84d 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -56,8 +56,10 @@ public function __construct( protected string $project, protected string $endpoint, protected string $key, - protected UtopiaDatabase $db - ) { + protected DataSource $dataSource, + protected ?UtopiaDatabase $dbForProject = null + ) + { $this->client = (new Client()) ->setEndpoint($endpoint) ->setProject($project) diff --git a/src/Migration/Sources/DataSource.php b/src/Migration/Sources/DataSource.php new file mode 100644 index 00000000..523704ea --- /dev/null +++ b/src/Migration/Sources/DataSource.php @@ -0,0 +1,9 @@ + Date: Sat, 15 Mar 2025 22:24:17 +1300 Subject: [PATCH 10/38] Add source reader interface --- src/Migration/Sources/Appwrite.php | 3 +++ src/Migration/Sources/Appwrite/Reader.php | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 src/Migration/Sources/Appwrite/Reader.php diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 0b73c84d..50fde961 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -38,6 +38,7 @@ use Utopia\Migration\Resources\Storage\Bucket; use Utopia\Migration\Resources\Storage\File; use Utopia\Migration\Source; +use Utopia\Migration\Sources\Appwrite\Reader; use Utopia\Migration\Transfer; class Appwrite extends Source @@ -52,6 +53,8 @@ class Appwrite extends Source private Functions $functions; + private Reader $database; + public function __construct( protected string $project, protected string $endpoint, diff --git a/src/Migration/Sources/Appwrite/Reader.php b/src/Migration/Sources/Appwrite/Reader.php new file mode 100644 index 00000000..4bbd79ac --- /dev/null +++ b/src/Migration/Sources/Appwrite/Reader.php @@ -0,0 +1,21 @@ + Date: Sat, 15 Mar 2025 22:24:39 +1300 Subject: [PATCH 11/38] Add database reader impl --- src/Migration/Sources/Appwrite/Reader.php | 53 ++- .../Appwrite/Reader/DatabaseReader.php | 307 ++++++++++++++++++ 2 files changed, 351 insertions(+), 9 deletions(-) create mode 100644 src/Migration/Sources/Appwrite/Reader/DatabaseReader.php diff --git a/src/Migration/Sources/Appwrite/Reader.php b/src/Migration/Sources/Appwrite/Reader.php index 4bbd79ac..3eeece71 100644 --- a/src/Migration/Sources/Appwrite/Reader.php +++ b/src/Migration/Sources/Appwrite/Reader.php @@ -2,6 +2,7 @@ namespace Utopia\Migration\Sources\Appwrite; +use Utopia\Database\Query; use Utopia\Migration\Resources\Database\Collection; use Utopia\Migration\Resources\Database\Database; @@ -9,13 +10,47 @@ interface Reader { public function report(array $resources, array &$report); - public function listDatabases(): array; - - public function listCollections(Database $database): array; - - public function listAttributes(Collection $collection): array; - - public function listIndexes(Collection $collection): array; - - public function listDocuments(Collection $collection): array; + /** + * List databases that match the given queries + * + * @param array $queries + * @return array + */ + public function listDatabases(array $queries = []): array; + + /** + * @param Database $resource + * @param array $queries + * @return array + */ + public function listCollections(Database $resource, array $queries = []): array; + + /** + * @param Collection $resource + * @param array $queries + * @return array + */ + public function listAttributes(Collection $resource, array $queries = []): array; + + /** + * @param Collection $resource + * @param array $queries + * @return array + */ + public function listIndexes(Collection $resource, array $queries = []): array; + + /** + * @param Collection $resource + * @param array $queries + * @return array + */ + public function listDocuments(Collection $resource, array $queries = []): array; + + /** + * @param Collection $resource + * @param string $documentId + * @param array $queries + * @return array + */ + public function getDocument(Collection $resource, string $documentId, array $queries = []): array; } \ No newline at end of file diff --git a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php new file mode 100644 index 00000000..b67b6fbe --- /dev/null +++ b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php @@ -0,0 +1,307 @@ +countResources('databases'); + } + + if (\in_array(Resource::TYPE_COLLECTION, $resources)) { + $report[Resource::TYPE_COLLECTION] = 0; + $databases = $this->listDatabases(); + foreach ($databases as $database) { + $collectionId = "database_{$database->getInternalId()}"; + + $report[Resource::TYPE_COLLECTION] += $this->countResources($collectionId); + } + } + + if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { + $report[Resource::TYPE_DOCUMENT] = 0; + $databases = $this->listDatabases(); + foreach ($databases as $database) { + $dbResource = new Database( + $database->getId(), + $database->getAttribute('name'), + $database->getCreatedAt(), + $database->getUpdatedAt(), + ); + + $collections = $this->listCollections($dbResource); + + foreach ($collections as $collection) { + $collectionId = "database_{$database->getInternal()}_collection_{$collection->getInternalId()}"; + + $report[Resource::TYPE_DOCUMENT] += $this->countResources($collectionId); + } + } + } + + if (\in_array(Resource::TYPE_ATTRIBUTE, $resources)) { + $report[Resource::TYPE_ATTRIBUTE] = 0; + $databases = $this->listDatabases(); + foreach ($databases as $database) { + $dbResource = new Database( + $database->getId(), + $database->getAttribute('name'), + $database->getCreatedAt(), + $database->getUpdatedAt(), + ); + + $collections = $this->listCollections($dbResource); + + foreach ($collections as $collection) { + $report[Resource::TYPE_ATTRIBUTE] += $this->countResources('attributes', [ + Query::equal('databaseInternalId', [$database->getInternalId()]), + Query::equal('collectionInternalId', [$collection->getInternalId()]), + ]); + } + } + } + + if (\in_array(Resource::TYPE_INDEX, $resources)) { + $report[Resource::TYPE_INDEX] = 0; + $databases = $this->listDatabases(); + foreach ($databases as $database) { + $dbResource = new Database( + $database->getId(), + $database->getAttribute('name'), + $database->getCreatedAt(), + $database->getUpdatedAt(), + ); + + $collections = $this->listCollections($dbResource); + + foreach ($collections as $collection) { + $report[Resource::TYPE_INDEX] += $this->countResources('indexes', [ + Query::equal('databaseInternalId', [$database->getInternalId()]), + Query::equal('collectionInternalId', [$collection->getInternalId()]), + ]); + } + } + } + } + + public function listDatabases(array $queries = []): array + { + return $this->dbForProject->find('databases', $queries); + } + + public function listCollections(Database $resource, array $queries = []): array + { + try { + $database = $this->dbForProject->getDocument('databases', $resource->getId()); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + try { + return $this->dbForProject->find( + 'database_' . $database->getInternalId(), + $queries + ); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } + } + + public function listAttributes(Collection $resource, array $queries = []): array + { + $database = $this->dbForProject->getDocument( + 'databases', + $resource->getDatabase()->getId(), + ); + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + $collection = $this->dbForProject->getDocument( + 'database_' . $database->getInternalId(), + $resource->getId(), + ); + + if ($collection->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Collection not found', + ); + } + + $queries[] = Query::equal('databaseInternalId', [$database->getInternalId()]); + $queries[] = Query::equal('collectionInternalId', [$collection->getInternalId()]); + + try { + return $this->dbForProject->find('attributes', $queries); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } + } + + public function listIndexes(Collection $resource, array $queries = []): array + { + $database = $this->dbForProject->getDocument( + 'databases', + $resource->getDatabase()->getId(), + ); + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + $collection = $this->dbForProject->getDocument( + 'database_' . $database->getInternalId(), + $resource->getId(), + ); + + if ($collection->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Collection not found', + ); + } + + $queries[] = Query::equal('databaseInternalId', [$database->getInternalId()]); + $queries[] = Query::equal('collectionInternalId', [$collection->getInternalId()]); + + try { + return $this->dbForProject->find('indexes', $queries); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } + } + + public function listDocuments(Collection $resource, array $queries = []): array + { + $database = $this->dbForProject->getDocument( + 'databases', + $resource->getDatabase()->getId(), + ); + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + $collection = $this->dbForProject->getDocument( + 'database_' . $database->getInternalId(), + $resource->getId(), + ); + + if ($collection->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Collection not found', + ); + } + + $collectionId = "database_{$database->getInternalId()}_collection_{$collection->getInternalId()}"; + + try { + return $this->dbForProject->find($collectionId, $queries); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } + } + + public function getDocument(Collection $resource, string $documentId, array $queries = []): array + { + return $this->dbForProject->getDocument( + $resource->getId(), + $documentId, + $queries + )->getArrayCopy(); + } + + /** + * @param string $collection + * @param array $queries + * @return int + * @throws DatabaseException + */ + private function countResources(string $collection, array $queries = []): int + { + return $this->dbForProject->count($collection, $queries); + } +} \ No newline at end of file From ec8da3fd9c8abbac12b03b7e85ebb286e4f8ca99 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sat, 15 Mar 2025 22:24:45 +1300 Subject: [PATCH 12/38] Add API reader impl --- .../Sources/Appwrite/Reader/APIReader.php | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 src/Migration/Sources/Appwrite/Reader/APIReader.php diff --git a/src/Migration/Sources/Appwrite/Reader/APIReader.php b/src/Migration/Sources/Appwrite/Reader/APIReader.php new file mode 100644 index 00000000..8cc315fc --- /dev/null +++ b/src/Migration/Sources/Appwrite/Reader/APIReader.php @@ -0,0 +1,164 @@ +database->list()['total']; + } + + if (\in_array(Resource::TYPE_COLLECTION, $resources)) { + $report[Resource::TYPE_COLLECTION] = 0; + $databases = $this->database->list()['databases']; + foreach ($databases as $database) { + $report[Resource::TYPE_COLLECTION] += $this->database->listCollections( + $database['$id'], + [Query::limit(1)] + )['total']; + } + } + + if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { + $report[Resource::TYPE_DOCUMENT] = 0; + $databases = $this->database->list()['databases']; + foreach ($databases as $database) { + $collections = $this->database->listCollections($database['$id'])['collections']; + foreach ($collections as $collection) { + $report[Resource::TYPE_DOCUMENT] += $this->database->listDocuments( + $database['$id'], + $collection['$id'], + [Query::limit(1)] + )['total']; + } + } + } + + if (\in_array(Resource::TYPE_ATTRIBUTE, $resources)) { + $report[Resource::TYPE_ATTRIBUTE] = 0; + $databases = $this->database->list()['databases']; + foreach ($databases as $database) { + $collections = $this->database->listCollections($database['$id'])['collections']; + foreach ($collections as $collection) { + $report[Resource::TYPE_ATTRIBUTE] += $this->database->listAttributes( + $database['$id'], + $collection['$id'] + )['total']; + } + } + } + + if (\in_array(Resource::TYPE_INDEX, $resources)) { + $report[Resource::TYPE_INDEX] = 0; + $databases = $this->database->list()['databases']; + foreach ($databases as $database) { + $collections = $this->database->listCollections($database['$id'])['collections']; + foreach ($collections as $collection) { + $report[Resource::TYPE_INDEX] += $this->database->listIndexes( + $database['$id'], + $collection['$id'] + )['total']; + } + } + } + } + + /** + * @throws AppwriteException + */ + public function listDatabases(array $queries = []): array + { + return $this->database->list($queries); + } + + /** + * @throws AppwriteException + */ + public function listCollections(Database $resource, array $queries = []): array + { + return $this->database->listCollections( + $resource->getId(), + $queries + )['collections']; + } + + /** + * @param Collection $resource + * @param array $queries + * @return array + * @throws AppwriteException + */ + public function listAttributes(Collection $resource, array $queries = []): array + { + return $this->database->listAttributes( + $resource->getDatabase()->getId(), + $resource->getId(), + $queries + )['attributes']; + } + + /** + * @param Collection $resource + * @param array $queries + * @return array + * @throws AppwriteException + */ + public function listIndexes(Collection $resource, array $queries = []): array + { + return $this->database->listIndexes( + $resource->getDatabase()->getId(), + $resource->getId(), + $queries + )['indexes']; + } + + + /** + * @param Collection $resource + * @param array $queries + * @return array + * @throws AppwriteException + */ + public function listDocuments(Collection $resource, array $queries = []): array + { + return $this->database->listDocuments( + $resource->getDatabase()->getId(), + $resource->getId(), + $queries + )['documents']; + } + + /** + * @param Collection $resource + * @param string $documentId + * @param array $queries + * @return array + * @throws AppwriteException + */ + public function getDocument(Collection $resource, string $documentId, array $queries = []): array + { + return $this->database->getDocument( + $resource->getDatabase()->getId(), + $resource->getId(), + $documentId, + $queries + ); + } +} \ No newline at end of file From a89b8c113d053fb0f7b4ee1ab82b265eb74da9c3 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sat, 15 Mar 2025 22:27:34 +1300 Subject: [PATCH 13/38] Init reader --- src/Migration/Sources/Appwrite.php | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 50fde961..e22cb1b0 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -5,12 +5,14 @@ use Appwrite\AppwriteException; use Appwrite\Client; use Appwrite\Query; +use Appwrite\Services\Databases; use Appwrite\Services\Functions; use Appwrite\Services\Storage; use Appwrite\Services\Teams; use Appwrite\Services\Users; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Exception as DatabaseException; +use Utopia\Database\Exception\Timeout; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Auth\Hash; @@ -39,6 +41,8 @@ use Utopia\Migration\Resources\Storage\File; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite\Reader; +use Utopia\Migration\Sources\Appwrite\Reader\APIReader; +use Utopia\Migration\Sources\Appwrite\Reader\DatabaseReader; use Utopia\Migration\Transfer; class Appwrite extends Source @@ -55,6 +59,9 @@ class Appwrite extends Source private Reader $database; + /** + * @throws \Exception + */ public function __construct( protected string $project, protected string $endpoint, @@ -75,6 +82,18 @@ public function __construct( $this->headers['X-Appwrite-Project'] = $this->project; $this->headers['X-Appwrite-Key'] = $this->key; + + switch ($this->dataSource) { + case DataSource::API: + $this->database = new APIReader(new Databases($this->client)); + break; + case DataSource::DATABASE: + if (\is_null($dbForProject)) { + throw new \Exception('Database is required for database source'); + } + $this->database = new DatabaseReader($dbForProject); + break; + } } public static function getName(): string From 4bd26f72f30d4c60f3ea5f5c138a2006bc47afae Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sat, 15 Mar 2025 22:28:14 +1300 Subject: [PATCH 14/38] Split report to domain functions --- src/Migration/Sources/Appwrite.php | 288 ++++++++++++----------------- 1 file changed, 114 insertions(+), 174 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index e22cb1b0..aab81f87 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -146,214 +146,154 @@ public function report(array $resources = []): array $resources = $this->getSupportedResources(); } - // Auth try { - $scope = 'users.read'; - if (\in_array(Resource::TYPE_USER, $resources)) { - $report[Resource::TYPE_USER] = $this->users->list()['total']; - } + $this->reportAuth($resources, $report); + $this->reportDatabases($resources, $report); + $this->reportStorage($resources, $report); + $this->reportFunctions($resources, $report); - $scope = 'teams.read'; - if (\in_array(Resource::TYPE_TEAM, $resources)) { - $report[Resource::TYPE_TEAM] = $this->teams->list()['total']; - } - - if (\in_array(Resource::TYPE_MEMBERSHIP, $resources)) { - $report[Resource::TYPE_MEMBERSHIP] = 0; - $teams = $this->teams->list()['teams']; - foreach ($teams as $team) { - $report[Resource::TYPE_MEMBERSHIP] += $this->teams->listMemberships( - $team['$id'], - [Query::limit(1)] - )['total']; - } - } - - // Databases - $scope = 'databases.read'; - if (\in_array(Resource::TYPE_DATABASE, $resources)) { - $report[Resource::TYPE_DATABASE] = $this->countResources('databases'); + $report['version'] = $this->call( + 'GET', + '/health/version', + [ + 'X-Appwrite-Key' => '', + 'X-Appwrite-Project' => '', + ] + )['version']; + } catch (\Throwable $e) { + if ($e->getCode() === 403) { + throw new \Exception("Missing required scopes."); + } else { + throw new \Exception($e->getMessage(), previous: $e); } + } - $scope = 'collections.read'; - if (\in_array(Resource::TYPE_COLLECTION, $resources)) { - $report[Resource::TYPE_COLLECTION] = 0; - $databases = $this->listDatabases(); - foreach ($databases as $database) { - $collectionId = "database_{$database->getInternalId()}"; + $this->previousReport = $report; - $report[Resource::TYPE_COLLECTION] += $this->countResources($collectionId); - } - } - - $scope = 'documents.read'; - if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { - $report[Resource::TYPE_DOCUMENT] = 0; - $databases = $this->listDatabases(); - foreach ($databases as $database) { - $dbResource = new Database( - $database->getId(), - $database->getAttribute('name'), - $database->getCreatedAt(), - $database->getUpdatedAt(), - ); + return $report; + } - $collections = $this->listCollections($dbResource); + /** + * @param array $resources + * @param array $report + * @throws AppwriteException + */ + private function reportAuth(array $resources, array &$report): void + { + if (\in_array(Resource::TYPE_USER, $resources)) { + $report[Resource::TYPE_USER] = $this->users->list()['total']; + } - foreach ($collections as $collection) { - $collectionId = "database_{$database->getInternal()}_collection_{$collection->getInternalId()}"; + if (\in_array(Resource::TYPE_TEAM, $resources)) { + $report[Resource::TYPE_TEAM] = $this->teams->list()['total']; + } - $report[Resource::TYPE_DOCUMENT] += $this->countResources($collectionId); - } - } + if (\in_array(Resource::TYPE_MEMBERSHIP, $resources)) { + $report[Resource::TYPE_MEMBERSHIP] = 0; + $teams = $this->teams->list()['teams']; + foreach ($teams as $team) { + $report[Resource::TYPE_MEMBERSHIP] += $this->teams->listMemberships( + $team['$id'], + [Query::limit(1)] + )['total']; } + } + } - $scope = 'attributes.read'; - if (\in_array(Resource::TYPE_ATTRIBUTE, $resources)) { - $report[Resource::TYPE_ATTRIBUTE] = 0; - $databases = $this->listDatabases(); - foreach ($databases as $database) { - $dbResource = new Database( - $database->getId(), - $database->getAttribute('name'), - $database->getCreatedAt(), - $database->getUpdatedAt(), - ); + /** + * @throws Exception + * @throws AppwriteException + */ + private function reportDatabases(array $resources, array &$report): void + { + $this->database->report($resources, $report); + } - $collections = $this->listCollections($dbResource); + /** + * @param array $resources + * @param array $report + * @throws AppwriteException + */ + private function reportStorage(array $resources, array &$report): void + { + if (\in_array(Resource::TYPE_BUCKET, $resources)) { + $report[Resource::TYPE_BUCKET] = $this->storage->listBuckets()['total']; + } - foreach ($collections as $collection) { - $report[Resource::TYPE_ATTRIBUTE] += $this->countResources('attributes', [ - Query::equal('databaseInternalId', [$database->getInternalId()]), - Query::equal('collectionInternalId', [$collection->getInternalId()]), - ]); - } - } - } + if (\in_array(Resource::TYPE_FILE, $resources)) { + $report[Resource::TYPE_FILE] = 0; + $report['size'] = 0; + $buckets = []; + $lastBucket = null; - $scope = 'indexes.read'; - if (\in_array(Resource::TYPE_INDEX, $resources)) { - $report[Resource::TYPE_INDEX] = 0; - $databases = $this->listDatabases(); - foreach ($databases as $database) { - $dbResource = new Database( - $database->getId(), - $database->getAttribute('name'), - $database->getCreatedAt(), - $database->getUpdatedAt(), - ); + while (true) { + $currentBuckets = $this->storage->listBuckets( + $lastBucket + ? [Query::cursorAfter($lastBucket)] + : [Query::limit(20)] + )['buckets']; - $collections = $this->listCollections($dbResource); + $buckets = array_merge($buckets, $currentBuckets); + $lastBucket = $buckets[count($buckets) - 1]['$id'] ?? null; - foreach ($collections as $collection) { - $report[Resource::TYPE_INDEX] += $this->countResources('indexes', [ - Query::equal('databaseInternalId', [$database->getInternalId()]), - Query::equal('collectionInternalId', [$collection->getInternalId()]), - ]); - } + if (count($currentBuckets) < 20) { + break; } } - // Storage - $scope = 'buckets.read'; - if (\in_array(Resource::TYPE_BUCKET, $resources)) { - $report[Resource::TYPE_BUCKET] = $this->storage->listBuckets()['total']; - } - - $scope = 'files.read'; - if (\in_array(Resource::TYPE_FILE, $resources)) { - $report[Resource::TYPE_FILE] = 0; - $report['size'] = 0; - $buckets = []; - $lastBucket = null; + foreach ($buckets as $bucket) { + $files = []; + $lastFile = null; while (true) { - $currentBuckets = $this->storage->listBuckets( - $lastBucket - ? [Query::cursorAfter($lastBucket)] + $currentFiles = $this->storage->listFiles( + $bucket['$id'], + $lastFile + ? [Query::cursorAfter($lastFile)] : [Query::limit(20)] - )['buckets']; + )['files']; - $buckets = array_merge($buckets, $currentBuckets); - $lastBucket = $buckets[count($buckets) - 1]['$id'] ?? null; + $files = array_merge($files, $currentFiles); + $lastFile = $files[count($files) - 1]['$id'] ?? null; - if (count($currentBuckets) < 20) { + if (count($currentFiles) < 20) { break; } } - foreach ($buckets as $bucket) { - $files = []; - $lastFile = null; - - while (true) { - $currentFiles = $this->storage->listFiles( - $bucket['$id'], - $lastFile - ? [Query::cursorAfter($lastFile)] - : [Query::limit(20)] - )['files']; - - $files = array_merge($files, $currentFiles); - $lastFile = $files[count($files) - 1]['$id'] ?? null; - - if (count($currentFiles) < 20) { - break; - } - } - - $report[Resource::TYPE_FILE] += count($files); - foreach ($files as $file) { - $report['size'] += $this->storage->getFile( - $bucket['$id'], - $file['$id'] - )['sizeOriginal']; - } + $report[Resource::TYPE_FILE] += count($files); + foreach ($files as $file) { + $report['size'] += $this->storage->getFile( + $bucket['$id'], + $file['$id'] + )['sizeOriginal']; } - $report['size'] = $report['size'] / 1000 / 1000; // MB - } - - // Functions - $scope = 'functions.read'; - if (\in_array(Resource::TYPE_FUNCTION, $resources)) { - $report[Resource::TYPE_FUNCTION] = $this->functions->list()['total']; } + $report['size'] = $report['size'] / 1000 / 1000; // MB + } + } - if (\in_array(Resource::TYPE_DEPLOYMENT, $resources)) { - $report[Resource::TYPE_DEPLOYMENT] = 0; - $functions = $this->functions->list()['functions']; - foreach ($functions as $function) { - if (!empty($function['deployment'])) { - $report[Resource::TYPE_DEPLOYMENT] += 1; - } - } - } + private function reportFunctions(array $resources, array &$report): void + { + if (\in_array(Resource::TYPE_FUNCTION, $resources)) { + $report[Resource::TYPE_FUNCTION] = $this->functions->list()['total']; + } - if (\in_array(Resource::TYPE_ENVIRONMENT_VARIABLE, $resources)) { - $report[Resource::TYPE_ENVIRONMENT_VARIABLE] = 0; - $functions = $this->functions->list()['functions']; - foreach ($functions as $function) { - $report[Resource::TYPE_ENVIRONMENT_VARIABLE] += $this->functions->listVariables($function['$id'])['total']; + if (\in_array(Resource::TYPE_DEPLOYMENT, $resources)) { + $report[Resource::TYPE_DEPLOYMENT] = 0; + $functions = $this->functions->list()['functions']; + foreach ($functions as $function) { + if (!empty($function['deployment'])) { + $report[Resource::TYPE_DEPLOYMENT] += 1; } } + } - $report['version'] = $this->call( - 'GET', - '/health/version', - [ - 'X-Appwrite-Key' => '', - 'X-Appwrite-Project' => '', - ] - )['version']; - - $this->previousReport = $report; - - return $report; - } catch (\Throwable $e) { - if ($e->getCode() === 403) { - throw new \Exception("Missing scope: $scope."); - } else { - throw new \Exception($e->getMessage(), previous: $e); + if (\in_array(Resource::TYPE_ENVIRONMENT_VARIABLE, $resources)) { + $report[Resource::TYPE_ENVIRONMENT_VARIABLE] = 0; + $functions = $this->functions->list()['functions']; + foreach ($functions as $function) { + $report[Resource::TYPE_ENVIRONMENT_VARIABLE] += $this->functions->listVariables($function['$id'])['total']; } } } From f22d3727f3195a20eb7390df9eec8555c3bf9f5d Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sat, 15 Mar 2025 22:30:12 +1300 Subject: [PATCH 15/38] Read from reader --- src/Migration/Sources/Appwrite.php | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index aab81f87..06f3016c 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -11,8 +11,6 @@ use Appwrite\Services\Teams; use Appwrite\Services\Users; use Utopia\Database\Database as UtopiaDatabase; -use Utopia\Database\Exception as DatabaseException; -use Utopia\Database\Exception\Timeout; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Auth\Hash; @@ -301,8 +299,8 @@ private function reportFunctions(array $resources, array &$report): void /** * Export Auth Resources * - * @param int $batchSize Max 100 - * @param array $resources + * @param int $batchSize Max 100 + * @param array $resources */ protected function exportGroupAuth(int $batchSize, array $resources): void { @@ -386,7 +384,7 @@ private function exportUsers(int $batchSize): void '', $user['emailVerification'] ?? false, $user['phoneVerification'] ?? false, - ! $user['status'], + !$user['status'], $user['prefs'] ?? [], ); @@ -599,7 +597,6 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void } } - /** * @param int $batchSize * @throws Exception @@ -622,7 +619,7 @@ private function exportDatabases(int $batchSize): void $queries[] = Query::cursorAfter($lastDatabase); } - $response = $this->listDatabases($queries); + $response = $this->database->listDatabases($queries); foreach ($response as $database) { $newDatabase = new Database( @@ -669,7 +666,7 @@ private function exportCollections(int $batchSize): void $queries[] = Query::cursorAfter($lastCollection); } - $response = $this->listCollections($database, $queries); + $response = $this->database->listCollections($database, $queries); foreach ($response as $collection) { $newCollection = new Collection( @@ -700,7 +697,6 @@ private function exportCollections(int $batchSize): void } } - /** * @param int $batchSize * @throws Exception @@ -720,7 +716,7 @@ private function exportAttributes(int $batchSize): void $queries[] = Query::cursorAfter($lastAttribute); } - $response = $this->listAttributes($collection, $queries); + $response = $this->database->listAttributes($collection, $queries); foreach ($response as $attribute) { if ( @@ -899,7 +895,7 @@ private function exportIndexes(int $batchSize): void $queries[] = Query::cursorAfter($lastIndex); } - $response = $this->listIndexes($collection, $queries); + $response = $this->database->listIndexes($collection, $queries); foreach ($response['indexes'] as $index) { $indexes[] = new Index( @@ -976,18 +972,18 @@ private function exportDocuments(int $batchSize): void $queries[] = Query::select($selects); - $response = $this->listDocuments($collection, $queries); + $response = $this->database->listDocuments($collection, $queries); foreach ($response as $document) { // HACK: Handle many to many - if (! empty($manyToMany)) { + if (!empty($manyToMany)) { $stack = ['$id']; // Adding $id because we can't select only relations foreach ($manyToMany as $relation) { - $stack[] = $relation.'.$id'; + $stack[] = $relation . '.$id'; } - $doc = $this->db->getDocument( - $collection->getId(), + $doc = $this->database->getDocument( + $collection, $document['$id'], [Query::select($stack)] ); @@ -1604,7 +1600,7 @@ private function exportDeploymentData(Func $func, array $deployment): void ); // Content-Length header was missing, file is less than max buffer size. - if (! array_key_exists('Content-Length', $responseHeaders)) { + if (!array_key_exists('Content-Length', $responseHeaders)) { $file = $this->call( 'GET', "/functions/{$func->getId()}/deployments/{$deployment['$id']}/download", From ef20d0500541309ff31dbec4a403ebe6469d4ac6 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sat, 15 Mar 2025 22:30:52 +1300 Subject: [PATCH 16/38] Update lock --- composer.lock | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/composer.lock b/composer.lock index b172e0c7..81c0b86a 100644 --- a/composer.lock +++ b/composer.lock @@ -191,16 +191,16 @@ }, { "name": "google/protobuf", - "version": "v4.30.0", + "version": "v4.30.1", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6" + "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/e1d66682f6836aa87820400f0aa07d9eb566feb6", - "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/f29ba8a30dfd940efb3a8a75dc44446539101f24", + "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24", "shasum": "" }, "require": { @@ -229,9 +229,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.0" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.1" }, - "time": "2025-03-04T22:54:49+00:00" + "time": "2025-03-13T21:08:17+00:00" }, { "name": "jean85/pretty-package-versions", @@ -2128,16 +2128,16 @@ }, { "name": "utopia-php/database", - "version": "0.61.0", + "version": "0.61.1", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "7961014d5ee50c05a45653c1b91e37fcf18aed3e" + "reference": "2e0165bd14a570ec151f400ed381108e81d15b94" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/7961014d5ee50c05a45653c1b91e37fcf18aed3e", - "reference": "7961014d5ee50c05a45653c1b91e37fcf18aed3e", + "url": "https://api.github.com/repos/utopia-php/database/zipball/2e0165bd14a570ec151f400ed381108e81d15b94", + "reference": "2e0165bd14a570ec151f400ed381108e81d15b94", "shasum": "" }, "require": { @@ -2178,9 +2178,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.61.0" + "source": "https://github.com/utopia-php/database/tree/0.61.1" }, - "time": "2025-03-06T03:13:06+00:00" + "time": "2025-03-14T01:19:38+00:00" }, { "name": "utopia-php/dsn", @@ -2565,16 +2565,16 @@ }, { "name": "laravel/pint", - "version": "v1.21.1", + "version": "v1.21.2", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86" + "reference": "370772e7d9e9da087678a0edf2b11b6960e40558" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/c44bffbb2334e90fba560933c45948fa4a3f3e86", - "reference": "c44bffbb2334e90fba560933c45948fa4a3f3e86", + "url": "https://api.github.com/repos/laravel/pint/zipball/370772e7d9e9da087678a0edf2b11b6960e40558", + "reference": "370772e7d9e9da087678a0edf2b11b6960e40558", "shasum": "" }, "require": { @@ -2585,9 +2585,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.70.2", - "illuminate/view": "^11.44.1", - "larastan/larastan": "^3.1.0", + "friendsofphp/php-cs-fixer": "^3.72.0", + "illuminate/view": "^11.44.2", + "larastan/larastan": "^3.2.0", "laravel-zero/framework": "^11.36.1", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3", @@ -2627,7 +2627,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-03-11T03:22:21+00:00" + "time": "2025-03-14T22:31:42+00:00" }, { "name": "myclabs/deep-copy", From eceaa6c00cf5bdf5047f93e12f6fb4410c56765b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sat, 15 Mar 2025 22:42:23 +1300 Subject: [PATCH 17/38] Default to API --- src/Migration/Sources/Appwrite.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 06f3016c..d3827920 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -64,7 +64,7 @@ public function __construct( protected string $project, protected string $endpoint, protected string $key, - protected DataSource $dataSource, + protected DataSource $dataSource = DataSource::API, protected ?UtopiaDatabase $dbForProject = null ) { From 3db6e7d597fa6067d392c82613b71d12d2172336 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sat, 15 Mar 2025 23:59:29 +1300 Subject: [PATCH 18/38] Fix ref --- src/Migration/Sources/Appwrite/Reader/DatabaseReader.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php index b67b6fbe..6ca957ea 100644 --- a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php +++ b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php @@ -47,7 +47,7 @@ public function report(array $resources, array &$report): void $collections = $this->listCollections($dbResource); foreach ($collections as $collection) { - $collectionId = "database_{$database->getInternal()}_collection_{$collection->getInternalId()}"; + $collectionId = "database_{$database->getInternalId()}_collection_{$collection->getInternalId()}"; $report[Resource::TYPE_DOCUMENT] += $this->countResources($collectionId); } From 63e0025122c24b13b5426be59ef1ba9057440f06 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sun, 16 Mar 2025 00:30:39 +1300 Subject: [PATCH 19/38] Flatten relationships from DB --- .../Sources/Appwrite/Reader/DatabaseReader.php | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php index 6ca957ea..1fe7ac1e 100644 --- a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php +++ b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php @@ -179,7 +179,7 @@ public function listAttributes(Collection $resource, array $queries = []): array $queries[] = Query::equal('collectionInternalId', [$collection->getInternalId()]); try { - return $this->dbForProject->find('attributes', $queries); + $attributes = $this->dbForProject->find('attributes', $queries); } catch (DatabaseException $e) { throw new Exception( resourceName: $resource->getName(), @@ -190,6 +190,21 @@ public function listAttributes(Collection $resource, array $queries = []): array previous: $e ); } + + foreach ($attributes as $attribute) { + if ($attribute['type'] !== UtopiaDatabase::VAR_RELATIONSHIP) { + continue; + } + + $options = $attribute['options']; + foreach ($options as $key => $value) { + $attribute[$key] = $value; + } + + unset($attribute['options']); + } + + return $attributes; } public function listIndexes(Collection $resource, array $queries = []): array From 245ed2dec523a376aa49302c6710ee4849853c47 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Sun, 16 Mar 2025 00:36:14 +1300 Subject: [PATCH 20/38] Fix extraneous key --- src/Migration/Sources/Appwrite.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index d3827920..8787e9cd 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -897,7 +897,7 @@ private function exportIndexes(int $batchSize): void $response = $this->database->listIndexes($collection, $queries); - foreach ($response['indexes'] as $index) { + foreach ($response as $index) { $indexes[] = new Index( 'unique()', $index['key'], From c5b8fe1199bccbdd6b603453682d454c02727d26 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Mar 2025 13:26:34 +1300 Subject: [PATCH 21/38] Fix using external ID for document fetch --- .../Appwrite/Reader/DatabaseReader.php | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php index 1fe7ac1e..c93fe800 100644 --- a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php +++ b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php @@ -302,8 +302,38 @@ public function listDocuments(Collection $resource, array $queries = []): array public function getDocument(Collection $resource, string $documentId, array $queries = []): array { - return $this->dbForProject->getDocument( + $database = $this->dbForProject->getDocument( + 'databases', + $resource->getDatabase()->getId(), + ); + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + $collection = $this->dbForProject->getDocument( + 'database_' . $database->getInternalId(), $resource->getId(), + ); + + if ($collection->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Collection not found', + ); + } + + $collectionId = "database_{$database->getInternalId()}_collection_{$collection->getInternalId()}"; + + return $this->dbForProject->getDocument( + $collectionId, $documentId, $queries )->getArrayCopy(); From 53006b9f47b548b2ade63dcb2ac13f959c8504f6 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Mar 2025 13:54:56 +1300 Subject: [PATCH 22/38] Fix CLI ref --- bin/MigrationCLI.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index 1b21d1d5..430ff606 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -15,6 +15,7 @@ use Utopia\Migration\Destinations\Local; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite; +use Utopia\Migration\Sources\DataSource; use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; @@ -214,7 +215,8 @@ public function getSource(): Source $_ENV['SOURCE_APPWRITE_TEST_PROJECT'], $_ENV['SOURCE_APPWRITE_TEST_ENDPOINT'], $_ENV['SOURCE_APPWRITE_TEST_KEY'], - $this->getDatabase(), + DataSource::DATABASE, + $this->getDatabase(), ); case 'supabase': return new Supabase( From 2803445b4ee89b81ca7c61d943d7a383afd31b6a Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Mar 2025 17:16:17 +1300 Subject: [PATCH 23/38] Fix query type --- bin/MigrationCLI.php | 2 +- src/Migration/Sources/Appwrite/Reader/DatabaseReader.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index 430ff606..ab56f97f 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -216,7 +216,7 @@ public function getSource(): Source $_ENV['SOURCE_APPWRITE_TEST_ENDPOINT'], $_ENV['SOURCE_APPWRITE_TEST_KEY'], DataSource::DATABASE, - $this->getDatabase(), + $this->getDatabase(), ); case 'supabase': return new Supabase( diff --git a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php index c93fe800..f56065e6 100644 --- a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php +++ b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php @@ -2,9 +2,9 @@ namespace Utopia\Migration\Sources\Appwrite\Reader; -use Appwrite\Query; use Utopia\Database\Database as UtopiaDatabase; use Utopia\Database\Exception as DatabaseException; +use Utopia\Database\Query; use Utopia\Migration\Exception; use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Collection; From 23c628875a08c5651071bcb6994fbbeafd43f505 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Mar 2025 17:21:48 +1300 Subject: [PATCH 24/38] Fix document return type --- src/Migration/Sources/Appwrite/Reader/DatabaseReader.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php index f56065e6..63717020 100644 --- a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php +++ b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php @@ -287,7 +287,7 @@ public function listDocuments(Collection $resource, array $queries = []): array $collectionId = "database_{$database->getInternalId()}_collection_{$collection->getInternalId()}"; try { - return $this->dbForProject->find($collectionId, $queries); + $documents = $this->dbForProject->find($collectionId, $queries); } catch (DatabaseException $e) { throw new Exception( resourceName: $resource->getName(), @@ -298,6 +298,10 @@ public function listDocuments(Collection $resource, array $queries = []): array previous: $e ); } + + return \array_map(function ($document) { + return $document->getArrayCopy(); + }, $documents); } public function getDocument(Collection $resource, string $documentId, array $queries = []): array From 5246c0df0304d5071499ce7da0bf7cce7e4c108b Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Mar 2025 18:54:30 +1300 Subject: [PATCH 25/38] Remove unused methods --- bin/MigrationCLI.php | 2 +- src/Migration/Sources/Appwrite.php | 235 ----------------------------- 2 files changed, 1 insertion(+), 236 deletions(-) diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index ab56f97f..bdc62661 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -313,7 +313,7 @@ function (mixed $value) { }, function (mixed $value) { if (is_null($value)) { - return null; + return; } return json_decode($value, true)['value']; diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 8787e9cd..1f0bc257 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -1023,241 +1023,6 @@ private function exportDocuments(int $batchSize): void } } - /** - * @throws Exception - * - * @returns array - */ - private function listDatabases(array $queries = []): array - { - try { - return $this->db->find('databases', $queries); - } catch (DatabaseException $e) { - throw new Exception( - resourceName: 'databases', - resourceGroup: Transfer::GROUP_DATABASES, - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ); - } - } - - /** - * @throws Exception - * - * @returns array - */ - private function listCollections(Database $resource, array $queries = []): array - { - try { - $database = $this->db->getDocument('databases', $resource->getId()); - } catch (DatabaseException $e) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ); - } - - if ($database->isEmpty()) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Database not found', - ); - } - - try { - return $this->db->find( - 'database_' . $database->getInternalId(), - $queries - ); - } catch (DatabaseException $e) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ); - } - } - - /** - * @param Collection $resource - * @param array $queries - * @return array - * @throws DatabaseException - * @throws Exception - */ - private function listAttributes(Collection $resource, array $queries = []): array - { - $database = $this->db->getDocument( - 'databases', - $resource->getDatabase()->getId(), - ); - - if ($database->isEmpty()) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Database not found', - ); - } - - $collection = $this->db->getDocument( - 'database_' . $database->getInternalId(), - $resource->getId(), - ); - - if ($collection->isEmpty()) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Collection not found', - ); - } - - $queries[] = Query::equal('databaseInternalId', [$database->getInternalId()]); - $queries[] = Query::equal('collectionInternalId', [$collection->getInternalId()]); - - try { - return $this->db->find('attributes', $queries); - } catch (DatabaseException $e) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ); - } - } - - /** - * @param Collection $resource - * @param array $queries - * @return array - * @throws DatabaseException - * @throws Exception - */ - private function listIndexes(Collection $resource, array $queries = []): array - { - $database = $this->db->getDocument( - 'databases', - $resource->getDatabase()->getId(), - ); - - if ($database->isEmpty()) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Database not found', - ); - } - - $collection = $this->db->getDocument( - 'database_' . $database->getInternalId(), - $resource->getId(), - ); - - if ($collection->isEmpty()) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Collection not found', - ); - } - - $queries[] = Query::equal('databaseInternalId', [$database->getInternalId()]); - $queries[] = Query::equal('collectionInternalId', [$collection->getInternalId()]); - - try { - return $this->db->find('indexes', $queries); - } catch (DatabaseException $e) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ); - } - } - - private function listDocuments(Collection $resource, array $queries = []): array - { - $database = $this->db->getDocument( - 'databases', - $resource->getDatabase()->getId(), - ); - - if ($database->isEmpty()) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Database not found', - ); - } - - $collection = $this->db->getDocument( - 'database_' . $database->getInternalId(), - $resource->getId(), - ); - - if ($collection->isEmpty()) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: 'Collection not found', - ); - } - - $collectionId = "database_{$database->getInternalId()}_collection_{$collection->getInternalId()}"; - - try { - return $this->db->find($collectionId, $queries); - } catch (DatabaseException $e) { - throw new Exception( - resourceName: $resource->getName(), - resourceGroup: $resource->getGroup(), - resourceId: $resource->getId(), - message: $e->getMessage(), - code: $e->getCode(), - previous: $e - ); - } - } - - /** - * @param string $collection - * @param array $queries - * @return int - * @throws Exception - */ - private function countResources(string $collection, array $queries = []): int - { - try { - return $this->db->count($collection, $queries); - } catch (DatabaseException $e) { - throw new Exception($e->getMessage(), $e->getCode(), $e); - } - } - protected function exportGroupStorage(int $batchSize, array $resources): void { try { From 04c7aad8c1d095d4e8608fbe7bd2a5462aaa02a0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Mar 2025 18:55:03 +1300 Subject: [PATCH 26/38] Add template query type to reader interface --- composer.lock | 12 +++---- src/Migration/Sources/Appwrite/Reader.php | 40 +++++++++++++++++++---- src/Migration/Sources/DataSource.php | 2 +- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/composer.lock b/composer.lock index 81c0b86a..93440d98 100644 --- a/composer.lock +++ b/composer.lock @@ -2128,16 +2128,16 @@ }, { "name": "utopia-php/database", - "version": "0.61.1", + "version": "0.61.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "2e0165bd14a570ec151f400ed381108e81d15b94" + "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/2e0165bd14a570ec151f400ed381108e81d15b94", - "reference": "2e0165bd14a570ec151f400ed381108e81d15b94", + "url": "https://api.github.com/repos/utopia-php/database/zipball/349fbdf4bc088f7775c7dfb8b80239a617a88436", + "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436", "shasum": "" }, "require": { @@ -2178,9 +2178,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.61.1" + "source": "https://github.com/utopia-php/database/tree/0.61.2" }, - "time": "2025-03-14T01:19:38+00:00" + "time": "2025-03-15T11:47:42+00:00" }, { "name": "utopia-php/dsn", diff --git a/src/Migration/Sources/Appwrite/Reader.php b/src/Migration/Sources/Appwrite/Reader.php index 3eeece71..2453e799 100644 --- a/src/Migration/Sources/Appwrite/Reader.php +++ b/src/Migration/Sources/Appwrite/Reader.php @@ -2,10 +2,13 @@ namespace Utopia\Migration\Sources\Appwrite; -use Utopia\Database\Query; +use Utopia\Migration\Resource; use Utopia\Migration\Resources\Database\Collection; use Utopia\Migration\Resources\Database\Database; +/** + * @template QueryType + */ interface Reader { public function report(array $resources, array &$report); @@ -13,7 +16,7 @@ public function report(array $resources, array &$report); /** * List databases that match the given queries * - * @param array $queries + * @param array $queries * @return array */ public function listDatabases(array $queries = []): array; @@ -27,21 +30,21 @@ public function listCollections(Database $resource, array $queries = []): array; /** * @param Collection $resource - * @param array $queries + * @param array $queries * @return array */ public function listAttributes(Collection $resource, array $queries = []): array; /** * @param Collection $resource - * @param array $queries + * @param array $queries * @return array */ public function listIndexes(Collection $resource, array $queries = []): array; /** * @param Collection $resource - * @param array $queries + * @param array $queries * @return array */ public function listDocuments(Collection $resource, array $queries = []): array; @@ -53,4 +56,29 @@ public function listDocuments(Collection $resource, array $queries = []): array; * @return array */ public function getDocument(Collection $resource, string $documentId, array $queries = []): array; -} \ No newline at end of file + + /** + * @param array $attributes + * @return QueryType|string + */ + public function querySelect(array $attributes): mixed; + + /** + * @param string $attribute + * @param array $values + * @return QueryType|string + */ + public function queryEqual(string $attribute, array $values): mixed; + + /** + * @param Resource|string $resource + * @return QueryType|string + */ + public function queryCursorAfter(Resource|string $resource): mixed; + + /** + * @param int $limit + * @return QueryType|string + */ + public function queryLimit(int $limit): mixed; +} diff --git a/src/Migration/Sources/DataSource.php b/src/Migration/Sources/DataSource.php index 523704ea..930cd5b9 100644 --- a/src/Migration/Sources/DataSource.php +++ b/src/Migration/Sources/DataSource.php @@ -6,4 +6,4 @@ enum DataSource { case API; case DATABASE; -} \ No newline at end of file +} From abb352875be4d23b027b46cb91afc2cf6f3cda42 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Mar 2025 18:55:24 +1300 Subject: [PATCH 27/38] Implement API query methods --- .../Sources/Appwrite/Reader/APIReader.php | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/Migration/Sources/Appwrite/Reader/APIReader.php b/src/Migration/Sources/Appwrite/Reader/APIReader.php index 8cc315fc..42cc7de5 100644 --- a/src/Migration/Sources/Appwrite/Reader/APIReader.php +++ b/src/Migration/Sources/Appwrite/Reader/APIReader.php @@ -10,9 +10,12 @@ use Utopia\Migration\Resources\Database\Database; use Utopia\Migration\Sources\Appwrite\Reader; +/** + * @implements Reader + */ class APIReader implements Reader { - public function __construct(private Databases $database) + public function __construct(private readonly Databases $database) { } @@ -161,4 +164,41 @@ public function getDocument(Collection $resource, string $documentId, array $que $queries ); } -} \ No newline at end of file + + /** + * @param array $attributes + * @return string + */ + public function querySelect(array $attributes): mixed + { + return Query::select($attributes); + } + + /** + * @param string $attribute + * @param array $values + * @return string + */ + public function queryEqual(string $attribute, array $values): mixed + { + return Query::equal($attribute, $values); + } + + /** + * @param Resource|string $resource + * @return string + */ + public function queryCursorAfter(Resource|string $resource): mixed + { + if (!\is_string($resource)) { + throw new \InvalidArgumentException('Querying with a cursor through the API requires a string resource ID'); + } + + return Query::cursorAfter($resource); + } + + public function queryLimit(int $limit): mixed + { + return Query::limit($limit); + } +} From af2f32114c251978227d28242743cb3649008522 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Mar 2025 18:55:48 +1300 Subject: [PATCH 28/38] Implement database query methods --- .../Appwrite/Reader/DatabaseReader.php | 69 ++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php index 63717020..af928c1f 100644 --- a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php +++ b/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php @@ -3,14 +3,21 @@ namespace Utopia\Migration\Sources\Appwrite\Reader; use Utopia\Database\Database as UtopiaDatabase; +use Utopia\Database\Document as UtopiaDocument; use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Query; use Utopia\Migration\Exception; use Utopia\Migration\Resource; +use Utopia\Migration\Resources\Database\Attribute; use Utopia\Migration\Resources\Database\Collection; use Utopia\Migration\Resources\Database\Database; +use Utopia\Migration\Resources\Database\Document; +use Utopia\Migration\Resources\Database\Index; use Utopia\Migration\Sources\Appwrite\Reader; +/** + * @implements Reader + */ class DatabaseReader implements Reader { public function __construct(private readonly UtopiaDatabase $dbForProject) @@ -343,6 +350,66 @@ public function getDocument(Collection $resource, string $documentId, array $que )->getArrayCopy(); } + /** + * @param array $attributes + * @return Query + */ + public function querySelect(array $attributes): mixed + { + return Query::select($attributes); + } + + /** + * @param string $attribute + * @param array $values + * @return Query + */ + public function queryEqual(string $attribute, array $values): mixed + { + return Query::equal($attribute, $values); + } + + /** + * @param Resource|string $resource + * @return Query + * @throws DatabaseException + * @throws Exception + */ + public function queryCursorAfter(mixed $resource): mixed + { + if (\is_string($resource)) { + throw new \InvalidArgumentException('Querying with a cursor through the database requires a resource reference'); + } + + switch ($resource::class) { + case Database::class: + $document = $this->dbForProject->getDocument('databases', $resource->getId()); + break; + case Collection::class: + $document = $this->dbForProject->getDocument('database_' . $resource->getDatabase()->getInternalId(), $resource->getId()); + break; + case Attribute::class: + $document = $this->dbForProject->getDocument('attributes', $resource->getId()); + break; + case Index::class: + $document = $this->dbForProject->getDocument('indexes', $resource->getId()); + break; + case Document::class: + $document = $this->getDocument($resource->getCollection(), $resource->getId()); + $document = new UtopiaDocument($document); + break; + default: + throw new \InvalidArgumentException('Unsupported resource type'); + } + + return Query::cursorAfter($document); + } + + public function queryLimit(int $limit): mixed + { + return Query::limit($limit); + } + /** * @param string $collection * @param array $queries @@ -353,4 +420,4 @@ private function countResources(string $collection, array $queries = []): int { return $this->dbForProject->count($collection, $queries); } -} \ No newline at end of file +} From 2ad219a815fa0f9b530cac855cecdacbaa17a46c Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Mar 2025 18:56:03 +1300 Subject: [PATCH 29/38] Implement new query methods --- src/Migration/Sources/Appwrite.php | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 1f0bc257..1f40f789 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -66,8 +66,7 @@ public function __construct( protected string $key, protected DataSource $dataSource = DataSource::API, protected ?UtopiaDatabase $dbForProject = null - ) - { + ) { $this->client = (new Client()) ->setEndpoint($endpoint) ->setProject($project) @@ -606,17 +605,17 @@ private function exportDatabases(int $batchSize): void $lastDatabase = null; while (true) { - $queries = [Query::limit($batchSize)]; + $queries = [$this->database->queryLimit($batchSize)]; if ($this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_DATABASE) { - $queries[] = Query::equal('$id', $this->rootResourceId); - $queries[] = Query::limit(1); + $queries[] = $this->database->queryEqual('$id', [$this->rootResourceId]); + $queries[] = $this->database->queryLimit(1); } $databases = []; if ($lastDatabase) { - $queries[] = Query::cursorAfter($lastDatabase); + $queries[] = $this->database->queryCursorAfter($lastDatabase); } $response = $this->database->listDatabases($queries); @@ -659,11 +658,11 @@ private function exportCollections(int $batchSize): void /** @var Database $database */ while (true) { - $queries = [Query::limit($batchSize)]; + $queries = [$this->database->queryLimit($batchSize)]; $collections = []; if ($lastCollection) { - $queries[] = Query::cursorAfter($lastCollection); + $queries[] = $this->database->queryCursorAfter($lastCollection); } $response = $this->database->listCollections($database, $queries); @@ -709,11 +708,11 @@ private function exportAttributes(int $batchSize): void $lastAttribute = null; while (true) { - $queries = [Query::limit($batchSize)]; + $queries = [$this->database->queryLimit($batchSize)]; $attributes = []; if ($lastAttribute) { - $queries[] = Query::cursorAfter($lastAttribute); + $queries[] = $this->database->queryCursorAfter($lastAttribute); } $response = $this->database->listAttributes($collection, $queries); @@ -888,11 +887,11 @@ private function exportIndexes(int $batchSize): void $lastIndex = null; while (true) { - $queries = [Query::limit($batchSize)]; + $queries = [$this->database->queryLimit($batchSize)]; $indexes = []; if ($lastIndex) { - $queries[] = Query::cursorAfter($lastIndex); + $queries[] = $this->database->queryCursorAfter($lastIndex); } $response = $this->database->listIndexes($collection, $queries); @@ -938,12 +937,12 @@ private function exportDocuments(int $batchSize): void $lastDocument = null; while (true) { - $queries = [Query::limit($batchSize)]; + $queries = [$this->database->queryLimit($batchSize)]; $documents = []; if ($lastDocument) { - $queries[] = Query::cursorAfter($lastDocument); + $queries[] = $this->database->queryCursorAfter($lastDocument); } $selects = ['*', '$id', '$permissions', '$updatedAt', '$createdAt']; // We want relations flat! From 7b2da8eb955164897d599fea718806e2178fe88f Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Mar 2025 19:04:30 +1300 Subject: [PATCH 30/38] Select query updates --- src/Migration/Sources/Appwrite.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 1f40f789..20c096a1 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -969,7 +969,7 @@ private function exportDocuments(int $batchSize): void } /** @var Attribute|Relationship $attribute */ - $queries[] = Query::select($selects); + $queries[] = $this->database->querySelect($selects); $response = $this->database->listDocuments($collection, $queries); @@ -984,7 +984,7 @@ private function exportDocuments(int $batchSize): void $doc = $this->database->getDocument( $collection, $document['$id'], - [Query::select($stack)] + [$this->database->querySelect($stack)] ); foreach ($manyToMany as $key) { From 21b4388517643ee61483825f06a4e7d58fe5a843 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Mon, 17 Mar 2025 19:26:38 +1300 Subject: [PATCH 31/38] Override database batch size per data source --- src/Migration/Sources/Appwrite.php | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 20c096a1..3602743b 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -129,6 +129,17 @@ public static function getSupportedResources(): array ]; } + /** + * @return int + */ + public function getDatabasesBatchSize(): int + { + return match ($this->dataSource) { + DataSource::API => 500, + DataSource::DATABASE => 1000, + }; + } + /** * @param array $resources * @return array @@ -1436,9 +1447,4 @@ private function exportDeploymentData(Func $func, array $deployment): void } } } - - public function getBatchSize(): int - { - return 500; - } } From 9eeb3a5f5e3b76b9ce83d5fd8f75eab3a696a2f0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Mar 2025 15:35:26 +1300 Subject: [PATCH 32/38] Fix pagination --- src/Migration/Sources/Appwrite.php | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 3602743b..12470dc1 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -646,7 +646,7 @@ private function exportDatabases(int $batchSize): void break; } - $lastDatabase = $databases[count($databases) - 1]->getId(); + $lastDatabase = $databases[count($databases) - 1]; $this->callback($databases); @@ -698,7 +698,7 @@ private function exportCollections(int $batchSize): void $this->callback($collections); - $lastCollection = $collections[count($collections) - 1]->getId(); + $lastCollection = $collections[count($collections) - 1]; if (count($collections) < $batchSize) { break; @@ -875,7 +875,7 @@ private function exportAttributes(int $batchSize): void $this->callback($attributes); - $lastAttribute = $attributes[count($attributes) - 1]->getId(); + $lastAttribute = $attributes[count($attributes) - 1]; if (count($attributes) < $batchSize) { break; @@ -927,7 +927,7 @@ private function exportIndexes(int $batchSize): void $this->callback($indexes); - $lastIndex = $indexes[count($indexes) - 1]->getId(); + $lastIndex = $indexes[count($indexes) - 1]; if (count($indexes) < $batchSize) { break; @@ -1014,14 +1014,15 @@ private function exportDocuments(int $batchSize): void unset($document['$collectionId']); unset($document['$databaseId']); - $documents[] = new Document( + $document = new Document( $id, $collection, $document, $permissions ); - - $lastDocument = $id; + + $documents[] = $document; + $lastDocument = $document; } $this->callback($documents); From 3828215038bd6a199dea111f9d81e3e0b841d3a8 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Mar 2025 16:05:50 +1300 Subject: [PATCH 33/38] Format --- src/Migration/Sources/Appwrite.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 12470dc1..6a5b574f 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -1020,7 +1020,7 @@ private function exportDocuments(int $batchSize): void $document, $permissions ); - + $documents[] = $document; $lastDocument = $document; } From a318b599363fdb392e48e7bd81a21cf43d2b7efb Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Mar 2025 16:12:13 +1300 Subject: [PATCH 34/38] Naming --- src/Migration/Sources/Appwrite.php | 4 +- .../Reader/{APIReader.php => API.php} | 10 ++-- .../{DatabaseReader.php => Database.php} | 46 +++++++++---------- 3 files changed, 30 insertions(+), 30 deletions(-) rename src/Migration/Sources/Appwrite/Reader/{APIReader.php => API.php} (97%) rename src/Migration/Sources/Appwrite/Reader/{DatabaseReader.php => Database.php} (90%) diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 6a5b574f..491de51b 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -39,8 +39,8 @@ use Utopia\Migration\Resources\Storage\File; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite\Reader; -use Utopia\Migration\Sources\Appwrite\Reader\APIReader; -use Utopia\Migration\Sources\Appwrite\Reader\DatabaseReader; +use Utopia\Migration\Sources\Appwrite\Reader\API as APIReader; +use Utopia\Migration\Sources\Appwrite\Reader\Database as DatabaseReader; use Utopia\Migration\Transfer; class Appwrite extends Source diff --git a/src/Migration/Sources/Appwrite/Reader/APIReader.php b/src/Migration/Sources/Appwrite/Reader/API.php similarity index 97% rename from src/Migration/Sources/Appwrite/Reader/APIReader.php rename to src/Migration/Sources/Appwrite/Reader/API.php index 42cc7de5..5cec16bc 100644 --- a/src/Migration/Sources/Appwrite/Reader/APIReader.php +++ b/src/Migration/Sources/Appwrite/Reader/API.php @@ -13,7 +13,7 @@ /** * @implements Reader */ -class APIReader implements Reader +class API implements Reader { public function __construct(private readonly Databases $database) { @@ -169,7 +169,7 @@ public function getDocument(Collection $resource, string $documentId, array $que * @param array $attributes * @return string */ - public function querySelect(array $attributes): mixed + public function querySelect(array $attributes): string { return Query::select($attributes); } @@ -179,7 +179,7 @@ public function querySelect(array $attributes): mixed * @param array $values * @return string */ - public function queryEqual(string $attribute, array $values): mixed + public function queryEqual(string $attribute, array $values): string { return Query::equal($attribute, $values); } @@ -188,7 +188,7 @@ public function queryEqual(string $attribute, array $values): mixed * @param Resource|string $resource * @return string */ - public function queryCursorAfter(Resource|string $resource): mixed + public function queryCursorAfter(Resource|string $resource): string { if (!\is_string($resource)) { throw new \InvalidArgumentException('Querying with a cursor through the API requires a string resource ID'); @@ -197,7 +197,7 @@ public function queryCursorAfter(Resource|string $resource): mixed return Query::cursorAfter($resource); } - public function queryLimit(int $limit): mixed + public function queryLimit(int $limit): string { return Query::limit($limit); } diff --git a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php b/src/Migration/Sources/Appwrite/Reader/Database.php similarity index 90% rename from src/Migration/Sources/Appwrite/Reader/DatabaseReader.php rename to src/Migration/Sources/Appwrite/Reader/Database.php index af928c1f..98c2f0da 100644 --- a/src/Migration/Sources/Appwrite/Reader/DatabaseReader.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -8,17 +8,17 @@ use Utopia\Database\Query; use Utopia\Migration\Exception; use Utopia\Migration\Resource; -use Utopia\Migration\Resources\Database\Attribute; -use Utopia\Migration\Resources\Database\Collection; -use Utopia\Migration\Resources\Database\Database; -use Utopia\Migration\Resources\Database\Document; -use Utopia\Migration\Resources\Database\Index; +use Utopia\Migration\Resources\Database\Attribute as AttributeResource; +use Utopia\Migration\Resources\Database\Collection as CollectionResource; +use Utopia\Migration\Resources\Database\Database as DatabaseResource; +use Utopia\Migration\Resources\Database\Document as DocumentResource; +use Utopia\Migration\Resources\Database\Index as IndexResource; use Utopia\Migration\Sources\Appwrite\Reader; /** * @implements Reader */ -class DatabaseReader implements Reader +class Database implements Reader { public function __construct(private readonly UtopiaDatabase $dbForProject) { @@ -44,7 +44,7 @@ public function report(array $resources, array &$report): void $report[Resource::TYPE_DOCUMENT] = 0; $databases = $this->listDatabases(); foreach ($databases as $database) { - $dbResource = new Database( + $dbResource = new DatabaseResource( $database->getId(), $database->getAttribute('name'), $database->getCreatedAt(), @@ -65,7 +65,7 @@ public function report(array $resources, array &$report): void $report[Resource::TYPE_ATTRIBUTE] = 0; $databases = $this->listDatabases(); foreach ($databases as $database) { - $dbResource = new Database( + $dbResource = new DatabaseResource( $database->getId(), $database->getAttribute('name'), $database->getCreatedAt(), @@ -87,7 +87,7 @@ public function report(array $resources, array &$report): void $report[Resource::TYPE_INDEX] = 0; $databases = $this->listDatabases(); foreach ($databases as $database) { - $dbResource = new Database( + $dbResource = new DatabaseResource( $database->getId(), $database->getAttribute('name'), $database->getCreatedAt(), @@ -111,7 +111,7 @@ public function listDatabases(array $queries = []): array return $this->dbForProject->find('databases', $queries); } - public function listCollections(Database $resource, array $queries = []): array + public function listCollections(DatabaseResource $resource, array $queries = []): array { try { $database = $this->dbForProject->getDocument('databases', $resource->getId()); @@ -152,7 +152,7 @@ public function listCollections(Database $resource, array $queries = []): array } } - public function listAttributes(Collection $resource, array $queries = []): array + public function listAttributes(CollectionResource $resource, array $queries = []): array { $database = $this->dbForProject->getDocument( 'databases', @@ -214,7 +214,7 @@ public function listAttributes(Collection $resource, array $queries = []): array return $attributes; } - public function listIndexes(Collection $resource, array $queries = []): array + public function listIndexes(CollectionResource $resource, array $queries = []): array { $database = $this->dbForProject->getDocument( 'databases', @@ -261,7 +261,7 @@ public function listIndexes(Collection $resource, array $queries = []): array } } - public function listDocuments(Collection $resource, array $queries = []): array + public function listDocuments(CollectionResource $resource, array $queries = []): array { $database = $this->dbForProject->getDocument( 'databases', @@ -311,7 +311,7 @@ public function listDocuments(Collection $resource, array $queries = []): array }, $documents); } - public function getDocument(Collection $resource, string $documentId, array $queries = []): array + public function getDocument(CollectionResource $resource, string $documentId, array $queries = []): array { $database = $this->dbForProject->getDocument( 'databases', @@ -354,7 +354,7 @@ public function getDocument(Collection $resource, string $documentId, array $que * @param array $attributes * @return Query */ - public function querySelect(array $attributes): mixed + public function querySelect(array $attributes): Query { return Query::select($attributes); } @@ -364,7 +364,7 @@ public function querySelect(array $attributes): mixed * @param array $values * @return Query */ - public function queryEqual(string $attribute, array $values): mixed + public function queryEqual(string $attribute, array $values): Query { return Query::equal($attribute, $values); } @@ -375,26 +375,26 @@ public function queryEqual(string $attribute, array $values): mixed * @throws DatabaseException * @throws Exception */ - public function queryCursorAfter(mixed $resource): mixed + public function queryCursorAfter(mixed $resource): Query { if (\is_string($resource)) { throw new \InvalidArgumentException('Querying with a cursor through the database requires a resource reference'); } switch ($resource::class) { - case Database::class: + case DatabaseResource::class: $document = $this->dbForProject->getDocument('databases', $resource->getId()); break; - case Collection::class: + case CollectionResource::class: $document = $this->dbForProject->getDocument('database_' . $resource->getDatabase()->getInternalId(), $resource->getId()); break; - case Attribute::class: + case AttributeResource::class: $document = $this->dbForProject->getDocument('attributes', $resource->getId()); break; - case Index::class: + case IndexResource::class: $document = $this->dbForProject->getDocument('indexes', $resource->getId()); break; - case Document::class: + case DocumentResource::class: $document = $this->getDocument($resource->getCollection(), $resource->getId()); $document = new UtopiaDocument($document); break; @@ -405,7 +405,7 @@ public function queryCursorAfter(mixed $resource): mixed return Query::cursorAfter($document); } - public function queryLimit(int $limit): mixed + public function queryLimit(int $limit): Query { return Query::limit($limit); } From 110cc870cd4274a13cac0d5bec890cb5539ba305 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Mar 2025 16:14:56 +1300 Subject: [PATCH 35/38] Interface doc --- src/Migration/Sources/Appwrite/Reader.php | 25 +++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/Migration/Sources/Appwrite/Reader.php b/src/Migration/Sources/Appwrite/Reader.php index 2453e799..0a8ee835 100644 --- a/src/Migration/Sources/Appwrite/Reader.php +++ b/src/Migration/Sources/Appwrite/Reader.php @@ -11,6 +11,13 @@ */ interface Reader { + /** + * Get information about the resources currently on the source + * + * @param array $resources + * @param array $report + * @return mixed + */ public function report(array $resources, array &$report); /** @@ -22,6 +29,8 @@ public function report(array $resources, array &$report); public function listDatabases(array $queries = []): array; /** + * List collections that match the given queries + * * @param Database $resource * @param array $queries * @return array @@ -29,6 +38,8 @@ public function listDatabases(array $queries = []): array; public function listCollections(Database $resource, array $queries = []): array; /** + * List attributes that match the given queries + * * @param Collection $resource * @param array $queries * @return array @@ -36,6 +47,8 @@ public function listCollections(Database $resource, array $queries = []): array; public function listAttributes(Collection $resource, array $queries = []): array; /** + * List indexes that match the given queries + * * @param Collection $resource * @param array $queries * @return array @@ -43,6 +56,8 @@ public function listAttributes(Collection $resource, array $queries = []): array public function listIndexes(Collection $resource, array $queries = []): array; /** + * List documents that match the given queries + * * @param Collection $resource * @param array $queries * @return array @@ -50,6 +65,8 @@ public function listIndexes(Collection $resource, array $queries = []): array; public function listDocuments(Collection $resource, array $queries = []): array; /** + * Get a document by its ID in the given collection + * * @param Collection $resource * @param string $documentId * @param array $queries @@ -58,12 +75,16 @@ public function listDocuments(Collection $resource, array $queries = []): array; public function getDocument(Collection $resource, string $documentId, array $queries = []): array; /** + * Return a query to select the given attributes + * * @param array $attributes * @return QueryType|string */ public function querySelect(array $attributes): mixed; /** + * Return a query to filter the given attributes + * * @param string $attribute * @param array $values * @return QueryType|string @@ -71,12 +92,16 @@ public function querySelect(array $attributes): mixed; public function queryEqual(string $attribute, array $values): mixed; /** + * Return a query to paginate after the given resource + * * @param Resource|string $resource * @return QueryType|string */ public function queryCursorAfter(Resource|string $resource): mixed; /** + * Return a query to limit the number of results + * * @param int $limit * @return QueryType|string */ From 4de72696b17d22428fcf5a83deee27a0277008cf Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Mar 2025 16:22:49 +1300 Subject: [PATCH 36/38] Format --- src/Migration/Sources/Appwrite/Reader.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Migration/Sources/Appwrite/Reader.php b/src/Migration/Sources/Appwrite/Reader.php index 0a8ee835..e03a6981 100644 --- a/src/Migration/Sources/Appwrite/Reader.php +++ b/src/Migration/Sources/Appwrite/Reader.php @@ -101,7 +101,7 @@ public function queryCursorAfter(Resource|string $resource): mixed; /** * Return a query to limit the number of results - * + * * @param int $limit * @return QueryType|string */ From f35772bea643e119e805baa4b629774c371e76cd Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Mar 2025 18:22:14 +1300 Subject: [PATCH 37/38] Fix collection pagination --- src/Migration/Sources/Appwrite/Reader/Database.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php index 98c2f0da..6da1944a 100644 --- a/src/Migration/Sources/Appwrite/Reader/Database.php +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -386,7 +386,8 @@ public function queryCursorAfter(mixed $resource): Query $document = $this->dbForProject->getDocument('databases', $resource->getId()); break; case CollectionResource::class: - $document = $this->dbForProject->getDocument('database_' . $resource->getDatabase()->getInternalId(), $resource->getId()); + $database = $this->dbForProject->getDocument('databases', $resource->getDatabase()->getId()); + $document = $this->dbForProject->getDocument('database_' . $database->getInternalId(), $resource->getId()); break; case AttributeResource::class: $document = $this->dbForProject->getDocument('attributes', $resource->getId()); From fa93acc57f16e30d4aac67f6f344fe0c809df7a0 Mon Sep 17 00:00:00 2001 From: Jake Barnby Date: Tue, 18 Mar 2025 19:38:59 +1300 Subject: [PATCH 38/38] Enum -> const --- bin/MigrationCLI.php | 3 +-- src/Migration/Sources/Appwrite.php | 19 ++++++++++++------- src/Migration/Sources/DataSource.php | 9 --------- 3 files changed, 13 insertions(+), 18 deletions(-) delete mode 100644 src/Migration/Sources/DataSource.php diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index bdc62661..3528186c 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -15,7 +15,6 @@ use Utopia\Migration\Destinations\Local; use Utopia\Migration\Source; use Utopia\Migration\Sources\Appwrite; -use Utopia\Migration\Sources\DataSource; use Utopia\Migration\Sources\Firebase; use Utopia\Migration\Sources\NHost; use Utopia\Migration\Sources\Supabase; @@ -215,7 +214,7 @@ public function getSource(): Source $_ENV['SOURCE_APPWRITE_TEST_PROJECT'], $_ENV['SOURCE_APPWRITE_TEST_ENDPOINT'], $_ENV['SOURCE_APPWRITE_TEST_KEY'], - DataSource::DATABASE, + Appwrite::SOURCE_DATABASE, $this->getDatabase(), ); case 'supabase': diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 491de51b..88503d78 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -45,6 +45,9 @@ class Appwrite extends Source { + public const SOURCE_API = 'api'; + public const SOURCE_DATABASE = 'database'; + protected Client $client; private Users $users; @@ -64,7 +67,7 @@ public function __construct( protected string $project, protected string $endpoint, protected string $key, - protected DataSource $dataSource = DataSource::API, + protected string $source = self::SOURCE_API, protected ?UtopiaDatabase $dbForProject = null ) { $this->client = (new Client()) @@ -80,16 +83,18 @@ public function __construct( $this->headers['X-Appwrite-Project'] = $this->project; $this->headers['X-Appwrite-Key'] = $this->key; - switch ($this->dataSource) { - case DataSource::API: + switch ($this->source) { + case static::SOURCE_API: $this->database = new APIReader(new Databases($this->client)); break; - case DataSource::DATABASE: + case static::SOURCE_DATABASE: if (\is_null($dbForProject)) { throw new \Exception('Database is required for database source'); } $this->database = new DatabaseReader($dbForProject); break; + default: + throw new \Exception('Unknown source'); } } @@ -134,9 +139,9 @@ public static function getSupportedResources(): array */ public function getDatabasesBatchSize(): int { - return match ($this->dataSource) { - DataSource::API => 500, - DataSource::DATABASE => 1000, + return match ($this->source) { + static::SOURCE_API => 500, + static::SOURCE_DATABASE => 1000, }; } diff --git a/src/Migration/Sources/DataSource.php b/src/Migration/Sources/DataSource.php deleted file mode 100644 index 930cd5b9..00000000 --- a/src/Migration/Sources/DataSource.php +++ /dev/null @@ -1,9 +0,0 @@ -