diff --git a/bin/MigrationCLI.php b/bin/MigrationCLI.php index 04c80a5b..3528186c 100644 --- a/bin/MigrationCLI.php +++ b/bin/MigrationCLI.php @@ -213,7 +213,9 @@ public function getSource(): Source return new Appwrite( $_ENV['SOURCE_APPWRITE_TEST_PROJECT'], $_ENV['SOURCE_APPWRITE_TEST_ENDPOINT'], - $_ENV['SOURCE_APPWRITE_TEST_KEY'] + $_ENV['SOURCE_APPWRITE_TEST_KEY'], + Appwrite::SOURCE_DATABASE, + $this->getDatabase(), ); case 'supabase': return new Supabase( @@ -361,21 +363,18 @@ function (mixed $value, Document $attribute) { } ); - $database = new Database( - new MariaDB(new PDO( - $_ENV['DESTINATION_APPWRITE_TEST_DSN'], - $_ENV['DESTINATION_APPWRITE_TEST_USER'], - $_ENV['DESTINATION_APPWRITE_TEST_PASSWORD'], - [ - PDO::ATTR_TIMEOUT => 3, - PDO::ATTR_PERSISTENT => true, - PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, - PDO::ATTR_EMULATE_PREPARES => true, - PDO::ATTR_STRINGIFY_FETCHES => true - ], - )), - new Cache(new None()) - ); + $database = new Database(new MariaDB(new PDO( + $_ENV['DESTINATION_APPWRITE_TEST_DSN'], + $_ENV['DESTINATION_APPWRITE_TEST_USER'], + $_ENV['DESTINATION_APPWRITE_TEST_PASSWORD'], + [ + PDO::ATTR_TIMEOUT => 3, + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true + ], + )), new Cache(new None())); $database ->setDatabase('appwrite') diff --git a/composer.lock b/composer.lock index a7ae9a3f..93440d98 100644 --- a/composer.lock +++ b/composer.lock @@ -191,16 +191,16 @@ }, { "name": "google/protobuf", - "version": "v4.30.0", + "version": "v4.30.1", "source": { "type": "git", "url": "https://github.com/protocolbuffers/protobuf-php.git", - "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6" + "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/e1d66682f6836aa87820400f0aa07d9eb566feb6", - "reference": "e1d66682f6836aa87820400f0aa07d9eb566feb6", + "url": "https://api.github.com/repos/protocolbuffers/protobuf-php/zipball/f29ba8a30dfd940efb3a8a75dc44446539101f24", + "reference": "f29ba8a30dfd940efb3a8a75dc44446539101f24", "shasum": "" }, "require": { @@ -229,9 +229,9 @@ "proto" ], "support": { - "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.0" + "source": "https://github.com/protocolbuffers/protobuf-php/tree/v4.30.1" }, - "time": "2025-03-04T22:54:49+00:00" + "time": "2025-03-13T21:08:17+00:00" }, { "name": "jean85/pretty-package-versions", @@ -636,16 +636,16 @@ }, { "name": "open-telemetry/exporter-otlp", - "version": "1.2.0", + "version": "1.2.1", "source": { "type": "git", "url": "https://github.com/opentelemetry-php/exporter-otlp.git", - "reference": "243d9657c44a06f740cf384f486afe954c2b725f" + "reference": "b7580440b7481a98da97aceabeb46e1b276c8747" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/243d9657c44a06f740cf384f486afe954c2b725f", - "reference": "243d9657c44a06f740cf384f486afe954c2b725f", + "url": "https://api.github.com/repos/opentelemetry-php/exporter-otlp/zipball/b7580440b7481a98da97aceabeb46e1b276c8747", + "reference": "b7580440b7481a98da97aceabeb46e1b276c8747", "shasum": "" }, "require": { @@ -696,7 +696,7 @@ "issues": "https://github.com/open-telemetry/opentelemetry-php/issues", "source": "https://github.com/open-telemetry/opentelemetry-php" }, - "time": "2025-01-08T23:50:03+00:00" + "time": "2025-03-06T23:21:56+00:00" }, { "name": "open-telemetry/gen-otlp-protobuf", @@ -2128,16 +2128,16 @@ }, { "name": "utopia-php/database", - "version": "0.61.0", + "version": "0.61.2", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "7961014d5ee50c05a45653c1b91e37fcf18aed3e" + "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/7961014d5ee50c05a45653c1b91e37fcf18aed3e", - "reference": "7961014d5ee50c05a45653c1b91e37fcf18aed3e", + "url": "https://api.github.com/repos/utopia-php/database/zipball/349fbdf4bc088f7775c7dfb8b80239a617a88436", + "reference": "349fbdf4bc088f7775c7dfb8b80239a617a88436", "shasum": "" }, "require": { @@ -2178,9 +2178,9 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/0.61.0" + "source": "https://github.com/utopia-php/database/tree/0.61.2" }, - "time": "2025-03-06T03:13:06+00:00" + "time": "2025-03-15T11:47:42+00:00" }, { "name": "utopia-php/dsn", @@ -2231,16 +2231,16 @@ }, { "name": "utopia-php/framework", - "version": "0.33.17", + "version": "0.33.19", "source": { "type": "git", "url": "https://github.com/utopia-php/http.git", - "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644" + "reference": "64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/http/zipball/73fac6fbce9f56282dba4e52a58cf836ec434644", - "reference": "73fac6fbce9f56282dba4e52a58cf836ec434644", + "url": "https://api.github.com/repos/utopia-php/http/zipball/64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0", + "reference": "64c7b7bb8a8595ffe875fa8d4b7705684dbf46c0", "shasum": "" }, "require": { @@ -2272,9 +2272,9 @@ ], "support": { "issues": "https://github.com/utopia-php/http/issues", - "source": "https://github.com/utopia-php/http/tree/0.33.17" + "source": "https://github.com/utopia-php/http/tree/0.33.19" }, - "time": "2025-02-24T17:35:48+00:00" + "time": "2025-03-06T11:37:49+00:00" }, { "name": "utopia-php/mongo", @@ -2338,22 +2338,24 @@ }, { "name": "utopia-php/storage", - "version": "0.18.9", + "version": "0.18.10", "source": { "type": "git", "url": "https://github.com/utopia-php/storage.git", - "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c" + "reference": "76f31158f4251abb207f7a9b16f7cb0bfdb3b39e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/storage/zipball/1cf455404e8700b3093fd73d74a38d41cdced90c", - "reference": "1cf455404e8700b3093fd73d74a38d41cdced90c", + "url": "https://api.github.com/repos/utopia-php/storage/zipball/76f31158f4251abb207f7a9b16f7cb0bfdb3b39e", + "reference": "76f31158f4251abb207f7a9b16f7cb0bfdb3b39e", "shasum": "" }, "require": { "ext-brotli": "*", + "ext-curl": "*", "ext-fileinfo": "*", "ext-lz4": "*", + "ext-simplexml": "*", "ext-snappy": "*", "ext-xz": "*", "ext-zlib": "*", @@ -2387,9 +2389,9 @@ ], "support": { "issues": "https://github.com/utopia-php/storage/issues", - "source": "https://github.com/utopia-php/storage/tree/0.18.9" + "source": "https://github.com/utopia-php/storage/tree/0.18.10" }, - "time": "2025-02-11T13:10:40+00:00" + "time": "2025-03-03T10:47:54+00:00" }, { "name": "utopia-php/system", @@ -2563,16 +2565,16 @@ }, { "name": "laravel/pint", - "version": "v1.21.0", + "version": "v1.21.2", "source": { "type": "git", "url": "https://github.com/laravel/pint.git", - "reference": "531fa0871fbde719c51b12afa3a443b8f4e4b425" + "reference": "370772e7d9e9da087678a0edf2b11b6960e40558" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/pint/zipball/531fa0871fbde719c51b12afa3a443b8f4e4b425", - "reference": "531fa0871fbde719c51b12afa3a443b8f4e4b425", + "url": "https://api.github.com/repos/laravel/pint/zipball/370772e7d9e9da087678a0edf2b11b6960e40558", + "reference": "370772e7d9e9da087678a0edf2b11b6960e40558", "shasum": "" }, "require": { @@ -2583,9 +2585,9 @@ "php": "^8.2.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^3.68.5", - "illuminate/view": "^11.42.0", - "larastan/larastan": "^3.0.4", + "friendsofphp/php-cs-fixer": "^3.72.0", + "illuminate/view": "^11.44.2", + "larastan/larastan": "^3.2.0", "laravel-zero/framework": "^11.36.1", "mockery/mockery": "^1.6.12", "nunomaduro/termwind": "^2.3", @@ -2625,7 +2627,7 @@ "issues": "https://github.com/laravel/pint/issues", "source": "https://github.com/laravel/pint" }, - "time": "2025-02-18T03:18:57+00:00" + "time": "2025-03-14T22:31:42+00:00" }, { "name": "myclabs/deep-copy", @@ -2940,16 +2942,16 @@ }, { "name": "phpstan/phpstan", - "version": "1.12.20", + "version": "1.12.21", "source": { "type": "git", "url": "https://github.com/phpstan/phpstan.git", - "reference": "3240b1972042c7f73cf1045e879ea5bd5f761bb7" + "reference": "14276fdef70575106a3392a4ed553c06a984df28" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/3240b1972042c7f73cf1045e879ea5bd5f761bb7", - "reference": "3240b1972042c7f73cf1045e879ea5bd5f761bb7", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/14276fdef70575106a3392a4ed553c06a984df28", + "reference": "14276fdef70575106a3392a4ed553c06a984df28", "shasum": "" }, "require": { @@ -2994,7 +2996,7 @@ "type": "github" } ], - "time": "2025-03-05T13:37:43+00:00" + "time": "2025-03-09T09:24:50+00:00" }, { "name": "phpunit/php-code-coverage", @@ -3321,16 +3323,16 @@ }, { "name": "phpunit/phpunit", - "version": "11.5.11", + "version": "11.5.12", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "3946ac38410be7440186c6e74584f31b15107fc7" + "reference": "d42785840519401ed2113292263795eb4c0f95da" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/3946ac38410be7440186c6e74584f31b15107fc7", - "reference": "3946ac38410be7440186c6e74584f31b15107fc7", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/d42785840519401ed2113292263795eb4c0f95da", + "reference": "d42785840519401ed2113292263795eb4c0f95da", "shasum": "" }, "require": { @@ -3351,7 +3353,7 @@ "phpunit/php-timer": "^7.0.1", "sebastian/cli-parser": "^3.0.2", "sebastian/code-unit": "^3.0.2", - "sebastian/comparator": "^6.3.0", + "sebastian/comparator": "^6.3.1", "sebastian/diff": "^6.0.2", "sebastian/environment": "^7.2.0", "sebastian/exporter": "^6.3.0", @@ -3402,7 +3404,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.11" + "source": "https://github.com/sebastianbergmann/phpunit/tree/11.5.12" }, "funding": [ { @@ -3418,7 +3420,7 @@ "type": "tidelift" } ], - "time": "2025-03-05T07:36:02+00:00" + "time": "2025-03-07T07:31:03+00:00" }, { "name": "sebastian/cli-parser", @@ -3592,16 +3594,16 @@ }, { "name": "sebastian/comparator", - "version": "6.3.0", + "version": "6.3.1", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/comparator.git", - "reference": "d4e47a769525c4dd38cea90e5dcd435ddbbc7115" + "reference": "24b8fbc2c8e201bb1308e7b05148d6ab393b6959" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/d4e47a769525c4dd38cea90e5dcd435ddbbc7115", - "reference": "d4e47a769525c4dd38cea90e5dcd435ddbbc7115", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/24b8fbc2c8e201bb1308e7b05148d6ab393b6959", + "reference": "24b8fbc2c8e201bb1308e7b05148d6ab393b6959", "shasum": "" }, "require": { @@ -3620,7 +3622,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-main": "6.2-dev" + "dev-main": "6.3-dev" } }, "autoload": { @@ -3660,7 +3662,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/comparator/issues", "security": "https://github.com/sebastianbergmann/comparator/security/policy", - "source": "https://github.com/sebastianbergmann/comparator/tree/6.3.0" + "source": "https://github.com/sebastianbergmann/comparator/tree/6.3.1" }, "funding": [ { @@ -3668,7 +3670,7 @@ "type": "github" } ], - "time": "2025-01-06T10:28:19+00:00" + "time": "2025-03-07T06:57:01+00:00" }, { "name": "sebastian/complexity", diff --git a/src/Migration/Destinations/Appwrite.php b/src/Migration/Destinations/Appwrite.php index e1b9eac2..1fd182e5 100644 --- a/src/Migration/Destinations/Appwrite.php +++ b/src/Migration/Destinations/Appwrite.php @@ -438,6 +438,7 @@ protected function createAttribute(Attribute $resource): bool 'databases', $resource->getCollection()->getDatabase()->getId(), ); + if ($database->isEmpty()) { throw new Exception( resourceName: $resource->getName(), @@ -451,6 +452,7 @@ protected function createAttribute(Attribute $resource): bool 'database_' . $database->getInternalId(), $resource->getCollection()->getId(), ); + if ($collection->isEmpty()) { throw new Exception( resourceName: $resource->getName(), diff --git a/src/Migration/Resources/Database/Attribute.php b/src/Migration/Resources/Database/Attribute.php index 826c2a9d..dab1d9a9 100644 --- a/src/Migration/Resources/Database/Attribute.php +++ b/src/Migration/Resources/Database/Attribute.php @@ -8,14 +8,14 @@ abstract class Attribute extends Resource { public const TYPE_STRING = 'string'; - public const TYPE_INTEGER = 'int'; - public const TYPE_FLOAT = 'float'; - public const TYPE_BOOLEAN = 'bool'; - public const TYPE_DATETIME = 'dateTime'; + public const TYPE_INTEGER = 'integer'; + public const TYPE_FLOAT = 'double'; + public const TYPE_BOOLEAN = 'boolean'; + public const TYPE_DATETIME = 'datetime'; public const TYPE_EMAIL = 'email'; public const TYPE_ENUM = 'enum'; - public const TYPE_IP = 'IP'; - public const TYPE_URL = 'URL'; + public const TYPE_IP = 'ip'; + public const TYPE_URL = 'url'; public const TYPE_RELATIONSHIP = 'relationship'; /** diff --git a/src/Migration/Source.php b/src/Migration/Source.php index 46333b61..fb4a146a 100644 --- a/src/Migration/Source.php +++ b/src/Migration/Source.php @@ -4,6 +4,8 @@ abstract class Source extends Target { + protected static int $defaultBatchSize = 100; + /** * @var callable(array): void $transferCallback */ @@ -14,6 +16,26 @@ abstract class Source extends Target */ public array $previousReport = []; + public function getAuthBatchSize(): int + { + return static::$defaultBatchSize; + } + + public function getDatabasesBatchSize(): int + { + return static::$defaultBatchSize; + } + + public function getStorageBatchSize(): int + { + return static::$defaultBatchSize; + } + + public function getFunctionsBatchSize(): int + { + return static::$defaultBatchSize; + } + /** * @param array $resources * @return void @@ -39,7 +61,7 @@ public function run(array $resources, callable $callback, string $rootResourceId $prunedResources = []; foreach ($returnedResources as $resource) { /** @var Resource $resource */ - if (! in_array($resource->getName(), $resources)) { + if (!in_array($resource->getName(), $resources)) { $resource->setStatus(Resource::STATUS_SKIPPED); } else { $prunedResources[] = $resource; @@ -56,24 +78,24 @@ public function run(array $resources, callable $callback, string $rootResourceId /** * Export Resources * - * @param array $resources Resources to export + * @param array $resources Resources to export */ public function exportResources(array $resources): void { - // Convert Resources back into their relevant groups - - $batchSize = $this->getBatchSize(); - $groups = []; foreach ($resources as $resource) { - if (\in_array($resource, Transfer::GROUP_AUTH_RESOURCES)) { - $groups[Transfer::GROUP_AUTH][] = $resource; - } elseif (\in_array($resource, Transfer::GROUP_DATABASES_RESOURCES)) { - $groups[Transfer::GROUP_DATABASES][] = $resource; - } elseif (\in_array($resource, Transfer::GROUP_STORAGE_RESOURCES)) { - $groups[Transfer::GROUP_STORAGE][] = $resource; - } elseif (\in_array($resource, Transfer::GROUP_FUNCTIONS_RESOURCES)) { - $groups[Transfer::GROUP_FUNCTIONS][] = $resource; + $mapping = [ + Transfer::GROUP_AUTH => Transfer::GROUP_AUTH_RESOURCES, + Transfer::GROUP_DATABASES => Transfer::GROUP_DATABASES_RESOURCES, + Transfer::GROUP_STORAGE => Transfer::GROUP_STORAGE_RESOURCES, + Transfer::GROUP_FUNCTIONS => Transfer::GROUP_FUNCTIONS_RESOURCES, + ]; + + foreach ($mapping as $group => $resources) { + if (\in_array($resource, $resources, true)) { + $groups[$group][] = $resource; + break; + } } } @@ -81,58 +103,53 @@ public function exportResources(array $resources): void return; } - // Send each group to the relevant export function foreach ($groups as $group => $resources) { switch ($group) { case Transfer::GROUP_AUTH: - $this->exportGroupAuth($batchSize, $resources); + $this->exportGroupAuth($this->getAuthBatchSize(), $resources); break; case Transfer::GROUP_DATABASES: - $this->exportGroupDatabases($batchSize, $resources); + $this->exportGroupDatabases($this->getDatabasesBatchSize(), $resources); break; case Transfer::GROUP_STORAGE: - $this->exportGroupStorage($batchSize, $resources); + $this->exportGroupStorage($this->getStorageBatchSize(), $resources); break; case Transfer::GROUP_FUNCTIONS: - $this->exportGroupFunctions($batchSize, $resources); + $this->exportGroupFunctions($this->getFunctionsBatchSize(), $resources); break; } } } - public function getBatchSize(): int - { - return 100; - } /** * Export Auth Group * - * @param int $batchSize Max 100 - * @param array $resources Resources to export + * @param int $batchSize + * @param array $resources Resources to export */ abstract protected function exportGroupAuth(int $batchSize, array $resources): void; /** * Export Databases Group * - * @param int $batchSize Max 100 - * @param array $resources Resources to export + * @param int $batchSize + * @param array $resources Resources to export */ abstract protected function exportGroupDatabases(int $batchSize, array $resources): void; /** * Export Storage Group * - * @param int $batchSize Max 5 - * @param array $resources Resources to export + * @param int $batchSize Max 5 + * @param array $resources Resources to export */ abstract protected function exportGroupStorage(int $batchSize, array $resources): void; /** * Export Functions Group * - * @param int $batchSize Max 100 - * @param array $resources Resources to export + * @param int $batchSize + * @param array $resources Resources to export */ abstract protected function exportGroupFunctions(int $batchSize, array $resources): void; } diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 6ab29796..88503d78 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -38,26 +38,37 @@ use Utopia\Migration\Resources\Storage\Bucket; use Utopia\Migration\Resources\Storage\File; use Utopia\Migration\Source; +use Utopia\Migration\Sources\Appwrite\Reader; +use Utopia\Migration\Sources\Appwrite\Reader\API as APIReader; +use Utopia\Migration\Sources\Appwrite\Reader\Database as DatabaseReader; use Utopia\Migration\Transfer; class Appwrite extends Source { + public const SOURCE_API = 'api'; + public const SOURCE_DATABASE = 'database'; + protected Client $client; private Users $users; private Teams $teams; - private Databases $database; - private Storage $storage; private Functions $functions; + private Reader $database; + + /** + * @throws \Exception + */ public function __construct( protected string $project, - string $endpoint, - protected string $key + protected string $endpoint, + protected string $key, + protected string $source = self::SOURCE_API, + protected ?UtopiaDatabase $dbForProject = null ) { $this->client = (new Client()) ->setEndpoint($endpoint) @@ -66,14 +77,25 @@ public function __construct( $this->users = new Users($this->client); $this->teams = new Teams($this->client); - $this->database = new Databases($this->client); $this->storage = new Storage($this->client); $this->functions = new Functions($this->client); - $this->endpoint = $endpoint; - $this->headers['X-Appwrite-Project'] = $this->project; $this->headers['X-Appwrite-Key'] = $this->key; + + switch ($this->source) { + case static::SOURCE_API: + $this->database = new APIReader(new Databases($this->client)); + break; + case static::SOURCE_DATABASE: + if (\is_null($dbForProject)) { + throw new \Exception('Database is required for database source'); + } + $this->database = new DatabaseReader($dbForProject); + break; + default: + throw new \Exception('Unknown source'); + } } public static function getName(): string @@ -113,7 +135,18 @@ public static function getSupportedResources(): array } /** - * @param array $resources + * @return int + */ + public function getDatabasesBatchSize(): int + { + return match ($this->source) { + static::SOURCE_API => 500, + static::SOURCE_DATABASE => 1000, + }; + } + + /** + * @param array $resources * @return array * * @throws \Exception @@ -126,193 +159,154 @@ public function report(array $resources = []): array $resources = $this->getSupportedResources(); } - // Auth try { - $scope = 'users.read'; - if (\in_array(Resource::TYPE_USER, $resources)) { - $report[Resource::TYPE_USER] = $this->users->list()['total']; - } + $this->reportAuth($resources, $report); + $this->reportDatabases($resources, $report); + $this->reportStorage($resources, $report); + $this->reportFunctions($resources, $report); - $scope = 'teams.read'; - if (\in_array(Resource::TYPE_TEAM, $resources)) { - $report[Resource::TYPE_TEAM] = $this->teams->list()['total']; + $report['version'] = $this->call( + 'GET', + '/health/version', + [ + 'X-Appwrite-Key' => '', + 'X-Appwrite-Project' => '', + ] + )['version']; + } catch (\Throwable $e) { + if ($e->getCode() === 403) { + throw new \Exception("Missing required scopes."); + } else { + throw new \Exception($e->getMessage(), previous: $e); } + } - if (\in_array(Resource::TYPE_MEMBERSHIP, $resources)) { - $report[Resource::TYPE_MEMBERSHIP] = 0; - $teams = $this->teams->list()['teams']; - foreach ($teams as $team) { - $report[Resource::TYPE_MEMBERSHIP] += $this->teams->listMemberships( - $team['$id'], - [Query::limit(1)] - )['total']; - } - } + $this->previousReport = $report; - // Databases - $scope = 'databases.read'; - if (\in_array(Resource::TYPE_DATABASE, $resources)) { - $report[Resource::TYPE_DATABASE] = $this->database->list()['total']; - } + return $report; + } - $scope = 'collections.read'; - if (\in_array(Resource::TYPE_COLLECTION, $resources)) { - $report[Resource::TYPE_COLLECTION] = 0; - $databases = $this->database->list()['databases']; - foreach ($databases as $database) { - $report[Resource::TYPE_COLLECTION] += $this->database->listCollections( - $database['$id'], - [Query::limit(1)] - )['total']; - } - } + /** + * @param array $resources + * @param array $report + * @throws AppwriteException + */ + private function reportAuth(array $resources, array &$report): void + { + if (\in_array(Resource::TYPE_USER, $resources)) { + $report[Resource::TYPE_USER] = $this->users->list()['total']; + } - $scope = 'documents.read'; - if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { - $report[Resource::TYPE_DOCUMENT] = 0; - $databases = $this->database->list()['databases']; - foreach ($databases as $database) { - $collections = $this->database->listCollections($database['$id'])['collections']; - foreach ($collections as $collection) { - $report[Resource::TYPE_DOCUMENT] += $this->database->listDocuments( - $database['$id'], - $collection['$id'], - [Query::limit(1)] - )['total']; - } - } - } + if (\in_array(Resource::TYPE_TEAM, $resources)) { + $report[Resource::TYPE_TEAM] = $this->teams->list()['total']; + } - $scope = 'attributes.read'; - if (\in_array(Resource::TYPE_ATTRIBUTE, $resources)) { - $report[Resource::TYPE_ATTRIBUTE] = 0; - $databases = $this->database->list()['databases']; - foreach ($databases as $database) { - $collections = $this->database->listCollections($database['$id'])['collections']; - foreach ($collections as $collection) { - $report[Resource::TYPE_ATTRIBUTE] += $this->database->listAttributes( - $database['$id'], - $collection['$id'] - )['total']; - } - } + if (\in_array(Resource::TYPE_MEMBERSHIP, $resources)) { + $report[Resource::TYPE_MEMBERSHIP] = 0; + $teams = $this->teams->list()['teams']; + foreach ($teams as $team) { + $report[Resource::TYPE_MEMBERSHIP] += $this->teams->listMemberships( + $team['$id'], + [Query::limit(1)] + )['total']; } + } + } - $scope = 'indexes.read'; - if (\in_array(Resource::TYPE_INDEX, $resources)) { - $report[Resource::TYPE_INDEX] = 0; - $databases = $this->database->list()['databases']; - foreach ($databases as $database) { - $collections = $this->database->listCollections($database['$id'])['collections']; - foreach ($collections as $collection) { - $report[Resource::TYPE_INDEX] += $this->database->listIndexes( - $database['$id'], - $collection['$id'] - )['total']; - } - } - } + /** + * @throws Exception + * @throws AppwriteException + */ + private function reportDatabases(array $resources, array &$report): void + { + $this->database->report($resources, $report); + } - // Storage - $scope = 'buckets.read'; - if (\in_array(Resource::TYPE_BUCKET, $resources)) { - $report[Resource::TYPE_BUCKET] = $this->storage->listBuckets()['total']; - } + /** + * @param array $resources + * @param array $report + * @throws AppwriteException + */ + private function reportStorage(array $resources, array &$report): void + { + if (\in_array(Resource::TYPE_BUCKET, $resources)) { + $report[Resource::TYPE_BUCKET] = $this->storage->listBuckets()['total']; + } - $scope = 'files.read'; - if (\in_array(Resource::TYPE_FILE, $resources)) { - $report[Resource::TYPE_FILE] = 0; - $report['size'] = 0; - $buckets = []; - $lastBucket = null; + if (\in_array(Resource::TYPE_FILE, $resources)) { + $report[Resource::TYPE_FILE] = 0; + $report['size'] = 0; + $buckets = []; + $lastBucket = null; - while (true) { - $currentBuckets = $this->storage->listBuckets( - $lastBucket - ? [Query::cursorAfter($lastBucket)] - : [Query::limit(20)] - )['buckets']; + while (true) { + $currentBuckets = $this->storage->listBuckets( + $lastBucket + ? [Query::cursorAfter($lastBucket)] + : [Query::limit(20)] + )['buckets']; - $buckets = array_merge($buckets, $currentBuckets); - $lastBucket = $buckets[count($buckets) - 1]['$id'] ?? null; + $buckets = array_merge($buckets, $currentBuckets); + $lastBucket = $buckets[count($buckets) - 1]['$id'] ?? null; - if (count($currentBuckets) < 20) { - break; - } + if (count($currentBuckets) < 20) { + break; } + } - foreach ($buckets as $bucket) { - $files = []; - $lastFile = null; - - while (true) { - $currentFiles = $this->storage->listFiles( - $bucket['$id'], - $lastFile - ? [Query::cursorAfter($lastFile)] - : [Query::limit(20)] - )['files']; + foreach ($buckets as $bucket) { + $files = []; + $lastFile = null; - $files = array_merge($files, $currentFiles); - $lastFile = $files[count($files) - 1]['$id'] ?? null; + while (true) { + $currentFiles = $this->storage->listFiles( + $bucket['$id'], + $lastFile + ? [Query::cursorAfter($lastFile)] + : [Query::limit(20)] + )['files']; - if (count($currentFiles) < 20) { - break; - } - } + $files = array_merge($files, $currentFiles); + $lastFile = $files[count($files) - 1]['$id'] ?? null; - $report[Resource::TYPE_FILE] += count($files); - foreach ($files as $file) { - $report['size'] += $this->storage->getFile( - $bucket['$id'], - $file['$id'] - )['sizeOriginal']; + if (count($currentFiles) < 20) { + break; } } - $report['size'] = $report['size'] / 1000 / 1000; // MB - } - // Functions - $scope = 'functions.read'; - if (\in_array(Resource::TYPE_FUNCTION, $resources)) { - $report[Resource::TYPE_FUNCTION] = $this->functions->list()['total']; - } - - if (\in_array(Resource::TYPE_DEPLOYMENT, $resources)) { - $report[Resource::TYPE_DEPLOYMENT] = 0; - $functions = $this->functions->list()['functions']; - foreach ($functions as $function) { - if (! empty($function['deployment'])) { - $report[Resource::TYPE_DEPLOYMENT] += 1; - } + $report[Resource::TYPE_FILE] += count($files); + foreach ($files as $file) { + $report['size'] += $this->storage->getFile( + $bucket['$id'], + $file['$id'] + )['sizeOriginal']; } } + $report['size'] = $report['size'] / 1000 / 1000; // MB + } + } - if (\in_array(Resource::TYPE_ENVIRONMENT_VARIABLE, $resources)) { - $report[Resource::TYPE_ENVIRONMENT_VARIABLE] = 0; - $functions = $this->functions->list()['functions']; - foreach ($functions as $function) { - $report[Resource::TYPE_ENVIRONMENT_VARIABLE] += $this->functions->listVariables($function['$id'])['total']; + private function reportFunctions(array $resources, array &$report): void + { + if (\in_array(Resource::TYPE_FUNCTION, $resources)) { + $report[Resource::TYPE_FUNCTION] = $this->functions->list()['total']; + } + + if (\in_array(Resource::TYPE_DEPLOYMENT, $resources)) { + $report[Resource::TYPE_DEPLOYMENT] = 0; + $functions = $this->functions->list()['functions']; + foreach ($functions as $function) { + if (!empty($function['deployment'])) { + $report[Resource::TYPE_DEPLOYMENT] += 1; } } + } - $report['version'] = $this->call( - 'GET', - '/health/version', - [ - 'X-Appwrite-Key' => '', - 'X-Appwrite-Project' => '', - ] - )['version']; - - $this->previousReport = $report; - - return $report; - } catch (\Throwable $e) { - if ($e->getCode() === 403) { - throw new \Exception("Missing scope: $scope."); - } else { - throw new \Exception($e->getMessage(), previous: $e); + if (\in_array(Resource::TYPE_ENVIRONMENT_VARIABLE, $resources)) { + $report[Resource::TYPE_ENVIRONMENT_VARIABLE] = 0; + $functions = $this->functions->list()['functions']; + foreach ($functions as $function) { + $report[Resource::TYPE_ENVIRONMENT_VARIABLE] += $this->functions->listVariables($function['$id'])['total']; } } } @@ -320,8 +314,8 @@ public function report(array $resources = []): array /** * Export Auth Resources * - * @param int $batchSize Max 100 - * @param array $resources + * @param int $batchSize Max 100 + * @param array $resources */ protected function exportGroupAuth(int $batchSize, array $resources): void { @@ -405,7 +399,7 @@ private function exportUsers(int $batchSize): void '', $user['emailVerification'] ?? false, $user['phoneVerification'] ?? false, - ! $user['status'], + !$user['status'], $user['prefs'] ?? [], ); @@ -619,301 +613,30 @@ protected function exportGroupDatabases(int $batchSize, array $resources): void } /** - * @throws AppwriteException - */ - private function exportDocuments(int $batchSize): void - { - $collections = $this->cache->get(Collection::getName()); - - foreach ($collections as $collection) { - /** @var Collection $collection */ - $lastDocument = null; - - while (true) { - $queries = [Query::limit($batchSize)]; - - $documents = []; - - if ($lastDocument) { - $queries[] = Query::cursorAfter($lastDocument); - } - - $selects = ['*', '$id', '$permissions', '$updatedAt', '$createdAt']; // We want relations flat! - $manyToMany = []; - - $attributes = $this->cache->get(Attribute::getName()); - foreach ($attributes as $attribute) { - /** @var Relationship $attribute */ - if ( - $attribute->getCollection()->getId() === $collection->getId() && - $attribute->getType() === Attribute::TYPE_RELATIONSHIP && - $attribute->getSide() === 'parent' && - $attribute->getRelationType() == 'manyToMany' - ) { - /** - * Blockers: - * we should use but Does not work properly: - * $selects[] = $attribute->getKey() . '.$id'; - * when selecting for a relation we get all relations not just the one we were asking. - * when selecting for a relation like select(*, relation.$id) , all relations get resolve - */ - $manyToMany[] = $attribute->getKey(); - } - } - /** @var Attribute|Relationship $attribute */ - - $queries[] = Query::select($selects); - - $response = $this->database->listDocuments( - $collection->getDatabase()->getId(), - $collection->getId(), - $queries - ); - - foreach ($response['documents'] as $document) { - // HACK: Handle many to many - if (! empty($manyToMany)) { - $stack = ['$id']; // Adding $id because we can't select only relations - foreach ($manyToMany as $relation) { - $stack[] = $relation.'.$id'; - } - - $doc = $this->database->getDocument( - $collection->getDatabase()->getId(), - $collection->getId(), - $document['$id'], - [Query::select($stack)] - ); - - foreach ($manyToMany as $key) { - $document[$key] = []; - foreach ($doc[$key] as $relationDocument) { - $document[$key][] = $relationDocument['$id']; - } - } - } - - $id = $document['$id']; - $permissions = $document['$permissions']; - - unset($document['$id']); - unset($document['$permissions']); - unset($document['$collectionId']); - unset($document['$databaseId']); - - // Certain Appwrite versions allowed for data to be required but null - // This isn't allowed in modern versions so we need to remove it by comparing their attributes and replacing it with default value. - $attributes = $this->cache->get(Attribute::getName()); - foreach ($attributes as $attribute) { - /** @var Attribute $attribute */ - if ($attribute->getCollection()->getId() !== $collection->getId()) { - continue; - } - - if ($attribute->isRequired() && ! isset($document[$attribute->getKey()])) { - switch ($attribute->getType()) { - case Attribute::TYPE_BOOLEAN: - $document[$attribute->getKey()] = false; - break; - case Attribute::TYPE_STRING: - $document[$attribute->getKey()] = ''; - break; - case Attribute::TYPE_INTEGER: - $document[$attribute->getKey()] = 0; - break; - case Attribute::TYPE_FLOAT: - $document[$attribute->getKey()] = 0.0; - break; - case Attribute::TYPE_DATETIME: - $document[$attribute->getKey()] = '1970-01-01 00:00:00.000'; - break; - case Attribute::TYPE_URL: - $document[$attribute->getKey()] = 'http://null'; - break; - } - } - } - - $documents[] = new Document( - $id, - $collection, - $document, - $permissions - ); - - $lastDocument = $id; - } - - $this->callback($documents); - - if (count($response['documents']) < $batchSize) { - break; - } - } - } - } - - /** - * @throws \Exception - */ - private function convertAttribute(array $value, Collection $collection): Attribute - { - switch ($value['type']) { - case 'string': - if (! isset($value['format'])) { - return new Text( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - size: $value['size'] ?? 0, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ); - } - - return match ($value['format']) { - 'email' => new Email( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - size: $value['size'] ?? 254, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ), - 'enum' => new Enum( - $value['key'], - $collection, - elements: $value['elements'], - required: $value['required'], - default: $value['default'], - array: $value['array'], - size: $value['size'] ?? UtopiaDatabase::LENGTH_KEY, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ), - 'url' => new URL( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - size: $value['size'] ?? 2000, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ), - 'ip' => new IP( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - size: $value['size'] ?? 39, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ), - default => new Text( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - size: $value['size'] ?? 0, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ), - }; - case 'boolean': - return new Boolean( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ); - case 'integer': - return new Integer( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - min: $value['min'] ?? null, - max: $value['max'] ?? null, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ); - case 'double': - return new Decimal( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - min: $value['min'] ?? null, - max: $value['max'] ?? null, - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ); - case 'relationship': - return new Relationship( - $value['key'], - $collection, - relatedCollection: $value['relatedCollection'], - relationType: $value['relationType'], - twoWay: $value['twoWay'], - twoWayKey: $value['twoWayKey'], - onDelete: $value['onDelete'], - side: $value['side'], - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ); - case 'datetime': - return new DateTime( - $value['key'], - $collection, - required: $value['required'], - default: $value['default'], - array: $value['array'], - createdAt: $value['$createdAt'] ?? '', - updatedAt: $value['$updatedAt'] ?? '', - ); - } - - throw new \Exception('Unknown attribute type: '.$value['type']); - } - - /** - * @throws AppwriteException + * @param int $batchSize + * @throws Exception */ private function exportDatabases(int $batchSize): void { - $this->database = new Databases($this->client); - $lastDatabase = null; while (true) { - $queries = [Query::limit($batchSize)]; + $queries = [$this->database->queryLimit($batchSize)]; if ($this->rootResourceId !== '' && $this->rootResourceType === Resource::TYPE_DATABASE) { - $queries[] = Query::equal('$id', $this->rootResourceId); - $queries[] = Query::limit(1); + $queries[] = $this->database->queryEqual('$id', [$this->rootResourceId]); + $queries[] = $this->database->queryLimit(1); } $databases = []; if ($lastDatabase) { - $queries[] = Query::cursorAfter($lastDatabase); + $queries[] = $this->database->queryCursorAfter($lastDatabase); } - $response = $this->database->list($queries); + $response = $this->database->listDatabases($queries); - foreach ($response['databases'] as $database) { + foreach ($response as $database) { $newDatabase = new Database( $database['$id'], $database['name'], @@ -928,7 +651,7 @@ private function exportDatabases(int $batchSize): void break; } - $lastDatabase = $databases[count($databases) - 1]->getId(); + $lastDatabase = $databases[count($databases) - 1]; $this->callback($databases); @@ -939,7 +662,8 @@ private function exportDatabases(int $batchSize): void } /** - * @throws AppwriteException + * @param int $batchSize + * @throws Exception */ private function exportCollections(int $batchSize): void { @@ -950,19 +674,16 @@ private function exportCollections(int $batchSize): void /** @var Database $database */ while (true) { - $queries = [Query::limit($batchSize)]; + $queries = [$this->database->queryLimit($batchSize)]; $collections = []; if ($lastCollection) { - $queries[] = Query::cursorAfter($lastCollection); + $queries[] = $this->database->queryCursorAfter($lastCollection); } - $response = $this->database->listCollections( - $database->getId(), - $queries - ); + $response = $this->database->listCollections($database, $queries); - foreach ($response['collections'] as $collection) { + foreach ($response as $collection) { $newCollection = new Collection( $database, $collection['name'], @@ -982,7 +703,7 @@ private function exportCollections(int $batchSize): void $this->callback($collections); - $lastCollection = $collections[count($collections) - 1]->getId(); + $lastCollection = $collections[count($collections) - 1]; if (count($collections) < $batchSize) { break; @@ -992,8 +713,8 @@ private function exportCollections(int $batchSize): void } /** - * @throws AppwriteException - * @throws \Exception + * @param int $batchSize + * @throws Exception */ private function exportAttributes(int $batchSize): void { @@ -1003,26 +724,152 @@ private function exportAttributes(int $batchSize): void $lastAttribute = null; while (true) { - $queries = [Query::limit($batchSize)]; + $queries = [$this->database->queryLimit($batchSize)]; $attributes = []; if ($lastAttribute) { - $queries[] = Query::cursorAfter($lastAttribute); + $queries[] = $this->database->queryCursorAfter($lastAttribute); } - $response = $this->database->listAttributes( - $collection->getDatabase()->getId(), - $collection->getId(), - $queries - ); + $response = $this->database->listAttributes($collection, $queries); - foreach ($response['attributes'] as $attribute) { - /** @var array $attribute */ - if ($attribute['type'] === 'relationship' && $attribute['side'] === 'child') { + foreach ($response as $attribute) { + if ( + $attribute['type'] === UtopiaDatabase::VAR_RELATIONSHIP + && $attribute['side'] === UtopiaDatabase::RELATION_SIDE_CHILD + ) { continue; } - $attr = $this->convertAttribute($attribute, $collection); + switch ($attribute['type']) { + case Attribute::TYPE_STRING: + $attr = match ($attribute['format']) { + Attribute::TYPE_EMAIL => new Email( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 254, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_ENUM => new Enum( + $attribute['key'], + $collection, + elements: $attribute['elements'], + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? UtopiaDatabase::LENGTH_KEY, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_URL => new URL( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 2000, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + Attribute::TYPE_IP => new IP( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 39, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + default => new Text( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + size: $attribute['size'] ?? 0, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ), + }; + + break; + case Attribute::TYPE_BOOLEAN: + $attr = new Boolean( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ); + break; + case Attribute::TYPE_INTEGER: + $attr = new Integer( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + min: $attribute['min'] ?? null, + max: $attribute['max'] ?? null, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ); + break; + case Attribute::TYPE_FLOAT: + $attr = new Decimal( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + min: $attribute['min'] ?? null, + max: $attribute['max'] ?? null, + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ); + break; + case Attribute::TYPE_RELATIONSHIP: + $attr = new Relationship( + $attribute['key'], + $collection, + relatedCollection: $attribute['relatedCollection'], + relationType: $attribute['relationType'], + twoWay: $attribute['twoWay'], + twoWayKey: $attribute['twoWayKey'], + onDelete: $attribute['onDelete'], + side: $attribute['side'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ); + break; + case Attribute::TYPE_DATETIME: + $attr = new DateTime( + $attribute['key'], + $collection, + required: $attribute['required'], + default: $attribute['default'], + array: $attribute['array'], + createdAt: $attribute['$createdAt'] ?? '', + updatedAt: $attribute['$updatedAt'] ?? '', + ); + break; + } + + if (!isset($attr)) { + throw new Exception( + resourceName: Resource::TYPE_ATTRIBUTE, + resourceGroup: Transfer::GROUP_DATABASES, + resourceId: $attribute['$id'], + message: 'Unknown attribute type: ' . $attribute['type'] + ); + } $attributes[] = $attr; } @@ -1033,7 +880,7 @@ private function exportAttributes(int $batchSize): void $this->callback($attributes); - $lastAttribute = $attributes[count($attributes) - 1]->getId(); + $lastAttribute = $attributes[count($attributes) - 1]; if (count($attributes) < $batchSize) { break; @@ -1043,7 +890,8 @@ private function exportAttributes(int $batchSize): void } /** - * @throws AppwriteException + * @param int $batchSize + * @throws Exception */ private function exportIndexes(int $batchSize): void { @@ -1055,20 +903,16 @@ private function exportIndexes(int $batchSize): void $lastIndex = null; while (true) { - $queries = [Query::limit($batchSize)]; + $queries = [$this->database->queryLimit($batchSize)]; $indexes = []; if ($lastIndex) { - $queries[] = Query::cursorAfter($lastIndex); + $queries[] = $this->database->queryCursorAfter($lastIndex); } - $response = $this->database->listIndexes( - $collection->getDatabase()->getId(), - $collection->getId(), - $queries - ); + $response = $this->database->listIndexes($collection, $queries); - foreach ($response['indexes'] as $index) { + foreach ($response as $index) { $indexes[] = new Index( 'unique()', $index['key'], @@ -1088,7 +932,7 @@ private function exportIndexes(int $batchSize): void $this->callback($indexes); - $lastIndex = $indexes[count($indexes) - 1]->getId(); + $lastIndex = $indexes[count($indexes) - 1]; if (count($indexes) < $batchSize) { break; @@ -1097,6 +941,104 @@ private function exportIndexes(int $batchSize): void } } + /** + * @throws Exception + */ + private function exportDocuments(int $batchSize): void + { + $collections = $this->cache->get(Collection::getName()); + + foreach ($collections as $collection) { + /** @var Collection $collection */ + $lastDocument = null; + + while (true) { + $queries = [$this->database->queryLimit($batchSize)]; + + $documents = []; + + if ($lastDocument) { + $queries[] = $this->database->queryCursorAfter($lastDocument); + } + + $selects = ['*', '$id', '$permissions', '$updatedAt', '$createdAt']; // We want relations flat! + $manyToMany = []; + + $attributes = $this->cache->get(Attribute::getName()); + foreach ($attributes as $attribute) { + /** @var Relationship $attribute */ + if ( + $attribute->getCollection()->getId() === $collection->getId() && + $attribute->getType() === Attribute::TYPE_RELATIONSHIP && + $attribute->getSide() === 'parent' && + $attribute->getRelationType() == 'manyToMany' + ) { + /** + * Blockers: + * we should use but Does not work properly: + * $selects[] = $attribute->getKey() . '.$id'; + * when selecting for a relation we get all relations not just the one we were asking. + * when selecting for a relation like select(*, relation.$id) , all relations get resolve + */ + $manyToMany[] = $attribute->getKey(); + } + } + /** @var Attribute|Relationship $attribute */ + + $queries[] = $this->database->querySelect($selects); + + $response = $this->database->listDocuments($collection, $queries); + + foreach ($response as $document) { + // HACK: Handle many to many + if (!empty($manyToMany)) { + $stack = ['$id']; // Adding $id because we can't select only relations + foreach ($manyToMany as $relation) { + $stack[] = $relation . '.$id'; + } + + $doc = $this->database->getDocument( + $collection, + $document['$id'], + [$this->database->querySelect($stack)] + ); + + foreach ($manyToMany as $key) { + $document[$key] = []; + foreach ($doc[$key] as $relationDocument) { + $document[$key][] = $relationDocument['$id']; + } + } + } + + $id = $document['$id']; + $permissions = $document['$permissions']; + + unset($document['$id']); + unset($document['$permissions']); + unset($document['$collectionId']); + unset($document['$databaseId']); + + $document = new Document( + $id, + $collection, + $document, + $permissions + ); + + $documents[] = $document; + $lastDocument = $document; + } + + $this->callback($documents); + + if (count($response) < $batchSize) { + break; + } + } + } + } + protected function exportGroupStorage(int $batchSize, array $resources): void { try { @@ -1439,7 +1381,7 @@ private function exportDeploymentData(Func $func, array $deployment): void ); // Content-Length header was missing, file is less than max buffer size. - if (! array_key_exists('Content-Length', $responseHeaders)) { + if (!array_key_exists('Content-Length', $responseHeaders)) { $file = $this->call( 'GET', "/functions/{$func->getId()}/deployments/{$deployment['$id']}/download", @@ -1511,9 +1453,4 @@ private function exportDeploymentData(Func $func, array $deployment): void } } } - - public function getBatchSize(): int - { - return 500; - } } diff --git a/src/Migration/Sources/Appwrite/Reader.php b/src/Migration/Sources/Appwrite/Reader.php new file mode 100644 index 00000000..e03a6981 --- /dev/null +++ b/src/Migration/Sources/Appwrite/Reader.php @@ -0,0 +1,109 @@ + $queries + * @return array + */ + public function listDatabases(array $queries = []): array; + + /** + * List collections that match the given queries + * + * @param Database $resource + * @param array $queries + * @return array + */ + public function listCollections(Database $resource, array $queries = []): array; + + /** + * List attributes that match the given queries + * + * @param Collection $resource + * @param array $queries + * @return array + */ + public function listAttributes(Collection $resource, array $queries = []): array; + + /** + * List indexes that match the given queries + * + * @param Collection $resource + * @param array $queries + * @return array + */ + public function listIndexes(Collection $resource, array $queries = []): array; + + /** + * List documents that match the given queries + * + * @param Collection $resource + * @param array $queries + * @return array + */ + public function listDocuments(Collection $resource, array $queries = []): array; + + /** + * Get a document by its ID in the given collection + * + * @param Collection $resource + * @param string $documentId + * @param array $queries + * @return array + */ + public function getDocument(Collection $resource, string $documentId, array $queries = []): array; + + /** + * Return a query to select the given attributes + * + * @param array $attributes + * @return QueryType|string + */ + public function querySelect(array $attributes): mixed; + + /** + * Return a query to filter the given attributes + * + * @param string $attribute + * @param array $values + * @return QueryType|string + */ + public function queryEqual(string $attribute, array $values): mixed; + + /** + * Return a query to paginate after the given resource + * + * @param Resource|string $resource + * @return QueryType|string + */ + public function queryCursorAfter(Resource|string $resource): mixed; + + /** + * Return a query to limit the number of results + * + * @param int $limit + * @return QueryType|string + */ + public function queryLimit(int $limit): mixed; +} diff --git a/src/Migration/Sources/Appwrite/Reader/API.php b/src/Migration/Sources/Appwrite/Reader/API.php new file mode 100644 index 00000000..5cec16bc --- /dev/null +++ b/src/Migration/Sources/Appwrite/Reader/API.php @@ -0,0 +1,204 @@ + + */ +class API implements Reader +{ + public function __construct(private readonly Databases $database) + { + } + + /** + * @throws AppwriteException + */ + public function report(array $resources, array &$report): void + { + if (\in_array(Resource::TYPE_DATABASE, $resources)) { + $report[Resource::TYPE_DATABASE] = $this->database->list()['total']; + } + + if (\in_array(Resource::TYPE_COLLECTION, $resources)) { + $report[Resource::TYPE_COLLECTION] = 0; + $databases = $this->database->list()['databases']; + foreach ($databases as $database) { + $report[Resource::TYPE_COLLECTION] += $this->database->listCollections( + $database['$id'], + [Query::limit(1)] + )['total']; + } + } + + if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { + $report[Resource::TYPE_DOCUMENT] = 0; + $databases = $this->database->list()['databases']; + foreach ($databases as $database) { + $collections = $this->database->listCollections($database['$id'])['collections']; + foreach ($collections as $collection) { + $report[Resource::TYPE_DOCUMENT] += $this->database->listDocuments( + $database['$id'], + $collection['$id'], + [Query::limit(1)] + )['total']; + } + } + } + + if (\in_array(Resource::TYPE_ATTRIBUTE, $resources)) { + $report[Resource::TYPE_ATTRIBUTE] = 0; + $databases = $this->database->list()['databases']; + foreach ($databases as $database) { + $collections = $this->database->listCollections($database['$id'])['collections']; + foreach ($collections as $collection) { + $report[Resource::TYPE_ATTRIBUTE] += $this->database->listAttributes( + $database['$id'], + $collection['$id'] + )['total']; + } + } + } + + if (\in_array(Resource::TYPE_INDEX, $resources)) { + $report[Resource::TYPE_INDEX] = 0; + $databases = $this->database->list()['databases']; + foreach ($databases as $database) { + $collections = $this->database->listCollections($database['$id'])['collections']; + foreach ($collections as $collection) { + $report[Resource::TYPE_INDEX] += $this->database->listIndexes( + $database['$id'], + $collection['$id'] + )['total']; + } + } + } + } + + /** + * @throws AppwriteException + */ + public function listDatabases(array $queries = []): array + { + return $this->database->list($queries); + } + + /** + * @throws AppwriteException + */ + public function listCollections(Database $resource, array $queries = []): array + { + return $this->database->listCollections( + $resource->getId(), + $queries + )['collections']; + } + + /** + * @param Collection $resource + * @param array $queries + * @return array + * @throws AppwriteException + */ + public function listAttributes(Collection $resource, array $queries = []): array + { + return $this->database->listAttributes( + $resource->getDatabase()->getId(), + $resource->getId(), + $queries + )['attributes']; + } + + /** + * @param Collection $resource + * @param array $queries + * @return array + * @throws AppwriteException + */ + public function listIndexes(Collection $resource, array $queries = []): array + { + return $this->database->listIndexes( + $resource->getDatabase()->getId(), + $resource->getId(), + $queries + )['indexes']; + } + + + /** + * @param Collection $resource + * @param array $queries + * @return array + * @throws AppwriteException + */ + public function listDocuments(Collection $resource, array $queries = []): array + { + return $this->database->listDocuments( + $resource->getDatabase()->getId(), + $resource->getId(), + $queries + )['documents']; + } + + /** + * @param Collection $resource + * @param string $documentId + * @param array $queries + * @return array + * @throws AppwriteException + */ + public function getDocument(Collection $resource, string $documentId, array $queries = []): array + { + return $this->database->getDocument( + $resource->getDatabase()->getId(), + $resource->getId(), + $documentId, + $queries + ); + } + + /** + * @param array $attributes + * @return string + */ + public function querySelect(array $attributes): string + { + return Query::select($attributes); + } + + /** + * @param string $attribute + * @param array $values + * @return string + */ + public function queryEqual(string $attribute, array $values): string + { + return Query::equal($attribute, $values); + } + + /** + * @param Resource|string $resource + * @return string + */ + public function queryCursorAfter(Resource|string $resource): string + { + if (!\is_string($resource)) { + throw new \InvalidArgumentException('Querying with a cursor through the API requires a string resource ID'); + } + + return Query::cursorAfter($resource); + } + + public function queryLimit(int $limit): string + { + return Query::limit($limit); + } +} diff --git a/src/Migration/Sources/Appwrite/Reader/Database.php b/src/Migration/Sources/Appwrite/Reader/Database.php new file mode 100644 index 00000000..6da1944a --- /dev/null +++ b/src/Migration/Sources/Appwrite/Reader/Database.php @@ -0,0 +1,424 @@ + + */ +class Database implements Reader +{ + public function __construct(private readonly UtopiaDatabase $dbForProject) + { + } + + public function report(array $resources, array &$report): void + { + if (\in_array(Resource::TYPE_DATABASE, $resources)) { + $report[Resource::TYPE_DATABASE] = $this->countResources('databases'); + } + + if (\in_array(Resource::TYPE_COLLECTION, $resources)) { + $report[Resource::TYPE_COLLECTION] = 0; + $databases = $this->listDatabases(); + foreach ($databases as $database) { + $collectionId = "database_{$database->getInternalId()}"; + + $report[Resource::TYPE_COLLECTION] += $this->countResources($collectionId); + } + } + + if (\in_array(Resource::TYPE_DOCUMENT, $resources)) { + $report[Resource::TYPE_DOCUMENT] = 0; + $databases = $this->listDatabases(); + foreach ($databases as $database) { + $dbResource = new DatabaseResource( + $database->getId(), + $database->getAttribute('name'), + $database->getCreatedAt(), + $database->getUpdatedAt(), + ); + + $collections = $this->listCollections($dbResource); + + foreach ($collections as $collection) { + $collectionId = "database_{$database->getInternalId()}_collection_{$collection->getInternalId()}"; + + $report[Resource::TYPE_DOCUMENT] += $this->countResources($collectionId); + } + } + } + + if (\in_array(Resource::TYPE_ATTRIBUTE, $resources)) { + $report[Resource::TYPE_ATTRIBUTE] = 0; + $databases = $this->listDatabases(); + foreach ($databases as $database) { + $dbResource = new DatabaseResource( + $database->getId(), + $database->getAttribute('name'), + $database->getCreatedAt(), + $database->getUpdatedAt(), + ); + + $collections = $this->listCollections($dbResource); + + foreach ($collections as $collection) { + $report[Resource::TYPE_ATTRIBUTE] += $this->countResources('attributes', [ + Query::equal('databaseInternalId', [$database->getInternalId()]), + Query::equal('collectionInternalId', [$collection->getInternalId()]), + ]); + } + } + } + + if (\in_array(Resource::TYPE_INDEX, $resources)) { + $report[Resource::TYPE_INDEX] = 0; + $databases = $this->listDatabases(); + foreach ($databases as $database) { + $dbResource = new DatabaseResource( + $database->getId(), + $database->getAttribute('name'), + $database->getCreatedAt(), + $database->getUpdatedAt(), + ); + + $collections = $this->listCollections($dbResource); + + foreach ($collections as $collection) { + $report[Resource::TYPE_INDEX] += $this->countResources('indexes', [ + Query::equal('databaseInternalId', [$database->getInternalId()]), + Query::equal('collectionInternalId', [$collection->getInternalId()]), + ]); + } + } + } + } + + public function listDatabases(array $queries = []): array + { + return $this->dbForProject->find('databases', $queries); + } + + public function listCollections(DatabaseResource $resource, array $queries = []): array + { + try { + $database = $this->dbForProject->getDocument('databases', $resource->getId()); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + try { + return $this->dbForProject->find( + 'database_' . $database->getInternalId(), + $queries + ); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } + } + + public function listAttributes(CollectionResource $resource, array $queries = []): array + { + $database = $this->dbForProject->getDocument( + 'databases', + $resource->getDatabase()->getId(), + ); + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + $collection = $this->dbForProject->getDocument( + 'database_' . $database->getInternalId(), + $resource->getId(), + ); + + if ($collection->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Collection not found', + ); + } + + $queries[] = Query::equal('databaseInternalId', [$database->getInternalId()]); + $queries[] = Query::equal('collectionInternalId', [$collection->getInternalId()]); + + try { + $attributes = $this->dbForProject->find('attributes', $queries); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } + + foreach ($attributes as $attribute) { + if ($attribute['type'] !== UtopiaDatabase::VAR_RELATIONSHIP) { + continue; + } + + $options = $attribute['options']; + foreach ($options as $key => $value) { + $attribute[$key] = $value; + } + + unset($attribute['options']); + } + + return $attributes; + } + + public function listIndexes(CollectionResource $resource, array $queries = []): array + { + $database = $this->dbForProject->getDocument( + 'databases', + $resource->getDatabase()->getId(), + ); + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + $collection = $this->dbForProject->getDocument( + 'database_' . $database->getInternalId(), + $resource->getId(), + ); + + if ($collection->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Collection not found', + ); + } + + $queries[] = Query::equal('databaseInternalId', [$database->getInternalId()]); + $queries[] = Query::equal('collectionInternalId', [$collection->getInternalId()]); + + try { + return $this->dbForProject->find('indexes', $queries); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } + } + + public function listDocuments(CollectionResource $resource, array $queries = []): array + { + $database = $this->dbForProject->getDocument( + 'databases', + $resource->getDatabase()->getId(), + ); + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + $collection = $this->dbForProject->getDocument( + 'database_' . $database->getInternalId(), + $resource->getId(), + ); + + if ($collection->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Collection not found', + ); + } + + $collectionId = "database_{$database->getInternalId()}_collection_{$collection->getInternalId()}"; + + try { + $documents = $this->dbForProject->find($collectionId, $queries); + } catch (DatabaseException $e) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: $e->getMessage(), + code: $e->getCode(), + previous: $e + ); + } + + return \array_map(function ($document) { + return $document->getArrayCopy(); + }, $documents); + } + + public function getDocument(CollectionResource $resource, string $documentId, array $queries = []): array + { + $database = $this->dbForProject->getDocument( + 'databases', + $resource->getDatabase()->getId(), + ); + + if ($database->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Database not found', + ); + } + + $collection = $this->dbForProject->getDocument( + 'database_' . $database->getInternalId(), + $resource->getId(), + ); + + if ($collection->isEmpty()) { + throw new Exception( + resourceName: $resource->getName(), + resourceGroup: $resource->getGroup(), + resourceId: $resource->getId(), + message: 'Collection not found', + ); + } + + $collectionId = "database_{$database->getInternalId()}_collection_{$collection->getInternalId()}"; + + return $this->dbForProject->getDocument( + $collectionId, + $documentId, + $queries + )->getArrayCopy(); + } + + /** + * @param array $attributes + * @return Query + */ + public function querySelect(array $attributes): Query + { + return Query::select($attributes); + } + + /** + * @param string $attribute + * @param array $values + * @return Query + */ + public function queryEqual(string $attribute, array $values): Query + { + return Query::equal($attribute, $values); + } + + /** + * @param Resource|string $resource + * @return Query + * @throws DatabaseException + * @throws Exception + */ + public function queryCursorAfter(mixed $resource): Query + { + if (\is_string($resource)) { + throw new \InvalidArgumentException('Querying with a cursor through the database requires a resource reference'); + } + + switch ($resource::class) { + case DatabaseResource::class: + $document = $this->dbForProject->getDocument('databases', $resource->getId()); + break; + case CollectionResource::class: + $database = $this->dbForProject->getDocument('databases', $resource->getDatabase()->getId()); + $document = $this->dbForProject->getDocument('database_' . $database->getInternalId(), $resource->getId()); + break; + case AttributeResource::class: + $document = $this->dbForProject->getDocument('attributes', $resource->getId()); + break; + case IndexResource::class: + $document = $this->dbForProject->getDocument('indexes', $resource->getId()); + break; + case DocumentResource::class: + $document = $this->getDocument($resource->getCollection(), $resource->getId()); + $document = new UtopiaDocument($document); + break; + default: + throw new \InvalidArgumentException('Unsupported resource type'); + } + + return Query::cursorAfter($document); + } + + public function queryLimit(int $limit): Query + { + return Query::limit($limit); + } + + /** + * @param string $collection + * @param array $queries + * @return int + * @throws DatabaseException + */ + private function countResources(string $collection, array $queries = []): int + { + return $this->dbForProject->count($collection, $queries); + } +} diff --git a/tests/Migration/Unit/Adapters/MockSource.php b/tests/Migration/Unit/Adapters/MockSource.php index 66f22aca..e257a60c 100644 --- a/tests/Migration/Unit/Adapters/MockSource.php +++ b/tests/Migration/Unit/Adapters/MockSource.php @@ -12,11 +12,11 @@ class MockSource extends Source public function pushMockResource(Resource $resource): void { - if (!key_exists($resource->getGroup(), $this->mockResources)) { + if (!array_key_exists($resource->getGroup(), $this->mockResources)) { $this->mockResources[$resource->getGroup()] = []; } - if (!key_exists($resource->getName(), $this->mockResources[$resource->getGroup()])) { + if (!array_key_exists($resource->getName(), $this->mockResources[$resource->getGroup()])) { $this->mockResources[$resource->getGroup()][$resource->getName()] = []; }