diff --git a/src/Migration/Sources/Appwrite.php b/src/Migration/Sources/Appwrite.php index 06f888bb..50c6b34c 100644 --- a/src/Migration/Sources/Appwrite.php +++ b/src/Migration/Sources/Appwrite.php @@ -199,18 +199,59 @@ public function report(array $resources = []): array */ private function reportAuth(array $resources, array &$report): void { + // check if we need to fetch teams! + $needTeams = !empty(array_intersect( + [Resource::TYPE_TEAM, Resource::TYPE_MEMBERSHIP], + $resources + )); + + $pageLimit = 25; + $teams = ['total' => 0, 'teams' => []]; + if (\in_array(Resource::TYPE_USER, $resources)) { - $report[Resource::TYPE_USER] = $this->users->list()['total']; + $report[Resource::TYPE_USER] = $this->users->list( + [Query::limit(1)] + )['total']; + } + + if ($needTeams) { + if (\in_array(Resource::TYPE_MEMBERSHIP, $resources)) { + $allTeams = []; + $lastTeam = null; + + while (true) { + $params = $lastTeam + // TODO: should we use offset here? + // this, realistically, shouldn't be too much ig + ? [Query::cursorAfter($lastTeam)] + : [Query::limit($pageLimit)]; + + $teamList = $this->teams->list($params); + + $totalTeams = $teamList['total']; + $currentTeams = $teamList['teams']; + + $allTeams = array_merge($allTeams, $currentTeams); + $lastTeam = $currentTeams[count($currentTeams) - 1]['$id'] ?? null; + + if (count($currentTeams) < $pageLimit) { + break; + } + } + $teams = ['total' => $totalTeams, 'teams' => $allTeams]; + } else { + $teamList = $this->teams->list([Query::limit(1)]); + $teams = ['total' => $teamList['total'], 'teams' => []]; + } } if (\in_array(Resource::TYPE_TEAM, $resources)) { - $report[Resource::TYPE_TEAM] = $this->teams->list()['total']; + $report[Resource::TYPE_TEAM] = $teams['total']; } if (\in_array(Resource::TYPE_MEMBERSHIP, $resources)) { $report[Resource::TYPE_MEMBERSHIP] = 0; - $teams = $this->teams->list()['teams']; - foreach ($teams as $team) { + foreach ($teams['teams'] as $team) { $report[Resource::TYPE_MEMBERSHIP] += $this->teams->listMemberships( $team['$id'], [Query::limit(1)] @@ -236,9 +277,14 @@ private function reportDatabases(array $resources, array &$report): void private function reportStorage(array $resources, array &$report): void { if (\in_array(Resource::TYPE_BUCKET, $resources)) { - $report[Resource::TYPE_BUCKET] = $this->storage->listBuckets()['total']; + // just fetch one bucket for the `total` + $report[Resource::TYPE_BUCKET] = $this->storage->listBuckets([ + Query::limit(1) + ])['total']; } + $pageLimit = 25; + if (\in_array(Resource::TYPE_FILE, $resources)) { $report[Resource::TYPE_FILE] = 0; $report['size'] = 0; @@ -249,58 +295,89 @@ private function reportStorage(array $resources, array &$report): void $currentBuckets = $this->storage->listBuckets( $lastBucket ? [Query::cursorAfter($lastBucket)] - : [Query::limit(20)] + : [Query::limit($pageLimit)] )['buckets']; $buckets = array_merge($buckets, $currentBuckets); $lastBucket = $buckets[count($buckets) - 1]['$id'] ?? null; - if (count($currentBuckets) < 20) { + if (count($currentBuckets) < $pageLimit) { break; } } foreach ($buckets as $bucket) { - $files = []; $lastFile = null; - while (true) { - $currentFiles = $this->storage->listFiles( + $files = $this->storage->listFiles( $bucket['$id'], $lastFile ? [Query::cursorAfter($lastFile)] - : [Query::limit(20)] + : [Query::limit($pageLimit)] )['files']; - $files = array_merge($files, $currentFiles); + $report[Resource::TYPE_FILE] += count($files); + foreach ($files as $file) { + // already includes the `sizeOriginal` + $report['size'] += $file['sizeOriginal'] ?? 0; + } + $lastFile = $files[count($files) - 1]['$id'] ?? null; - if (count($currentFiles) < 20) { + if (count($files) < $pageLimit) { break; } } - - $report[Resource::TYPE_FILE] += count($files); - foreach ($files as $file) { - $report['size'] += $this->storage->getFile( - $bucket['$id'], - $file['$id'] - )['sizeOriginal']; - } } + $report['size'] = $report['size'] / 1000 / 1000; // MB } } private function reportFunctions(array $resources, array &$report): void { + $pageLimit = 25; + $needVarsOrDeployments = ( + \in_array(Resource::TYPE_DEPLOYMENT, $resources) || + \in_array(Resource::TYPE_ENVIRONMENT_VARIABLE, $resources) + ); + + $functions = []; + $totalFunctions = 0; + + if (!$needVarsOrDeployments && \in_array(Resource::TYPE_FUNCTION, $resources)) { + // Only function count needed, short-circuit + $funcList = $this->functions->list([Query::limit(1)]); + $report[Resource::TYPE_FUNCTION] = $funcList['total']; + return; + } + + if ($needVarsOrDeployments) { + $lastFunction = null; + while (true) { + $params = $lastFunction + ? [Query::cursorAfter($lastFunction)] + : [Query::limit($pageLimit)]; + + $funcList = $this->functions->list($params); + + $totalFunctions = $funcList['total']; + $currentFunctions = $funcList['functions']; + $functions = array_merge($functions, $currentFunctions); + + $lastFunction = $currentFunctions[count($currentFunctions) - 1]['$id'] ?? null; + if (count($currentFunctions) < $pageLimit) { + break; + } + } + } + if (\in_array(Resource::TYPE_FUNCTION, $resources)) { - $report[Resource::TYPE_FUNCTION] = $this->functions->list()['total']; + $report[Resource::TYPE_FUNCTION] = $totalFunctions; } if (\in_array(Resource::TYPE_DEPLOYMENT, $resources)) { $report[Resource::TYPE_DEPLOYMENT] = 0; - $functions = $this->functions->list()['functions']; foreach ($functions as $function) { if (!empty($function['deploymentId'])) { $report[Resource::TYPE_DEPLOYMENT] += 1; @@ -310,9 +387,9 @@ private function reportFunctions(array $resources, array &$report): void if (\in_array(Resource::TYPE_ENVIRONMENT_VARIABLE, $resources)) { $report[Resource::TYPE_ENVIRONMENT_VARIABLE] = 0; - $functions = $this->functions->list()['functions']; foreach ($functions as $function) { - $report[Resource::TYPE_ENVIRONMENT_VARIABLE] += $this->functions->listVariables($function['$id'])['total']; + // function model contains `vars`, we don't need to fetch the list again. + $report[Resource::TYPE_ENVIRONMENT_VARIABLE] += count($function['vars'] ?? []); } } } @@ -1297,7 +1374,7 @@ private function exportFunctions(int $batchSize): void $function['events'], $function['schedule'], $function['timeout'], - $function['deploymentId'], + $function['deploymentId'] ?? '', $function['entrypoint'] ); diff --git a/src/Migration/Sources/Appwrite/Reader/API.php b/src/Migration/Sources/Appwrite/Reader/API.php index 1708d416..97d70e24 100644 --- a/src/Migration/Sources/Appwrite/Reader/API.php +++ b/src/Migration/Sources/Appwrite/Reader/API.php @@ -61,18 +61,46 @@ public function report(array $resources, array &$report): mixed foreach ($databases as $database) { $databaseId = $database['$id']; - /* $tablesResponse = $this->tables->list(...); */ - $tablesResponse = $this->database->listCollections($databaseId); - $tables = $tablesResponse['collections']; + $tables = []; + $pageLimit = 25; + $lastTable = null; + + while (true) { + /* $currentTables = $this->tables->list(...); */ + $currentTables = $this->database->listCollections( + $databaseId, + $lastTable + ? [Query::cursorAfter($lastTable)] + : [Query::limit($pageLimit)] + )['collections']; /* ['tables'] */ + + $tables = array_merge($tables, $currentTables); + $lastTable = $tables[count($tables) - 1]['$id'] ?? null; + + if (count($currentTables) < $pageLimit) { + break; + } + } if (Resource::isSupported(Resource::TYPE_TABLE, $resources)) { - $report[Resource::TYPE_TABLE] += $tablesResponse['total']; + $report[Resource::TYPE_TABLE] += count($tables); } if (Resource::isSupported([Resource::TYPE_ROW, Resource::TYPE_COLUMN, Resource::TYPE_INDEX], $resources)) { foreach ($tables as $table) { $tableId = $table['$id']; + if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) { + // a table already returns a list of attributes + $report[Resource::TYPE_COLUMN] += count($table['columns'] ?? $table['attributes'] ?? []); + } + + if (in_array(Resource::TYPE_INDEX, $resources)) { + // a table already returns a list of indexes + $report[Resource::TYPE_INDEX] += count($table['indexes'] ?? []); + } + + // this one's a bit heavy if the number of tables are high! if (Resource::isSupported(Resource::TYPE_ROW, $resources)) { /* $rowsResponse = $this->tables->listRows(...) */ $rowsResponse = $this->database->listDocuments( @@ -80,19 +108,8 @@ public function report(array $resources, array &$report): mixed $tableId, [Query::limit(1)] ); - $report[Resource::TYPE_ROW] += $rowsResponse['total']; - } - if (Resource::isSupported(Resource::TYPE_COLUMN, $resources)) { - /* $columnsResponse = $this->tables->listColumns(...); */ - $columnsResponse = $this->database->listAttributes($databaseId, $tableId); - $report[Resource::TYPE_COLUMN] += $columnsResponse['total']; - } - - if (in_array(Resource::TYPE_INDEX, $resources)) { - /* $indexesResponse = $this->tables->listIndexes(...); */ - $indexesResponse = $this->database->listIndexes($databaseId, $tableId); - $report[Resource::TYPE_INDEX] += $indexesResponse['total']; + $report[Resource::TYPE_ROW] += $rowsResponse['total']; } } }