Merge pull request #124 from utopia-php/log-projects

Add: logs

Author: abnegate

src/Migration/Sources/Appwrite.php

@@ -10,6 +10,7 @@
 use Appwrite\Services\Storage;
 use Appwrite\Services\Teams;
 use Appwrite\Services\Users;
+use Utopia\Console;
 use Utopia\Database\Database as UtopiaDatabase;
 use Utopia\Database\DateTime as UtopiaDateTime;
 use Utopia\Migration\Exception;
@@ -74,6 +75,12 @@ class Appwrite extends Source
     public const SOURCE_API = 'api';
     public const SOURCE_DATABASE = 'database';
 
+    // Debug logging for specific projects
+    public static array $debugProjects = [
+        '67ec0369002bd8a96885' => 'SimpMusic#Maxrave',
+        '6838382d0014e002589c' => 'Fastwrite#DocuTrust',
+    ];
+
     protected Client $client;
 
     private Reader $reader;
@@ -120,7 +127,7 @@ public function __construct(
 
         $this->reader = match ($this->source) {
             static::SOURCE_API => new APIReader(new Databases($this->client)),
-            static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabasesDB),
+            static::SOURCE_DATABASE => new DatabaseReader($this->dbForProject, $this->getDatabasesDB, $this->project),
             default => throw new \Exception('Unknown source'),
         };
 
@@ -131,6 +138,19 @@ public static function getName(): string
         return 'Appwrite';
     }
 
+    /**
+     * Log migration debug info for tracked projects
+     */
+    private function logDebugTrackedProject(string $message): void
+    {
+        $projectTag = self::$debugProjects[$this->project] ?? null;
+        if ($projectTag === null) {
+            return;
+        }
+
+        Console::info("MIGRATIONS-SOURCE-$projectTag: $message");
+    }
+
     /**
      * @return array<string>
      */
@@ -972,11 +992,22 @@ private function exportIndexes(string $entityType, int $batchSize): void
     private function exportRecords(string $entityName, string $fieldName, int $batchSize): void
     {
         $entities = $this->cache->get($entityName);
+
+        $this->logDebugTrackedProject("exportRecords started | Entity: $entityName | Tables count: " . count($entities));
+
         foreach ($entities as $table) {
             /** @var Table $table */
             $lastRow = null;
+            $iterationCount = 0;
+
+            $this->logDebugTrackedProject("Starting table export | Table: {$table->getName()} | ID: {$table->getId()}");
 
             while (true) {
+                $iterationCount++;
+
+                $memUsage = round(memory_get_usage(true) / 1024 / 1024, 2);
+                $this->logDebugTrackedProject("Table: {$table->getName()} | Iteration: $iterationCount | Memory: {$memUsage}MB | LastRow: " . ($lastRow ? $lastRow->getId() : 'null'));
+
                 $queries = [
                     $this->reader->queryLimit($batchSize),
                     ...$this->queries,
@@ -1009,8 +1040,14 @@ private function exportRecords(string $entityName, string $fieldName, int $batch
 
                 $queries[] = $this->reader->querySelect($selects);
 
+                $timestamp = microtime(true);
+                $this->logDebugTrackedProject("BEFORE listRows() | Table: {$table->getName()} | Batch: $batchSize | Timestamp: {$timestamp}");
+
                 $response = $this->reader->listRows($table, $queries);
 
+                $timestamp = microtime(true);
+                $this->logDebugTrackedProject("AFTER listRows() | Table: {$table->getName()} | Rows: " . count($response) . " | Timestamp: $timestamp");
+
                 foreach ($response as $row) {
                     // HACK: Handle many to many (only for schema-based databases)
                     if (!empty($manyToMany)) {
@@ -1069,13 +1106,24 @@ private function exportRecords(string $entityName, string $fieldName, int $batch
                     $lastRow = $row;
                 }
 
+                $this->logDebugTrackedProject("Processed rows from response | Table: {$table->getName()} | Rows in batch: " . count($rows));
+
+                $this->logDebugTrackedProject("BEFORE callback() | Table: {$table->getName()} | Rows: " . count($rows));
+
                 $this->callback($rows);
 
+                $this->logDebugTrackedProject("AFTER callback() | Table: {$table->getName()}");
+
                 if (count($response) < $batchSize) {
+                    $this->logDebugTrackedProject("Table export completed | Table: {$table->getName()} | Response count: " . count($response) . " < Batch size: $batchSize");
                     break;
                 }
             }
+
+            $this->logDebugTrackedProject("Finished table export | Table: {$table->getName()} | Total iterations: {$iterationCount}");
         }
+
+        $this->logDebugTrackedProject("exportRecords completed | Entity: {$entityName}");
     }
 
     protected function exportGroupStorage(int $batchSize, array $resources): void

src/Migration/Sources/Appwrite/Reader/Database.php

@@ -2,6 +2,7 @@
 
 namespace Utopia\Migration\Sources\Appwrite\Reader;
 
+use Utopia\Console;
 use Utopia\Database\Database as UtopiaDatabase;
 use Utopia\Database\Document as UtopiaDocument;
 use Utopia\Database\Exception as DatabaseException;
@@ -15,6 +16,7 @@
 use Utopia\Migration\Resources\Database\Index as IndexResource;
 use Utopia\Migration\Resources\Database\Row as RowResource;
 use Utopia\Migration\Resources\Database\Table as TableResource;
+use Utopia\Migration\Sources\Appwrite;
 use Utopia\Migration\Sources\Appwrite\Reader;
 
 /**
@@ -29,11 +31,29 @@ class Database implements Reader
 
     public function __construct(
         private readonly UtopiaDatabase $dbForProject,
-        ?callable $getDatabasesDB = null
+        ?callable $getDatabasesDB = null,
+        private readonly ?string $projectId = null
     ) {
         $this->getDatabasesDB = $getDatabasesDB;
     }
 
+    /**
+     * Log debug message for tracked projects
+     */
+    private function logDebug(string $message): void
+    {
+        if ($this->projectId === null) {
+            return;
+        }
+
+        $projectTag = Appwrite::$debugProjects[$this->projectId] ?? null;
+        if ($projectTag === null) {
+            return;
+        }
+
+        Console::info("MIGRATIONS-READER-$projectTag: $message");
+    }
+
     /**
      * Get the appropriate database instance for the given database DSN
      */
@@ -347,7 +367,9 @@ public function listRows(TableResource $resource, array $queries = []): array
         $dbInstance = $this->getDatabase($resource->getDatabase()->getDatabase());
 
         try {
+            $this->logDebug("BEFORE dbInstance->find() | TableID: $tableId | Queries: " . count($queries));
             $rows = $dbInstance->find($tableId, $queries);
+            $this->logDebug("AFTER dbInstance->find() | TableID: $tableId | Rows returned: " . count($rows));
         } catch (DatabaseException $e) {
             throw new Exception(
                 resourceName: $resource->getName(),