From b10a5b441e8390a863b0cee8dbbfbc5acbc9f59b Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 2 Jul 2025 11:12:52 +0530 Subject: [PATCH 1/5] add: support for array attributes. --- src/Migration/Sources/CSV.php | 40 +++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index aee145d9..03e35a1b 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -135,31 +135,40 @@ private function exportDocuments(int $batchSize): void } } + $arrayKeys = []; $attributeTypes = []; $manyToManyKeys = []; foreach ($attributes as $attribute) { $key = $attribute['key']; + $type = $attribute['type']; + $isArray = $attribute['array'] ?? false; + $relationSide = $attribute['side'] ?? ''; + $relationType = $attribute['relationType'] ?? ''; if ( - $attribute['type'] === Attribute::TYPE_RELATIONSHIP && - ($attribute['side'] ?? '') === UtopiaDatabase::RELATION_SIDE_CHILD + $type === Attribute::TYPE_RELATIONSHIP && + $relationSide === UtopiaDatabase::RELATION_SIDE_CHILD ) { continue; } - $attributeTypes[$key] = $attribute['type']; + $attributeTypes[$key] = $type; if ( - $attribute['type'] === Attribute::TYPE_RELATIONSHIP && - ($attribute['relationType'] ?? '') === 'manyToMany' && - ($attribute['side'] ?? '') === 'parent' + $type === Attribute::TYPE_RELATIONSHIP && + $relationType === 'manyToMany' && + $relationSide === 'parent' ) { $manyToManyKeys[] = $key; } + + if ($isArray && $type !== Attribute::TYPE_RELATIONSHIP) { + $arrayKeys[] = $key; + } } - $this->withCSVStream(function ($stream) use ($attributeTypes, $manyToManyKeys, $collection, $batchSize) { + $this->withCSVStream(function ($stream) use ($attributeTypes, $manyToManyKeys, $arrayKeys, $collection, $batchSize) { $headers = fgetcsv($stream); if (! is_array($headers) || count($headers) === 0) { return; @@ -197,6 +206,23 @@ private function exportDocuments(int $batchSize): void continue; } + if (in_array($key, $arrayKeys, true)) { + $arrayValues = str_contains($parsedValue, ',') + ? array_map('trim', explode(',', $parsedValue)) + : [$parsedValue]; + + $parsedData[$key] = array_map(function($item) use ($type) { + return match ($type) { + Attribute::TYPE_INTEGER => is_numeric($item) ? (int) $item : null, + Attribute::TYPE_FLOAT => is_numeric($item) ? (float) $item : null, + Attribute::TYPE_BOOLEAN => filter_var($item, FILTER_VALIDATE_BOOLEAN), + default => $item, + }; + }, $arrayValues); + + continue; + } + $parsedData[$key] = match ($type) { Attribute::TYPE_INTEGER => is_numeric($parsedValue) ? (int) $parsedValue : null, Attribute::TYPE_FLOAT => is_numeric($parsedValue) ? (float) $parsedValue : null, From d028977c75f1b234980bac0826ea2b8ad3100981 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 2 Jul 2025 11:42:12 +0530 Subject: [PATCH 2/5] update: account for empty array key types. --- src/Migration/Sources/CSV.php | 45 +++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index 03e35a1b..d540fd68 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -194,32 +194,41 @@ private function exportDocuments(int $batchSize): void $parsedValue = trim($value); $type = $attributeTypes[$key] ?? null; - if (! isset($type) || $parsedValue === '') { + if (! isset($type)) { continue; } if (in_array($key, $manyToManyKeys, true)) { - $parsedData[$key] = str_contains($parsedValue, ',') - ? array_map('trim', explode(',', $parsedValue)) - : [$parsedValue]; - + $parsedData[$key] = $parsedValue === '' + ? [] + : array_values( + array_filter( + array_map( + 'trim', + explode(',', $parsedValue) + ) + ) + ); continue; } if (in_array($key, $arrayKeys, true)) { - $arrayValues = str_contains($parsedValue, ',') - ? array_map('trim', explode(',', $parsedValue)) - : [$parsedValue]; - - $parsedData[$key] = array_map(function($item) use ($type) { - return match ($type) { - Attribute::TYPE_INTEGER => is_numeric($item) ? (int) $item : null, - Attribute::TYPE_FLOAT => is_numeric($item) ? (float) $item : null, - Attribute::TYPE_BOOLEAN => filter_var($item, FILTER_VALIDATE_BOOLEAN), - default => $item, - }; - }, $arrayValues); - + if ($parsedValue === '') { + $parsedData[$key] = []; + } else { + $arrayValues = str_contains($parsedValue, ',') + ? array_map('trim', explode(',', $parsedValue)) + : [$parsedValue]; + + $parsedData[$key] = array_map(function ($item) use ($type) { + return match ($type) { + Attribute::TYPE_INTEGER => is_numeric($item) ? (int) $item : null, + Attribute::TYPE_FLOAT => is_numeric($item) ? (float) $item : null, + Attribute::TYPE_BOOLEAN => filter_var($item, FILTER_VALIDATE_BOOLEAN), + default => $item, + }; + }, $arrayValues); + } continue; } From d2121f6e6f7c25fb27cfe07255f48be4e76fda0a Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 2 Jul 2025 11:43:32 +0530 Subject: [PATCH 3/5] fix: check on non-empty values only. --- src/Migration/Sources/CSV.php | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index d540fd68..27cb75ae 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -232,12 +232,14 @@ private function exportDocuments(int $batchSize): void continue; } - $parsedData[$key] = match ($type) { - Attribute::TYPE_INTEGER => is_numeric($parsedValue) ? (int) $parsedValue : null, - Attribute::TYPE_FLOAT => is_numeric($parsedValue) ? (float) $parsedValue : null, - Attribute::TYPE_BOOLEAN => filter_var($parsedValue, FILTER_VALIDATE_BOOLEAN), - default => $parsedValue, - }; + if ($parsedValue !== '') { + $parsedData[$key] = match ($type) { + Attribute::TYPE_INTEGER => is_numeric($parsedValue) ? (int)$parsedValue : null, + Attribute::TYPE_FLOAT => is_numeric($parsedValue) ? (float)$parsedValue : null, + Attribute::TYPE_BOOLEAN => filter_var($parsedValue, FILTER_VALIDATE_BOOLEAN), + default => $parsedValue, + }; + } } $documentId = $parsedData['$id'] ?? 'unique()'; From 4d029df62ff908d3357ff4c4f944cc68d743d632 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 2 Jul 2025 11:44:02 +0530 Subject: [PATCH 4/5] lint. --- src/Migration/Sources/CSV.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index 27cb75ae..48aea673 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -234,8 +234,8 @@ private function exportDocuments(int $batchSize): void if ($parsedValue !== '') { $parsedData[$key] = match ($type) { - Attribute::TYPE_INTEGER => is_numeric($parsedValue) ? (int)$parsedValue : null, - Attribute::TYPE_FLOAT => is_numeric($parsedValue) ? (float)$parsedValue : null, + Attribute::TYPE_INTEGER => is_numeric($parsedValue) ? (int) $parsedValue : null, + Attribute::TYPE_FLOAT => is_numeric($parsedValue) ? (float) $parsedValue : null, Attribute::TYPE_BOOLEAN => filter_var($parsedValue, FILTER_VALIDATE_BOOLEAN), default => $parsedValue, }; From d5bc659a181697bc30748153270c0b5f795e3587 Mon Sep 17 00:00:00 2001 From: Darshan Date: Wed, 2 Jul 2025 12:01:52 +0530 Subject: [PATCH 5/5] address comment for csv escaping. --- src/Migration/Sources/CSV.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Migration/Sources/CSV.php b/src/Migration/Sources/CSV.php index 48aea673..ee7ce5de 100644 --- a/src/Migration/Sources/CSV.php +++ b/src/Migration/Sources/CSV.php @@ -216,9 +216,8 @@ private function exportDocuments(int $batchSize): void if ($parsedValue === '') { $parsedData[$key] = []; } else { - $arrayValues = str_contains($parsedValue, ',') - ? array_map('trim', explode(',', $parsedValue)) - : [$parsedValue]; + $arrayValues = str_getcsv($parsedValue); + $arrayValues = array_map('trim', $arrayValues); $parsedData[$key] = array_map(function ($item) use ($type) { return match ($type) {