Handle non utf chars

src/Database/Adapter.php

@@ -1464,4 +1464,11 @@ public function enableAlterLocks(bool $enable): self
 
         return $this;
     }
+
+    /**
+     * Handle non utf characters supported?
+     *
+     * @return bool
+     */
+    abstract public function getSupportNonUtcCharacters(): bool;
 }

src/Database/Adapter/MariaDB.php

@@ -7,6 +7,7 @@
 use Utopia\Database\Database;
 use Utopia\Database\Document;
 use Utopia\Database\Exception as DatabaseException;
+use Utopia\Database\Exception\Character as CharacterException;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Exception\Limit as LimitException;
 use Utopia\Database\Exception\NotFound as NotFoundException;
@@ -1838,6 +1839,10 @@ public function getInternalIndexesKeys(): array
 
     protected function processException(PDOException $e): \Exception
     {
+        if ($e->getCode() === '22007' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1366) {
+            return new CharacterException('Invalid character', $e->getCode(), $e);
+        }
+
         // Timeout
         if ($e->getCode() === '70100' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1969) {
             return new TimeoutException('Query timed out', $e->getCode(), $e);
@@ -2230,4 +2235,9 @@ public function getSupportForAlterLocks(): bool
     {
         return true;
     }
+
+    public function getSupportNonUtcCharacters(): bool
+    {
+        return true;
+    }
 }

src/Database/Adapter/Mongo.php

@@ -3221,4 +3221,9 @@ public function getSupportForAlterLocks(): bool
     {
         return false;
     }
+
+    public function getSupportNonUtcCharacters(): bool
+    {
+        return false;
+    }
 }

src/Database/Adapter/MySQL.php

@@ -5,6 +5,7 @@
 use PDOException;
 use Utopia\Database\Database;
 use Utopia\Database\Exception as DatabaseException;
+use Utopia\Database\Exception\Character as CharacterException;
 use Utopia\Database\Exception\Dependency as DependencyException;
 use Utopia\Database\Exception\Structure as StructureException;
 use Utopia\Database\Exception\Timeout as TimeoutException;
@@ -147,6 +148,10 @@ public function getSupportForCastIndexArray(): bool
 
     protected function processException(PDOException $e): \Exception
     {
+        if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 1366) {
+            return new CharacterException('Invalid character', $e->getCode(), $e);
+        }
+
         // Timeout
         if ($e->getCode() === 'HY000' && isset($e->errorInfo[1]) && $e->errorInfo[1] === 3024) {
             return new TimeoutException('Query timed out', $e->getCode(), $e);

src/Database/Adapter/Pool.php

@@ -642,4 +642,9 @@ public function getSupportForAlterLocks(): bool
     {
         return $this->delegate(__FUNCTION__, \func_get_args());
     }
+
+    public function getSupportNonUtcCharacters(): bool
+    {
+        return $this->delegate(__FUNCTION__, \func_get_args());
+    }
 }

src/Database/Adapter/Postgres.php

@@ -2738,4 +2738,9 @@ protected function bindOperatorParams(\PDOStatement|PDOStatementProxy $stmt, Ope
                 break;
         }
     }
+
+    public function getSupportNonUtcCharacters(): bool
+    {
+        return false;
+    }
 }

src/Database/Adapter/SQLite.php

@@ -1876,4 +1876,9 @@ public function getSupportForAlterLocks(): bool
     {
         return false;
     }
+
+    public function getSupportNonUtcCharacters(): bool
+    {
+        return false;
+    }
 }

src/Database/Database.php

@@ -9359,4 +9359,13 @@ private function rollbackAttributeMetadata(Document $collection, array $attribut
         $collection->setAttribute('attributes', \array_values($filteredAttributes));
     }
 
+    /**
+     * @param string $string
+     * @return string
+     */
+    public function sanitizeNonUtf8(string $string): string
+    {
+        return $string;
+    }
+
 }

src/Database/Exception/Character.php

@@ -0,0 +1,9 @@
+<?php
+
+namespace Utopia\Database\Exception;
+
+use Utopia\Database\Exception;
+
+class Character extends Exception
+{
+}

tests/e2e/Adapter/Base.php

@@ -23,8 +23,8 @@
 
 abstract class Base extends TestCase
 {
-    use CollectionTests;
-    use CustomDocumentTypeTests;
+    //    use CollectionTests;
+    //    use CustomDocumentTypeTests;
     use DocumentTests;
     use AttributeTests;
     use IndexTests;

tests/e2e/Adapter/Scopes/DocumentTests.php

@@ -6,10 +6,10 @@
 use Throwable;
 use Utopia\Database\Adapter\SQL;
 use Utopia\Database\Database;
-use Utopia\Database\DateTime;
 use Utopia\Database\Document;
 use Utopia\Database\Exception as DatabaseException;
 use Utopia\Database\Exception\Authorization as AuthorizationException;
+use Utopia\Database\Exception\Character as CharacterException;
 use Utopia\Database\Exception\Conflict as ConflictException;
 use Utopia\Database\Exception\Duplicate as DuplicateException;
 use Utopia\Database\Exception\Limit as LimitException;
@@ -22,6 +22,41 @@
 
 trait DocumentTests
 {
+    public function testNonUtfChars(): void
+    {
+        /** @var Database $database */
+        $database = $this->getDatabase();
+
+        if (!$database->getAdapter()->getSupportNonUtcCharacters()) {
+            $this->expectNotToPerformAssertions();
+            return;
+        }
+
+        $database->createCollection(__FUNCTION__);
+        $this->assertEquals(true, $database->createAttribute(__FUNCTION__, 'title', Database::VAR_STRING, 128, true));
+
+        $nonUtfString = "Hello\x00World\xC3\x28\xFF\xFE\xA0Test\x00End";
+
+        try {
+            $database->createDocument(__FUNCTION__, new Document([
+                'title' => $nonUtfString,
+            ]));
+            $this->fail('Failed to throw exception');
+        } catch (Throwable $e) {
+            $this->assertTrue($e instanceof CharacterException);
+        }
+
+        $nonUtfString = iconv('UTF-8', 'UTF-8//IGNORE', $nonUtfString); // removes invalid UTF-8
+        $nonUtfString = str_replace("\0", '', $nonUtfString); // remove null bytes
+
+        $document = $database->createDocument(__FUNCTION__, new Document([
+            'title' => $nonUtfString,
+        ]));
+
+        $this->assertFalse($document->isEmpty());
+        $this->assertEquals('HelloWorld(TestEnd', $document->getAttribute('title'));
+    }
+
     public function testBigintSequence(): void
     {
         /** @var Database $database */