From 776ef1223a8214abfd48ea57fd4a55d0415cd551 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 20 Jan 2025 16:28:12 +0900 Subject: [PATCH 1/5] Implement First iteration of SUM --- src/Database/Adapter.php | 20 ++++ src/Database/Adapter/MariaDB.php | 72 +++++++++++- src/Database/Adapter/SQL.php | 4 +- src/Database/Query.php | 14 +++ src/Database/Validator/Queries.php | 1 + src/Database/Validator/Queries/Document.php | 2 +- src/Database/Validator/Queries/Documents.php | 2 + src/Database/Validator/Query/Base.php | 1 + src/Database/Validator/Query/Sum.php | 100 ++++++++++++++++ tests/e2e/Adapter/Base.php | 113 +++++++++++++++++++ tests/unit/QueryTest.php | 7 ++ tests/unit/Validator/Query/SumTest.php | 58 ++++++++++ 12 files changed, 386 insertions(+), 8 deletions(-) create mode 100644 src/Database/Validator/Query/Sum.php create mode 100644 tests/unit/Validator/Query/SumTest.php diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 6e7cc9f9e..5e1ef815b 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -956,6 +956,26 @@ protected function getAttributeSelections(array $queries): array return $selections; } + /** + * Get all sum attributes from queries + * + * @param Query[] $queries + * @return string[] + */ + protected function getAttributeSums(array $queries): array + { + $selections = []; + + foreach ($queries as $query) { + switch ($query->getMethod()) { + case Query::TYPE_SUM: + $selections[] = $query->getValues(); + } + } + + return $selections; + } + /** * Filter Keys * diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index b5d2956f1..1f4785e5c 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1899,9 +1899,12 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $sqlLimit .= \is_null($offset) ? '' : ' OFFSET :offset'; $selections = $this->getAttributeSelections($queries); + $sumSelections = $this->getAttributeSums($queries); + + $sqlSelection = !empty($sumSelections) ? $this->getAttributeSumProjection($sumSelections, 'table_main') : $this->getAttributeProjection($selections, 'table_main'); $sql = " - SELECT {$this->getAttributeProjection($selections, 'table_main')} + SELECT {$sqlSelection} FROM {$this->getSQLTable($name)} AS table_main {$sqlWhere} {$sqlOrder} @@ -1979,6 +1982,10 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, unset($results[$index]['_permissions']); } + if (!empty($sumSelections)) { + $results[$index] = $this->filterSumAttributes($results[$index], 'table_main'); + } + $results[$index] = new Document($results[$index]); } @@ -2138,10 +2145,11 @@ public function sum(string $collection, string $attribute, array $queries = [], * * @param array $selections * @param string $prefix + * @param bool $addMetadata * @return mixed * @throws Exception */ - protected function getAttributeProjection(array $selections, string $prefix = ''): mixed + protected function getAttributeProjection(array $selections, string $prefix = '', bool $addMetadata = true): mixed { if (empty($selections) || \in_array('*', $selections)) { if (!empty($prefix)) { @@ -2151,10 +2159,12 @@ protected function getAttributeProjection(array $selections, string $prefix = '' } // Remove $id, $permissions and $collection if present since it is always selected by default - $selections = \array_diff($selections, ['$id', '$permissions', '$collection']); + if ($addMetadata) { + $selections = \array_diff($selections, ['$id', '$permissions', '$collection']); - $selections[] = '_uid'; - $selections[] = '_permissions'; + $selections[] = '_uid'; + $selections[] = '_permissions'; + } if (\in_array('$internalId', $selections)) { $selections[] = '_id'; @@ -2182,6 +2192,58 @@ protected function getAttributeProjection(array $selections, string $prefix = '' return \implode(', ', $selections); } + /** + * Get the SQL sum projection given the selected attributes + * + * @param array $sumSelections + * @return string + */ + protected function getAttributeSumProjection(array $sumSelections, string $prefix = ''): string + { + $sqlQuery = []; + + foreach ($sumSelections as $sumSelection) { + if (is_array($sumSelection)) { + foreach ($sumSelection as &$selection) { + $selection = "`{$prefix}`.`{$this->filter($selection)}`"; + } + $queryData = implode('+', $sumSelection); + + $sqlQuery[] = "SUM({$queryData})"; + } else { + $sqlQuery[] = "SUM(`{$prefix}`.`{$this->filter($sumSelection)}`)"; + } + } + + return \implode(', ', $sqlQuery); + } + + /** + * Convert the data received from the sum query back to the original attribute names + * + * @param array $results + * @return array + */ + protected function filterSumAttributes(array $results, string $prefix): array + { + $newResults = []; + + foreach ($results as $key => $value) { + // Remove SUM( from the beginning and ) from the end + $newKey = \preg_replace('/^SUM\(|\)$/', '', $key); + + // Remove any remaining backticks + $newKey = \str_replace('`', '', $newKey); + + // Remove the prefixes + $newKey = \str_replace("{$prefix}.", '', $newKey); + + $newResults[$newKey] = $value; + } + + return $newResults; + } + /** * Get SQL Condition * diff --git a/src/Database/Adapter/SQL.php b/src/Database/Adapter/SQL.php index ee69d5919..6e674032f 100644 --- a/src/Database/Adapter/SQL.php +++ b/src/Database/Adapter/SQL.php @@ -859,7 +859,7 @@ public function getSupportForRelationships(): bool */ protected function bindConditionValue(mixed $stmt, Query $query): void { - if ($query->getMethod() == Query::TYPE_SELECT) { + if ($query->getMethod() == Query::TYPE_SELECT || $query->getMethod() == Query::TYPE_SUM) { return; } @@ -1119,7 +1119,7 @@ public function getSQLConditions(array $queries = [], string $separator = 'AND') $conditions = []; foreach ($queries as $query) { - if ($query->getMethod() === Query::TYPE_SELECT) { + if ($query->getMethod() === Query::TYPE_SELECT || $query->getMethod() === Query::TYPE_SUM) { continue; } diff --git a/src/Database/Query.php b/src/Database/Query.php index 7388453b2..006586a16 100644 --- a/src/Database/Query.php +++ b/src/Database/Query.php @@ -23,6 +23,7 @@ class Query public const TYPE_ENDS_WITH = 'endsWith'; public const TYPE_SELECT = 'select'; + public const TYPE_SUM = 'sum'; // Order methods public const TYPE_ORDER_DESC = 'orderDesc'; @@ -53,6 +54,7 @@ class Query self::TYPE_STARTS_WITH, self::TYPE_ENDS_WITH, self::TYPE_SELECT, + self::TYPE_SUM, self::TYPE_ORDER_DESC, self::TYPE_ORDER_ASC, self::TYPE_LIMIT, @@ -214,6 +216,7 @@ public static function isMethod(string $value): bool self::TYPE_ENDS_WITH, self::TYPE_OR, self::TYPE_AND, + self::TYPE_SUM, self::TYPE_SELECT => true, default => false, }; @@ -459,6 +462,17 @@ public static function select(array $attributes): self return new self(self::TYPE_SELECT, values: $attributes); } + /** + * Helper method to create Query with sum method + * + * @param array $attributes + * @return Query + */ + public static function sum(array $attributes): self + { + return new self(self::TYPE_SUM, values: $attributes); + } + /** * Helper method to create Query with orderDesc method * diff --git a/src/Database/Validator/Queries.php b/src/Database/Validator/Queries.php index b1d67aad0..de097f3e9 100644 --- a/src/Database/Validator/Queries.php +++ b/src/Database/Validator/Queries.php @@ -80,6 +80,7 @@ public function isValid($value): bool $method = $query->getMethod(); $methodType = match ($method) { Query::TYPE_SELECT => Base::METHOD_TYPE_SELECT, + Query::TYPE_SUM => Base::METHOD_TYPE_SUM, Query::TYPE_LIMIT => Base::METHOD_TYPE_LIMIT, Query::TYPE_OFFSET => Base::METHOD_TYPE_OFFSET, Query::TYPE_CURSOR_AFTER, diff --git a/src/Database/Validator/Queries/Document.php b/src/Database/Validator/Queries/Document.php index 41c9f3f9b..776d508cf 100644 --- a/src/Database/Validator/Queries/Document.php +++ b/src/Database/Validator/Queries/Document.php @@ -35,7 +35,7 @@ public function __construct(array $attributes) ]); $validators = [ - new Select($attributes), + new Select($attributes) ]; parent::__construct($validators); diff --git a/src/Database/Validator/Queries/Documents.php b/src/Database/Validator/Queries/Documents.php index abce8694f..e16ce8428 100644 --- a/src/Database/Validator/Queries/Documents.php +++ b/src/Database/Validator/Queries/Documents.php @@ -12,6 +12,7 @@ use Utopia\Database\Validator\Query\Offset; use Utopia\Database\Validator\Query\Order; use Utopia\Database\Validator\Query\Select; +use Utopia\Database\Validator\Query\Sum; class Documents extends IndexedQueries { @@ -66,6 +67,7 @@ public function __construct( ), new Order($attributes), new Select($attributes), + new Sum($attributes), ]; parent::__construct($attributes, $indexes, $validators); diff --git a/src/Database/Validator/Query/Base.php b/src/Database/Validator/Query/Base.php index a37fdd65a..ef419e4cc 100644 --- a/src/Database/Validator/Query/Base.php +++ b/src/Database/Validator/Query/Base.php @@ -12,6 +12,7 @@ abstract class Base extends Validator public const METHOD_TYPE_ORDER = 'order'; public const METHOD_TYPE_FILTER = 'filter'; public const METHOD_TYPE_SELECT = 'select'; + public const METHOD_TYPE_SUM = 'sum'; protected string $message = 'Invalid query'; diff --git a/src/Database/Validator/Query/Sum.php b/src/Database/Validator/Query/Sum.php new file mode 100644 index 000000000..18b50bf61 --- /dev/null +++ b/src/Database/Validator/Query/Sum.php @@ -0,0 +1,100 @@ + + */ + protected array $schema = []; + + /** + * List of internal attributes + * + * @var array + */ + protected const INTERNAL_ATTRIBUTES = [ + '$id', + '$internalId', + '$createdAt', + '$updatedAt', + '$permissions', + '$collection', + ]; + + /** + * @param array $attributes + */ + public function __construct(array $attributes = []) + { + foreach ($attributes as $attribute) { + $this->schema[$attribute->getAttribute('key', $attribute->getAttribute('$id'))] = $attribute->getArrayCopy(); + } + } + + /** + * Is valid. + * + * Returns true if method is TYPE_SUM selections are valid + * + * Otherwise, returns false + * + * @param Query $value + * @return bool + */ + public function isValid($value): bool + { + if (!$value instanceof Query) { + return false; + } + + if ($value->getMethod() !== Query::TYPE_SUM) { + return false; + } + + $internalKeys = \array_map( + fn ($attr) => $attr['$id'], + Database::INTERNAL_ATTRIBUTES + ); + + foreach ($value->getValues() as $attribute) { + if (\str_contains($attribute, '.')) { + //special symbols with `dots` + if (isset($this->schema[$attribute])) { + continue; + } + + // For relationships, just validate the top level. + // Will validate each nested level during the recursive calls. + $attribute = \explode('.', $attribute)[0]; + } + + // Skip internal attributes + if (\in_array($attribute, $internalKeys)) { + continue; + } + + if (!isset($this->schema[$attribute]) && $attribute !== '*') { + $this->message = 'Attribute not found in schema: ' . $attribute; + return false; + } + + // Check if attribute is an numeric value + if ($this->schema[$attribute]['type'] !== Database::VAR_INTEGER && $this->schema[$attribute]['type'] !== Database::VAR_FLOAT) { + $this->message = 'Attribute is not a numeric value: ' . $attribute; + return false; + } + } + return true; + } + + public function getMethodType(): string + { + return self::METHOD_TYPE_SUM; + } +} diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 19ca3602d..980000050 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -17210,4 +17210,117 @@ public function testEvents(): void $database->delete('hellodb'); }); } + + public function testSumQueries() + { + $database = static::getDatabase(); + + $database->createCollection( + 'testSumQueries', + attributes: [ + new Document([ + '$id' => ID::custom('integer'), + 'type' => Database::VAR_INTEGER, + 'size' => 64, + 'required' => true, + ]), + new Document([ + '$id' => ID::custom('float'), + 'type' => Database::VAR_FLOAT, + 'size' => 64, + 'required' => true, + ]), + new Document([ + '$id' => ID::custom('string'), + 'type' => Database::VAR_STRING, + 'size' => 64, + 'required' => true, + ]), + ], + permissions: [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ], + documentSecurity: false + ); + + $database->createDocuments('testSumQueries', [ + new Document([ + 'integer' => 10, + 'float' => 10.5, + 'string' => 'test', + ]), + new Document([ + 'integer' => 20, + 'float' => 20.5, + 'string' => 'test', + ]), + ]); + + // Check sum of integer attribute + $documents = $database->find('testSumQueries', [ + Query::sum(['integer']), + ]); + + $this->assertCount(1, $documents); + $document = $documents[0]; + + $this->assertEquals(30, $document->getAttribute('integer')); + + // Check sum of float attribute + $documents = $database->find('testSumQueries', [ + Query::sum(['float']), + ]); + + $this->assertEquals(31, $documents[0]->getAttribute('float')); + + // Check sum of multiple attributes + $documents = $database->find('testSumQueries', [ + Query::sum(['integer', 'float']), + ]); + + $this->assertEquals(61, $documents[0]->getAttribute('integer+float')); + + // Check sum of multiple attributes separately + $documents = $database->find('testSumQueries', [ + Query::sum(['integer']), + Query::sum(['float']), + ]); + + $this->assertEquals(30, $documents[0]->getAttribute('integer')); + $this->assertEquals(31, $documents[0]->getAttribute('float')); + + // Expect Fail: Sum of non-numeric attribute + try { + $database->find('testSumQueries', [ + Query::sum(['string']), + ]); + $this->fail('Failed to throw exception'); + } catch (QueryException $e) { + $this->assertEquals('Invalid query: Attribute is not a numeric value: string', $e->getMessage()); + } + + // Expect Fail: Sum of non-existent attribute + try { + $database->find('testSumQueries', [ + Query::sum(['nonExistent']), + ]); + $this->fail('Failed to throw exception'); + } catch (QueryException $e) { + $this->assertEquals('Invalid query: Attribute not found in schema: nonExistent', $e->getMessage()); + } + + // Expect Fail: Mix Select and Sum + try { + $database->find('testSumQueries', [ + Query::sum(['integer']), + Query::select(['string']), + ]); + $this->fail('Failed to throw exception'); + } catch (QueryException $e) { + $this->assertEquals('Invalid query: Cannot mix sum and select queries', $e->getMessage()); + } + } } diff --git a/tests/unit/QueryTest.php b/tests/unit/QueryTest.php index d9ad6cd93..3875c24ff 100644 --- a/tests/unit/QueryTest.php +++ b/tests/unit/QueryTest.php @@ -198,6 +198,11 @@ public function testParse(): void $this->assertEquals(1, $query->getAttribute()); $this->assertEquals(['Hello /\ World'], $query->getValues()); + $query = Query::parse(Query::sum(['title', 'director'])->toString()); + $this->assertEquals('sum', $query->getMethod()); + $this->assertEquals(null, $query->getAttribute()); + $this->assertEquals(['title', 'director'], $query->getValues()); + $json = Query::or([ Query::equal('actors', ['Brad Pitt']), Query::equal('actors', ['Johnny Depp']) @@ -262,6 +267,7 @@ public function testIsMethod(): void $this->assertTrue(Query::isMethod('isNotNull')); $this->assertTrue(Query::isMethod('between')); $this->assertTrue(Query::isMethod('select')); + $this->assertTrue(Query::isMethod('sum')); $this->assertTrue(Query::isMethod('or')); $this->assertTrue(Query::isMethod('and')); @@ -283,6 +289,7 @@ public function testIsMethod(): void $this->assertTrue(Query::isMethod(QUERY::TYPE_IS_NOT_NULL)); $this->assertTrue(Query::isMethod(QUERY::TYPE_BETWEEN)); $this->assertTrue(Query::isMethod(QUERY::TYPE_SELECT)); + $this->assertTrue(Query::isMethod(QUERY::TYPE_SUM)); $this->assertTrue(Query::isMethod(QUERY::TYPE_OR)); $this->assertTrue(Query::isMethod(QUERY::TYPE_AND)); diff --git a/tests/unit/Validator/Query/SumTest.php b/tests/unit/Validator/Query/SumTest.php new file mode 100644 index 000000000..06e4c6d2b --- /dev/null +++ b/tests/unit/Validator/Query/SumTest.php @@ -0,0 +1,58 @@ +validator = new Sum( + attributes: [ + new Document([ + '$id' => 'value', + 'key' => 'value', + 'type' => Database::VAR_INTEGER, + 'array' => false, + ]), + new Document([ + '$id' => 'valueFloat', + 'key' => 'valueFloat', + 'type' => Database::VAR_FLOAT, + 'array' => false, + ]), + new Document([ + '$id' => 'valueStr', + 'key' => 'valueStr', + 'type' => Database::VAR_STRING, + 'array' => false, + ]), + ], + ); + } + + public function testValueSuccess(): void + { + $this->assertTrue($this->validator->isValid(Query::sum(['value']))); + $this->assertTrue($this->validator->isValid(Query::sum(['valueFloat']))); + } + + public function testValueFailure(): void + { + $this->assertFalse($this->validator->isValid(Query::limit(1))); + $this->assertEquals('Invalid query', $this->validator->getDescription()); + $this->assertFalse($this->validator->isValid(Query::sum(['valueStr']))); + } +} From 319f4b80b3f70156ae3b761737437b09571ecb98 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 27 Jan 2025 10:33:03 +0900 Subject: [PATCH 2/5] Get MariaDB fully working --- src/Database/Adapter.php | 7 +- src/Database/Adapter/MariaDB.php | 41 +++-- src/Database/Database.php | 11 +- src/Database/Validator/Queries.php | 12 ++ tests/e2e/Adapter/Base.php | 265 +++++++++++++++++------------ 5 files changed, 196 insertions(+), 140 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 5e1ef815b..c3b5cfdc1 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -960,16 +960,15 @@ protected function getAttributeSelections(array $queries): array * Get all sum attributes from queries * * @param Query[] $queries - * @return string[] + * @return array> */ protected function getAttributeSums(array $queries): array { $selections = []; foreach ($queries as $query) { - switch ($query->getMethod()) { - case Query::TYPE_SUM: - $selections[] = $query->getValues(); + if ($query->getMethod() === Query::TYPE_SUM) { + $selections[] = $query->getValues(); } } diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 1f4785e5c..189b4e29c 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1901,16 +1901,20 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $selections = $this->getAttributeSelections($queries); $sumSelections = $this->getAttributeSums($queries); - $sqlSelection = !empty($sumSelections) ? $this->getAttributeSumProjection($sumSelections, 'table_main') : $this->getAttributeProjection($selections, 'table_main'); - $sql = " - SELECT {$sqlSelection} + SELECT {$this->getAttributeProjection($selections, 'table_main')} FROM {$this->getSQLTable($name)} AS table_main {$sqlWhere} {$sqlOrder} - {$sqlLimit}; + {$sqlLimit} "; + if (!empty($sumSelections)) { + $sql = "SELECT {$this->getSumQueries($sumSelections)} FROM ({$sql}) table_sum;"; + } else { + $sql .= ';'; + } + $sql = $this->trigger(Database::EVENT_DOCUMENT_FIND, $sql); $stmt = $this->getPDO()->prepare($sql); @@ -2145,11 +2149,10 @@ public function sum(string $collection, string $attribute, array $queries = [], * * @param array $selections * @param string $prefix - * @param bool $addMetadata * @return mixed * @throws Exception */ - protected function getAttributeProjection(array $selections, string $prefix = '', bool $addMetadata = true): mixed + protected function getAttributeProjection(array $selections, string $prefix = ''): mixed { if (empty($selections) || \in_array('*', $selections)) { if (!empty($prefix)) { @@ -2159,12 +2162,10 @@ protected function getAttributeProjection(array $selections, string $prefix = '' } // Remove $id, $permissions and $collection if present since it is always selected by default - if ($addMetadata) { - $selections = \array_diff($selections, ['$id', '$permissions', '$collection']); + $selections = \array_diff($selections, ['$id', '$permissions', '$collection']); - $selections[] = '_uid'; - $selections[] = '_permissions'; - } + $selections[] = '_uid'; + $selections[] = '_permissions'; if (\in_array('$internalId', $selections)) { $selections[] = '_id'; @@ -2195,24 +2196,20 @@ protected function getAttributeProjection(array $selections, string $prefix = '' /** * Get the SQL sum projection given the selected attributes * - * @param array $sumSelections + * @param array $sumSelections * @return string */ - protected function getAttributeSumProjection(array $sumSelections, string $prefix = ''): string + protected function getSumQueries(array $sumSelections): string { $sqlQuery = []; foreach ($sumSelections as $sumSelection) { - if (is_array($sumSelection)) { - foreach ($sumSelection as &$selection) { - $selection = "`{$prefix}`.`{$this->filter($selection)}`"; - } - $queryData = implode('+', $sumSelection); - - $sqlQuery[] = "SUM({$queryData})"; - } else { - $sqlQuery[] = "SUM(`{$prefix}`.`{$this->filter($sumSelection)}`)"; + foreach ($sumSelection as &$selection) { + $selection = "`{$this->filter($selection)}`"; } + $queryData = implode('+', $sumSelection); + + $sqlQuery[] = "SUM({$queryData})"; } return \implode(', ', $sqlQuery); diff --git a/src/Database/Database.php b/src/Database/Database.php index fe79a5719..040646465 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -5396,6 +5396,7 @@ public function find(string $collection, array $queries = [], string $forPermiss $selections = $this->validateSelections($collection, $selects); $nestedSelections = []; + $skipDecode = false; foreach ($queries as $index => &$query) { switch ($query->getMethod()) { @@ -5430,6 +5431,12 @@ public function find(string $collection, array $queries = [], string $forPermiss } $query->setValues(\array_values($values)); break; + case Query::TYPE_SUM: + $skipDecode = true; + if (\str_contains($query->getAttribute(), '.')) { + unset($queries[$index]); + } + break; default: if (\str_contains($query->getAttribute(), '.')) { unset($queries[$index]); @@ -5459,7 +5466,9 @@ public function find(string $collection, array $queries = [], string $forPermiss $node = $this->silent(fn () => $this->populateDocumentRelationships($collection, $node, $nestedSelections)); } $node = $this->casting($collection, $node); - $node = $this->decode($collection, $node, $selections); + if (!$skipDecode) { + $node = $this->decode($collection, $node, $selections); + } if (!$node->isEmpty()) { $node->setAttribute('$collection', $collection->getId()); diff --git a/src/Database/Validator/Queries.php b/src/Database/Validator/Queries.php index de097f3e9..2373e7e9d 100644 --- a/src/Database/Validator/Queries.php +++ b/src/Database/Validator/Queries.php @@ -61,6 +61,18 @@ public function isValid($value): bool return false; } + $methodsUsed = array_map( + fn (Query|string $query) => $query instanceof Query ? $query->getMethod() : $query, + $value + ); + + $methodsUsed = array_unique($methodsUsed); + + if (in_array(Query::TYPE_SELECT, $methodsUsed) && in_array(Query::TYPE_SUM, $methodsUsed)) { + $this->message = 'Invalid query: Cannot mix select and sum queries'; + return false; + } + foreach ($value as $query) { if (!$query instanceof Query) { try { diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index 980000050..f98ef2b98 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -17102,6 +17102,158 @@ public function testUpdateDocumentsRelationships(): void } } + public function testSumQueries(): void + { + $database = static::getDatabase(); + + $database->createCollection( + 'testSumQueries', + attributes: [ + new Document([ + '$id' => ID::custom('integer'), + 'type' => Database::VAR_INTEGER, + 'size' => 64, + 'required' => true, + ]), + new Document([ + '$id' => ID::custom('float'), + 'type' => Database::VAR_FLOAT, + 'size' => 64, + 'required' => true, + ]), + new Document([ + '$id' => ID::custom('string'), + 'type' => Database::VAR_STRING, + 'size' => 64, + 'required' => true, + ]), + ], + permissions: [ + Permission::read(Role::any()), + Permission::create(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ], + documentSecurity: false + ); + + $documents = $database->createDocuments('testSumQueries', [ + new Document([ + '$id' => 'doc1', + 'integer' => 10, + 'float' => 10.5, + 'string' => 'test1', + ]), + new Document([ + '$id' => 'doc2', + 'integer' => 20, + 'float' => 20.5, + 'string' => 'test2', + ]), + ]); + $document = $documents[0]; + + // Check sum of integer attribute + $documents = $database->find('testSumQueries', [ + Query::sum(['integer']), + ]); + + $this->assertCount(1, $documents); + $this->assertEquals(30, $documents[0]->getAttribute('integer')); + $this->assertEquals(['integer'], array_keys($documents[0]->getAttributes())); + + // Check sum of float attribute + $documents = $database->find('testSumQueries', [ + Query::sum(['float']), + ]); + + $this->assertEquals(31, $documents[0]->getAttribute('float')); + + // Check sum of multiple attributes + $documents = $database->find('testSumQueries', [ + Query::sum(['integer', 'float']), + ]); + + $this->assertEquals(61, $documents[0]->getAttribute('integer+float')); + + // Check sum of multiple attributes separately + $documents = $database->find('testSumQueries', [ + Query::sum(['integer']), + Query::sum(['float']), + ]); + + $this->assertEquals(30, $documents[0]->getAttribute('integer')); + $this->assertEquals(31, $documents[0]->getAttribute('float')); + + // Expect Fail: Sum of non-numeric attribute + try { + $database->find('testSumQueries', [ + Query::sum(['string']), + ]); + $this->fail('Failed to throw exception'); + } catch (QueryException $e) { + $this->assertEquals('Invalid query: Attribute is not a numeric value: string', $e->getMessage()); + } + + // Expect Fail: Sum of non-existent attribute + try { + $database->find('testSumQueries', [ + Query::sum(['nonExistent']), + ]); + $this->fail('Failed to throw exception'); + } catch (QueryException $e) { + $this->assertEquals('Invalid query: Attribute not found in schema: nonExistent', $e->getMessage()); + } + + // Expect Fail: Mix Select and Sum + try { + $database->find('testSumQueries', [ + Query::sum(['integer']), + Query::select(['string']), + ]); + $this->fail('Failed to throw exception'); + } catch (QueryException $e) { + $this->assertEquals('Invalid query: Cannot mix select and sum queries', $e->getMessage()); + } + + /** + * Test sum alongside other queries + */ + $documents = $database->find('testSumQueries', [ + Query::sum(['integer']), + Query::limit(1), + ]); + + $this->assertCount(1, $documents); + $this->assertEquals(10, $documents[0]->getAttribute('integer')); + + $documents = $database->find('testSumQueries', [ + Query::sum(['integer']), + Query::offset(1), + ]); + + $this->assertCount(1, $documents); + $this->assertEquals(20, $documents[0]->getAttribute('integer')); + + $documents = $database->find('testSumQueries', [ + Query::sum(['integer']), + Query::cursorAfter($document), + ]); + + $this->assertCount(1, $documents); + $this->assertEquals(20, $documents[0]->getAttribute('integer')); + + $documents = $database->find('testSumQueries', [ + Query::sum(['integer']), + Query::endsWith('string', 'st2'), + ]); + + $this->assertCount(1, $documents); + $this->assertEquals(20, $documents[0]->getAttribute('integer')); + + $database->deleteCollection('testSumQueries'); + } + public function testEvents(): void { Authorization::skip(function () { @@ -17210,117 +17362,4 @@ public function testEvents(): void $database->delete('hellodb'); }); } - - public function testSumQueries() - { - $database = static::getDatabase(); - - $database->createCollection( - 'testSumQueries', - attributes: [ - new Document([ - '$id' => ID::custom('integer'), - 'type' => Database::VAR_INTEGER, - 'size' => 64, - 'required' => true, - ]), - new Document([ - '$id' => ID::custom('float'), - 'type' => Database::VAR_FLOAT, - 'size' => 64, - 'required' => true, - ]), - new Document([ - '$id' => ID::custom('string'), - 'type' => Database::VAR_STRING, - 'size' => 64, - 'required' => true, - ]), - ], - permissions: [ - Permission::read(Role::any()), - Permission::create(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()) - ], - documentSecurity: false - ); - - $database->createDocuments('testSumQueries', [ - new Document([ - 'integer' => 10, - 'float' => 10.5, - 'string' => 'test', - ]), - new Document([ - 'integer' => 20, - 'float' => 20.5, - 'string' => 'test', - ]), - ]); - - // Check sum of integer attribute - $documents = $database->find('testSumQueries', [ - Query::sum(['integer']), - ]); - - $this->assertCount(1, $documents); - $document = $documents[0]; - - $this->assertEquals(30, $document->getAttribute('integer')); - - // Check sum of float attribute - $documents = $database->find('testSumQueries', [ - Query::sum(['float']), - ]); - - $this->assertEquals(31, $documents[0]->getAttribute('float')); - - // Check sum of multiple attributes - $documents = $database->find('testSumQueries', [ - Query::sum(['integer', 'float']), - ]); - - $this->assertEquals(61, $documents[0]->getAttribute('integer+float')); - - // Check sum of multiple attributes separately - $documents = $database->find('testSumQueries', [ - Query::sum(['integer']), - Query::sum(['float']), - ]); - - $this->assertEquals(30, $documents[0]->getAttribute('integer')); - $this->assertEquals(31, $documents[0]->getAttribute('float')); - - // Expect Fail: Sum of non-numeric attribute - try { - $database->find('testSumQueries', [ - Query::sum(['string']), - ]); - $this->fail('Failed to throw exception'); - } catch (QueryException $e) { - $this->assertEquals('Invalid query: Attribute is not a numeric value: string', $e->getMessage()); - } - - // Expect Fail: Sum of non-existent attribute - try { - $database->find('testSumQueries', [ - Query::sum(['nonExistent']), - ]); - $this->fail('Failed to throw exception'); - } catch (QueryException $e) { - $this->assertEquals('Invalid query: Attribute not found in schema: nonExistent', $e->getMessage()); - } - - // Expect Fail: Mix Select and Sum - try { - $database->find('testSumQueries', [ - Query::sum(['integer']), - Query::select(['string']), - ]); - $this->fail('Failed to throw exception'); - } catch (QueryException $e) { - $this->assertEquals('Invalid query: Cannot mix sum and select queries', $e->getMessage()); - } - } } From ccccf7bb6e6cf3b804df5498ac0053f2594b4f71 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 27 Jan 2025 10:49:51 +0900 Subject: [PATCH 3/5] Further clean up code --- src/Database/Adapter/MariaDB.php | 48 ++++++-------------------------- 1 file changed, 8 insertions(+), 40 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 189b4e29c..e6b0ba13e 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -1986,10 +1986,6 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, unset($results[$index]['_permissions']); } - if (!empty($sumSelections)) { - $results[$index] = $this->filterSumAttributes($results[$index], 'table_main'); - } - $results[$index] = new Document($results[$index]); } @@ -2196,49 +2192,21 @@ protected function getAttributeProjection(array $selections, string $prefix = '' /** * Get the SQL sum projection given the selected attributes * - * @param array $sumSelections + * @param array> $attributeGroups * @return string */ - protected function getSumQueries(array $sumSelections): string - { - $sqlQuery = []; - - foreach ($sumSelections as $sumSelection) { - foreach ($sumSelection as &$selection) { - $selection = "`{$this->filter($selection)}`"; - } - $queryData = implode('+', $sumSelection); - - $sqlQuery[] = "SUM({$queryData})"; - } - - return \implode(', ', $sqlQuery); - } - - /** - * Convert the data received from the sum query back to the original attribute names - * - * @param array $results - * @return array - */ - protected function filterSumAttributes(array $results, string $prefix): array + protected function getSumQueries(array $attributeGroups): string { - $newResults = []; - - foreach ($results as $key => $value) { - // Remove SUM( from the beginning and ) from the end - $newKey = \preg_replace('/^SUM\(|\)$/', '', $key); - - // Remove any remaining backticks - $newKey = \str_replace('`', '', $newKey); + $sumExpressions = []; - // Remove the prefixes - $newKey = \str_replace("{$prefix}.", '', $newKey); + foreach ($attributeGroups as $attributeGroup) { + $columnAlias = \implode('+', $attributeGroup); + $sumExpression = implode('+', array_map(fn ($attribute) => "`{$this->filter($attribute)}`", $attributeGroup)); - $newResults[$newKey] = $value; + $sumExpressions[] = "SUM({$sumExpression}) AS `{$columnAlias}`"; } - return $newResults; + return \implode(', ', $sumExpressions); } /** From 4d26d0a1a43504b645f0794006f0775a34394158 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 27 Jan 2025 10:59:41 +0900 Subject: [PATCH 4/5] Implement PostgreSQL support --- src/Database/Adapter/MariaDB.php | 8 ++++++-- src/Database/Adapter/Postgres.php | 33 ++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 4e271c94a..4d561bb26 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -2442,9 +2442,9 @@ protected function getAttributeProjection(array $selections, string $prefix = '' } /** - * Get the SQL sum projection given the selected attributes + * Get the SQL sum queries given the selected attributes * - * @param array> $attributeGroups + * @param array|string> $attributeGroups * @return string */ protected function getSumQueries(array $attributeGroups): string @@ -2452,6 +2452,10 @@ protected function getSumQueries(array $attributeGroups): string $sumExpressions = []; foreach ($attributeGroups as $attributeGroup) { + if (\is_string($attributeGroup)) { + $attributeGroup = [$attributeGroup]; + } + $columnAlias = \implode('+', $attributeGroup); $sumExpression = implode('+', array_map(fn ($attribute) => "`{$this->filter($attribute)}`", $attributeGroup)); diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index c2b886776..57df2998f 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -1917,15 +1917,22 @@ public function find(string $collection, array $queries = [], ?int $limit = 25, $sqlLimit = \is_null($limit) ? '' : 'LIMIT :limit'; $sqlLimit .= \is_null($offset) ? '' : ' OFFSET :offset'; $selections = $this->getAttributeSelections($queries); + $sumSelections = $this->getAttributeSums($queries); $sql = " SELECT {$this->getAttributeProjection($selections, 'table_main')} FROM {$this->getSQLTable($name)} as table_main {$sqlWhere} {$sqlOrder} - {$sqlLimit}; + {$sqlLimit} "; + if (!empty($sumSelections)) { + $sql = "SELECT {$this->getSumQueries($sumSelections)} FROM ({$sql}) table_sum;"; + } else { + $sql .= ';'; + } + $sql = $this->trigger(Database::EVENT_DOCUMENT_FIND, $sql); $stmt = $this->getPDO()->prepare($sql); @@ -2201,6 +2208,30 @@ protected function getAttributeProjection(array $selections, string $prefix = '' return \implode(', ', $selections); } + /** + * Get the SQL sum queries given the selected attributes + * + * @param array> $attributeGroups + * @return string + */ + protected function getSumQueries(array $attributeGroups): string + { + $sumExpressions = []; + + foreach ($attributeGroups as $attributeGroup) { + if (\is_string($attributeGroup)) { + $attributeGroup = [$attributeGroup]; + } + + $columnAlias = \implode('+', $attributeGroup); + $sumExpression = implode('+', array_map(fn ($attribute) => "\"{$this->filter($attribute)}\"", $attributeGroup)); + + $sumExpressions[] = "SUM({$sumExpression}) AS \"{$columnAlias}\""; + } + + return \implode(', ', $sumExpressions); + } + /** * Get SQL Condition From 661cd1d65a0ef59e705cd5eb671f614e44c8a042 Mon Sep 17 00:00:00 2001 From: Bradley Schofield Date: Mon, 27 Jan 2025 12:02:03 +0900 Subject: [PATCH 5/5] Add SUM Query feature flag and disable mongo support --- src/Database/Adapter.php | 7 +++++++ src/Database/Adapter/MariaDB.php | 10 ++++++++++ src/Database/Adapter/Mongo.php | 5 +++++ src/Database/Adapter/Postgres.php | 10 ++++++++++ tests/e2e/Adapter/Base.php | 5 +++++ 5 files changed, 37 insertions(+) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index b6e40d66c..b16ecdd23 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -900,6 +900,13 @@ abstract public function getSupportForCastIndexArray(): bool; */ abstract public function getSupportForUpserts(): bool; + /** + * Are sum queries supported? + * + * @return bool + */ + abstract public function getSupportForSum(): bool; + /** * Get current attribute count from collection document * diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 4d561bb26..e665a968b 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -2645,6 +2645,16 @@ public function getSupportForUpserts(): bool return true; } + /** + * Are sum queries supported? + * + * @return bool + */ + public function getSupportForSum(): bool + { + return true; + } + /** * Set max execution time * @param int $milliseconds diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index c5f0522af..7a1f97b8d 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1820,6 +1820,11 @@ public function getSupportForUpserts(): bool return false; } + public function getSupportForSum(): bool + { + return false; + } + /** * Get current attribute count from collection document * diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 57df2998f..c71de02fc 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2480,6 +2480,16 @@ public function getSupportForUpserts(): bool return false; } + /** + * Are sum queries supported? + * + * @return bool + */ + public function getSupportForSum(): bool + { + return true; + } + /** * @return string */ diff --git a/tests/e2e/Adapter/Base.php b/tests/e2e/Adapter/Base.php index e1c5e0886..91da329ff 100644 --- a/tests/e2e/Adapter/Base.php +++ b/tests/e2e/Adapter/Base.php @@ -17525,6 +17525,11 @@ public function testUpdateDocumentsRelationships(): void public function testSumQueries(): void { + if (!static::getDatabase()->getAdapter()->getSupportForSum()) { + $this->expectNotToPerformAssertions(); + return; + } + $database = static::getDatabase(); $database->createCollection(