diff --git a/system/BaseModel.php b/system/BaseModel.php index b9d21d5a974c..38d1529cc0ce 100644 --- a/system/BaseModel.php +++ b/system/BaseModel.php @@ -586,6 +586,22 @@ abstract public function countAllResults(bool $reset = true, bool $test = false) */ abstract public function chunk(int $size, Closure $userFunc); + /** + * Loops over records in batches, allowing you to operate on each chunk at a time. + * This method works only with DB calls. + * + * This method calls the `$userFunc` with the chunk, instead of a single record as in `chunk()`. + * This allows you to operate on multiple records at once, which can be more efficient for certain operations. + * + * @param Closure(list>|list): mixed $userFunc + * + * @returns void + * + * @throws DataException + * @throws InvalidArgumentException if $size is not a positive integer + */ + abstract public function chunkRows(int $size, Closure $userFunc): void; + /** * Fetches the row of database. * diff --git a/system/Model.php b/system/Model.php index 307d66a52cb6..e157d4a65e01 100644 --- a/system/Model.php +++ b/system/Model.php @@ -525,17 +525,10 @@ public function countAllResults(bool $reset = true, bool $test = false) return $this->builder()->testMode($test)->countAllResults($reset); } - /** - * {@inheritDoc} - * - * Works with `$this->builder` to get the Compiled select to - * determine the rows to operate on. - * This method works only with dbCalls. - */ - public function chunk(int $size, Closure $userFunc) + private function iterateChunks(int $size): \Generator { if ($size <= 0) { - throw new InvalidArgumentException('chunk() requires a positive integer for the $size argument.'); + throw new InvalidArgumentException('$size must be a positive integer.'); } $total = $this->builder()->countAllResults(false); @@ -557,6 +550,20 @@ public function chunk(int $size, Closure $userFunc) continue; } + yield $rows; + } + } + + /** + * {@inheritDoc} + * + * Works with `$this->builder` to get the Compiled select to + * determine the rows to operate on. + * This method works only with dbCalls. + */ + public function chunk(int $size, Closure $userFunc) + { + foreach ($this->iterateChunks($size) as $rows) { foreach ($rows as $row) { if ($userFunc($row) === false) { return; @@ -565,6 +572,22 @@ public function chunk(int $size, Closure $userFunc) } } + /** + * {@inheritDoc} + * + * Works with `$this->builder` to get the Compiled select to + * determine the rows to operate on. + * This method works only with dbCalls. + */ + public function chunkRows(int $size, Closure $userFunc): void + { + foreach ($this->iterateChunks($size) as $rows) { + if ($userFunc($rows) === false) { + return; + } + } + } + /** * Provides a shared instance of the Query Builder. * diff --git a/tests/system/Models/MiscellaneousModelTest.php b/tests/system/Models/MiscellaneousModelTest.php index e95a709e61bd..e590e5dbc207 100644 --- a/tests/system/Models/MiscellaneousModelTest.php +++ b/tests/system/Models/MiscellaneousModelTest.php @@ -44,7 +44,7 @@ public function testChunk(): void public function testChunkThrowsOnZeroSize(): void { $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('chunk() requires a positive integer for the $size argument.'); + $this->expectExceptionMessage('$size must be a positive integer.'); $this->createModel(UserModel::class)->chunk(0, static function ($row): void {}); } @@ -52,7 +52,7 @@ public function testChunkThrowsOnZeroSize(): void public function testChunkThrowsOnNegativeSize(): void { $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('chunk() requires a positive integer for the $size argument.'); + $this->expectExceptionMessage('$size must be a positive integer.'); $this->createModel(UserModel::class)->chunk(-1, static function ($row): void {}); } @@ -97,6 +97,76 @@ public function testChunkEmptyTable(): void $this->assertSame(0, $rowCount); } + public function testChunkRows(): void + { + $chunkCount = 0; + $numRowsInChunk = []; + + $this->createModel(UserModel::class)->chunkRows(2, static function ($rows) use (&$chunkCount, &$numRowsInChunk): void { + $chunkCount++; + $numRowsInChunk[] = count($rows); + }); + + $this->assertSame(2, $chunkCount); + $this->assertSame([2, 2], $numRowsInChunk); + } + + public function testChunkRowsThrowsOnZeroSize(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('$size must be a positive integer.'); + + $this->createModel(UserModel::class)->chunkRows(0, static function ($row): void {}); + } + + public function testChunkRowsThrowsOnNegativeSize(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('$size must be a positive integer.'); + + $this->createModel(UserModel::class)->chunkRows(-1, static function ($row): void {}); + } + + public function testChunkRowsEarlyExit(): void + { + $rowCount = 0; + + $this->createModel(UserModel::class)->chunkRows(2, static function ($rows) use (&$rowCount): bool { + $rowCount++; + + return false; + }); + + $this->assertSame(1, $rowCount); + } + + public function testChunkRowsDoesNotRunExtraQuery(): void + { + $queryCount = 0; + $listener = static function () use (&$queryCount): void { + $queryCount++; + }; + + Events::on('DBQuery', $listener); + $this->createModel(UserModel::class)->chunkRows(4, static function ($rows): void {}); + Events::removeListener('DBQuery', $listener); + + $this->assertSame(2, $queryCount); + } + + public function testChunkRowsEmptyTable(): void + { + $this->db->table('user')->truncate(); + + $rowCount = 0; + + $this->createModel(UserModel::class)->chunkRows(2, static function ($row) use (&$rowCount): void { + $rowCount++; + }); + + $this->assertSame(0, $rowCount); + } + public function testCanCreateAndSaveEntityClasses(): void { $entity = $this->createModel(EntityModel::class)->where('name', 'Developer')->first(); diff --git a/user_guide_src/source/changelogs/v4.8.0.rst b/user_guide_src/source/changelogs/v4.8.0.rst index b45b1fda6b16..8467a7e1dcc5 100644 --- a/user_guide_src/source/changelogs/v4.8.0.rst +++ b/user_guide_src/source/changelogs/v4.8.0.rst @@ -71,6 +71,8 @@ Others Model ===== +- Added new ``chunkRows()`` method to ``CodeIgniter\Model`` for processing large datasets in smaller chunks. + Libraries ========= diff --git a/user_guide_src/source/models/model.rst b/user_guide_src/source/models/model.rst index f1383b45f803..01418cc396d0 100644 --- a/user_guide_src/source/models/model.rst +++ b/user_guide_src/source/models/model.rst @@ -900,14 +900,27 @@ Processing Large Amounts of Data ================================ Sometimes, you need to process large amounts of data and would run the risk of running out of memory. -To make this simpler, you may use the chunk() method to get smaller chunks of data that you can then +This is best used during cronjobs, data exports, or other large tasks. To make this simpler, you can +process the data in smaller, manageable pieces using the methods below. + +chunk() +------- + +You may use the ``chunk()`` method to get smaller chunks of data that you can then do your work on. The first parameter is the number of rows to retrieve in a single chunk. The second parameter is a Closure that will be called for each row of data. -This is best used during cronjobs, data exports, or other large tasks. - .. literalinclude:: model/049.php +chunkRows() +----------- + +.. versionadded:: 4.8.0 + +On the other hand, if you want the entire chunk to be passed to the Closure at once, you can use the ``chunkRows()`` method. + +.. literalinclude:: model/064.php + .. _model-events-callbacks: Working with Query Builder diff --git a/user_guide_src/source/models/model/064.php b/user_guide_src/source/models/model/064.php new file mode 100644 index 000000000000..9afa04e32bb0 --- /dev/null +++ b/user_guide_src/source/models/model/064.php @@ -0,0 +1,6 @@ +chunkRows(100, static function ($rows) { + // do something. + // $rows is an array of rows representing chunk of 100 items. +});