From af48e9b226079ea441d7b08cd95dc66d898fe3e3 Mon Sep 17 00:00:00 2001 From: Daniel Jones Date: Wed, 11 Nov 2026 22:58:20 +0000 Subject: [PATCH 1/3] Fill a store's indexes in one pass, an item at a time Warming the Stache made every entry of a collection, kept them all, and then had each index read all of them. Items are now made and cached one at a time, and every index that takes its value from a single item is filled from one pass over them. Entries are let go of once they have been indexed. Entries that need another entry to be indexed (a localization, an entry in a structure, anything whose values go looking for other entries) are indexed in a second pass, read back one at a time. A structure's uri, parent and order indexes are still built the way they were, once the rest are there. Term associations are built from the entries' own indexes rather than by loading every entry that has a term. --- .../Exceptions/ItemsNotReadyException.php | 12 + src/Stache/Indexes/Index.php | 33 +++ src/Stache/Indexes/Terms/Associations.php | 32 +-- src/Stache/Stache.php | 12 +- src/Stache/Stores/CollectionEntriesStore.php | 31 +++ src/Stache/Stores/Store.php | 226 ++++++++++++++++-- 6 files changed, 305 insertions(+), 41 deletions(-) create mode 100644 src/Stache/Exceptions/ItemsNotReadyException.php diff --git a/src/Stache/Exceptions/ItemsNotReadyException.php b/src/Stache/Exceptions/ItemsNotReadyException.php new file mode 100644 index 00000000000..1a45e9f78ce --- /dev/null +++ b/src/Stache/Exceptions/ItemsNotReadyException.php @@ -0,0 +1,12 @@ +addMessage("Updating index: {$this->store->key()}/{$this->name}", 'stache'); + // Most indexes are filled in a single pass over the store's items, along with + // any others that are waiting to be built, rather than each reading every item. + if ($this->store->fillIndexes($this)) { + return $this; + } + $this->items = $this->getItems(); $this->cache(); @@ -102,6 +108,33 @@ public function update() return $this; } + /** + * Sets and caches items that have been worked out elsewhere. + */ + public function fill(array $items) + { + $this->items = $items; + + $this->cache(); + + return $this; + } + + /** + * Runs a callback as if an index were loading, for work that + * fills indexes without going through load(). + */ + public static function whileLoading(string $key, callable $callback) + { + static::$loadingStack[] = $key; + + try { + return $callback(); + } finally { + array_pop(static::$loadingStack); + } + } + public function isCached() { return Stache::cacheStore()->has($this->cacheKey()); diff --git a/src/Stache/Indexes/Terms/Associations.php b/src/Stache/Indexes/Terms/Associations.php index 606c2a674fb..0ab637fdc2e 100644 --- a/src/Stache/Indexes/Terms/Associations.php +++ b/src/Stache/Indexes/Terms/Associations.php @@ -2,6 +2,7 @@ namespace Statamic\Stache\Indexes\Terms; +use Statamic\Facades\Stache; use Statamic\Facades\Taxonomy; use Statamic\Stache\Indexes\Index; use Statamic\Support\Str; @@ -11,25 +12,28 @@ class Associations extends Index private ?array $grouped = null; private ?array $groupedFrom = null; + /** + * What each entry has in the taxonomy's field is already in its collection's + * index for that field, so the entries themselves don't need to be loaded. + */ public function getItems() { return Taxonomy::findByHandle($handle = $this->store->childKey()) ->collections() ->flatMap(function ($collection) use ($handle) { - return $collection->queryEntries() - ->where($handle, '<>', null) - ->get() - ->flatMap(function ($entry) use ($handle) { - return collect($entry->value($handle)) - ->map(function ($value) use ($entry) { - return [ - 'value' => $value, - 'slug' => Str::slug($value), - 'entry' => $entry->id(), - 'collection' => $entry->collectionHandle(), - 'site' => $entry->locale(), - ]; - }); + $store = Stache::store('entries')->store($collection->handle()); + $sites = $store->index('site'); + + return $store->index($handle)->items() + ->reject(fn ($values) => $values == null) + ->flatMap(function ($values, $id) use ($collection, $sites) { + return collect($values)->map(fn ($value) => [ + 'value' => $value, + 'slug' => Str::slug($value), + 'entry' => (string) $id, + 'collection' => $collection->handle(), + 'site' => $sites->get($id), + ]); })->all(); })->all(); } diff --git a/src/Stache/Stache.php b/src/Stache/Stache.php index 1b25b25591c..10f864741b7 100644 --- a/src/Stache/Stache.php +++ b/src/Stache/Stache.php @@ -134,11 +134,13 @@ public function warm() $stores = $this->stores()->except($this->exclude); - if ($this->shouldUseParallelWarming($stores)) { - $this->warmInParallel($stores); - } else { - $stores->each->warm(); - } + Store::duringWarm(function () use ($stores) { + if ($this->shouldUseParallelWarming($stores)) { + $this->warmInParallel($stores); + } else { + $stores->each->warm(); + } + }); $this->stopTimer(); diff --git a/src/Stache/Stores/CollectionEntriesStore.php b/src/Stache/Stores/CollectionEntriesStore.php index a5be77798da..3e0633fb24a 100644 --- a/src/Stache/Stores/CollectionEntriesStore.php +++ b/src/Stache/Stores/CollectionEntriesStore.php @@ -163,6 +163,37 @@ public function makeItemFromFile($path, $contents) return $entry; } + protected function canIndexWithoutOtherItems($item): bool + { + // A localization falls back to its origin's values, and an entry + // in a structure gets its uri, parent and order from the tree. + // Asked of the entry's own property, because hasOrigin() goes looking for the origin. + $hasOrigin = (bool) Closure::bind(fn () => $this->origin, $item, EntryInstance::class)(); + + return ! $hasOrigin && ! $this->findCollection($item->collectionHandle())?->hasStructure(); + } + + protected function indexNeedsOtherIndexes($name): bool + { + // Where an entry sits in a tree is found by querying the collection's + // entries, which needs the rest of this store's indexes to be there. + return in_array($name, ['uri', 'parent', 'order']) && $this->collection()?->hasStructure(); + } + + protected function releaseItem($key, $item): void + { + // A structure's uri, parent and order indexes are still to be built from these. + if ($this->collection()?->hasStructure()) { + return; + } + + // Entries are kept so that uris can be built and indexes don't have to read them back + // from the cache. One that has been indexed is needed for neither, and one that is + // waiting to be indexed is read back then, rather than all of them being held. + Blink::store('structure-entries')->forget($key); + $this->snapshots()->forget($key); + } + protected function extractAttributesFromPath($path) { $site = Site::default()->handle(); diff --git a/src/Stache/Stores/Store.php b/src/Stache/Stores/Store.php index ee31e4236a9..98af79de9b0 100644 --- a/src/Stache/Stores/Store.php +++ b/src/Stache/Stores/Store.php @@ -3,10 +3,12 @@ namespace Statamic\Stache\Stores; use Facades\Statamic\Stache\Traverser; +use ReflectionMethod; use Statamic\Facades\File; use Statamic\Facades\Path; use Statamic\Facades\Stache; use Statamic\Stache\Exceptions\DuplicateKeyException; +use Statamic\Stache\Exceptions\ItemsNotReadyException; use Statamic\Stache\Indexes; use Statamic\Stache\Indexes\Index; use Statamic\Support\Arr; @@ -26,6 +28,11 @@ abstract class Store protected $shouldCacheFileItems = false; protected $modified; protected $keys; + private ?array $filling = null; + private bool $makingItems = false; + private ?string $filledDuringWarm = null; + private static array $buildsItsOwnItems = []; + private static ?string $warmRun = null; /** * @return string @@ -310,30 +317,20 @@ public function paths() return $this->paths = collect($paths); } - $files = Traverser::filter([$this, 'getItemFilter'])->traverse($this); - - $fileItems = $files->map(function ($timestamp, $path) { - return [ - 'item' => $item = $this->makeItemFromFile($path, File::get($path)), - 'key' => $this->getItemKey($item), - 'path' => $path, - ]; - }); - - $items = $fileItems->reject(function ($item) { - try { - $this->keys()->add($item['key'], $item['path']); - } catch (DuplicateKeyException $e) { - $isDuplicate = true; - Stache::duplicates()->track($this, $e->getKey(), $e->getPath()); - } + // An index value being worked out as the items are made can't be given the items. + if ($this->makingItems && $this->filling !== null) { + throw new ItemsNotReadyException; + } - return $isDuplicate ?? false; - }); + $files = Traverser::filter([$this, 'getItemFilter'])->traverse($this); - $items->each(fn ($item) => $this->cacheItem($item['item'])); + $this->makingItems = true; - $paths = $items->pluck('path', 'key'); + try { + $paths = collect($this->makeAndCacheItemsOneAtATime($files)); + } finally { + $this->makingItems = false; + } $this->cachePaths($paths); @@ -416,11 +413,196 @@ public function clear() Stache::cacheStore()->forget("stache::timestamps::{$this->key()}"); } + /** + * Each file is turned into an item and cached before the next one is read, so the + * items don't all have to be in memory at once. While indexes are being filled, + * an item that needs no other item to be indexed is indexed here and let go of. + */ + private function makeAndCacheItemsOneAtATime($files): array + { + $paths = []; + $indexAsMade = true; + + foreach ($files as $path => $timestamp) { + $item = $this->makeItemFromFile($path, File::get($path)); + $key = $this->getItemKey($item); + + try { + $this->keys()->add($key, $path); + } catch (DuplicateKeyException $e) { + Stache::duplicates()->track($this, $e->getKey(), $e->getPath()); + + continue; + } + + $this->cacheItem($item); + $paths[$key] = $path; + + if ($this->filling === null) { + continue; + } + + if ($indexAsMade && $this->canIndexWithoutOtherItems($item)) { + try { + $this->takeIndexValues($key, $item); + } catch (ItemsNotReadyException $e) { + // Something an index needs (a mount, a computed value) went looking for another + // item of this store. The rest are likely to as well, so they wait their turn. + $indexAsMade = false; + } + } + + $this->releaseItem($key, $item); + } + + return $paths; + } + + /** + * Whether an item's index values can be worked out before the rest of the store's items exist. + */ + protected function canIndexWithoutOtherItems($item): bool + { + return false; + } + + /** + * Whether an index needs the store's other indexes to be there before it can be built. + */ + protected function indexNeedsOtherIndexes($name): bool + { + return false; + } + + /** + * Called while indexes are being filled, once an item has been indexed or has been + * cached to be indexed later, for a store that would otherwise hold on to its items. + */ + protected function releaseItem($key, $item): void + { + // + } + + private function takeIndexValues($key, $item): void + { + foreach ($this->filling['indexes'] as $name => $index) { + $this->filling['values'][$name][$key] = $index->getItemValue($item); + } + + $this->filling['done'][$key] = true; + } + + /** + * The indexes that get their values from one item at a time, and so can be filled + * in a single pass over the items. An index that works out its items some + * other way, or that needs the other indexes first, builds itself. + */ + private function indexesFilledTogether($indexes) + { + return $indexes->filter(function ($index, $name) { + return $index instanceof Indexes\Value + && ! $this->indexNeedsOtherIndexes($name) + && (static::$buildsItsOwnItems[$index::class] ??= (new ReflectionMethod($index, 'getItems'))->class !== Indexes\Value::class) === false; + }); + } + + /** + * Fills indexes from a single pass over the store's items, reading one item at a time, + * rather than each index reading every item for itself. + * + * Asked for by an index that needs building, it fills that index along with any others + * that haven't been built yet. Returns false if that index has to build itself. + */ + public function fillIndexes(?Index $for = null): bool + { + // An index asked for while the pass is running is one that an item's value depends on. + if ($this->filling !== null) { + return false; + } + + // A store that gathers its items in its own way is left to do so. + if ((new ReflectionMethod($this, 'getItemsFromFiles'))->class !== self::class) { + return false; + } + + $indexes = $this->indexesFilledTogether($this->resolveIndexes()); + + $all = $indexes->count(); + + if ($for) { + if (! $indexes->contains(fn ($index) => $index === $for)) { + return false; + } + + $indexes = $indexes->filter(fn ($index) => $index === $for || ! $index->isCached()); + } + + $this->filling = ['indexes' => $indexes->all(), 'values' => $indexes->map(fn () => [])->all(), 'done' => []]; + + try { + // On a cold Stache this is where the items are made, and most are indexed as they go. + $paths = $this->paths(); + + // The rest need other items to exist before they can be indexed. They all do now. + Index::whileLoading($this->key().'/*', function () use ($paths) { + foreach ($paths as $key => $path) { + if (isset($this->filling['done'][$key])) { + continue; + } + + if ($item = $this->getItem($key)) { + $this->takeIndexValues($key, $item); + $this->releaseItem($key, $item); + } + } + }); + + $filled = $this->filling; + } finally { + $this->filling = null; + } + + foreach ($indexes as $name => $index) { + // In the order of the paths, as it would be if the index had read every item itself. + $index->fill( + $paths->filter(fn ($path, $key) => isset($filled['done'][$key])) + ->map(fn ($path, $key) => $filled['values'][$name][$key]) + ->all() + ); + } + + // Only a pass that filled every one of them saves a warm from doing so. + $this->filledDuringWarm = $indexes->count() === $all ? self::$warmRun : null; + + return true; + } + + /** + * Runs a warm of several stores. A store that another one needed + * along the way will know it has already been dealt with. + */ + public static function duringWarm(callable $callback) + { + self::$warmRun = uniqid('', true); + + try { + return $callback(); + } finally { + self::$warmRun = null; + } + } + public function warm() { + $indexes = $this->resolveIndexes(); + + // Another store warmed before this one may have needed these + // indexes, and they'll have been filled then. + $filled = ($this->filledDuringWarm && $this->filledDuringWarm === self::$warmRun) || $this->fillIndexes(); + $this->shouldCacheFileItems = true; - $this->resolveIndexes()->each->update(); + ($filled ? $indexes->diffKeys($this->indexesFilledTogether($indexes)) : $indexes)->each->update(); $this->shouldCacheFileItems = false; $this->fileItems = null; From 855e68ca0b77f1c13c9a058d380ca2383be8a3b9 Mon Sep 17 00:00:00 2001 From: Daniel Jones Date: Wed, 11 Nov 2026 22:58:20 +0000 Subject: [PATCH 2/3] Test filling indexes in one pass --- tests/Stache/FillingIndexesTest.php | 199 +++++++++++++++++++++++ tests/Stache/Stores/EntriesStoreTest.php | 65 ++++++++ 2 files changed, 264 insertions(+) create mode 100644 tests/Stache/FillingIndexesTest.php diff --git a/tests/Stache/FillingIndexesTest.php b/tests/Stache/FillingIndexesTest.php new file mode 100644 index 00000000000..dad71e6d26d --- /dev/null +++ b/tests/Stache/FillingIndexesTest.php @@ -0,0 +1,199 @@ +store($collection); + } + + private function createContent() + { + $this->setSites([ + 'en' => ['url' => 'http://localhost/', 'locale' => 'en_US'], + 'fr' => ['url' => 'http://localhost/fr/', 'locale' => 'fr_FR'], + ]); + + Taxonomy::make('tags')->sites(['en', 'fr'])->save(); + + Collection::make('blog')->routes('blog/{slug}')->dated(true)->taxonomies(['tags'])->sites(['en', 'fr'])->save(); + Collection::make('pages')->routes('{parent_uri}/{slug}')->structureContents(['root' => true])->sites(['en', 'fr'])->save(); + + EntryFactory::id('one')->locale('en')->collection('blog')->slug('one')->date('2024-01-01')->data(['title' => 'One', 'tags' => ['alfa', 'bravo']])->create(); + EntryFactory::id('two')->locale('en')->collection('blog')->slug('two')->date('2024-02-01')->data(['title' => 'Two', 'tags' => ['bravo']])->create(); + EntryFactory::id('un')->locale('fr')->collection('blog')->slug('un')->date('2024-01-01')->origin('one')->data([])->create(); + + EntryFactory::id('home')->locale('en')->collection('pages')->slug('home')->data(['title' => 'Home'])->create(); + EntryFactory::id('about')->locale('en')->collection('pages')->slug('about')->data(['title' => 'About'])->create(); + EntryFactory::id('a-propos')->locale('fr')->collection('pages')->slug('a-propos')->origin('about')->data(['title' => 'À propos'])->create(); + + Stache::clear(); + } + + #[Test] + public function asking_for_one_index_fills_the_others_in_the_same_pass() + { + $this->createContent(); + + $store = $this->store('blog'); + + $this->assertFalse($store->resolveIndexes()->get('title')->isCached()); + + $store->index('slug'); + + $this->assertTrue($store->resolveIndexes()->get('title')->isCached()); + $this->assertTrue($store->resolveIndexes()->get('uri')->isCached()); + $this->assertTrue($store->resolveIndexes()->get('tags')->isCached()); + } + + #[Test] + public function indexes_hold_what_they_would_if_each_had_read_every_entry_for_itself() + { + $this->createContent(); + + Stache::warm(); + + $store = $this->store('blog'); + + foreach ($store->resolveIndexes() as $name => $index) { + $expected = $index->getItems(); + + $this->assertSame(array_keys($expected), $index->items()->keys()->all(), $name); + $this->assertEquals($expected, $index->items()->all(), $name); + } + + $this->assertEquals(['one' => '/blog/one', 'two' => '/blog/two', 'un' => '/blog/un'], $store->index('uri')->items()->all()); + } + + #[Test] + public function a_localization_is_indexed_with_the_values_it_takes_from_its_origin() + { + $this->createContent(); + + $this->assertEquals('One', $this->store('blog')->index('title')->get('un')); + $this->assertEquals(['alfa', 'bravo'], $this->store('blog')->index('tags')->get('un')); + $this->assertEquals('one', $this->store('blog')->index('origin')->get('un')); + + $this->assertEquals(['un'], Entry::query()->where('collection', 'blog')->where('site', 'fr')->where('title', 'One')->get()->map->id()->all()); + } + + #[Test] + public function entries_of_a_collection_mounted_on_another_entry_are_indexed_once_every_entry_exists() + { + $this->createContent(); + + // Building a uri looks the mount up, which goes through every collection's entries. + Collection::findByHandle('blog')->routes('{mount}/{slug}')->mount('home')->save(); + Stache::clear(); + + $store = $this->store('blog'); + + $this->assertEquals('/home/one', $store->index('uri')->get('one')); + $this->assertEquals('/home/two', $store->index('uri')->get('two')); + $this->assertEquals(['one' => 'One', 'two' => 'Two', 'un' => 'One'], $store->index('title')->items()->all()); + $this->assertNull(Blink::store('structure-entries')->get('one')); + } + + #[Test] + public function term_associations_are_built_from_the_entries_indexes() + { + $this->createContent(); + + $associations = Stache::store('terms')->store('tags')->index('associations'); + + $this->assertEquals(['one', 'un'], collect($associations->forSlug('alfa'))->pluck('entry')->sort()->values()->all()); + $this->assertEquals(['one', 'two', 'un'], collect($associations->forSlug('bravo'))->pluck('entry')->sort()->values()->all()); + $this->assertEquals(['en', 'fr'], collect($associations->forSlug('alfa'))->pluck('site')->sort()->values()->all()); + } + + #[Test] + public function entries_arent_kept_in_memory_once_they_have_been_indexed() + { + $this->createContent(); + Blink::store('structure-entries')->flush(); + + $this->store('blog')->index('slug'); + + $this->assertNull(Blink::store('structure-entries')->get('one')); + $this->assertNull(Blink::store('structure-entries')->get('un')); + + // A structure's uri, parent and order indexes are built from its entries afterwards. + $this->store('pages')->index('slug'); + + $this->assertNotNull(Blink::store('structure-entries')->get('about')); + } + + #[Test] + public function rebuilding_one_index_leaves_the_others_alone() + { + $this->createContent(); + + $store = $this->store('blog'); + $store->index('slug'); + + $title = $store->resolveIndexes()->get('title'); + Stache::cacheStore()->forever($title->cacheKey(), ['untouched' => true]); + + $store->resolveIndexes()->get('slug')->update(); + + $this->assertEquals(['untouched' => true], Stache::cacheStore()->get($title->cacheKey())); + $this->assertEquals(['one' => 'one', 'two' => 'two', 'un' => 'un'], $store->resolveIndexes()->get('slug')->items()->all()); + } + + #[Test] + public function warming_again_in_the_same_process_fills_the_indexes_again() + { + $this->createContent(); + + Stache::warm(); + $this->assertEquals('One', $this->store('blog')->index('title')->get('one')); + + // Something that leaves the indexes behind what is cached. + $this->cacheWithTitle('one', 'Changed'); + + Stache::warm(); + $this->assertEquals('Changed', $this->store('blog')->index('title')->get('one')); + $this->assertEquals('Changed', $this->store('blog')->index('title')->get('un')); + } + + #[Test] + public function a_store_another_store_needed_during_a_warm_isnt_filled_twice() + { + $this->createContent(); + + $made = 0; + \Illuminate\Support\Facades\Event::listen(\Illuminate\Cache\Events\KeyWritten::class, function ($event) use (&$made) { + if ($event->key === 'stache::indexes::entries::blog::title') { + $made++; + } + }); + + // The terms are warmed before the entries, and their associations need the entries' indexes. + Stache::warm(); + + $this->assertEquals(1, $made); + } + + private function cacheWithTitle($id, $title) + { + $store = $this->store('blog'); + $key = (fn () => $this->getItemCacheKey($id))->call($store); + + Stache::cacheStore()->forever($key, Stache::cacheStore()->get($key)->set('title', $title)); + Blink::flush(); + } +} diff --git a/tests/Stache/Stores/EntriesStoreTest.php b/tests/Stache/Stores/EntriesStoreTest.php index 1e22fe1f940..dace07d4380 100644 --- a/tests/Stache/Stores/EntriesStoreTest.php +++ b/tests/Stache/Stores/EntriesStoreTest.php @@ -576,6 +576,71 @@ public function snapshots_are_not_kept_for_a_later_job_in_the_same_worker() $this->assertEquals('Saved by another process', $item->get('title')); } + #[Test] + public function filling_indexes_in_one_pass_gives_what_each_index_builds_for_itself() + { + foreach (['alphabetical', 'blog', 'numeric', 'pages'] as $collection) { + $store = $this->parent->store($collection); + + $store->warm(); + + foreach ($store->resolveIndexes() as $name => $index) { + $expected = $index->getItems(); + + $this->assertNotEmpty($expected, "{$collection}/{$name}"); + $this->assertSame(array_keys($expected), $index->items()->keys()->all(), "{$collection}/{$name}"); + $this->assertEquals($expected, $index->items()->all(), "{$collection}/{$name}"); + } + } + } + + #[Test] + public function a_queue_worker_fills_indexes_from_what_is_cached_at_the_time() + { + $this->useSerializingCacheStore(); + + $store = $this->parent->store('blog'); + + // A job needs an index, so the store's indexes are filled. + $this->assertEquals('Christmas', $store->index('title')->get('blog-christmas')); + + // Another process saves the entry, and its indexes are cleared to be built again. + $this->saveAsAnotherProcessWould($store, 'blog-christmas', 'Saved by another process'); + + // The worker moves on to its next job. Blink is left as it was. + Stache::stores()->each->resetMemoizedState(); + app('stache.indexes')->each->resetMemoizedState(); + + $this->assertEquals('Saved by another process', $store->index('title')->get('blog-christmas')); + } + + #[Test] + public function a_later_request_in_the_same_process_fills_indexes_from_what_is_cached_at_the_time() + { + $this->useSerializingCacheStore(); + + $store = $this->parent->store('blog'); + $this->assertEquals('Christmas', $store->index('title')->get('blog-christmas')); + + $this->saveAsAnotherProcessWould($store, 'blog-christmas', 'Saved by another process'); + + // The next request gets a new Blink (Octane clears resolved facades), but the same store and indexes. + Blink::swap(new \Statamic\Support\Blink); + $store->resolveIndexes()->each->resetMemoizedState(); + + $this->assertEquals('Saved by another process', $store->index('title')->get('blog-christmas')); + } + + private function saveAsAnotherProcessWould($store, $id, $title) + { + $key = (fn () => $this->getItemCacheKey($id))->call($store); + + Stache::cacheStore()->forever($key, Stache::cacheStore()->get($key)->set('title', $title)); + + // The paths are kept under the path index's key, and haven't changed. + $store->resolveIndexes()->except('path')->each(fn ($index) => Stache::cacheStore()->forget($index->cacheKey())); + } + private function useSerializingCacheStore() { config([ From b81fb5fdd390e8be8fd9926605fe3865b80a4e05 Mon Sep 17 00:00:00 2001 From: Daniel Jones Date: Wed, 11 Nov 2026 23:15:57 +0000 Subject: [PATCH 3/3] Let an index that was asked for too early be asked for again A computed value that queries its own collection asks for an index while the entries are still being made. That entry waits for the second pass, but the index was left marked as loaded with nothing in it, so the second pass read it as empty. Warming also filled indexes when updating them was disabled, and a parallel warm's workers didn't know a warm was running. --- src/Stache/Indexes/Index.php | 6 ++++++ src/Stache/Stache.php | 2 +- src/Stache/Stores/Store.php | 4 ++++ tests/Stache/FillingIndexesTest.php | 28 ++++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/Stache/Indexes/Index.php b/src/Stache/Indexes/Index.php index 3b8dfd650de..1b055c23c2f 100644 --- a/src/Stache/Indexes/Index.php +++ b/src/Stache/Indexes/Index.php @@ -3,6 +3,7 @@ namespace Statamic\Stache\Indexes; use Statamic\Facades\Stache; +use Statamic\Stache\Exceptions\ItemsNotReadyException; abstract class Index { @@ -80,6 +81,11 @@ public function load() } $this->store->cacheIndexUsage($this); + } catch (ItemsNotReadyException $e) { + // Whatever asked will ask again once the items are there, and should get them. + $this->loaded = false; + + throw $e; } finally { array_pop(static::$loadingStack); } diff --git a/src/Stache/Stache.php b/src/Stache/Stache.php index 10f864741b7..2012d752fe5 100644 --- a/src/Stache/Stache.php +++ b/src/Stache/Stache.php @@ -299,7 +299,7 @@ protected function warmInParallel($stores) $closures = $chunks->map(function ($chunk) { return function () use ($chunk) { - return $chunk->each->warm()->keys()->all(); + return Store::duringWarm(fn () => $chunk->each->warm()->keys()->all()); }; })->all(); diff --git a/src/Stache/Stores/Store.php b/src/Stache/Stores/Store.php index 98af79de9b0..d2c259c6d12 100644 --- a/src/Stache/Stores/Store.php +++ b/src/Stache/Stores/Store.php @@ -515,6 +515,10 @@ private function indexesFilledTogether($indexes) */ public function fillIndexes(?Index $for = null): bool { + if (! Stache::shouldUpdateIndexes()) { + return false; + } + // An index asked for while the pass is running is one that an item's value depends on. if ($this->filling !== null) { return false; diff --git a/tests/Stache/FillingIndexesTest.php b/tests/Stache/FillingIndexesTest.php index dad71e6d26d..86d1e2f6a6f 100644 --- a/tests/Stache/FillingIndexesTest.php +++ b/tests/Stache/FillingIndexesTest.php @@ -108,6 +108,34 @@ public function entries_of_a_collection_mounted_on_another_entry_are_indexed_onc $this->assertNull(Blink::store('structure-entries')->get('one')); } + #[Test] + public function a_computed_value_that_queries_the_collection_is_indexed_once_every_entry_exists() + { + // Registered first, as an app would in a service provider. + Collection::computed('blog', 'in_this_site', function ($entry) { + return Entry::query()->where('collection', 'blog')->where('site', $entry->locale())->count(); + }); + + $this->createContent(); + + $store = $this->store('blog'); + $store->cacheIndexUsage('in_this_site'); + + $this->assertEquals(['one' => 2, 'two' => 2, 'un' => 1], $store->index('in_this_site')->items()->all()); + $this->assertEquals(['one' => 'en', 'two' => 'en', 'un' => 'fr'], $store->index('site')->items()->all()); + } + + #[Test] + public function warming_leaves_indexes_alone_when_updating_them_is_disabled() + { + $this->createContent(); + + Stache::disableUpdatingIndexes(); + Stache::warm(); + + $this->assertFalse($this->store('blog')->resolveIndexes()->get('title')->isCached()); + } + #[Test] public function term_associations_are_built_from_the_entries_indexes() {