diff --git a/src/Stache/Exceptions/ItemsNotReadyException.php b/src/Stache/Exceptions/ItemsNotReadyException.php new file mode 100644 index 00000000000..1a45e9f78ce --- /dev/null +++ b/src/Stache/Exceptions/ItemsNotReadyException.php @@ -0,0 +1,12 @@ +store->cacheIndexUsage($this); + } catch (ItemsNotReadyException $e) { + // Whatever asked will ask again once the items are there, and should get them. + $this->loaded = false; + + throw $e; } finally { array_pop(static::$loadingStack); } @@ -95,6 +101,12 @@ public function update() debugbar()->addMessage("Updating index: {$this->store->key()}/{$this->name}", 'stache'); + // Most indexes are filled in a single pass over the store's items, along with + // any others that are waiting to be built, rather than each reading every item. + if ($this->store->fillIndexes($this)) { + return $this; + } + $this->items = $this->getItems(); $this->cache(); @@ -102,6 +114,33 @@ public function update() return $this; } + /** + * Sets and caches items that have been worked out elsewhere. + */ + public function fill(array $items) + { + $this->items = $items; + + $this->cache(); + + return $this; + } + + /** + * Runs a callback as if an index were loading, for work that + * fills indexes without going through load(). + */ + public static function whileLoading(string $key, callable $callback) + { + static::$loadingStack[] = $key; + + try { + return $callback(); + } finally { + array_pop(static::$loadingStack); + } + } + public function isCached() { return Stache::cacheStore()->has($this->cacheKey()); diff --git a/src/Stache/Indexes/Terms/Associations.php b/src/Stache/Indexes/Terms/Associations.php index 606c2a674fb..0ab637fdc2e 100644 --- a/src/Stache/Indexes/Terms/Associations.php +++ b/src/Stache/Indexes/Terms/Associations.php @@ -2,6 +2,7 @@ namespace Statamic\Stache\Indexes\Terms; +use Statamic\Facades\Stache; use Statamic\Facades\Taxonomy; use Statamic\Stache\Indexes\Index; use Statamic\Support\Str; @@ -11,25 +12,28 @@ class Associations extends Index private ?array $grouped = null; private ?array $groupedFrom = null; + /** + * What each entry has in the taxonomy's field is already in its collection's + * index for that field, so the entries themselves don't need to be loaded. + */ public function getItems() { return Taxonomy::findByHandle($handle = $this->store->childKey()) ->collections() ->flatMap(function ($collection) use ($handle) { - return $collection->queryEntries() - ->where($handle, '<>', null) - ->get() - ->flatMap(function ($entry) use ($handle) { - return collect($entry->value($handle)) - ->map(function ($value) use ($entry) { - return [ - 'value' => $value, - 'slug' => Str::slug($value), - 'entry' => $entry->id(), - 'collection' => $entry->collectionHandle(), - 'site' => $entry->locale(), - ]; - }); + $store = Stache::store('entries')->store($collection->handle()); + $sites = $store->index('site'); + + return $store->index($handle)->items() + ->reject(fn ($values) => $values == null) + ->flatMap(function ($values, $id) use ($collection, $sites) { + return collect($values)->map(fn ($value) => [ + 'value' => $value, + 'slug' => Str::slug($value), + 'entry' => (string) $id, + 'collection' => $collection->handle(), + 'site' => $sites->get($id), + ]); })->all(); })->all(); } diff --git a/src/Stache/Stache.php b/src/Stache/Stache.php index 1b25b25591c..2012d752fe5 100644 --- a/src/Stache/Stache.php +++ b/src/Stache/Stache.php @@ -134,11 +134,13 @@ public function warm() $stores = $this->stores()->except($this->exclude); - if ($this->shouldUseParallelWarming($stores)) { - $this->warmInParallel($stores); - } else { - $stores->each->warm(); - } + Store::duringWarm(function () use ($stores) { + if ($this->shouldUseParallelWarming($stores)) { + $this->warmInParallel($stores); + } else { + $stores->each->warm(); + } + }); $this->stopTimer(); @@ -297,7 +299,7 @@ protected function warmInParallel($stores) $closures = $chunks->map(function ($chunk) { return function () use ($chunk) { - return $chunk->each->warm()->keys()->all(); + return Store::duringWarm(fn () => $chunk->each->warm()->keys()->all()); }; })->all(); diff --git a/src/Stache/Stores/CollectionEntriesStore.php b/src/Stache/Stores/CollectionEntriesStore.php index a5be77798da..3e0633fb24a 100644 --- a/src/Stache/Stores/CollectionEntriesStore.php +++ b/src/Stache/Stores/CollectionEntriesStore.php @@ -163,6 +163,37 @@ public function makeItemFromFile($path, $contents) return $entry; } + protected function canIndexWithoutOtherItems($item): bool + { + // A localization falls back to its origin's values, and an entry + // in a structure gets its uri, parent and order from the tree. + // Asked of the entry's own property, because hasOrigin() goes looking for the origin. + $hasOrigin = (bool) Closure::bind(fn () => $this->origin, $item, EntryInstance::class)(); + + return ! $hasOrigin && ! $this->findCollection($item->collectionHandle())?->hasStructure(); + } + + protected function indexNeedsOtherIndexes($name): bool + { + // Where an entry sits in a tree is found by querying the collection's + // entries, which needs the rest of this store's indexes to be there. + return in_array($name, ['uri', 'parent', 'order']) && $this->collection()?->hasStructure(); + } + + protected function releaseItem($key, $item): void + { + // A structure's uri, parent and order indexes are still to be built from these. + if ($this->collection()?->hasStructure()) { + return; + } + + // Entries are kept so that uris can be built and indexes don't have to read them back + // from the cache. One that has been indexed is needed for neither, and one that is + // waiting to be indexed is read back then, rather than all of them being held. + Blink::store('structure-entries')->forget($key); + $this->snapshots()->forget($key); + } + protected function extractAttributesFromPath($path) { $site = Site::default()->handle(); diff --git a/src/Stache/Stores/Store.php b/src/Stache/Stores/Store.php index ee31e4236a9..d2c259c6d12 100644 --- a/src/Stache/Stores/Store.php +++ b/src/Stache/Stores/Store.php @@ -3,10 +3,12 @@ namespace Statamic\Stache\Stores; use Facades\Statamic\Stache\Traverser; +use ReflectionMethod; use Statamic\Facades\File; use Statamic\Facades\Path; use Statamic\Facades\Stache; use Statamic\Stache\Exceptions\DuplicateKeyException; +use Statamic\Stache\Exceptions\ItemsNotReadyException; use Statamic\Stache\Indexes; use Statamic\Stache\Indexes\Index; use Statamic\Support\Arr; @@ -26,6 +28,11 @@ abstract class Store protected $shouldCacheFileItems = false; protected $modified; protected $keys; + private ?array $filling = null; + private bool $makingItems = false; + private ?string $filledDuringWarm = null; + private static array $buildsItsOwnItems = []; + private static ?string $warmRun = null; /** * @return string @@ -310,30 +317,20 @@ public function paths() return $this->paths = collect($paths); } - $files = Traverser::filter([$this, 'getItemFilter'])->traverse($this); - - $fileItems = $files->map(function ($timestamp, $path) { - return [ - 'item' => $item = $this->makeItemFromFile($path, File::get($path)), - 'key' => $this->getItemKey($item), - 'path' => $path, - ]; - }); - - $items = $fileItems->reject(function ($item) { - try { - $this->keys()->add($item['key'], $item['path']); - } catch (DuplicateKeyException $e) { - $isDuplicate = true; - Stache::duplicates()->track($this, $e->getKey(), $e->getPath()); - } + // An index value being worked out as the items are made can't be given the items. + if ($this->makingItems && $this->filling !== null) { + throw new ItemsNotReadyException; + } - return $isDuplicate ?? false; - }); + $files = Traverser::filter([$this, 'getItemFilter'])->traverse($this); - $items->each(fn ($item) => $this->cacheItem($item['item'])); + $this->makingItems = true; - $paths = $items->pluck('path', 'key'); + try { + $paths = collect($this->makeAndCacheItemsOneAtATime($files)); + } finally { + $this->makingItems = false; + } $this->cachePaths($paths); @@ -416,11 +413,200 @@ public function clear() Stache::cacheStore()->forget("stache::timestamps::{$this->key()}"); } + /** + * Each file is turned into an item and cached before the next one is read, so the + * items don't all have to be in memory at once. While indexes are being filled, + * an item that needs no other item to be indexed is indexed here and let go of. + */ + private function makeAndCacheItemsOneAtATime($files): array + { + $paths = []; + $indexAsMade = true; + + foreach ($files as $path => $timestamp) { + $item = $this->makeItemFromFile($path, File::get($path)); + $key = $this->getItemKey($item); + + try { + $this->keys()->add($key, $path); + } catch (DuplicateKeyException $e) { + Stache::duplicates()->track($this, $e->getKey(), $e->getPath()); + + continue; + } + + $this->cacheItem($item); + $paths[$key] = $path; + + if ($this->filling === null) { + continue; + } + + if ($indexAsMade && $this->canIndexWithoutOtherItems($item)) { + try { + $this->takeIndexValues($key, $item); + } catch (ItemsNotReadyException $e) { + // Something an index needs (a mount, a computed value) went looking for another + // item of this store. The rest are likely to as well, so they wait their turn. + $indexAsMade = false; + } + } + + $this->releaseItem($key, $item); + } + + return $paths; + } + + /** + * Whether an item's index values can be worked out before the rest of the store's items exist. + */ + protected function canIndexWithoutOtherItems($item): bool + { + return false; + } + + /** + * Whether an index needs the store's other indexes to be there before it can be built. + */ + protected function indexNeedsOtherIndexes($name): bool + { + return false; + } + + /** + * Called while indexes are being filled, once an item has been indexed or has been + * cached to be indexed later, for a store that would otherwise hold on to its items. + */ + protected function releaseItem($key, $item): void + { + // + } + + private function takeIndexValues($key, $item): void + { + foreach ($this->filling['indexes'] as $name => $index) { + $this->filling['values'][$name][$key] = $index->getItemValue($item); + } + + $this->filling['done'][$key] = true; + } + + /** + * The indexes that get their values from one item at a time, and so can be filled + * in a single pass over the items. An index that works out its items some + * other way, or that needs the other indexes first, builds itself. + */ + private function indexesFilledTogether($indexes) + { + return $indexes->filter(function ($index, $name) { + return $index instanceof Indexes\Value + && ! $this->indexNeedsOtherIndexes($name) + && (static::$buildsItsOwnItems[$index::class] ??= (new ReflectionMethod($index, 'getItems'))->class !== Indexes\Value::class) === false; + }); + } + + /** + * Fills indexes from a single pass over the store's items, reading one item at a time, + * rather than each index reading every item for itself. + * + * Asked for by an index that needs building, it fills that index along with any others + * that haven't been built yet. Returns false if that index has to build itself. + */ + public function fillIndexes(?Index $for = null): bool + { + if (! Stache::shouldUpdateIndexes()) { + return false; + } + + // An index asked for while the pass is running is one that an item's value depends on. + if ($this->filling !== null) { + return false; + } + + // A store that gathers its items in its own way is left to do so. + if ((new ReflectionMethod($this, 'getItemsFromFiles'))->class !== self::class) { + return false; + } + + $indexes = $this->indexesFilledTogether($this->resolveIndexes()); + + $all = $indexes->count(); + + if ($for) { + if (! $indexes->contains(fn ($index) => $index === $for)) { + return false; + } + + $indexes = $indexes->filter(fn ($index) => $index === $for || ! $index->isCached()); + } + + $this->filling = ['indexes' => $indexes->all(), 'values' => $indexes->map(fn () => [])->all(), 'done' => []]; + + try { + // On a cold Stache this is where the items are made, and most are indexed as they go. + $paths = $this->paths(); + + // The rest need other items to exist before they can be indexed. They all do now. + Index::whileLoading($this->key().'/*', function () use ($paths) { + foreach ($paths as $key => $path) { + if (isset($this->filling['done'][$key])) { + continue; + } + + if ($item = $this->getItem($key)) { + $this->takeIndexValues($key, $item); + $this->releaseItem($key, $item); + } + } + }); + + $filled = $this->filling; + } finally { + $this->filling = null; + } + + foreach ($indexes as $name => $index) { + // In the order of the paths, as it would be if the index had read every item itself. + $index->fill( + $paths->filter(fn ($path, $key) => isset($filled['done'][$key])) + ->map(fn ($path, $key) => $filled['values'][$name][$key]) + ->all() + ); + } + + // Only a pass that filled every one of them saves a warm from doing so. + $this->filledDuringWarm = $indexes->count() === $all ? self::$warmRun : null; + + return true; + } + + /** + * Runs a warm of several stores. A store that another one needed + * along the way will know it has already been dealt with. + */ + public static function duringWarm(callable $callback) + { + self::$warmRun = uniqid('', true); + + try { + return $callback(); + } finally { + self::$warmRun = null; + } + } + public function warm() { + $indexes = $this->resolveIndexes(); + + // Another store warmed before this one may have needed these + // indexes, and they'll have been filled then. + $filled = ($this->filledDuringWarm && $this->filledDuringWarm === self::$warmRun) || $this->fillIndexes(); + $this->shouldCacheFileItems = true; - $this->resolveIndexes()->each->update(); + ($filled ? $indexes->diffKeys($this->indexesFilledTogether($indexes)) : $indexes)->each->update(); $this->shouldCacheFileItems = false; $this->fileItems = null; diff --git a/tests/Stache/FillingIndexesTest.php b/tests/Stache/FillingIndexesTest.php new file mode 100644 index 00000000000..86d1e2f6a6f --- /dev/null +++ b/tests/Stache/FillingIndexesTest.php @@ -0,0 +1,227 @@ +store($collection); + } + + private function createContent() + { + $this->setSites([ + 'en' => ['url' => 'http://localhost/', 'locale' => 'en_US'], + 'fr' => ['url' => 'http://localhost/fr/', 'locale' => 'fr_FR'], + ]); + + Taxonomy::make('tags')->sites(['en', 'fr'])->save(); + + Collection::make('blog')->routes('blog/{slug}')->dated(true)->taxonomies(['tags'])->sites(['en', 'fr'])->save(); + Collection::make('pages')->routes('{parent_uri}/{slug}')->structureContents(['root' => true])->sites(['en', 'fr'])->save(); + + EntryFactory::id('one')->locale('en')->collection('blog')->slug('one')->date('2024-01-01')->data(['title' => 'One', 'tags' => ['alfa', 'bravo']])->create(); + EntryFactory::id('two')->locale('en')->collection('blog')->slug('two')->date('2024-02-01')->data(['title' => 'Two', 'tags' => ['bravo']])->create(); + EntryFactory::id('un')->locale('fr')->collection('blog')->slug('un')->date('2024-01-01')->origin('one')->data([])->create(); + + EntryFactory::id('home')->locale('en')->collection('pages')->slug('home')->data(['title' => 'Home'])->create(); + EntryFactory::id('about')->locale('en')->collection('pages')->slug('about')->data(['title' => 'About'])->create(); + EntryFactory::id('a-propos')->locale('fr')->collection('pages')->slug('a-propos')->origin('about')->data(['title' => 'À propos'])->create(); + + Stache::clear(); + } + + #[Test] + public function asking_for_one_index_fills_the_others_in_the_same_pass() + { + $this->createContent(); + + $store = $this->store('blog'); + + $this->assertFalse($store->resolveIndexes()->get('title')->isCached()); + + $store->index('slug'); + + $this->assertTrue($store->resolveIndexes()->get('title')->isCached()); + $this->assertTrue($store->resolveIndexes()->get('uri')->isCached()); + $this->assertTrue($store->resolveIndexes()->get('tags')->isCached()); + } + + #[Test] + public function indexes_hold_what_they_would_if_each_had_read_every_entry_for_itself() + { + $this->createContent(); + + Stache::warm(); + + $store = $this->store('blog'); + + foreach ($store->resolveIndexes() as $name => $index) { + $expected = $index->getItems(); + + $this->assertSame(array_keys($expected), $index->items()->keys()->all(), $name); + $this->assertEquals($expected, $index->items()->all(), $name); + } + + $this->assertEquals(['one' => '/blog/one', 'two' => '/blog/two', 'un' => '/blog/un'], $store->index('uri')->items()->all()); + } + + #[Test] + public function a_localization_is_indexed_with_the_values_it_takes_from_its_origin() + { + $this->createContent(); + + $this->assertEquals('One', $this->store('blog')->index('title')->get('un')); + $this->assertEquals(['alfa', 'bravo'], $this->store('blog')->index('tags')->get('un')); + $this->assertEquals('one', $this->store('blog')->index('origin')->get('un')); + + $this->assertEquals(['un'], Entry::query()->where('collection', 'blog')->where('site', 'fr')->where('title', 'One')->get()->map->id()->all()); + } + + #[Test] + public function entries_of_a_collection_mounted_on_another_entry_are_indexed_once_every_entry_exists() + { + $this->createContent(); + + // Building a uri looks the mount up, which goes through every collection's entries. + Collection::findByHandle('blog')->routes('{mount}/{slug}')->mount('home')->save(); + Stache::clear(); + + $store = $this->store('blog'); + + $this->assertEquals('/home/one', $store->index('uri')->get('one')); + $this->assertEquals('/home/two', $store->index('uri')->get('two')); + $this->assertEquals(['one' => 'One', 'two' => 'Two', 'un' => 'One'], $store->index('title')->items()->all()); + $this->assertNull(Blink::store('structure-entries')->get('one')); + } + + #[Test] + public function a_computed_value_that_queries_the_collection_is_indexed_once_every_entry_exists() + { + // Registered first, as an app would in a service provider. + Collection::computed('blog', 'in_this_site', function ($entry) { + return Entry::query()->where('collection', 'blog')->where('site', $entry->locale())->count(); + }); + + $this->createContent(); + + $store = $this->store('blog'); + $store->cacheIndexUsage('in_this_site'); + + $this->assertEquals(['one' => 2, 'two' => 2, 'un' => 1], $store->index('in_this_site')->items()->all()); + $this->assertEquals(['one' => 'en', 'two' => 'en', 'un' => 'fr'], $store->index('site')->items()->all()); + } + + #[Test] + public function warming_leaves_indexes_alone_when_updating_them_is_disabled() + { + $this->createContent(); + + Stache::disableUpdatingIndexes(); + Stache::warm(); + + $this->assertFalse($this->store('blog')->resolveIndexes()->get('title')->isCached()); + } + + #[Test] + public function term_associations_are_built_from_the_entries_indexes() + { + $this->createContent(); + + $associations = Stache::store('terms')->store('tags')->index('associations'); + + $this->assertEquals(['one', 'un'], collect($associations->forSlug('alfa'))->pluck('entry')->sort()->values()->all()); + $this->assertEquals(['one', 'two', 'un'], collect($associations->forSlug('bravo'))->pluck('entry')->sort()->values()->all()); + $this->assertEquals(['en', 'fr'], collect($associations->forSlug('alfa'))->pluck('site')->sort()->values()->all()); + } + + #[Test] + public function entries_arent_kept_in_memory_once_they_have_been_indexed() + { + $this->createContent(); + Blink::store('structure-entries')->flush(); + + $this->store('blog')->index('slug'); + + $this->assertNull(Blink::store('structure-entries')->get('one')); + $this->assertNull(Blink::store('structure-entries')->get('un')); + + // A structure's uri, parent and order indexes are built from its entries afterwards. + $this->store('pages')->index('slug'); + + $this->assertNotNull(Blink::store('structure-entries')->get('about')); + } + + #[Test] + public function rebuilding_one_index_leaves_the_others_alone() + { + $this->createContent(); + + $store = $this->store('blog'); + $store->index('slug'); + + $title = $store->resolveIndexes()->get('title'); + Stache::cacheStore()->forever($title->cacheKey(), ['untouched' => true]); + + $store->resolveIndexes()->get('slug')->update(); + + $this->assertEquals(['untouched' => true], Stache::cacheStore()->get($title->cacheKey())); + $this->assertEquals(['one' => 'one', 'two' => 'two', 'un' => 'un'], $store->resolveIndexes()->get('slug')->items()->all()); + } + + #[Test] + public function warming_again_in_the_same_process_fills_the_indexes_again() + { + $this->createContent(); + + Stache::warm(); + $this->assertEquals('One', $this->store('blog')->index('title')->get('one')); + + // Something that leaves the indexes behind what is cached. + $this->cacheWithTitle('one', 'Changed'); + + Stache::warm(); + $this->assertEquals('Changed', $this->store('blog')->index('title')->get('one')); + $this->assertEquals('Changed', $this->store('blog')->index('title')->get('un')); + } + + #[Test] + public function a_store_another_store_needed_during_a_warm_isnt_filled_twice() + { + $this->createContent(); + + $made = 0; + \Illuminate\Support\Facades\Event::listen(\Illuminate\Cache\Events\KeyWritten::class, function ($event) use (&$made) { + if ($event->key === 'stache::indexes::entries::blog::title') { + $made++; + } + }); + + // The terms are warmed before the entries, and their associations need the entries' indexes. + Stache::warm(); + + $this->assertEquals(1, $made); + } + + private function cacheWithTitle($id, $title) + { + $store = $this->store('blog'); + $key = (fn () => $this->getItemCacheKey($id))->call($store); + + Stache::cacheStore()->forever($key, Stache::cacheStore()->get($key)->set('title', $title)); + Blink::flush(); + } +} diff --git a/tests/Stache/Stores/EntriesStoreTest.php b/tests/Stache/Stores/EntriesStoreTest.php index 1e22fe1f940..dace07d4380 100644 --- a/tests/Stache/Stores/EntriesStoreTest.php +++ b/tests/Stache/Stores/EntriesStoreTest.php @@ -576,6 +576,71 @@ public function snapshots_are_not_kept_for_a_later_job_in_the_same_worker() $this->assertEquals('Saved by another process', $item->get('title')); } + #[Test] + public function filling_indexes_in_one_pass_gives_what_each_index_builds_for_itself() + { + foreach (['alphabetical', 'blog', 'numeric', 'pages'] as $collection) { + $store = $this->parent->store($collection); + + $store->warm(); + + foreach ($store->resolveIndexes() as $name => $index) { + $expected = $index->getItems(); + + $this->assertNotEmpty($expected, "{$collection}/{$name}"); + $this->assertSame(array_keys($expected), $index->items()->keys()->all(), "{$collection}/{$name}"); + $this->assertEquals($expected, $index->items()->all(), "{$collection}/{$name}"); + } + } + } + + #[Test] + public function a_queue_worker_fills_indexes_from_what_is_cached_at_the_time() + { + $this->useSerializingCacheStore(); + + $store = $this->parent->store('blog'); + + // A job needs an index, so the store's indexes are filled. + $this->assertEquals('Christmas', $store->index('title')->get('blog-christmas')); + + // Another process saves the entry, and its indexes are cleared to be built again. + $this->saveAsAnotherProcessWould($store, 'blog-christmas', 'Saved by another process'); + + // The worker moves on to its next job. Blink is left as it was. + Stache::stores()->each->resetMemoizedState(); + app('stache.indexes')->each->resetMemoizedState(); + + $this->assertEquals('Saved by another process', $store->index('title')->get('blog-christmas')); + } + + #[Test] + public function a_later_request_in_the_same_process_fills_indexes_from_what_is_cached_at_the_time() + { + $this->useSerializingCacheStore(); + + $store = $this->parent->store('blog'); + $this->assertEquals('Christmas', $store->index('title')->get('blog-christmas')); + + $this->saveAsAnotherProcessWould($store, 'blog-christmas', 'Saved by another process'); + + // The next request gets a new Blink (Octane clears resolved facades), but the same store and indexes. + Blink::swap(new \Statamic\Support\Blink); + $store->resolveIndexes()->each->resetMemoizedState(); + + $this->assertEquals('Saved by another process', $store->index('title')->get('blog-christmas')); + } + + private function saveAsAnotherProcessWould($store, $id, $title) + { + $key = (fn () => $this->getItemCacheKey($id))->call($store); + + Stache::cacheStore()->forever($key, Stache::cacheStore()->get($key)->set('title', $title)); + + // The paths are kept under the path index's key, and haven't changed. + $store->resolveIndexes()->except('path')->each(fn ($index) => Stache::cacheStore()->forget($index->cacheKey())); + } + private function useSerializingCacheStore() { config([