[6.x] Build Stache indexes in one pass without holding every entry

src/Stache/Exceptions/ItemsNotReadyException.php

@@ -0,0 +1,12 @@
+<?php
+
+namespace Statamic\Stache\Exceptions;
+
+use Exception;
+
+/**
+ * Thrown when a store's items are asked for while they are still being made.
+ */
+class ItemsNotReadyException extends Exception
+{
+}

src/Stache/Indexes/Index.php

@@ -3,6 +3,7 @@
 namespace Statamic\Stache\Indexes;
 
 use Statamic\Facades\Stache;
+use Statamic\Stache\Exceptions\ItemsNotReadyException;
 
 abstract class Index
 {
@@ -80,6 +81,11 @@ public function load()
             }
 
             $this->store->cacheIndexUsage($this);
+        } catch (ItemsNotReadyException $e) {
+            // Whatever asked will ask again once the items are there, and should get them.
+            $this->loaded = false;
+
+            throw $e;
         } finally {
             array_pop(static::$loadingStack);
         }
@@ -95,13 +101,46 @@ public function update()
 
         debugbar()->addMessage("Updating index: {$this->store->key()}/{$this->name}", 'stache');
 
+        // Most indexes are filled in a single pass over the store's items, along with
+        // any others that are waiting to be built, rather than each reading every item.
+        if ($this->store->fillIndexes($this)) {
+            return $this;
+        }
+
         $this->items = $this->getItems();
 
         $this->cache();
 
         return $this;
     }
 
+    /**
+     * Sets and caches items that have been worked out elsewhere.
+     */
+    public function fill(array $items)
+    {
+        $this->items = $items;
+
+        $this->cache();
+
+        return $this;
+    }
+
+    /**
+     * Runs a callback as if an index were loading, for work that
+     * fills indexes without going through load().
+     */
+    public static function whileLoading(string $key, callable $callback)
+    {
+        static::$loadingStack[] = $key;
+
+        try {
+            return $callback();
+        } finally {
+            array_pop(static::$loadingStack);
+        }
+    }
+
     public function isCached()
     {
         return Stache::cacheStore()->has($this->cacheKey());

src/Stache/Indexes/Terms/Associations.php

@@ -2,6 +2,7 @@
 
 namespace Statamic\Stache\Indexes\Terms;
 
+use Statamic\Facades\Stache;
 use Statamic\Facades\Taxonomy;
 use Statamic\Stache\Indexes\Index;
 use Statamic\Support\Str;
@@ -11,25 +12,28 @@ class Associations extends Index
     private ?array $grouped = null;
     private ?array $groupedFrom = null;
 
+    /**
+     * What each entry has in the taxonomy's field is already in its collection's
+     * index for that field, so the entries themselves don't need to be loaded.
+     */
     public function getItems()
     {
         return Taxonomy::findByHandle($handle = $this->store->childKey())
             ->collections()
             ->flatMap(function ($collection) use ($handle) {
-                return $collection->queryEntries()
-                    ->where($handle, '<>', null)
-                    ->get()
-                    ->flatMap(function ($entry) use ($handle) {
-                        return collect($entry->value($handle))
-                            ->map(function ($value) use ($entry) {
-                                return [
-                                    'value' => $value,
-                                    'slug' => Str::slug($value),
-                                    'entry' => $entry->id(),
-                                    'collection' => $entry->collectionHandle(),
-                                    'site' => $entry->locale(),
-                                ];
-                            });
+                $store = Stache::store('entries')->store($collection->handle());
+                $sites = $store->index('site');
+
+                return $store->index($handle)->items()
+                    ->reject(fn ($values) => $values == null)
+                    ->flatMap(function ($values, $id) use ($collection, $sites) {
+                        return collect($values)->map(fn ($value) => [
+                            'value' => $value,
+                            'slug' => Str::slug($value),
+                            'entry' => (string) $id,
+                            'collection' => $collection->handle(),
+                            'site' => $sites->get($id),
+                        ]);
                     })->all();
             })->all();
     }

src/Stache/Stache.php

@@ -134,11 +134,13 @@ public function warm()
 
         $stores = $this->stores()->except($this->exclude);
 
-        if ($this->shouldUseParallelWarming($stores)) {
-            $this->warmInParallel($stores);
-        } else {
-            $stores->each->warm();
-        }
+        Store::duringWarm(function () use ($stores) {
+            if ($this->shouldUseParallelWarming($stores)) {
+                $this->warmInParallel($stores);
+            } else {
+                $stores->each->warm();
+            }
+        });
 
         $this->stopTimer();
 
@@ -297,7 +299,7 @@ protected function warmInParallel($stores)
 
             $closures = $chunks->map(function ($chunk) {
                 return function () use ($chunk) {
-                    return $chunk->each->warm()->keys()->all();
+                    return Store::duringWarm(fn () => $chunk->each->warm()->keys()->all());
                 };
             })->all();
 

src/Stache/Stores/CollectionEntriesStore.php

@@ -163,6 +163,37 @@ public function makeItemFromFile($path, $contents)
         return $entry;
     }
 
+    protected function canIndexWithoutOtherItems($item): bool
+    {
+        // A localization falls back to its origin's values, and an entry
+        // in a structure gets its uri, parent and order from the tree.
+        // Asked of the entry's own property, because hasOrigin() goes looking for the origin.
+        $hasOrigin = (bool) Closure::bind(fn () => $this->origin, $item, EntryInstance::class)();
+
+        return ! $hasOrigin && ! $this->findCollection($item->collectionHandle())?->hasStructure();
+    }
+
+    protected function indexNeedsOtherIndexes($name): bool
+    {
+        // Where an entry sits in a tree is found by querying the collection's
+        // entries, which needs the rest of this store's indexes to be there.
+        return in_array($name, ['uri', 'parent', 'order']) && $this->collection()?->hasStructure();
+    }
+
+    protected function releaseItem($key, $item): void
+    {
+        // A structure's uri, parent and order indexes are still to be built from these.
+        if ($this->collection()?->hasStructure()) {
+            return;
+        }
+
+        // Entries are kept so that uris can be built and indexes don't have to read them back
+        // from the cache. One that has been indexed is needed for neither, and one that is
+        // waiting to be indexed is read back then, rather than all of them being held.
+        Blink::store('structure-entries')->forget($key);
+        $this->snapshots()->forget($key);
+    }
+
     protected function extractAttributesFromPath($path)
     {
         $site = Site::default()->handle();