diff --git a/CHANGELOG.md b/CHANGELOG.md index 98d6b9b..e0d4fcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,27 @@ All notable changes to this project will be documented in this file. +## [2.0.1] - 2026-09-22 + +### Added + +- `CacheTags::withoutRecording()`, for navigation and other shared-layout + furniture a site assembles in its own PHP. Statamic's `{{ nav }}` tag was + already recorded as the navigation rather than as every entry in it; a menu + built by hand had no equivalent, so each of its entries became a dependency of + every page carrying it. +- `cache-invalidation:prune`, which drops graph rows for URLs that are no longer + in the static cache. Nothing removed them before — rows are only ever replaced + by rendering the same URL again — so the graph grew without bound. + +### Fixed + +- The sqlite graph now reclaims its file. `DELETE` moves pages onto sqlite's + freelist without shortening the file, and under WAL a `VACUUM` alone is not + enough either, so a graph that had been large once stayed large on disk: one + site was carrying a 251 MB file holding 18 MB of rows. Flushing and pruning + now `VACUUM` and checkpoint. + ## [2.0.0] - 2026-09-15 Invalidation is now derived from what pages actually read, instead of from rules diff --git a/README.md b/README.md index dc95cd0..099d6eb 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,10 @@ php artisan cache-invalidation:stats # preview with that, clear with this. php artisan cache-invalidation:clear api:reviews +# Drop graph rows for URLs that are no longer cached, and shrink the +# sqlite file to fit. Safe to schedule; --dry-run reports first. +php artisan cache-invalidation:prune + # Deploy check — exits non-zero when invalidation cannot work. php artisan cache-invalidation:doctor @@ -168,6 +172,48 @@ php artisan cache-invalidation:selftest `CACHE_INVALIDATION_DEBUG=true` adds an `X-Cache-Tags` header as pages are cached, so you can read a page's dependencies in devtools. +## Navigation you build yourself + +Statamic's `{{ nav }}` tag is handled already: the addon records the navigation +itself rather than every entry in it, so renaming a menu item does not clear the +whole site. + +A menu assembled in your own PHP has no such treatment, and every entry it reads +becomes a dependency of every page carrying it. Wrap it: + +```php +use RoxDigital\CacheInvalidation\Facades\CacheTags; + +CacheTags::add('nav:main'); + +$menu = CacheTags::withoutRecording(fn () => $this->buildMenu()); +``` + +Saving the navigation now clears the pages that render it; saving one page it +links to does not. The same applies to any furniture in a shared layout built +from content — a footer, a "latest posts" strip. + +The trade-off is deliberate: that menu item keeps its old title on already-cached +pages until something clears them. `cache-invalidation:why` on any page shows +whether the treatment took, since the per-entry tags disappear. + +## Keeping the graph bounded + +The graph only learns about a URL by rendering it, so nothing removes a URL that +quietly falls out of the static cache. Over months the graph comes to describe +mostly pages that are no longer cached. + +`cache-invalidation:stats` reports the gap. When it grows, prune: + +```php +// routes/console.php +Schedule::command('cache-invalidation:prune')->weekly(); +``` + +On sqlite this also reclaims the file. Deleting rows alone moves pages onto +sqlite's freelist and never shortens the file, so a graph that has been large +once stays large on disk; pruning and flushing both `VACUUM` afterwards. + ## Configuration Nothing needs setting. Publish it only to change a default: diff --git a/src/CacheTags.php b/src/CacheTags.php index bac2fa6..914e048 100644 --- a/src/CacheTags.php +++ b/src/CacheTags.php @@ -42,6 +42,31 @@ public function add(string ...$tags): void $this->recorder->add(...$tags); } + /** + * Run a callback without recording anything it reads. + * + * For page furniture that is built out of content but is not the page's + * content: a navigation, a footer, a "latest posts" strip in the shared + * layout. Left alone, every entry those touch becomes a dependency of every + * page that carries them, so renaming one menu item clears the whole site. + * + * Pair it with add() to leave a single honest dependency in its place: + * + * CacheTags::add('nav:main'); + * + * $menu = CacheTags::withoutRecording(fn () => $this->buildMenu()); + * + * Saving the navigation then clears the pages that render it, while saving + * one page it links to does not. The trade-off is deliberate: that menu item + * shows its old title on already-cached pages until they are cleared for some + * other reason. Statamic's own {{ nav }} tag is handled this way internally; + * this is for navigation a site builds itself. + */ + public function withoutRecording(callable $callback): mixed + { + return $this->recorder->suppressed($callback); + } + /** * Clear every cached URL carrying any of the given tags. * diff --git a/src/CachedUrls.php b/src/CachedUrls.php index 3de8c8c..92a11fe 100644 --- a/src/CachedUrls.php +++ b/src/CachedUrls.php @@ -21,6 +21,18 @@ public function __construct( private readonly Cacher $cacher, ) {} + /** + * Whether the configured cacher can enumerate what it holds at all. + * + * Matters to anything that treats all() as the whole truth: a cacher that + * cannot enumerate returns an empty list, which is indistinguishable from an + * empty cache and would make a pruner delete the entire graph. + */ + public function supported(): bool + { + return $this->cacher instanceof AbstractCacher; + } + /** * @return list */ diff --git a/src/Console/PruneCommand.php b/src/Console/PruneCommand.php new file mode 100644 index 0000000..debda9c --- /dev/null +++ b/src/Console/PruneCommand.php @@ -0,0 +1,83 @@ +supported()) { + $this->components->warn( + 'The configured cacher cannot list what it holds, so there is no safe set of URLs to keep.' + ); + + return self::FAILURE; + } + + $keep = $cached->all(); + $before = $graph->stats(); + + // Worked out here rather than behind another contract method: urls() and + // prune() between them already say everything a caller needs. + $keepSet = array_flip($keep); + + $stale = count(array_filter( + $graph->urls(), + static fn (string $url): bool => ! isset($keepSet[$url]), + )); + + if ($stale === 0) { + $this->components->info( + sprintf('Nothing to prune. %d tracked URL(s), all still cached.', $before['urls']) + ); + + return self::SUCCESS; + } + + if ($this->option('dry-run')) { + $this->components->info(sprintf( + '%d of %d tracked URL(s) are no longer cached and would be pruned.', + $stale, + $before['urls'], + )); + + return self::SUCCESS; + } + + $removed = $graph->prune($keep); + $graph->compact(); + + $after = $graph->stats(); + + $this->components->info(sprintf( + 'Pruned %d URL(s) and %d row(s). Graph now holds %d URL(s) across %d row(s).', + $before['urls'] - $after['urls'], + $removed, + $after['urls'], + $after['rows'], + )); + + return self::SUCCESS; + } +} diff --git a/src/Facades/CacheTags.php b/src/Facades/CacheTags.php index e996cf8..0fd57d8 100644 --- a/src/Facades/CacheTags.php +++ b/src/Facades/CacheTags.php @@ -9,6 +9,7 @@ /** * @method static void add(string ...$tags) + * @method static mixed withoutRecording(callable $callback) * @method static int invalidate(string ...$tags) * @method static list urlsFor(string ...$tags) * diff --git a/src/Graph/ClearGraphWhenCacheCleared.php b/src/Graph/ClearGraphWhenCacheCleared.php index babe55e..a815824 100644 --- a/src/Graph/ClearGraphWhenCacheCleared.php +++ b/src/Graph/ClearGraphWhenCacheCleared.php @@ -27,5 +27,11 @@ public function __construct( public function handle(StaticCacheCleared $event): void { $this->graph->flush(); + + // The DELETE alone leaves a SQLite file the size of the graph at its + // largest, which is how a site that has been running for months ends up + // with a mostly empty multi-hundred-megabyte file. This is the one moment + // the graph is known to be empty, so compacting is at its cheapest. + $this->graph->compact(); } } diff --git a/src/Graph/DependencyGraph.php b/src/Graph/DependencyGraph.php index 1488b69..46758b5 100644 --- a/src/Graph/DependencyGraph.php +++ b/src/Graph/DependencyGraph.php @@ -54,6 +54,30 @@ public function untracked(array $urls): array; */ public function urls(): array; + /** + * Drop every URL that is not in the given set, and the rows behind it. + * + * The graph only ever learns about a URL by rendering it, so nothing removes + * a URL that quietly fell out of the static cache. Left alone the graph grows + * without bound, and `untracked()` pays for rows describing pages that are + * no longer cached. Callers pass the URLs that are currently cached; anything + * else is garbage. + * + * @param list $keepUrls + * @return int Rows removed. + */ + public function prune(array $keepUrls): int; + + /** + * Reclaim storage freed by prune() or flush(). + * + * Separate from both because it is the expensive half: SQLite's DELETE only + * moves pages onto the freelist, so a graph that has been flushed keeps its + * old size on disk for ever. A no-op for drivers that manage their own + * storage. + */ + public function compact(): void; + public function flush(): void; /** diff --git a/src/Graph/NullGraph.php b/src/Graph/NullGraph.php index fc29ce3..27616bb 100644 --- a/src/Graph/NullGraph.php +++ b/src/Graph/NullGraph.php @@ -39,6 +39,13 @@ public function urls(): array return []; } + public function prune(array $keepUrls): int + { + return 0; + } + + public function compact(): void {} + public function flush(): void {} public function stats(): array diff --git a/src/Graph/SqlGraph.php b/src/Graph/SqlGraph.php index 309b1b4..8072d50 100644 --- a/src/Graph/SqlGraph.php +++ b/src/Graph/SqlGraph.php @@ -103,6 +103,43 @@ public function urls(): array return $this->query()->distinct()->orderBy('url')->pluck('url')->all(); } + public function prune(array $keepUrls): int + { + $keep = []; + + foreach ($keepUrls as $url) { + $keep[$this->hash($url)] = true; + } + + // Worked out in PHP rather than as a NOT IN, because chunking a NOT IN + // would make each chunk delete everything the other chunks kept. + $stale = []; + + foreach ($this->query()->distinct()->pluck('url_hash') as $hash) { + if (! isset($keep[$hash])) { + $stale[] = $hash; + } + } + + if ($stale === []) { + return 0; + } + + $removed = 0; + + foreach (array_chunk($stale, self::CHUNK) as $chunk) { + $removed += $this->query()->whereIn('url_hash', $chunk)->delete(); + } + + return $removed; + } + + /** + * A server-backed database reuses freed pages itself, so there is nothing + * useful to do here. SQLite overrides this. + */ + public function compact(): void {} + public function flush(): void { $this->query()->delete(); diff --git a/src/Graph/SqliteGraph.php b/src/Graph/SqliteGraph.php index 2cb3e12..34df639 100644 --- a/src/Graph/SqliteGraph.php +++ b/src/Graph/SqliteGraph.php @@ -38,6 +38,32 @@ protected function table(): string return 'dependencies'; } + /** + * SQLite's DELETE moves pages onto the freelist and never shortens the file, + * so a graph that has been flushed or pruned keeps the size of its largest + * ever state. Left alone a busy site ends up with a file that is almost + * entirely free pages, and every query still walks it. + * + * VACUUM rewrites the file around the rows that remain. It cannot run inside + * a transaction, hence the guard. + */ + public function compact(): void + { + $connection = $this->connection(); + + if ($connection->transactionLevel() > 0) { + return; + } + + $connection->statement('VACUUM'); + + // The connection runs in WAL mode (see ServiceProvider), where VACUUM + // rewrites the database inside the write-ahead log. Without a truncating + // checkpoint the main file keeps its old length on disk, so nothing has + // actually been reclaimed. + $connection->select('PRAGMA wal_checkpoint(TRUNCATE)'); + } + /** * Runs once per process. CREATE ... IF NOT EXISTS rather than a migration so * that installing the addon requires no artisan step. diff --git a/src/ServiceProvider.php b/src/ServiceProvider.php index bcc2538..23cdd40 100644 --- a/src/ServiceProvider.php +++ b/src/ServiceProvider.php @@ -15,6 +15,7 @@ use RoxDigital\CacheInvalidation\Console\AffectedCommand; use RoxDigital\CacheInvalidation\Console\ClearCommand; use RoxDigital\CacheInvalidation\Console\DoctorCommand; +use RoxDigital\CacheInvalidation\Console\PruneCommand; use RoxDigital\CacheInvalidation\Console\SelfTestCommand; use RoxDigital\CacheInvalidation\Console\StatsCommand; use RoxDigital\CacheInvalidation\Console\WhyCommand; @@ -60,6 +61,7 @@ class ServiceProvider extends AddonServiceProvider AffectedCommand::class, ClearCommand::class, DoctorCommand::class, + PruneCommand::class, SelfTestCommand::class, StatsCommand::class, WhyCommand::class, diff --git a/tests/Graph/PrunesAndCompactsTest.php b/tests/Graph/PrunesAndCompactsTest.php new file mode 100644 index 0000000..d24bfc3 --- /dev/null +++ b/tests/Graph/PrunesAndCompactsTest.php @@ -0,0 +1,103 @@ +graph->record('https://site.test/kept', ['entry:1', 'global:footer']); + $this->graph->record('https://site.test/gone', ['entry:2', 'global:footer']); + + $removed = $this->graph->prune(['https://site.test/kept']); + + $this->assertSame(2, $removed); + $this->assertSame(['https://site.test/kept'], $this->graph->urls()); + $this->assertSame(['entry:1', 'global:footer'], $this->graph->tagsFor('https://site.test/kept')); + $this->assertSame([], $this->graph->tagsFor('https://site.test/gone')); + } + + #[Test] + public function it_keeps_everything_when_every_url_is_still_cached(): void + { + $this->graph->record('https://site.test/a', ['entry:1']); + $this->graph->record('https://site.test/b', ['entry:2']); + + $removed = $this->graph->prune(['https://site.test/a', 'https://site.test/b']); + + $this->assertSame(0, $removed); + $this->assertSame(['https://site.test/a', 'https://site.test/b'], $this->graph->urls()); + } + + #[Test] + public function pruning_against_an_empty_cache_empties_the_graph(): void + { + $this->graph->record('https://site.test/a', ['entry:1']); + + $this->assertSame(1, $this->graph->prune([])); + $this->assertSame([], $this->graph->urls()); + } + + /** + * The reason prune() resolves the stale set in PHP: chunking a NOT IN would + * make every chunk delete what the other chunks were keeping. + */ + #[Test] + public function it_prunes_correctly_across_more_urls_than_one_query_chunk(): void + { + $urls = []; + + for ($i = 0; $i < 1200; $i++) { + $urls[] = $url = "https://site.test/page-{$i}"; + $this->graph->record($url, ['entry:'.$i]); + } + + $keep = array_slice($urls, 0, 600); + + $this->graph->prune($keep); + + $this->assertSame(600, count($this->graph->urls())); + $this->assertSame(['entry:0'], $this->graph->tagsFor($urls[0])); + $this->assertSame([], $this->graph->tagsFor($urls[1199])); + } + + #[Test] + public function compacting_reclaims_the_file_and_leaves_the_rows_alone(): void + { + for ($i = 0; $i < 4000; $i++) { + $this->graph->record("https://site.test/page-{$i}", ['entry:'.$i, 'global:footer']); + } + + $this->graph->prune(['https://site.test/page-0']); + + $bloated = $this->sqliteSize(); + + $this->graph->compact(); + + $this->assertLessThan($bloated, $this->sqliteSize(), 'VACUUM should shorten the file.'); + $this->assertSame(['entry:0', 'global:footer'], $this->graph->tagsFor('https://site.test/page-0')); + } + + #[Test] + public function compacting_an_untouched_graph_is_harmless(): void + { + $this->graph->record('https://site.test/a', ['entry:1']); + + $this->graph->compact(); + + $this->assertSame(['entry:1'], $this->graph->tagsFor('https://site.test/a')); + } + + private function sqliteSize(): int + { + clearstatcache(true, $path = $this->sqlitePath()); + + return is_file($path) ? (int) filesize($path) : 0; + } +} diff --git a/tests/Recording/WithoutRecordingTest.php b/tests/Recording/WithoutRecordingTest.php new file mode 100644 index 0000000..e436a8c --- /dev/null +++ b/tests/Recording/WithoutRecordingTest.php @@ -0,0 +1,102 @@ +save(); + Nav::make('main_nav')->title('Main')->save(); + + Entry::make()->collection('pages')->id('one')->slug('one')->save(); + Entry::make()->collection('pages')->id('two')->slug('two')->save(); + } + + #[Test] + public function it_drops_everything_read_inside_the_callback(): void + { + $tags = $this->tagsRecordedDuring(function (): void { + CacheTags::withoutRecording(function (): void { + Entry::find('one'); + Entry::find('two'); + }); + }); + + $this->assertSame([], $tags); + } + + #[Test] + public function a_tag_added_outside_the_callback_survives(): void + { + $tags = $this->tagsRecordedDuring(function (): void { + CacheTags::add('nav:main_nav'); + + CacheTags::withoutRecording(fn () => Entry::find('one')); + }); + + $this->assertSame(['nav:main_nav'], $tags); + } + + #[Test] + public function recording_resumes_after_the_callback(): void + { + $tags = $this->tagsRecordedDuring(function (): void { + CacheTags::withoutRecording(fn () => Entry::find('one')); + + Entry::find('two'); + }); + + $this->assertSame(['entry:two'], $tags); + } + + #[Test] + public function it_returns_what_the_callback_returns(): void + { + $this->assertSame('menu', CacheTags::withoutRecording(fn (): string => 'menu')); + } + + #[Test] + public function nesting_does_not_resume_recording_early(): void + { + $tags = $this->tagsRecordedDuring(function (): void { + CacheTags::withoutRecording(function (): void { + CacheTags::withoutRecording(fn () => Entry::find('one')); + + // Still inside the outer suppression, so this must not record. + Entry::find('two'); + }); + }); + + $this->assertSame([], $tags); + } + + #[Test] + public function an_exception_still_restores_recording(): void + { + $tags = $this->tagsRecordedDuring(function (): void { + try { + CacheTags::withoutRecording(function (): void { + throw new \RuntimeException('menu blew up'); + }); + } catch (\RuntimeException) { + // swallowed on purpose + } + + Entry::find('two'); + }); + + $this->assertSame(['entry:two'], $tags); + } +}