From 46d29770e2872c0c93c3ebd6da551852f3304787 Mon Sep 17 00:00:00 2001 From: Ajay D'Souza Date: Wed, 21 Oct 2026 01:39:09 +0000 Subject: [PATCH] Keep the newest unread articles on the device for offline reading The service worker now syncs offline copies whenever a logged-in page opens online. It reads /offline/articles (the newest unread articles with a readable copy, 30 by default, set under Settings), fetches each reader page that is new or has changed along with its stored images and the built CSS and JS, and deletes anything no longer listed, so archiving or deleting an article removes its copy at the next sync. Offline, a kept article opens as normal with a notice that it is the saved copy, and any other page falls back to an offline page listing what is available. Reading progress made offline is held in localStorage and sent when the connection returns. Pages are still fetched from the network first, so online behaviour is unchanged. Nothing stays on the device after logging out: the logout form clears the copies before it submits, the response sends Clear-Site-Data, and a sync that finds the session gone deletes them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014DPvSBmPyHN2QVmXsQQVKe --- README.md | 21 ++ .../Auth/AuthenticatedSessionController.php | 5 +- app/Http/Controllers/OfflineController.php | 71 +++++ app/Services/Settings.php | 6 + config/docket.php | 9 + public/offline.html | 18 +- public/offline.js | 46 +++ public/sw.js | 261 +++++++++++++++++- resources/js/app.js | 2 + resources/js/offline.js | 142 ++++++++++ resources/js/reader.js | 41 ++- resources/views/auth/verify-email.blade.php | 2 +- resources/views/items/show.blade.php | 4 + resources/views/layouts/app.blade.php | 4 +- resources/views/partials/pwa.blade.php | 4 + routes/web.php | 4 + tests/Feature/OfflineReadingTest.php | 124 +++++++++ tests/Feature/SettingsTest.php | 1 + 18 files changed, 733 insertions(+), 32 deletions(-) create mode 100644 app/Http/Controllers/OfflineController.php create mode 100644 public/offline.js create mode 100644 resources/js/offline.js create mode 100644 tests/Feature/OfflineReadingTest.php diff --git a/README.md b/README.md index 43277ab..51123d2 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,9 @@ Laravel 13 · PHP 8.3 · SQLite · Breeze (Blade) · Tailwind · Alpine. - **Reader**: serif or sans, adjustable size, light, sepia or dark; remembered per browser. Your scroll position is saved as you read and restored when you come back, on any device. +- **Offline reading**: the installed app keeps the newest unread articles + (30 by default), with their stored images, on each device, and refreshes + them whenever it is opened online. - **Pocket import**: upload Pocket's export CSV, or use `php artisan docket:import-pocket`. Dates, tags and unread/archive status are kept; extraction is queued at a gentle rate. Re-running it is safe. @@ -93,6 +96,24 @@ for a new item and `200` when the link was already saved, with the item requests a minute. Tokens are Laravel Sanctum personal access tokens; only a hash is stored, and revoking one on the Settings page stops it at once. +## Reading offline + +Install Docket on the device (**Add to Home screen** on Android or iOS, or +the install button in a desktop browser) and open it while online. Every +time a logged-in page opens with a connection, the service worker updates +its copies of the newest unread articles (**Settings → Articles kept for +offline reading**, 30 by default; 0 keeps none) and their stored images. + +Offline, those articles open as normal from the list on the offline page or +from any link to them. Your place in an article is remembered and sent when +the connection comes back. Archiving, favourites, tags and search need a +connection. Images that were never downloaded to the server (past the +per-article limits, or SVG) stay remote and don't show offline. + +An article leaves the device at the next sync once it is archived or +deleted, or drops out of the newest unread. Logging out deletes every +offline copy in that browser. + ## Coming from Pocket Pocket's export is a ZIP with one or more CSV files (`part_000000.csv`, diff --git a/app/Http/Controllers/Auth/AuthenticatedSessionController.php b/app/Http/Controllers/Auth/AuthenticatedSessionController.php index 23c9ad6..fd87b20 100644 --- a/app/Http/Controllers/Auth/AuthenticatedSessionController.php +++ b/app/Http/Controllers/Auth/AuthenticatedSessionController.php @@ -42,6 +42,9 @@ public function destroy(Request $request): RedirectResponse $request->session()->regenerateToken(); - return redirect('/'); + // Drop the offline copies of articles (and the service worker + // holding them) from this browser. The logout form also clears + // them before submitting, for browsers that ignore this header. + return redirect('/')->header('Clear-Site-Data', '"cache", "storage"'); } } diff --git a/app/Http/Controllers/OfflineController.php b/app/Http/Controllers/OfflineController.php new file mode 100644 index 0000000..485f8d8 --- /dev/null +++ b/app/Http/Controllers/OfflineController.php @@ -0,0 +1,71 @@ +withoutBodies() + ->addSelect('items.content_html') + ->unread() + ->whereNotNull('items.content_html') + ->where('items.content_html', '!=', '') + ->sorted('newest') + ->limit($limit) + ->get(); + + return response()->json( + [ + 'items' => $items->map(fn (Item $item): array => [ + 'url' => route('items.show', $item, false), + 'version' => (string) $item->updated_at?->getTimestamp(), + 'title' => $item->displayTitle(), + 'site' => $item->site_name ?: $item->host(), + 'minutes' => (int) $item->reading_time, + 'images' => $this->images($item), + ])->values()->all(), + ], + 200, + ['Cache-Control' => 'no-store'] + ); + } + + /** + * The stored images the article's HTML points at. Remote images are + * left out: they come from other sites and can't be kept reliably. + * + * @return list + */ + protected function images(Item $item): array + { + // The route's own prefix, so this follows any change to the URL. + $prefix = substr(route('items.image', ['item' => $item, 'filename' => 'x'], false), 0, -1); + + preg_match_all( + '#src="('.preg_quote($prefix, '#').ImageLocaliser::FILENAME_PATTERN.')"#', + (string) $item->content_html, + $matches + ); + + return array_values(array_unique($matches[1])); + } +} diff --git a/app/Services/Settings.php b/app/Services/Settings.php index 40bc8f9..06ae405 100644 --- a/app/Services/Settings.php +++ b/app/Services/Settings.php @@ -80,6 +80,12 @@ public static function fields(): array 'type' => 'int', 'rules' => ['required', 'integer', 'min:0', 'max:500'], ], + 'offline.articles' => [ + 'label' => 'Articles kept for offline reading', + 'help' => 'The installed app keeps this many of the newest unread articles on each device, refreshed whenever it is opened online. Set to 0 to keep none.', + 'type' => 'int', + 'rules' => ['required', 'integer', 'min:0', 'max:200'], + ], 'import.spacing_seconds' => [ 'label' => 'Import spacing (seconds)', 'help' => 'Gap between extraction jobs queued by a Pocket import, so a large import fetches gently.', diff --git a/config/docket.php b/config/docket.php index a2cb84a..490b8b3 100644 --- a/config/docket.php +++ b/config/docket.php @@ -83,6 +83,15 @@ 'time_budget' => 120, ], + /* + | Offline reading: the installed app keeps this many of the newest unread + | articles, with their stored images, on the device, and refreshes them + | whenever it is opened online. 0 turns it off and clears the copies. + */ + 'offline' => [ + 'articles' => 30, + ], + /* | Reading time is word count divided by this, rounded up. */ diff --git a/public/offline.html b/public/offline.html index 5e84ee0..640a903 100644 --- a/public/offline.html +++ b/public/offline.html @@ -5,15 +5,25 @@ Docket — offline

You’re offline

-

Docket needs a connection. Anything you were sharing hasn’t been saved yet — share it again when you’re back online.

+

Docket needs a connection for everything except the articles saved on this device. Anything you were sharing hasn’t been saved yet — share it again when you’re back online.

+
    +

    + diff --git a/public/offline.js b/public/offline.js new file mode 100644 index 0000000..714dff4 --- /dev/null +++ b/public/offline.js @@ -0,0 +1,46 @@ +// Lists the articles the service worker (sw.js) has saved on this device, +// on the page shown when Docket is opened without a connection. A plain +// file rather than an inline script, so it runs under any +// Content-Security-Policy. +(function () { + var list = document.getElementById('articles'); + var summary = document.getElementById('summary'); + var synced = document.getElementById('synced'); + + if (!('caches' in window)) { + return; + } + + caches.open('docket-articles') + .then(function (cache) { return cache.match('/__docket/offline-index'); }) + .then(function (response) { return response ? response.json() : { items: [] }; }) + .then(function (index) { + if (!index.items.length) { + return; + } + + summary.textContent = 'These unread articles are saved on this device. Everything else in Docket needs a connection.'; + + index.items.forEach(function (item) { + var link = document.createElement('a'); + var title = document.createElement('strong'); + var meta = document.createElement('span'); + + link.href = item.url; + title.textContent = item.title; + meta.textContent = item.site + (item.minutes > 0 ? ' · ' + item.minutes + ' min read' : ''); + + link.appendChild(title); + link.appendChild(meta); + + var row = document.createElement('li'); + row.appendChild(link); + list.appendChild(row); + }); + + if (index.syncedAt) { + synced.textContent = 'Last updated ' + new Date(index.syncedAt).toLocaleString('en-GB', { dateStyle: 'medium', timeStyle: 'short' }) + '.'; + } + }) + .catch(function () {}); +})(); diff --git a/public/sw.js b/public/sw.js index e60710a..b4d14ce 100644 --- a/public/sw.js +++ b/public/sw.js @@ -1,12 +1,38 @@ -// Minimal service worker: enough for installability. Everything behind -// auth is useless offline, so requests just pass through to the network, -// with a tiny cached fallback page for offline navigations. -const CACHE = 'docket-v1'; +// Service worker: installability, plus offline copies of the newest unread +// articles. +// +// Whenever an app page is opened online it asks this worker to sync. A sync +// reads /offline/articles (the list the server wants kept), fetches any +// reader page that is new or has changed, with its stored images and the +// built CSS and JS it uses, and deletes everything that is no longer on the +// list. Offline, a reader page in the list is served from that copy and any +// other page falls back to /offline.html, which lists what is available. +// +// Pages are always fetched from the network first, so online behaviour is +// unchanged. Nothing is kept while logged out: a sync that finds the session +// gone deletes the copies, and logging out clears them as well. +const SHELL = 'docket-shell-v2'; +const ARTICLES = 'docket-articles'; +const ASSETS = 'docket-assets'; +const KEEP = [SHELL, ARTICLES, ASSETS]; + const OFFLINE_URL = '/offline.html'; +const SHELL_FILES = [OFFLINE_URL, '/offline.js']; +const LIST_URL = '/offline/articles'; + +// Where the sync records what it kept, for /offline.html and the next sync. +// Never requested from the server. +const INDEX_KEY = '/__docket/offline-index'; + +const IMAGE = /^\/items\/\d+\/images\/[a-f0-9]{40}\.(?:jpg|png|gif|webp|avif)$/; +const ASSET = /\/build\/assets\/[^"'\s)]+/g; + +// A sync is skipped if one finished this recently. +const MIN_INTERVAL = 30 * 1000; self.addEventListener('install', (event) => { event.waitUntil( - caches.open(CACHE).then((cache) => cache.add(OFFLINE_URL)) + caches.open(SHELL).then((cache) => cache.addAll(SHELL_FILES)) ); self.skipWaiting(); }); @@ -14,16 +40,231 @@ self.addEventListener('install', (event) => { self.addEventListener('activate', (event) => { event.waitUntil( caches.keys().then((keys) => - Promise.all(keys.filter((k) => k !== CACHE).map((k) => caches.delete(k))) + Promise.all(keys.filter((k) => !KEEP.includes(k)).map((k) => caches.delete(k))) ) ); self.clients.claim(); }); self.addEventListener('fetch', (event) => { - if (event.request.mode === 'navigate') { - event.respondWith( - fetch(event.request).catch(() => caches.match(OFFLINE_URL)) - ); + const request = event.request; + const url = new URL(request.url); + + if (request.method !== 'GET' || url.origin !== self.location.origin) { + return; + } + + if (request.mode === 'navigate') { + event.respondWith(navigate(request, url)); + } else if (SHELL_FILES.includes(url.pathname)) { + event.respondWith(fetch(request).catch(() => caches.match(request))); + } else if (url.pathname.startsWith('/build/')) { + event.respondWith(cacheFirst(ASSETS, request)); + } else if (IMAGE.test(url.pathname)) { + event.respondWith(cached(ARTICLES, url.pathname).then((hit) => hit || fetch(request))); } }); + +self.addEventListener('message', (event) => { + const type = event.data && event.data.type; + + if (type === 'sync') { + event.waitUntil(sync()); + } else if (type === 'clear') { + event.waitUntil(clear()); + } +}); + +async function navigate(request, url) { + try { + return await fetch(request); + } catch (error) { + return (await cached(ARTICLES, url.pathname)) + || (await caches.match(OFFLINE_URL)) + || Response.error(); + } +} + +async function cacheFirst(name, request) { + const hit = await caches.match(request); + + if (hit) { + return hit; + } + + const response = await fetch(request); + + if (response.ok) { + const cache = await caches.open(name); + await cache.put(request, response.clone()); + } + + return response; +} + +async function cached(name, path) { + const cache = await caches.open(name); + + return cache.match(path); +} + +// A same-origin GET that only counts if it really came back: a redirect +// here means the session has gone and the login page answered instead. +async function get(path) { + try { + const response = await fetch(path, { credentials: 'same-origin', cache: 'no-store' }); + + return response.ok && !response.redirected ? response : null; + } catch (error) { + return null; + } +} + +async function clear() { + await caches.delete(ARTICLES); + await caches.delete(ASSETS); +} + +let running = null; +let finishedAt = 0; + +function sync() { + if (!running && Date.now() - finishedAt >= MIN_INTERVAL) { + running = runSync().finally(() => { + running = null; + finishedAt = Date.now(); + }); + } + + return running || Promise.resolve(); +} + +async function runSync() { + let response; + + try { + response = await fetch(LIST_URL, { + credentials: 'same-origin', + cache: 'no-store', + headers: { Accept: 'application/json' }, + }); + } catch (error) { + // Offline or the server is down: keep what is already here. + return; + } + + if (response.status === 401 || response.status === 419 || response.redirected) { + await clear(); + + return; + } + + if (!response.ok) { + return; + } + + const { items } = await response.json(); + const cache = await caches.open(ARTICLES); + const previous = await readIndex(cache); + const kept = []; + + for (const item of items) { + const before = previous.get(item.url); + let stored = Boolean(before && await cache.match(item.url)); + let version = stored ? before.version : null; + let assets = stored ? before.assets : []; + + if (version !== item.version) { + const page = await get(item.url); + + if (page) { + assets = [...new Set((await page.clone().text()).match(ASSET) || [])]; + await cache.put(item.url, page); + stored = true; + version = item.version; + } + } + + // A page that failed to refresh keeps its older copy (and older + // version, so the next sync tries again), if it has one. + if (!stored) { + continue; + } + + for (const src of item.images) { + if (!(await cache.match(src))) { + const image = await get(src); + + if (image) { + await cache.put(src, image); + } + } + } + + kept.push({ ...item, version, assets }); + } + + await storeAssets(kept); + await prune(cache, kept); + + await cache.put(INDEX_KEY, new Response( + JSON.stringify({ syncedAt: new Date().toISOString(), items: kept }), + { headers: { 'Content-Type': 'application/json' } } + )); +} + +async function readIndex(cache) { + const map = new Map(); + + try { + const response = await cache.match(INDEX_KEY); + const index = response ? await response.json() : { items: [] }; + + for (const item of index.items) { + map.set(item.url, item); + } + } catch (error) { + // A damaged index only means everything is fetched again. + } + + return map; +} + +// The built CSS and JS the kept pages load. A deploy changes their hashed +// names, so the copies an older page needs are kept until that page is +// refreshed, and anything no kept page uses is dropped. +async function storeAssets(kept) { + const wanted = new Set(kept.flatMap((item) => item.assets)); + const cache = await caches.open(ASSETS); + + for (const path of wanted) { + if (!(await cache.match(path))) { + const asset = await get(path); + + if (asset) { + await cache.put(path, asset); + } + } + } + + for (const request of await cache.keys()) { + if (!wanted.has(new URL(request.url).pathname)) { + await cache.delete(request); + } + } +} + +async function prune(cache, kept) { + const wanted = new Set([INDEX_KEY]); + + for (const item of kept) { + wanted.add(item.url); + item.images.forEach((src) => wanted.add(src)); + } + + for (const request of await cache.keys()) { + if (!wanted.has(new URL(request.url).pathname)) { + await cache.delete(request); + } + } +} diff --git a/resources/js/app.js b/resources/js/app.js index 6a7176f..91ed592 100644 --- a/resources/js/app.js +++ b/resources/js/app.js @@ -5,6 +5,7 @@ import Alpine from '@alpinejs/csp'; import behaviours from './behaviours'; import bulkSelection from './bulk-selection'; +import offline from './offline'; import reader from './reader'; window.Alpine = Alpine; @@ -13,5 +14,6 @@ Alpine.data('bulkSelection', bulkSelection); Alpine.data('reader', reader); behaviours(document); +offline(document); Alpine.start(); diff --git a/resources/js/offline.js b/resources/js/offline.js new file mode 100644 index 0000000..23b3bee --- /dev/null +++ b/resources/js/offline.js @@ -0,0 +1,142 @@ +/** + * The page side of offline reading (the work is done in public/sw.js). + * + * - Asks the service worker to sync the offline copies whenever a logged-in + * page opens online, or the connection comes back. Logged-in pages carry + * ; the login page doesn't. + * - Keeps reading progress made offline in localStorage and sends it once + * there is a connection again. + * - Clears the offline copies before a form marked data-clears-offline + * (log out) submits. + */ +export const ARTICLES_CACHE = 'docket-articles'; +export const PENDING_KEY = 'docket.pendingProgress'; + +function loggedIn() { + return document.querySelector('meta[name="docket-offline"]') !== null; +} + +export function requestSync() { + if (!loggedIn() || !navigator.onLine || !('serviceWorker' in navigator)) { + return; + } + + navigator.serviceWorker.ready + .then((registration) => registration.active?.postMessage({ type: 'sync' })) + .catch(() => {}); +} + +function readPending() { + try { + return JSON.parse(window.localStorage.getItem(PENDING_KEY) || '{}'); + } catch { + return {}; + } +} + +function writePending(pending) { + try { + if (Object.keys(pending).length === 0) { + window.localStorage.removeItem(PENDING_KEY); + } else { + window.localStorage.setItem(PENDING_KEY, JSON.stringify(pending)); + } + } catch { + // Storage unavailable: offline progress is lost, nothing else. + } +} + +/** + * Remember progress that couldn't be sent, keyed by its progress URL. + */ +export function queueProgress(url, progress) { + writePending({ ...readPending(), [url]: progress }); +} + +/** + * Progress made offline for this article and not yet sent, if any. It is + * newer than the position in an offline copy of the page. + */ +export function pendingProgress(url) { + const value = readPending()[url]; + + return typeof value === 'number' ? value : null; +} + +/** + * Drop queued progress for one article once newer progress has been sent. + */ +export function forgetProgress(url) { + const pending = readPending(); + + if (url in pending) { + delete pending[url]; + writePending(pending); + } +} + +export function postProgress(url, progress) { + const token = document.querySelector('meta[name="csrf-token"]')?.content ?? ''; + const body = new FormData(); + body.append('_token', token); + body.append('progress', String(progress)); + + return fetch(url, { + method: 'POST', + body, + credentials: 'same-origin', + headers: { Accept: 'application/json', 'X-Requested-With': 'XMLHttpRequest' }, + }); +} + +/** + * Send any progress queued while offline. Entries the server rejects + * outright (the article was deleted) are dropped; a network failure keeps + * them for next time. + */ +export function flushProgress() { + if (!loggedIn() || !navigator.onLine) { + return; + } + + Object.entries(readPending()).forEach(([url, progress]) => { + postProgress(url, progress) + .then((response) => { + if (response.ok || (response.status >= 400 && response.status < 500 && response.status !== 419)) { + const pending = readPending(); + + if (pending[url] === progress) { + delete pending[url]; + writePending(pending); + } + } + }) + .catch(() => {}); + }); +} + +export default function offline(root) { + root.addEventListener('submit', (event) => { + const form = event.target; + + if (!(form instanceof HTMLFormElement) || !form.hasAttribute('data-clears-offline') || !('caches' in window)) { + return; + } + + event.preventDefault(); + + navigator.serviceWorker?.controller?.postMessage({ type: 'clear' }); + + Promise.all([window.caches.delete(ARTICLES_CACHE), window.caches.delete('docket-assets')]) + .catch(() => {}) + .finally(() => form.submit()); + }); + + window.addEventListener('online', () => { + requestSync(); + flushProgress(); + }); + + requestSync(); + flushProgress(); +} diff --git a/resources/js/reader.js b/resources/js/reader.js index d550886..19bd81d 100644 --- a/resources/js/reader.js +++ b/resources/js/reader.js @@ -3,6 +3,8 @@ * are per browser) and reading progress (sent to the server, so it * follows you between devices). */ +import { forgetProgress, pendingProgress, postProgress, queueProgress } from './offline'; + export const STORAGE_KEY = 'docket.reader'; export const DEFAULTS = { theme: 'light', family: 'serif', size: 20 }; export const SIZES = { min: 14, max: 32, step: 2 }; @@ -37,6 +39,7 @@ export function applyPreferences(prefs, root = document.documentElement) { export default () => ({ prefs: loadPreferences(), panel: false, + offline: !navigator.onLine, progressUrl: '', savedProgress: 0, lastSent: null, @@ -44,7 +47,9 @@ export default () => ({ init() { this.progressUrl = this.$el.dataset.progressUrl || ''; - this.savedProgress = Number(this.$el.dataset.progress) || 0; + // Progress made offline beats the position baked into the page, + // which may be an older offline copy. + this.savedProgress = pendingProgress(this.progressUrl) ?? (Number(this.$el.dataset.progress) || 0); applyPreferences(this.prefs); @@ -71,6 +76,8 @@ export default () => ({ this.lastSent = this.savedProgress; + window.addEventListener('online', () => { this.offline = false; }); + window.addEventListener('offline', () => { this.offline = true; }); window.addEventListener('scroll', () => this.schedule(), { passive: true }); document.addEventListener('visibilitychange', () => { if (document.visibilityState === 'hidden') { @@ -111,24 +118,30 @@ export default () => ({ this.lastSent = progress; - const token = document.querySelector('meta[name="csrf-token"]')?.content ?? ''; - const body = new FormData(); - body.append('_token', token); - body.append('progress', String(progress)); + // Offline, keep it for offline.js to send when the connection is + // back (sendBeacon would accept it and then lose it). + if (!navigator.onLine) { + queueProgress(this.progressUrl, progress); + + return; + } // sendBeacon survives the page being closed; fetch is used - // otherwise so failures don't go unnoticed in the console. + // otherwise so a failure can be queued for later. if (leaving && navigator.sendBeacon) { - navigator.sendBeacon(this.progressUrl, body); + const body = new FormData(); + body.append('_token', document.querySelector('meta[name="csrf-token"]')?.content ?? ''); + body.append('progress', String(progress)); - return; + if (navigator.sendBeacon(this.progressUrl, body)) { + forgetProgress(this.progressUrl); + + return; + } } - fetch(this.progressUrl, { - method: 'POST', - body, - credentials: 'same-origin', - headers: { Accept: 'application/json', 'X-Requested-With': 'XMLHttpRequest' }, - }).catch(() => {}); + postProgress(this.progressUrl, progress) + .then(() => forgetProgress(this.progressUrl)) + .catch(() => queueProgress(this.progressUrl, progress)); }, }); diff --git a/resources/views/auth/verify-email.blade.php b/resources/views/auth/verify-email.blade.php index 3843739..960e877 100644 --- a/resources/views/auth/verify-email.blade.php +++ b/resources/views/auth/verify-email.blade.php @@ -14,7 +14,7 @@ -
    + @csrf
    @@ -124,7 +124,7 @@ class="fixed inset-x-0 top-14 z-30 border-b border-slate-800 bg-slate-900 px-3 p
    {{ Auth::user()->name }} -
    + @csrf
    diff --git a/resources/views/partials/pwa.blade.php b/resources/views/partials/pwa.blade.php index 8b9e4ef..2b31ca8 100644 --- a/resources/views/partials/pwa.blade.php +++ b/resources/views/partials/pwa.blade.php @@ -4,6 +4,10 @@ +@auth + {{-- Logged in: resources/js/offline.js keeps the offline copies in step. --}} + +@endauth