diff --git a/README.md b/README.md index 43277ab..51123d2 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,9 @@ Laravel 13 · PHP 8.3 · SQLite · Breeze (Blade) · Tailwind · Alpine. - **Reader**: serif or sans, adjustable size, light, sepia or dark; remembered per browser. Your scroll position is saved as you read and restored when you come back, on any device. +- **Offline reading**: the installed app keeps the newest unread articles + (30 by default), with their stored images, on each device, and refreshes + them whenever it is opened online. - **Pocket import**: upload Pocket's export CSV, or use `php artisan docket:import-pocket`. Dates, tags and unread/archive status are kept; extraction is queued at a gentle rate. Re-running it is safe. @@ -93,6 +96,24 @@ for a new item and `200` when the link was already saved, with the item requests a minute. Tokens are Laravel Sanctum personal access tokens; only a hash is stored, and revoking one on the Settings page stops it at once. +## Reading offline + +Install Docket on the device (**Add to Home screen** on Android or iOS, or +the install button in a desktop browser) and open it while online. Every +time a logged-in page opens with a connection, the service worker updates +its copies of the newest unread articles (**Settings → Articles kept for +offline reading**, 30 by default; 0 keeps none) and their stored images. + +Offline, those articles open as normal from the list on the offline page or +from any link to them. Your place in an article is remembered and sent when +the connection comes back. Archiving, favourites, tags and search need a +connection. Images that were never downloaded to the server (past the +per-article limits, or SVG) stay remote and don't show offline. + +An article leaves the device at the next sync once it is archived or +deleted, or drops out of the newest unread. Logging out deletes every +offline copy in that browser. + ## Coming from Pocket Pocket's export is a ZIP with one or more CSV files (`part_000000.csv`, diff --git a/app/Http/Controllers/Auth/AuthenticatedSessionController.php b/app/Http/Controllers/Auth/AuthenticatedSessionController.php index 23c9ad6..fd87b20 100644 --- a/app/Http/Controllers/Auth/AuthenticatedSessionController.php +++ b/app/Http/Controllers/Auth/AuthenticatedSessionController.php @@ -42,6 +42,9 @@ public function destroy(Request $request): RedirectResponse $request->session()->regenerateToken(); - return redirect('/'); + // Drop the offline copies of articles (and the service worker + // holding them) from this browser. The logout form also clears + // them before submitting, for browsers that ignore this header. + return redirect('/')->header('Clear-Site-Data', '"cache", "storage"'); } } diff --git a/app/Http/Controllers/OfflineController.php b/app/Http/Controllers/OfflineController.php new file mode 100644 index 0000000..485f8d8 --- /dev/null +++ b/app/Http/Controllers/OfflineController.php @@ -0,0 +1,71 @@ +withoutBodies() + ->addSelect('items.content_html') + ->unread() + ->whereNotNull('items.content_html') + ->where('items.content_html', '!=', '') + ->sorted('newest') + ->limit($limit) + ->get(); + + return response()->json( + [ + 'items' => $items->map(fn (Item $item): array => [ + 'url' => route('items.show', $item, false), + 'version' => (string) $item->updated_at?->getTimestamp(), + 'title' => $item->displayTitle(), + 'site' => $item->site_name ?: $item->host(), + 'minutes' => (int) $item->reading_time, + 'images' => $this->images($item), + ])->values()->all(), + ], + 200, + ['Cache-Control' => 'no-store'] + ); + } + + /** + * The stored images the article's HTML points at. Remote images are + * left out: they come from other sites and can't be kept reliably. + * + * @return list + */ + protected function images(Item $item): array + { + // The route's own prefix, so this follows any change to the URL. + $prefix = substr(route('items.image', ['item' => $item, 'filename' => 'x'], false), 0, -1); + + preg_match_all( + '#src="('.preg_quote($prefix, '#').ImageLocaliser::FILENAME_PATTERN.')"#', + (string) $item->content_html, + $matches + ); + + return array_values(array_unique($matches[1])); + } +} diff --git a/app/Services/Settings.php b/app/Services/Settings.php index 40bc8f9..06ae405 100644 --- a/app/Services/Settings.php +++ b/app/Services/Settings.php @@ -80,6 +80,12 @@ public static function fields(): array 'type' => 'int', 'rules' => ['required', 'integer', 'min:0', 'max:500'], ], + 'offline.articles' => [ + 'label' => 'Articles kept for offline reading', + 'help' => 'The installed app keeps this many of the newest unread articles on each device, refreshed whenever it is opened online. Set to 0 to keep none.', + 'type' => 'int', + 'rules' => ['required', 'integer', 'min:0', 'max:200'], + ], 'import.spacing_seconds' => [ 'label' => 'Import spacing (seconds)', 'help' => 'Gap between extraction jobs queued by a Pocket import, so a large import fetches gently.', diff --git a/config/docket.php b/config/docket.php index a2cb84a..490b8b3 100644 --- a/config/docket.php +++ b/config/docket.php @@ -83,6 +83,15 @@ 'time_budget' => 120, ], + /* + | Offline reading: the installed app keeps this many of the newest unread + | articles, with their stored images, on the device, and refreshes them + | whenever it is opened online. 0 turns it off and clears the copies. + */ + 'offline' => [ + 'articles' => 30, + ], + /* | Reading time is word count divided by this, rounded up. */ diff --git a/public/offline.html b/public/offline.html index 5e84ee0..640a903 100644 --- a/public/offline.html +++ b/public/offline.html @@ -5,15 +5,25 @@ Docket — offline

You’re offline

-

Docket needs a connection. Anything you were sharing hasn’t been saved yet — share it again when you’re back online.

+

Docket needs a connection for everything except the articles saved on this device. Anything you were sharing hasn’t been saved yet — share it again when you’re back online.

+ +

+ diff --git a/public/offline.js b/public/offline.js new file mode 100644 index 0000000..714dff4 --- /dev/null +++ b/public/offline.js @@ -0,0 +1,46 @@ +// Lists the articles the service worker (sw.js) has saved on this device, +// on the page shown when Docket is opened without a connection. A plain +// file rather than an inline script, so it runs under any +// Content-Security-Policy. +(function () { + var list = document.getElementById('articles'); + var summary = document.getElementById('summary'); + var synced = document.getElementById('synced'); + + if (!('caches' in window)) { + return; + } + + caches.open('docket-articles') + .then(function (cache) { return cache.match('/__docket/offline-index'); }) + .then(function (response) { return response ? response.json() : { items: [] }; }) + .then(function (index) { + if (!index.items.length) { + return; + } + + summary.textContent = 'These unread articles are saved on this device. Everything else in Docket needs a connection.'; + + index.items.forEach(function (item) { + var link = document.createElement('a'); + var title = document.createElement('strong'); + var meta = document.createElement('span'); + + link.href = item.url; + title.textContent = item.title; + meta.textContent = item.site + (item.minutes > 0 ? ' · ' + item.minutes + ' min read' : ''); + + link.appendChild(title); + link.appendChild(meta); + + var row = document.createElement('li'); + row.appendChild(link); + list.appendChild(row); + }); + + if (index.syncedAt) { + synced.textContent = 'Last updated ' + new Date(index.syncedAt).toLocaleString('en-GB', { dateStyle: 'medium', timeStyle: 'short' }) + '.'; + } + }) + .catch(function () {}); +})(); diff --git a/public/sw.js b/public/sw.js index e60710a..b4d14ce 100644 --- a/public/sw.js +++ b/public/sw.js @@ -1,12 +1,38 @@ -// Minimal service worker: enough for installability. Everything behind -// auth is useless offline, so requests just pass through to the network, -// with a tiny cached fallback page for offline navigations. -const CACHE = 'docket-v1'; +// Service worker: installability, plus offline copies of the newest unread +// articles. +// +// Whenever an app page is opened online it asks this worker to sync. A sync +// reads /offline/articles (the list the server wants kept), fetches any +// reader page that is new or has changed, with its stored images and the +// built CSS and JS it uses, and deletes everything that is no longer on the +// list. Offline, a reader page in the list is served from that copy and any +// other page falls back to /offline.html, which lists what is available. +// +// Pages are always fetched from the network first, so online behaviour is +// unchanged. Nothing is kept while logged out: a sync that finds the session +// gone deletes the copies, and logging out clears them as well. +const SHELL = 'docket-shell-v2'; +const ARTICLES = 'docket-articles'; +const ASSETS = 'docket-assets'; +const KEEP = [SHELL, ARTICLES, ASSETS]; + const OFFLINE_URL = '/offline.html'; +const SHELL_FILES = [OFFLINE_URL, '/offline.js']; +const LIST_URL = '/offline/articles'; + +// Where the sync records what it kept, for /offline.html and the next sync. +// Never requested from the server. +const INDEX_KEY = '/__docket/offline-index'; + +const IMAGE = /^\/items\/\d+\/images\/[a-f0-9]{40}\.(?:jpg|png|gif|webp|avif)$/; +const ASSET = /\/build\/assets\/[^"'\s)]+/g; + +// A sync is skipped if one finished this recently. +const MIN_INTERVAL = 30 * 1000; self.addEventListener('install', (event) => { event.waitUntil( - caches.open(CACHE).then((cache) => cache.add(OFFLINE_URL)) + caches.open(SHELL).then((cache) => cache.addAll(SHELL_FILES)) ); self.skipWaiting(); }); @@ -14,16 +40,231 @@ self.addEventListener('install', (event) => { self.addEventListener('activate', (event) => { event.waitUntil( caches.keys().then((keys) => - Promise.all(keys.filter((k) => k !== CACHE).map((k) => caches.delete(k))) + Promise.all(keys.filter((k) => !KEEP.includes(k)).map((k) => caches.delete(k))) ) ); self.clients.claim(); }); self.addEventListener('fetch', (event) => { - if (event.request.mode === 'navigate') { - event.respondWith( - fetch(event.request).catch(() => caches.match(OFFLINE_URL)) - ); + const request = event.request; + const url = new URL(request.url); + + if (request.method !== 'GET' || url.origin !== self.location.origin) { + return; + } + + if (request.mode === 'navigate') { + event.respondWith(navigate(request, url)); + } else if (SHELL_FILES.includes(url.pathname)) { + event.respondWith(fetch(request).catch(() => caches.match(request))); + } else if (url.pathname.startsWith('/build/')) { + event.respondWith(cacheFirst(ASSETS, request)); + } else if (IMAGE.test(url.pathname)) { + event.respondWith(cached(ARTICLES, url.pathname).then((hit) => hit || fetch(request))); } }); + +self.addEventListener('message', (event) => { + const type = event.data && event.data.type; + + if (type === 'sync') { + event.waitUntil(sync()); + } else if (type === 'clear') { + event.waitUntil(clear()); + } +}); + +async function navigate(request, url) { + try { + return await fetch(request); + } catch (error) { + return (await cached(ARTICLES, url.pathname)) + || (await caches.match(OFFLINE_URL)) + || Response.error(); + } +} + +async function cacheFirst(name, request) { + const hit = await caches.match(request); + + if (hit) { + return hit; + } + + const response = await fetch(request); + + if (response.ok) { + const cache = await caches.open(name); + await cache.put(request, response.clone()); + } + + return response; +} + +async function cached(name, path) { + const cache = await caches.open(name); + + return cache.match(path); +} + +// A same-origin GET that only counts if it really came back: a redirect +// here means the session has gone and the login page answered instead. +async function get(path) { + try { + const response = await fetch(path, { credentials: 'same-origin', cache: 'no-store' }); + + return response.ok && !response.redirected ? response : null; + } catch (error) { + return null; + } +} + +async function clear() { + await caches.delete(ARTICLES); + await caches.delete(ASSETS); +} + +let running = null; +let finishedAt = 0; + +function sync() { + if (!running && Date.now() - finishedAt >= MIN_INTERVAL) { + running = runSync().finally(() => { + running = null; + finishedAt = Date.now(); + }); + } + + return running || Promise.resolve(); +} + +async function runSync() { + let response; + + try { + response = await fetch(LIST_URL, { + credentials: 'same-origin', + cache: 'no-store', + headers: { Accept: 'application/json' }, + }); + } catch (error) { + // Offline or the server is down: keep what is already here. + return; + } + + if (response.status === 401 || response.status === 419 || response.redirected) { + await clear(); + + return; + } + + if (!response.ok) { + return; + } + + const { items } = await response.json(); + const cache = await caches.open(ARTICLES); + const previous = await readIndex(cache); + const kept = []; + + for (const item of items) { + const before = previous.get(item.url); + let stored = Boolean(before && await cache.match(item.url)); + let version = stored ? before.version : null; + let assets = stored ? before.assets : []; + + if (version !== item.version) { + const page = await get(item.url); + + if (page) { + assets = [...new Set((await page.clone().text()).match(ASSET) || [])]; + await cache.put(item.url, page); + stored = true; + version = item.version; + } + } + + // A page that failed to refresh keeps its older copy (and older + // version, so the next sync tries again), if it has one. + if (!stored) { + continue; + } + + for (const src of item.images) { + if (!(await cache.match(src))) { + const image = await get(src); + + if (image) { + await cache.put(src, image); + } + } + } + + kept.push({ ...item, version, assets }); + } + + await storeAssets(kept); + await prune(cache, kept); + + await cache.put(INDEX_KEY, new Response( + JSON.stringify({ syncedAt: new Date().toISOString(), items: kept }), + { headers: { 'Content-Type': 'application/json' } } + )); +} + +async function readIndex(cache) { + const map = new Map(); + + try { + const response = await cache.match(INDEX_KEY); + const index = response ? await response.json() : { items: [] }; + + for (const item of index.items) { + map.set(item.url, item); + } + } catch (error) { + // A damaged index only means everything is fetched again. + } + + return map; +} + +// The built CSS and JS the kept pages load. A deploy changes their hashed +// names, so the copies an older page needs are kept until that page is +// refreshed, and anything no kept page uses is dropped. +async function storeAssets(kept) { + const wanted = new Set(kept.flatMap((item) => item.assets)); + const cache = await caches.open(ASSETS); + + for (const path of wanted) { + if (!(await cache.match(path))) { + const asset = await get(path); + + if (asset) { + await cache.put(path, asset); + } + } + } + + for (const request of await cache.keys()) { + if (!wanted.has(new URL(request.url).pathname)) { + await cache.delete(request); + } + } +} + +async function prune(cache, kept) { + const wanted = new Set([INDEX_KEY]); + + for (const item of kept) { + wanted.add(item.url); + item.images.forEach((src) => wanted.add(src)); + } + + for (const request of await cache.keys()) { + if (!wanted.has(new URL(request.url).pathname)) { + await cache.delete(request); + } + } +} diff --git a/resources/js/app.js b/resources/js/app.js index 6a7176f..91ed592 100644 --- a/resources/js/app.js +++ b/resources/js/app.js @@ -5,6 +5,7 @@ import Alpine from '@alpinejs/csp'; import behaviours from './behaviours'; import bulkSelection from './bulk-selection'; +import offline from './offline'; import reader from './reader'; window.Alpine = Alpine; @@ -13,5 +14,6 @@ Alpine.data('bulkSelection', bulkSelection); Alpine.data('reader', reader); behaviours(document); +offline(document); Alpine.start(); diff --git a/resources/js/offline.js b/resources/js/offline.js new file mode 100644 index 0000000..23b3bee --- /dev/null +++ b/resources/js/offline.js @@ -0,0 +1,142 @@ +/** + * The page side of offline reading (the work is done in public/sw.js). + * + * - Asks the service worker to sync the offline copies whenever a logged-in + * page opens online, or the connection comes back. Logged-in pages carry + * ; the login page doesn't. + * - Keeps reading progress made offline in localStorage and sends it once + * there is a connection again. + * - Clears the offline copies before a form marked data-clears-offline + * (log out) submits. + */ +export const ARTICLES_CACHE = 'docket-articles'; +export const PENDING_KEY = 'docket.pendingProgress'; + +function loggedIn() { + return document.querySelector('meta[name="docket-offline"]') !== null; +} + +export function requestSync() { + if (!loggedIn() || !navigator.onLine || !('serviceWorker' in navigator)) { + return; + } + + navigator.serviceWorker.ready + .then((registration) => registration.active?.postMessage({ type: 'sync' })) + .catch(() => {}); +} + +function readPending() { + try { + return JSON.parse(window.localStorage.getItem(PENDING_KEY) || '{}'); + } catch { + return {}; + } +} + +function writePending(pending) { + try { + if (Object.keys(pending).length === 0) { + window.localStorage.removeItem(PENDING_KEY); + } else { + window.localStorage.setItem(PENDING_KEY, JSON.stringify(pending)); + } + } catch { + // Storage unavailable: offline progress is lost, nothing else. + } +} + +/** + * Remember progress that couldn't be sent, keyed by its progress URL. + */ +export function queueProgress(url, progress) { + writePending({ ...readPending(), [url]: progress }); +} + +/** + * Progress made offline for this article and not yet sent, if any. It is + * newer than the position in an offline copy of the page. + */ +export function pendingProgress(url) { + const value = readPending()[url]; + + return typeof value === 'number' ? value : null; +} + +/** + * Drop queued progress for one article once newer progress has been sent. + */ +export function forgetProgress(url) { + const pending = readPending(); + + if (url in pending) { + delete pending[url]; + writePending(pending); + } +} + +export function postProgress(url, progress) { + const token = document.querySelector('meta[name="csrf-token"]')?.content ?? ''; + const body = new FormData(); + body.append('_token', token); + body.append('progress', String(progress)); + + return fetch(url, { + method: 'POST', + body, + credentials: 'same-origin', + headers: { Accept: 'application/json', 'X-Requested-With': 'XMLHttpRequest' }, + }); +} + +/** + * Send any progress queued while offline. Entries the server rejects + * outright (the article was deleted) are dropped; a network failure keeps + * them for next time. + */ +export function flushProgress() { + if (!loggedIn() || !navigator.onLine) { + return; + } + + Object.entries(readPending()).forEach(([url, progress]) => { + postProgress(url, progress) + .then((response) => { + if (response.ok || (response.status >= 400 && response.status < 500 && response.status !== 419)) { + const pending = readPending(); + + if (pending[url] === progress) { + delete pending[url]; + writePending(pending); + } + } + }) + .catch(() => {}); + }); +} + +export default function offline(root) { + root.addEventListener('submit', (event) => { + const form = event.target; + + if (!(form instanceof HTMLFormElement) || !form.hasAttribute('data-clears-offline') || !('caches' in window)) { + return; + } + + event.preventDefault(); + + navigator.serviceWorker?.controller?.postMessage({ type: 'clear' }); + + Promise.all([window.caches.delete(ARTICLES_CACHE), window.caches.delete('docket-assets')]) + .catch(() => {}) + .finally(() => form.submit()); + }); + + window.addEventListener('online', () => { + requestSync(); + flushProgress(); + }); + + requestSync(); + flushProgress(); +} diff --git a/resources/js/reader.js b/resources/js/reader.js index d550886..19bd81d 100644 --- a/resources/js/reader.js +++ b/resources/js/reader.js @@ -3,6 +3,8 @@ * are per browser) and reading progress (sent to the server, so it * follows you between devices). */ +import { forgetProgress, pendingProgress, postProgress, queueProgress } from './offline'; + export const STORAGE_KEY = 'docket.reader'; export const DEFAULTS = { theme: 'light', family: 'serif', size: 20 }; export const SIZES = { min: 14, max: 32, step: 2 }; @@ -37,6 +39,7 @@ export function applyPreferences(prefs, root = document.documentElement) { export default () => ({ prefs: loadPreferences(), panel: false, + offline: !navigator.onLine, progressUrl: '', savedProgress: 0, lastSent: null, @@ -44,7 +47,9 @@ export default () => ({ init() { this.progressUrl = this.$el.dataset.progressUrl || ''; - this.savedProgress = Number(this.$el.dataset.progress) || 0; + // Progress made offline beats the position baked into the page, + // which may be an older offline copy. + this.savedProgress = pendingProgress(this.progressUrl) ?? (Number(this.$el.dataset.progress) || 0); applyPreferences(this.prefs); @@ -71,6 +76,8 @@ export default () => ({ this.lastSent = this.savedProgress; + window.addEventListener('online', () => { this.offline = false; }); + window.addEventListener('offline', () => { this.offline = true; }); window.addEventListener('scroll', () => this.schedule(), { passive: true }); document.addEventListener('visibilitychange', () => { if (document.visibilityState === 'hidden') { @@ -111,24 +118,30 @@ export default () => ({ this.lastSent = progress; - const token = document.querySelector('meta[name="csrf-token"]')?.content ?? ''; - const body = new FormData(); - body.append('_token', token); - body.append('progress', String(progress)); + // Offline, keep it for offline.js to send when the connection is + // back (sendBeacon would accept it and then lose it). + if (!navigator.onLine) { + queueProgress(this.progressUrl, progress); + + return; + } // sendBeacon survives the page being closed; fetch is used - // otherwise so failures don't go unnoticed in the console. + // otherwise so a failure can be queued for later. if (leaving && navigator.sendBeacon) { - navigator.sendBeacon(this.progressUrl, body); + const body = new FormData(); + body.append('_token', document.querySelector('meta[name="csrf-token"]')?.content ?? ''); + body.append('progress', String(progress)); - return; + if (navigator.sendBeacon(this.progressUrl, body)) { + forgetProgress(this.progressUrl); + + return; + } } - fetch(this.progressUrl, { - method: 'POST', - body, - credentials: 'same-origin', - headers: { Accept: 'application/json', 'X-Requested-With': 'XMLHttpRequest' }, - }).catch(() => {}); + postProgress(this.progressUrl, progress) + .then(() => forgetProgress(this.progressUrl)) + .catch(() => queueProgress(this.progressUrl, progress)); }, }); diff --git a/resources/views/auth/verify-email.blade.php b/resources/views/auth/verify-email.blade.php index 3843739..960e877 100644 --- a/resources/views/auth/verify-email.blade.php +++ b/resources/views/auth/verify-email.blade.php @@ -14,7 +14,7 @@ -
+ @csrf
@@ -124,7 +124,7 @@ class="fixed inset-x-0 top-14 z-30 border-b border-slate-800 bg-slate-900 px-3 p
{{ Auth::user()->name }} -
+ @csrf
diff --git a/resources/views/partials/pwa.blade.php b/resources/views/partials/pwa.blade.php index 8b9e4ef..2b31ca8 100644 --- a/resources/views/partials/pwa.blade.php +++ b/resources/views/partials/pwa.blade.php @@ -4,6 +4,10 @@ +@auth + {{-- Logged in: resources/js/offline.js keeps the offline copies in step. --}} + +@endauth