From 217a8645f23f593128011123063e107792344a98 Mon Sep 17 00:00:00 2001 From: Decidetto <5384177+Decidetto@users.noreply.github.com> Date: Sun, 6 Sep 2026 12:17:32 +0200 Subject: [PATCH 1/3] feat(mobile): add offline English and Chinese dictionaries --- dictionary-packs/LICENSE-WIKTIONARY.txt | 22 + dictionary-packs/LICENSE-WORDNET-3.1.txt | 44 ++ dictionary-packs/README.md | 105 +++ dictionary-packs/manifest.json | 31 + packages/app-expo/package.json | 2 + .../reader/DefinitionSheet.test.tsx | 350 ++++++++++ .../src/components/reader/DefinitionSheet.tsx | 409 ++++++++++++ .../reader/SelectionPopover.test.tsx | 121 ++++ .../components/reader/SelectionPopover.tsx | 27 +- .../reader/definition-controller.test.ts | 189 ++++++ .../reader/definition-controller.ts | 136 ++++ .../src/config/dictionary-config.test.ts | 75 +++ .../app-expo/src/config/dictionary-config.ts | 8 + .../src/config/dictionary-manifest.json | 31 + .../src/lib/dictionary/dictionary-database.ts | 81 +++ .../dictionary-lookup-service.test.ts | 266 ++++++++ .../dictionary/dictionary-lookup-service.ts | 163 +++++ .../dictionary-pack-manager.test.ts | 613 ++++++++++++++++++ .../lib/dictionary/dictionary-pack-manager.ts | 436 +++++++++++++ .../dictionary-pack-platform.test.ts | 151 +++++ .../dictionary/dictionary-pack-platform.ts | 219 +++++++ .../lib/dictionary/dictionary-runtime.test.ts | 170 +++++ .../src/lib/dictionary/dictionary-runtime.ts | 29 + .../app-expo/src/navigation/RootNavigator.tsx | 3 + .../app-expo/src/screens/ProfileScreen.tsx | 6 + .../app-expo/src/screens/ReaderScreen.tsx | 22 + .../reader/dictionary-reader-contract.test.ts | 27 + .../DictionarySettingsScreen.test.tsx | 325 ++++++++++ .../settings/DictionarySettingsScreen.tsx | 344 ++++++++++ .../settings/dictionary-locales.test.ts | 107 +++ .../src/stores/dictionary-store.test.ts | 332 ++++++++++ .../app-expo/src/stores/dictionary-store.ts | 191 ++++++ packages/app-expo/src/stores/index.ts | 3 + packages/cli/package.json | 2 + packages/cli/scripts/build-dictionary-pack.ts | 184 ++++++ packages/cli/scripts/convert-wordnet.ts | 55 ++ .../en-unnormalizable-canonical.jsonl | 1 + .../src/dictionary/fixtures/en-wordnet.jsonl | 2 + packages/cli/src/dictionary/fixtures/en.jsonl | 1 + .../src/dictionary/fixtures/test-license.txt | 1 + .../fixtures/wordnet-invalid/adj.exc | 1 + .../fixtures/wordnet-invalid/adv.exc | 1 + .../fixtures/wordnet-invalid/data.adj | 1 + .../fixtures/wordnet-invalid/data.adv | 1 + .../fixtures/wordnet-invalid/data.noun | 1 + .../fixtures/wordnet-invalid/data.verb | 1 + .../fixtures/wordnet-invalid/noun.exc | 1 + .../fixtures/wordnet-invalid/verb.exc | 1 + .../src/dictionary/fixtures/wordnet/adj.exc | 1 + .../src/dictionary/fixtures/wordnet/adv.exc | 1 + .../src/dictionary/fixtures/wordnet/data.adj | 2 + .../src/dictionary/fixtures/wordnet/data.adv | 1 + .../src/dictionary/fixtures/wordnet/data.noun | 7 + .../src/dictionary/fixtures/wordnet/data.verb | 7 + .../src/dictionary/fixtures/wordnet/noun.exc | 1 + .../src/dictionary/fixtures/wordnet/verb.exc | 4 + .../dictionary/fixtures/zh-redirects.jsonl | 12 + packages/cli/src/dictionary/fixtures/zh.jsonl | 1 + .../cli/src/dictionary/pack-builder.test.ts | 404 ++++++++++++ packages/cli/src/dictionary/pack-builder.ts | 379 +++++++++++ packages/cli/src/dictionary/schema.ts | 32 + .../src/dictionary/wordnet-converter.test.ts | 165 +++++ .../cli/src/dictionary/wordnet-converter.ts | 296 +++++++++ packages/core/package.json | 2 + packages/core/src/dictionary/index.ts | 15 + packages/core/src/dictionary/manifest.test.ts | 173 +++++ packages/core/src/dictionary/manifest.ts | 68 ++ .../core/src/dictionary/selection.test.ts | 32 + packages/core/src/dictionary/selection.ts | 29 + packages/core/src/dictionary/types.ts | 70 ++ packages/core/src/i18n/locales/en/reader.json | 44 ++ .../core/src/i18n/locales/zh-TW/reader.json | 44 ++ packages/core/src/i18n/locales/zh/reader.json | 44 ++ packages/core/src/index.ts | 3 + pnpm-lock.yaml | 26 + 75 files changed, 7150 insertions(+), 5 deletions(-) create mode 100644 dictionary-packs/LICENSE-WIKTIONARY.txt create mode 100644 dictionary-packs/LICENSE-WORDNET-3.1.txt create mode 100644 dictionary-packs/README.md create mode 100644 dictionary-packs/manifest.json create mode 100644 packages/app-expo/src/components/reader/DefinitionSheet.test.tsx create mode 100644 packages/app-expo/src/components/reader/DefinitionSheet.tsx create mode 100644 packages/app-expo/src/components/reader/SelectionPopover.test.tsx create mode 100644 packages/app-expo/src/components/reader/definition-controller.test.ts create mode 100644 packages/app-expo/src/components/reader/definition-controller.ts create mode 100644 packages/app-expo/src/config/dictionary-config.test.ts create mode 100644 packages/app-expo/src/config/dictionary-config.ts create mode 100644 packages/app-expo/src/config/dictionary-manifest.json create mode 100644 packages/app-expo/src/lib/dictionary/dictionary-database.ts create mode 100644 packages/app-expo/src/lib/dictionary/dictionary-lookup-service.test.ts create mode 100644 packages/app-expo/src/lib/dictionary/dictionary-lookup-service.ts create mode 100644 packages/app-expo/src/lib/dictionary/dictionary-pack-manager.test.ts create mode 100644 packages/app-expo/src/lib/dictionary/dictionary-pack-manager.ts create mode 100644 packages/app-expo/src/lib/dictionary/dictionary-pack-platform.test.ts create mode 100644 packages/app-expo/src/lib/dictionary/dictionary-pack-platform.ts create mode 100644 packages/app-expo/src/lib/dictionary/dictionary-runtime.test.ts create mode 100644 packages/app-expo/src/lib/dictionary/dictionary-runtime.ts create mode 100644 packages/app-expo/src/screens/reader/dictionary-reader-contract.test.ts create mode 100644 packages/app-expo/src/screens/settings/DictionarySettingsScreen.test.tsx create mode 100644 packages/app-expo/src/screens/settings/DictionarySettingsScreen.tsx create mode 100644 packages/app-expo/src/screens/settings/dictionary-locales.test.ts create mode 100644 packages/app-expo/src/stores/dictionary-store.test.ts create mode 100644 packages/app-expo/src/stores/dictionary-store.ts create mode 100644 packages/cli/scripts/build-dictionary-pack.ts create mode 100644 packages/cli/scripts/convert-wordnet.ts create mode 100644 packages/cli/src/dictionary/fixtures/en-unnormalizable-canonical.jsonl create mode 100644 packages/cli/src/dictionary/fixtures/en-wordnet.jsonl create mode 100644 packages/cli/src/dictionary/fixtures/en.jsonl create mode 100644 packages/cli/src/dictionary/fixtures/test-license.txt create mode 100644 packages/cli/src/dictionary/fixtures/wordnet-invalid/adj.exc create mode 100644 packages/cli/src/dictionary/fixtures/wordnet-invalid/adv.exc create mode 100644 packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adj create mode 100644 packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adv create mode 100644 packages/cli/src/dictionary/fixtures/wordnet-invalid/data.noun create mode 100644 packages/cli/src/dictionary/fixtures/wordnet-invalid/data.verb create mode 100644 packages/cli/src/dictionary/fixtures/wordnet-invalid/noun.exc create mode 100644 packages/cli/src/dictionary/fixtures/wordnet-invalid/verb.exc create mode 100644 packages/cli/src/dictionary/fixtures/wordnet/adj.exc create mode 100644 packages/cli/src/dictionary/fixtures/wordnet/adv.exc create mode 100644 packages/cli/src/dictionary/fixtures/wordnet/data.adj create mode 100644 packages/cli/src/dictionary/fixtures/wordnet/data.adv create mode 100644 packages/cli/src/dictionary/fixtures/wordnet/data.noun create mode 100644 packages/cli/src/dictionary/fixtures/wordnet/data.verb create mode 100644 packages/cli/src/dictionary/fixtures/wordnet/noun.exc create mode 100644 packages/cli/src/dictionary/fixtures/wordnet/verb.exc create mode 100644 packages/cli/src/dictionary/fixtures/zh-redirects.jsonl create mode 100644 packages/cli/src/dictionary/fixtures/zh.jsonl create mode 100644 packages/cli/src/dictionary/pack-builder.test.ts create mode 100644 packages/cli/src/dictionary/pack-builder.ts create mode 100644 packages/cli/src/dictionary/schema.ts create mode 100644 packages/cli/src/dictionary/wordnet-converter.test.ts create mode 100644 packages/cli/src/dictionary/wordnet-converter.ts create mode 100644 packages/core/src/dictionary/index.ts create mode 100644 packages/core/src/dictionary/manifest.test.ts create mode 100644 packages/core/src/dictionary/manifest.ts create mode 100644 packages/core/src/dictionary/selection.test.ts create mode 100644 packages/core/src/dictionary/selection.ts create mode 100644 packages/core/src/dictionary/types.ts diff --git a/dictionary-packs/LICENSE-WIKTIONARY.txt b/dictionary-packs/LICENSE-WIKTIONARY.txt new file mode 100644 index 000000000..025091d4c --- /dev/null +++ b/dictionary-packs/LICENSE-WIKTIONARY.txt @@ -0,0 +1,22 @@ +Chinese Wiktionary pack attribution + +The ReadAny Chinese offline dictionary pack contains transformed material from +the Chinese-language Wiktionary dump dated 2026-09-01: +https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2 + +Human-readable attribution page: +https://zh.wiktionary.org/wiki/Wiktionary:%E7%89%88%E6%9D%83%E4%BF%A1%E6%81%AF + +Wiktionary contributors license this material under Creative Commons +Attribution-ShareAlike 4.0 International (CC BY-SA 4.0): +https://creativecommons.org/licenses/by-sa/4.0/ + +Attribution: Wiktionary contributors. + +ReadAny used Wiktextract and its deterministic pack builder to retain headwords, +parts of speech, Chinese pronunciation when present, Simplified/Traditional +forms, and non-empty definitions. It removed examples, translations, audio, +images, and other fields not represented by the ReadAny dictionary schema. + +ReadAny's transformed SQLite pack is distributed under CC BY-SA 4.0. Neither +the Wikimedia Foundation nor Wiktionary contributors endorse ReadAny. diff --git a/dictionary-packs/LICENSE-WORDNET-3.1.txt b/dictionary-packs/LICENSE-WORDNET-3.1.txt new file mode 100644 index 000000000..38b04c622 --- /dev/null +++ b/dictionary-packs/LICENSE-WORDNET-3.1.txt @@ -0,0 +1,44 @@ +WordNet 3.1 license + +This software and database is being provided to you, the LICENSEE, by +Princeton University under the following license. By obtaining, using +and/or copying this software and database, you agree that you have +read, understood, and will comply with these terms and conditions.: + +Permission to use, copy, modify and distribute this software and +database and its documentation for any purpose and without fee or +royalty is hereby granted, provided that you agree to comply with +the following copyright notice and statements, including the disclaimer, +and that the same appear on ALL copies of the software, database and +documentation, including modifications that you make for internal +use or for distribution. + +WordNet 3.1 Copyright 2011 by Princeton University. All rights reserved. + +THIS SOFTWARE AND DATABASE IS PROVIDED "AS IS" AND PRINCETON +UNIVERSITY MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PRINCETON +UNIVERSITY MAKES NO REPRESENTATIONS OR WARRANTIES OF MERCHANT- +ABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE +OF THE LICENSED SOFTWARE, DATABASE OR DOCUMENTATION WILL NOT +INFRINGE ANY THIRD PARTY PATENTS, COPYRIGHTS, TRADEMARKS OR +OTHER RIGHTS. + +The name of Princeton University or Princeton may not be used in +advertising or publicity pertaining to distribution of the software +and/or database. Title to copyright in this software, database and +any associated documentation shall at all times remain with +Princeton University and LICENSEE agrees to preserve same. + +The license text above is reproduced from the header embedded in the official +WordNet 3.1 database files distributed at: +https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz + +Human-readable WordNet license page: +https://wordnet.princeton.edu/license-and-commercial-use + +Creator attribution: WordNet 3.1 Copyright 2011 by Princeton University. All +rights reserved. + +ReadAny transforms WordNet data into a SQLite lookup pack. Princeton University +does not endorse ReadAny or this transformation. diff --git a/dictionary-packs/README.md b/dictionary-packs/README.md new file mode 100644 index 000000000..e524c2601 --- /dev/null +++ b/dictionary-packs/README.md @@ -0,0 +1,105 @@ +# ReadAny offline dictionary packs v1 + +These are optional, one-time downloads used for on-device definition lookup. +They are not bundled into the app and lookup never falls back to AI or a web +definition service. + +## Exact user-download sizes + +| Language | Coverage/source | Entries | Bytes | MiB | SHA-256 | +| --- | --- | ---: | ---: | ---: | --- | +| English | WordNet 3.1 | 154,404 | 29,069,312 | 27.722656 | `90adbeab5ee325b31f2e34bbfa7c5b699932c900bfd1efdfc62eabd9e004ee0e` | +| Chinese | Chinese Wiktionary 2026-09-01 | 77,256 | 11,866,112 | 11.316406 | `d50218459f78a5e7bcba819fbb7db699271bdaf30757dc8b2443cb1e37bc11b1` | + +Downloading both packs is 40,935,424 bytes (39.039062 MiB). Each real SQLite +pack is below the 150 MiB hard stop used for this release. The release URLs in +`manifest.json` point to the published `dictionary-packs-v1` assets. The initial +packs are hosted by contributor cha1latte; they are data downloads, not app updates. +The app reads manifest updates from the official repository after this feature +is merged and falls back to its bundled manifest when offline. Maintainers can +move assets by updating the manifest URLs while preserving the verified hashes. +An alternate manifest can be selected at build time with +`EXPO_PUBLIC_DICTIONARY_MANIFEST_URL`. No binary packs are checked into this PR. + +WordNet gives strong ordinary English vocabulary in a compact download, but has +less slang, newer language, proper-name coverage, and obscure material than the +full English Wiktionary source that was rejected for this initial release after +its pinned extraction projected a multi-day run. + +## Pinned sources and provenance + +### English + +- Official source: `https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz` +- Archive bytes: 16,358,468 +- Archive SHA-256: `3f7d8be8ef6ecc7167d39b10d66954ec734280b5bdcd57f7d9eafe429d11c22a` +- Source edition: WordNet 3.1 +- Source date: 2011-05-26, recorded from the archive's top-level directory and + exception-file timestamps; WordNet 3.1 remains the authoritative edition identity +- License: WordNet 3.1 License; see `LICENSE-WORDNET-3.1.txt` +- Converted JSONL: 154,404 records, 206,143 definitions, 66,893 aliases +- Converted JSONL bytes: 30,477,132 +- Converted JSONL SHA-256: `65f632bef5671e85f649f510eef13d90d752d11dcedc252de65f48a6d2f0d359` + +### Chinese + +- Official source: `https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2` +- Dump bytes: 284,272,376 +- Dump SHA-256: `516d870bb8ddaa461e013cfb0d477229501c9b8f97fd56c8a7098f42e3586a22` +- Source edition/date: Chinese Wiktionary, 2026-09-01 +- License: CC BY-SA 4.0; see `LICENSE-WIKTIONARY.txt` +- Extractor: Wiktextract commit `1939b1f8b1ae5d6989b8cbaea91c639b1b5dcbef` +- Extracted JSONL: 245,518 language-matching data records, of which 80,623 + contain at least one non-empty gloss +- Accepted SQLite entries: 77,256; senses: 105,182; lookup rows: 129,666 +- Wiktextract hard redirects and same-language soft redirects contribute 10,113 + rank-1 lookup rows across 7,801 aliases that were absent from the prior pack. + Redirect chains resolve only to existing canonical entries; cycles, missing + targets, and unsupported-language aliases do not create lookup rows. + +## Deterministic transformation + +The WordNet converter reads `data.noun`, `data.verb`, `data.adj`, and `data.adv` +using the complete official database-row grammar: fixed-width offsets and +counts, hexadecimal lexical IDs, every pointer record, verb frame counts and +records, and exact pre-gloss token consumption. It converts +underscore-separated lemmas to display spaces, removes adjective syntactic +markers, groups definitions by lemma and part of speech, and omits quoted usage +examples. It attaches aliases from the four WordNet exception files. For +single-token nouns without an official noun exception, it uses conservative +plural rules: consonant-`y` to `-ies`, sibilants to `-es`, and otherwise `-s`. +For verbs it adds only third-person singular: `do`/`go` and sibilants use +`-es`, consonant-`y` uses `-ies`, and other verbs use `-s`; the official +exceptions supply irregular `be` and `have` forms. Thus `goes`, `does`, `has`, +`is`, and `tattoos` are present while `tattooes` is absent. It does not guess +past tense, gerunds, participles, or comparative/superlative forms. + +The shared pack builder retains: + +- language, canonical headword, and part of speech; +- every non-empty definition; +- WordNet exception and conservative regular aliases for English; +- the first available `zh-pron` and Simplified/Traditional aliases for Chinese; +- Chinese Wiktextract hard/soft redirects, including chains whose final target + is an existing same-language canonical entry; +- canonical lookup rank 0 and alias rank 1; and +- source edition/date, distinct source-archive, asset, and human-readable + attribution URLs, transformation identity, creator attribution, and the + complete source-specific license/notice text in SQLite metadata. + +The v1 schema has no examples, translations, audio, images, or AI-generated +data. WordNet semantic relations and verb frames are also not represented. + +## Reproduction commands + +```powershell +pnpm --filter @readany/cli dictionary:convert-wordnet -- --input-directory ./dictionary-source/wordnet-3.1\dict --output ./dictionary-source/en-wordnet-3.1.jsonl + +pnpm --filter @readany/cli dictionary:build -- --language en --input ./dictionary-source/en-wordnet-3.1.jsonl --output ./dictionary-source/readany-dictionary-en-v1.sqlite --version 1.0.0 --source-edition wordnet-3.1 --license "WordNet 3.1 License" --source-date 2011-05-26 --source-archive-url https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz --attribution-url https://wordnet.princeton.edu/license-and-commercial-use --license-file ./dictionary-packs/LICENSE-WORDNET-3.1.txt --creator-attribution "WordNet 3.1 Copyright 2011 by Princeton University. All rights reserved." --asset-url https://github.com/cha1latte/ReadAny/releases/download/dictionary-packs-v1/readany-dictionary-en-v1.sqlite --descriptor ./dictionary-source/en-descriptor.json + +pnpm --filter @readany/cli dictionary:build -- --language zh --input ./dictionary-source/zh-20260901.jsonl --output ./dictionary-source/readany-dictionary-zh-v1.sqlite --version 1.0.0 --source-edition zhwiktionary --license "CC BY-SA 4.0" --source-date 2026-09-01 --source-archive-url https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2 --attribution-url https://zh.wiktionary.org/wiki/Wiktionary:%E7%89%88%E6%9D%83%E4%BF%A1%E6%81%AF --license-file ./dictionary-packs/LICENSE-WIKTIONARY.txt --creator-attribution "Wiktionary contributors." --asset-url https://github.com/cha1latte/ReadAny/releases/download/dictionary-packs-v1/readany-dictionary-zh-v1.sqlite --descriptor ./dictionary-source/zh-descriptor.json +``` + +Before publication, each descriptor must exactly match an independent local +file-size and SHA-256 check. `manifest.json` and the app's bundled manifest must +remain byte-identical and must pass the shared strict manifest parser. diff --git a/dictionary-packs/manifest.json b/dictionary-packs/manifest.json new file mode 100644 index 000000000..08150a75b --- /dev/null +++ b/dictionary-packs/manifest.json @@ -0,0 +1,31 @@ +{ + "manifestVersion": 1, + "packs": { + "en": { + "language": "en", + "version": "1.0.0", + "schemaVersion": 1, + "sourceEdition": "wordnet-3.1", + "sourceDumpDate": "2011-05-26", + "sizeBytes": 29069312, + "sha256": "90adbeab5ee325b31f2e34bbfa7c5b699932c900bfd1efdfc62eabd9e004ee0e", + "url": "https://github.com/cha1latte/ReadAny/releases/download/dictionary-packs-v1/readany-dictionary-en-v1.sqlite", + "sourceArchiveUrl": "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + "attributionUrl": "https://wordnet.princeton.edu/license-and-commercial-use", + "license": "WordNet 3.1 License" + }, + "zh": { + "language": "zh", + "version": "1.0.0", + "schemaVersion": 1, + "sourceEdition": "zhwiktionary", + "sourceDumpDate": "2026-09-01", + "sizeBytes": 11866112, + "sha256": "d50218459f78a5e7bcba819fbb7db699271bdaf30757dc8b2443cb1e37bc11b1", + "url": "https://github.com/cha1latte/ReadAny/releases/download/dictionary-packs-v1/readany-dictionary-zh-v1.sqlite", + "sourceArchiveUrl": "https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2", + "attributionUrl": "https://zh.wiktionary.org/wiki/Wiktionary:%E7%89%88%E6%9D%83%E4%BF%A1%E6%81%AF", + "license": "CC BY-SA 4.0" + } + } +} diff --git a/packages/app-expo/package.json b/packages/app-expo/package.json index 8117f1158..abacc3906 100644 --- a/packages/app-expo/package.json +++ b/packages/app-expo/package.json @@ -115,11 +115,13 @@ "@babel/core": "^7.25.0", "@types/pako": "^2.0.4", "@types/react": "^19.1.17", + "@types/react-test-renderer": "19.1.0", "babel-plugin-module-resolver": "^5.0.2", "babel-plugin-transform-import-meta": "^2.3.2", "eas-cli": "^18.11.0", "esbuild": "^0.27.3", "react-native-svg-transformer": "^1.5.3", + "react-test-renderer": "19.1.0", "typescript": "^5.9.3", "vitest": "^4.1.2" } diff --git a/packages/app-expo/src/components/reader/DefinitionSheet.test.tsx b/packages/app-expo/src/components/reader/DefinitionSheet.test.tsx new file mode 100644 index 000000000..5fa7fb416 --- /dev/null +++ b/packages/app-expo/src/components/reader/DefinitionSheet.test.tsx @@ -0,0 +1,350 @@ +import type { DictionaryEntry, DictionaryPackDescriptor } from "@readany/core/dictionary"; +import i18n, { i18nReady } from "@readany/core/i18n"; +import React from "react"; +import TestRenderer, { act } from "react-test-renderer"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { DefinitionSheet } from "./DefinitionSheet"; +import { DefinitionController, type DefinitionState } from "./definition-controller"; + +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = + true; +(globalThis as typeof globalThis & { React: typeof React }).React = React; + +vi.mock("react-native", async () => { + const ReactModule = await import("react"); + const host = (name: string) => + function HostComponent(props: Record) { + return ReactModule.createElement(name, props, props.children as React.ReactNode); + }; + return { + ActivityIndicator: host("ActivityIndicator"), + Modal: host("Modal"), + Pressable: host("Pressable"), + ScrollView: host("ScrollView"), + StyleSheet: { create: (styles: unknown) => styles }, + Text: host("Text"), + TouchableOpacity: host("TouchableOpacity"), + View: host("View"), + }; +}); + +vi.mock("@/components/ui/Icon", () => ({ + XIcon: () => null, +})); + +vi.mock("@/stores", () => ({ + useDictionaryStore: Object.assign(() => undefined, { + getState: () => ({ + install: async () => {}, + manifest: null, + }), + }), +})); + +vi.mock("@/styles/theme", () => ({ + fontSize: { base: 16, lg: 18, sm: 14, xs: 12 }, + fontWeight: { medium: "500", semibold: "600" }, + radius: { full: 999, lg: 8, xl: 12 }, + useColors: () => ({ + background: "background", + border: "border", + card: "card", + destructive: "destructive", + foreground: "foreground", + muted: "muted", + mutedForeground: "mutedForeground", + primary: "primary", + primaryForeground: "primaryForeground", + }), +})); + +vi.mock("react-native-safe-area-context", () => ({ + useSafeAreaInsets: () => ({ bottom: 24 }), +})); + +const descriptor: DictionaryPackDescriptor = { + language: "en", + version: "1.0.0", + schemaVersion: 1, + sourceEdition: "enwiktionary", + sourceDumpDate: "2026-09-01", + sizeBytes: 1_572_864, + sha256: "a".repeat(64), + url: "https://example.test/readany-dictionary-en.sqlite", + sourceArchiveUrl: "https://example.test/en-source.xml.bz2", + attributionUrl: "https://en.wiktionary.org", + license: "CC BY-SA 4.0", +}; + +const englishEntry: DictionaryEntry = { + id: 1, + language: "en", + headword: "desire", + pronunciation: "/dɪˈzaɪəɹ/", + partOfSpeech: "noun", + senses: [ + { order: 0, definition: "A strong wish." }, + { order: 1, definition: "To want strongly." }, + ], +}; + +const chineseEntry: DictionaryEntry = { + id: 2, + language: "zh", + headword: "閱讀", + simplified: "阅读", + traditional: "閱讀", + pronunciation: "yuèdú", + partOfSpeech: "动词", + senses: [{ order: 0, definition: "看书或文章。" }], +}; + +function createController(state: DefinitionState, lookup = vi.fn(async () => [])) { + const controller = new DefinitionController({ + lookup, + install: vi.fn(async () => {}), + getDescriptor: () => descriptor, + }); + controller.state = state; + vi.spyOn(controller, "open").mockResolvedValue(); + return { controller, lookup }; +} + +function isHostType(node: TestRenderer.ReactTestInstance, type: string): boolean { + return node.type === type; +} + +function textContent(renderer: TestRenderer.ReactTestRenderer): string { + return renderer.root + .findAll((node) => isHostType(node, "Text")) + .flatMap((node) => node.children) + .join(""); +} + +function pressButton(renderer: TestRenderer.ReactTestRenderer, label: string): void { + const button = renderer.root + .findAll((node) => isHostType(node, "TouchableOpacity")) + .find((node) => node.findAll((child) => child.children.includes(label)).length > 0); + if (!button) throw new Error(`Button ${label} was not found`); + button.props.onPress(); +} + +describe("DefinitionSheet", () => { + beforeEach(async () => { + await i18nReady; + await act(async () => { + await i18n.changeLanguage("en"); + }); + }); + + it("consumes Traditional Chinese sheet, download-template, and accessibility translations", async () => { + await act(async () => { + await i18n.changeLanguage("zh-TW"); + }); + const chineseDescriptor: DictionaryPackDescriptor = { + ...descriptor, + language: "zh", + sourceEdition: "zhwiktionary", + }; + const { controller } = createController({ + kind: "missing-pack", + language: "zh", + descriptor: chineseDescriptor, + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + expect(textContent(renderer)).toContain("釋義"); + expect(textContent(renderer)).toContain("下載中文字典（1.5 MB）即可離線查詢釋義。"); + expect(renderer.root.findAllByProps({ accessibilityLabel: "關閉釋義" }).length).toBeGreaterThan( + 0, + ); + expect(renderer.root.findByProps({ accessibilityLabel: "下載中文字典" })).toBeTruthy(); + }); + + it("renders English headword, IPA, part of speech, and numbered senses", async () => { + const { controller } = createController({ + kind: "result", + displayText: "desire", + entries: [englishEntry], + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + expect(textContent(renderer)).toContain("desire"); + expect(textContent(renderer)).toContain("/dɪˈzaɪəɹ/"); + expect(textContent(renderer)).toContain("noun"); + expect(textContent(renderer)).toContain("1. A strong wish."); + expect(textContent(renderer)).toContain("2. To want strongly."); + }); + + it("renders Chinese simplified and traditional forms, pinyin, and Chinese senses", async () => { + const { controller } = createController({ + kind: "result", + displayText: "閱讀", + entries: [chineseEntry], + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + expect(textContent(renderer)).toContain("阅读"); + expect(textContent(renderer)).toContain("閱讀"); + expect(textContent(renderer)).toContain("yuèdú"); + expect(textContent(renderer)).toContain("动词"); + expect(textContent(renderer)).toContain("1. 看书或文章。"); + }); + + it("renders the missing pack size and a Download action", async () => { + const { controller } = createController({ kind: "missing-pack", language: "en", descriptor }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + expect(textContent(renderer)).toContain("1.5 MB"); + expect(textContent(renderer)).toContain("Download"); + }); + + it("disables Download and renders progress while the pack is downloading", async () => { + const { controller } = createController({ + kind: "downloading", + language: "en", + progress: 0.37, + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + const download = renderer.root + .findAll((node) => isHostType(node, "TouchableOpacity")) + .find((node) => node.findAll((child) => child.children.includes("Download")).length > 0); + expect(download?.props.disabled).toBe(true); + expect(textContent(renderer)).toContain("37%"); + }); + + it("renders the exact no-match guidance", async () => { + const { controller } = createController({ kind: "no-match", displayText: "unknown" }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + expect(textContent(renderer)).toContain("No definition found. Try selecting a single word."); + }); + + it("renders Retry and Manage Dictionaries for a recoverable error", async () => { + const calls: string[] = []; + const onClose = vi.fn(() => calls.push("close")); + const onManageDictionaries = vi.fn(() => calls.push("manage")); + const { controller } = createController({ kind: "error", message: "database unavailable" }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + await act(async () => { + pressButton(renderer, "Retry"); + await Promise.resolve(); + pressButton(renderer, "Manage Dictionaries"); + }); + + expect(textContent(renderer)).toContain("Dictionary lookup failed. Try again."); + expect(calls).toEqual(["close", "manage"]); + expect(onClose).toHaveBeenCalledTimes(1); + expect(onManageDictionaries).toHaveBeenCalledTimes(1); + }); + + it("closes the modal without launching another lookup", async () => { + const onClose = vi.fn(); + const { controller, lookup } = createController({ kind: "no-match", displayText: "desire" }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + const lookupCountBeforeClose = lookup.mock.calls.length; + await act(async () => { + renderer.root.findAll((node) => isHostType(node, "Modal"))[0]?.props.onRequestClose(); + }); + + expect(onClose).toHaveBeenCalledTimes(1); + expect(lookup).toHaveBeenCalledTimes(lookupCountBeforeClose); + }); +}); diff --git a/packages/app-expo/src/components/reader/DefinitionSheet.tsx b/packages/app-expo/src/components/reader/DefinitionSheet.tsx new file mode 100644 index 000000000..d113383b6 --- /dev/null +++ b/packages/app-expo/src/components/reader/DefinitionSheet.tsx @@ -0,0 +1,409 @@ +import { XIcon } from "@/components/ui/Icon"; +import { useDictionaryStore } from "@/stores"; +import { type ThemeColors, fontSize, fontWeight, radius, useColors } from "@/styles/theme"; +import type { DictionaryEntry } from "@readany/core/dictionary"; +import { useEffect, useRef, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { + ActivityIndicator, + Modal, + Pressable, + ScrollView, + StyleSheet, + Text, + TouchableOpacity, + View, +} from "react-native"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { + DefinitionController, + type DefinitionControllerDependencies, + type DefinitionState, +} from "./definition-controller"; + +interface DefinitionSheetProps { + visible: boolean; + text: string; + onClose: () => void; + onManageDictionaries: () => void; + /** Injected by tests or an application-level dictionary composition. */ + controller?: DefinitionController; +} + +export function DefinitionSheet({ + visible, + text, + onClose, + onManageDictionaries, + controller: suppliedController, +}: DefinitionSheetProps) { + const { t } = useTranslation(); + const colors = useColors(); + const insets = useSafeAreaInsets(); + const s = makeStyles(colors); + const fallbackController = useRef(null); + if (!suppliedController && !fallbackController.current) { + fallbackController.current = new DefinitionController(defaultDependencies()); + } + const controller = suppliedController ?? fallbackController.current; + if (!controller) throw new Error("Definition controller was not initialized"); + const [state, setState] = useState(controller.state); + + useEffect(() => controller.subscribe(setState), [controller]); + + useEffect(() => { + if (visible) { + void controller.open(text); + } else { + controller.close(); + } + }, [controller, text, visible]); + + const close = () => onClose(); + const manageDictionaries = () => { + onClose(); + onManageDictionaries(); + }; + + return ( + + + + + + {t("dictionary.title")} + + + + + + {renderState(state, controller, manageDictionaries, s, colors.primary, t)} + + + + ); +} + +function renderState( + state: DefinitionState, + controller: DefinitionController, + onManageDictionaries: () => void, + s: ReturnType, + primaryColor: string, + t: ReturnType["t"], +) { + switch (state.kind) { + case "idle": + return null; + case "loading": + return ( + + + {t("dictionary.loadingDefinition")} + + ); + case "unsupported": + return {t("dictionary.unsupportedSelection")}; + case "missing-pack": + return ; + case "downloading": + return ; + case "no-match": + return {t("dictionary.noDefinitionFound")}; + case "error": + return ( + + {t("dictionary.lookupError")} + + void controller.retry()} + accessibilityRole="button" + accessibilityLabel={t("dictionary.retryLookup")} + > + {t("dictionary.retry")} + + + {t("dictionary.manageDictionaries")} + + + + ); + case "result": + return ( + + {state.displayText} + {state.entries.map((entry) => ( + + ))} + + ); + } +} + +function EntryView({ entry, s }: { entry: DictionaryEntry; s: ReturnType }) { + const forms = + entry.language === "zh" + ? [entry.simplified ?? entry.headword, entry.traditional] + : [entry.headword]; + const uniqueForms = [...new Set(forms.filter((form): form is string => Boolean(form)))]; + + return ( + + + {uniqueForms.map((form) => ( + + {form} + + ))} + {entry.pronunciation ? {entry.pronunciation} : null} + + {entry.partOfSpeech ? {entry.partOfSpeech} : null} + + {entry.senses.map((sense, index) => ( + + {index + 1}. {sense.definition} + + ))} + + + ); +} + +function PackDownload({ + state, + controller, + s, + t, +}: { + state: + | Extract + | Extract; + controller: DefinitionController; + s: ReturnType; + t: ReturnType["t"]; +}) { + const downloading = state.kind === "downloading"; + const descriptor = state.kind === "missing-pack" ? state.descriptor : undefined; + const language = t(`dictionary.${state.language === "en" ? "english" : "chinese"}`); + const progress = downloading ? Math.round(state.progress * 100) : null; + return ( + + + {downloading + ? t("dictionary.downloadingDefinition", { language, progress }) + : t("dictionary.downloadDefinition", { + language, + size: formatBytes(descriptor?.sizeBytes ?? 0), + })} + + void controller.download()} + accessibilityRole="button" + accessibilityLabel={ + downloading + ? t("dictionary.downloadingAccessibility", { language }) + : t("dictionary.downloadAccessibility", { language }) + } + > + {t("dictionary.download")} + + + ); +} + +function defaultDependencies(): DefinitionControllerDependencies { + return { + lookup: (text) => useDictionaryStore.getState().lookup(text), + install: async (descriptor, onProgress) => { + const unsubscribe = useDictionaryStore.subscribe((state) => { + const status = state.packs[descriptor.language]; + if (status.state === "downloading") onProgress(status.progress); + }); + onProgress(0); + try { + await useDictionaryStore.getState().install(descriptor.language); + onProgress(1); + } finally { + unsubscribe(); + } + }, + getDescriptor: (language) => useDictionaryStore.getState().manifest?.packs[language], + }; +} + +function formatBytes(bytes: number): string { + if (bytes < 1024 * 1024) return `${Math.max(0, Math.round(bytes / 1024))} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +const makeStyles = (colors: ThemeColors) => + StyleSheet.create({ + backdrop: { + flex: 1, + backgroundColor: "rgba(0,0,0,0.3)", + }, + container: { + position: "absolute", + bottom: 0, + left: 0, + right: 0, + maxHeight: "60%", + backgroundColor: colors.background, + borderTopLeftRadius: radius.xl, + borderTopRightRadius: radius.xl, + }, + handle: { + width: 40, + height: 4, + marginTop: 8, + alignSelf: "center", + borderRadius: 2, + backgroundColor: colors.muted, + }, + header: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + paddingHorizontal: 16, + paddingVertical: 12, + borderBottomWidth: 1, + borderBottomColor: colors.border, + }, + title: { + fontSize: fontSize.base, + fontWeight: fontWeight.semibold, + color: colors.foreground, + }, + closeButton: { + width: 32, + height: 32, + alignItems: "center", + justifyContent: "center", + borderRadius: radius.full, + }, + content: { + padding: 16, + }, + loadingWrap: { + flexDirection: "row", + alignItems: "center", + gap: 8, + paddingVertical: 16, + }, + statusText: { + fontSize: fontSize.sm, + lineHeight: 20, + color: colors.mutedForeground, + }, + downloadWrap: { + gap: 12, + paddingVertical: 16, + }, + errorWrap: { + gap: 12, + paddingVertical: 16, + }, + errorText: { + fontSize: fontSize.sm, + lineHeight: 20, + color: colors.destructive, + }, + actions: { + flexDirection: "row", + flexWrap: "wrap", + gap: 8, + }, + primaryButton: { + alignSelf: "flex-start", + paddingHorizontal: 12, + paddingVertical: 8, + borderRadius: radius.lg, + backgroundColor: colors.primary, + }, + primaryButtonText: { + fontSize: fontSize.sm, + fontWeight: fontWeight.medium, + color: colors.primaryForeground, + }, + secondaryButton: { + alignSelf: "flex-start", + paddingHorizontal: 12, + paddingVertical: 8, + borderWidth: 1, + borderColor: colors.border, + borderRadius: radius.lg, + backgroundColor: colors.card, + }, + secondaryButtonText: { + fontSize: fontSize.sm, + fontWeight: fontWeight.medium, + color: colors.foreground, + }, + buttonDisabled: { + opacity: 0.55, + }, + results: { + gap: 12, + }, + selectedText: { + fontSize: fontSize.xs, + color: colors.mutedForeground, + }, + entry: { + gap: 8, + paddingBottom: 16, + borderBottomWidth: 1, + borderBottomColor: colors.border, + }, + entryHeading: { + flexDirection: "row", + flexWrap: "wrap", + alignItems: "baseline", + gap: 8, + }, + headword: { + fontSize: fontSize.lg, + fontWeight: fontWeight.semibold, + color: colors.foreground, + }, + pronunciation: { + fontSize: fontSize.sm, + color: colors.mutedForeground, + }, + partOfSpeech: { + alignSelf: "flex-start", + paddingHorizontal: 8, + paddingVertical: 3, + borderRadius: radius.lg, + overflow: "hidden", + fontSize: fontSize.xs, + color: colors.primary, + backgroundColor: colors.muted, + }, + senses: { + gap: 6, + }, + sense: { + fontSize: fontSize.base, + lineHeight: 23, + color: colors.foreground, + }, + }); diff --git a/packages/app-expo/src/components/reader/SelectionPopover.test.tsx b/packages/app-expo/src/components/reader/SelectionPopover.test.tsx new file mode 100644 index 000000000..83efe8e26 --- /dev/null +++ b/packages/app-expo/src/components/reader/SelectionPopover.test.tsx @@ -0,0 +1,121 @@ +import i18n, { i18nReady } from "@readany/core/i18n"; +import React from "react"; +import TestRenderer, { act } from "react-test-renderer"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { SelectionPopover } from "./SelectionPopover"; + +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = + true; +(globalThis as typeof globalThis & { React: typeof React }).React = React; + +vi.mock("expo-clipboard", () => ({ setStringAsync: vi.fn() })); + +vi.mock("@/components/ui/Icon", () => ({ + BookOpenIcon: () => null, + CopyIcon: () => null, + HighlighterIcon: () => null, + LanguagesIcon: () => null, + NotebookPenIcon: () => null, + SparklesIcon: () => null, + Trash2Icon: () => null, + Volume2Icon: () => null, + XIcon: () => null, +})); + +vi.mock("@/components/ui/RichTextEditor", () => ({ RichTextEditor: () => null })); + +vi.mock("@/styles/theme", () => ({ + fontSize: { base: 16, lg: 18, sm: 14 }, + fontWeight: { medium: "500", semibold: "600" }, + radius: { lg: 8, xl: 12, xxl: 16 }, + spacing: { sm: 8, md: 12, lg: 16 }, + useColors: () => ({ + border: "border", + card: "card", + destructive: "destructive", + foreground: "foreground", + muted: "muted", + mutedForeground: "mutedForeground", + primary: "primary", + primaryForeground: "primaryForeground", + }), + withOpacity: () => "transparent", +})); + +vi.mock("@readany/core/types", () => ({ + HIGHLIGHT_COLORS: ["yellow"], + HIGHLIGHT_COLOR_HEX: { yellow: "#ffff00" }, +})); + +vi.mock("react-native", async () => { + const ReactModule = await import("react"); + const host = (name: string) => + function HostComponent(props: Record) { + return ReactModule.createElement(name, props, props.children as React.ReactNode); + }; + return { + KeyboardAvoidingView: host("KeyboardAvoidingView"), + Modal: host("Modal"), + Platform: { OS: "android" }, + StyleSheet: { absoluteFill: {}, absoluteFillObject: {}, create: (styles: unknown) => styles }, + Text: host("Text"), + TouchableOpacity: host("TouchableOpacity"), + View: host("View"), + useWindowDimensions: () => ({ height: 800, width: 400 }), + }; +}); + +const selection = { + cfi: "epubcfi(/6/2)", + text: "selected dictionary word", + position: { x: 200, y: 100, selectionTop: 90, selectionBottom: 110 }, +}; + +describe("SelectionPopover", () => { + beforeEach(async () => { + await i18nReady; + await act(async () => { + await i18n.changeLanguage("zh-TW"); + }); + }); + it("passes the selected text to Define and dismisses the selection", async () => { + const onDefine = vi.fn(); + const onDismiss = vi.fn(); + let renderer!: TestRenderer.ReactTestRenderer; + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + const define = renderer.root.findByProps({ accessibilityLabel: "查詞" }); + act(() => define.props.onPress()); + + expect(onDefine).toHaveBeenCalledWith(selection.text); + expect(onDismiss).toHaveBeenCalledTimes(1); + }); + + it("does not render Define when no definition handler is supplied", async () => { + let renderer!: TestRenderer.ReactTestRenderer; + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + expect(renderer.root.findAllByProps({ accessibilityLabel: "查詞" })).toHaveLength(0); + }); +}); diff --git a/packages/app-expo/src/components/reader/SelectionPopover.tsx b/packages/app-expo/src/components/reader/SelectionPopover.tsx index daa748117..6fc5143ab 100644 --- a/packages/app-expo/src/components/reader/SelectionPopover.tsx +++ b/packages/app-expo/src/components/reader/SelectionPopover.tsx @@ -1,4 +1,5 @@ import { + BookOpenIcon, CopyIcon, HighlighterIcon, LanguagesIcon, @@ -55,6 +56,7 @@ interface Props { onAIChat: () => void; onSpeak?: (text: string, cfi: string) => void; onNote?: (text: string, cfi: string) => void; + onDefine?: (text: string) => void; onTranslate?: (text: string) => void; onRemoveHighlight?: () => void; existingHighlight?: { id: string; color: HighlightColor; note?: string } | null; @@ -69,6 +71,7 @@ export function SelectionPopover({ onAIChat, onSpeak, onNote, + onDefine, onTranslate, onRemoveHighlight, existingHighlight, @@ -99,10 +102,7 @@ export function SelectionPopover({ }, [selection.cfi, hasExistingHighlight]); const buttonCount = - 4 + - (onNote ? 1 : 0) + - (onTranslate ? 1 : 0) + - (onSpeak ? 1 : 0); + 4 + (onNote ? 1 : 0) + (onDefine ? 1 : 0) + (onTranslate ? 1 : 0) + (onSpeak ? 1 : 0); const colorRowItemCount = HIGHLIGHT_COLORS.length + (canRemoveHighlight ? 2 : 0); const colorRowWidth = showColors ? HIGHLIGHT_COLORS.length * COLOR_DOT_SIZE + @@ -183,6 +183,13 @@ export function SelectionPopover({ onDismiss(); }, [selection.text, onTranslate, onDismiss]); + const handleDefine = useCallback(() => { + if (onDefine) { + onDefine(selection.text); + } + onDismiss(); + }, [selection.text, onDefine, onDismiss]); + const handleRemove = useCallback(() => { if (onRemoveHighlight) { onRemoveHighlight(); @@ -255,6 +262,17 @@ export function SelectionPopover({ + {onDefine && ( + + + + )} + {onTranslate && ( @@ -270,7 +288,6 @@ export function SelectionPopover({ )} - diff --git a/packages/app-expo/src/components/reader/definition-controller.test.ts b/packages/app-expo/src/components/reader/definition-controller.test.ts new file mode 100644 index 000000000..7de68af7a --- /dev/null +++ b/packages/app-expo/src/components/reader/definition-controller.test.ts @@ -0,0 +1,189 @@ +import type { + DictionaryEntry, + DictionaryLanguage, + DictionaryPackDescriptor, +} from "@readany/core/dictionary"; +import { describe, expect, it, vi } from "vitest"; +import { DefinitionController } from "./definition-controller"; + +const descriptor: DictionaryPackDescriptor = { + language: "en", + version: "1.0.0", + schemaVersion: 1, + sourceEdition: "enwiktionary", + sourceDumpDate: "2026-09-01", + sizeBytes: 1_572_864, + sha256: "a".repeat(64), + url: "https://example.test/readany-dictionary-en.sqlite", + sourceArchiveUrl: "https://example.test/en-source.xml.bz2", + attributionUrl: "https://en.wiktionary.org", + license: "CC BY-SA 4.0", +}; + +const entry: DictionaryEntry = { + id: 1, + language: "en", + headword: "desire", + pronunciation: "/dɪˈzaɪəɹ/", + partOfSpeech: "noun", + senses: [{ order: 0, definition: "A strong wish." }], +}; + +function lookupError(code: string, message = code): Error & { code: string } { + return Object.assign(new Error(message), { code }); +} + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, reject, resolve }; +} + +function createController(options?: { + lookup?: (text: string) => Promise; + install?: ( + pack: DictionaryPackDescriptor, + onProgress: (progress: number) => void, + ) => Promise; + getDescriptor?: (language: DictionaryLanguage) => DictionaryPackDescriptor | undefined; +}) { + return new DefinitionController({ + lookup: options?.lookup ?? vi.fn(async () => [entry]), + install: options?.install ?? vi.fn(async () => {}), + getDescriptor: options?.getDescriptor ?? vi.fn(() => descriptor), + }); +} + +describe("DefinitionController", () => { + it("opens a supported selection into its local result", async () => { + const lookup = vi.fn(async () => [entry]); + const controller = createController({ lookup }); + + await controller.open(" Desire "); + + expect(lookup).toHaveBeenCalledWith(" Desire "); + expect(controller.state).toEqual({ + kind: "result", + displayText: "Desire", + entries: [entry], + }); + }); + + it("offers the matching pack when the local pack is absent", async () => { + const getDescriptor = vi.fn(() => descriptor); + const controller = createController({ + lookup: async () => { + throw lookupError("pack-not-installed"); + }, + getDescriptor, + }); + + await controller.open("desire"); + + expect(getDescriptor).toHaveBeenCalledWith("en"); + expect(controller.state).toEqual({ kind: "missing-pack", language: "en", descriptor }); + }); + + it("reports install progress then automatically retries the original selection", async () => { + const lookup = vi + .fn<(text: string) => Promise>() + .mockRejectedValueOnce(lookupError("pack-not-installed")) + .mockResolvedValueOnce([entry]); + const install = vi.fn( + async (_pack: DictionaryPackDescriptor, onProgress: (value: number) => void) => { + onProgress(0.37); + }, + ); + const controller = createController({ lookup, install }); + + await controller.open("desire"); + const downloading = controller.download(); + + expect(controller.state).toEqual({ kind: "downloading", language: "en", progress: 0.37 }); + await downloading; + + expect(install).toHaveBeenCalledWith(descriptor, expect.any(Function)); + expect(lookup).toHaveBeenNthCalledWith(2, "desire"); + expect(controller.state).toEqual({ + kind: "result", + displayText: "desire", + entries: [entry], + }); + }); + + it("shows no-match for a supported local lookup with no entries", async () => { + const controller = createController({ lookup: async () => [] }); + + await controller.open("unknown"); + + expect(controller.state).toEqual({ kind: "no-match", displayText: "unknown" }); + }); + + it("rejects an unsupported selection before invoking lookup", async () => { + const lookup = vi.fn(async () => [entry]); + const controller = createController({ lookup }); + + await controller.open("read閱讀"); + + expect(lookup).not.toHaveBeenCalled(); + expect(controller.state).toEqual({ kind: "unsupported", reason: "mixed-script" }); + }); + + it("retries a recoverable local lookup error", async () => { + const lookup = vi + .fn<(text: string) => Promise>() + .mockRejectedValueOnce(new Error("database temporarily unavailable")) + .mockResolvedValueOnce([entry]); + const controller = createController({ lookup }); + + await controller.open("desire"); + expect(controller.state).toEqual({ + kind: "error", + message: "database temporarily unavailable", + }); + + await controller.retry(); + + expect(lookup).toHaveBeenCalledTimes(2); + expect(controller.state).toEqual({ + kind: "result", + displayText: "desire", + entries: [entry], + }); + }); + + it("suppresses a stale lookup when the selected text changes", async () => { + const first = deferred(); + const lookup = vi.fn((text: string) => + text === "first" ? first.promise : Promise.resolve([entry]), + ); + const controller = createController({ lookup }); + + const firstOpen = controller.open("first"); + await controller.open("second"); + first.resolve([]); + await firstOpen; + + expect(controller.state).toEqual({ + kind: "result", + displayText: "second", + entries: [entry], + }); + }); + + it("closes by invalidating in-flight work and resetting to idle", async () => { + const pending = deferred(); + const controller = createController({ lookup: () => pending.promise }); + + const opening = controller.open("desire"); + controller.close(); + pending.resolve([entry]); + await opening; + + expect(controller.state).toEqual({ kind: "idle" }); + }); +}); diff --git a/packages/app-expo/src/components/reader/definition-controller.ts b/packages/app-expo/src/components/reader/definition-controller.ts new file mode 100644 index 000000000..1ea773877 --- /dev/null +++ b/packages/app-expo/src/components/reader/definition-controller.ts @@ -0,0 +1,136 @@ +import { + type DictionaryEntry, + type DictionaryLanguage, + type DictionaryPackDescriptor, + prepareDictionarySelection, +} from "@readany/core/dictionary"; + +export type DefinitionState = + | { kind: "idle" } + | { kind: "loading"; displayText: string } + | { kind: "unsupported"; reason: string } + | { kind: "missing-pack"; language: DictionaryLanguage; descriptor: DictionaryPackDescriptor } + | { kind: "downloading"; language: DictionaryLanguage; progress: number } + | { kind: "result"; displayText: string; entries: DictionaryEntry[] } + | { kind: "no-match"; displayText: string } + | { kind: "error"; message: string }; + +export interface DefinitionControllerDependencies { + lookup(text: string): Promise; + install( + descriptor: DictionaryPackDescriptor, + onProgress: (progress: number) => void, + ): Promise; + getDescriptor(language: DictionaryLanguage): DictionaryPackDescriptor | undefined; +} + +type DefinitionStateListener = (state: DefinitionState) => void; + +interface DictionaryError extends Error { + code?: string; +} + +export class DefinitionController { + private requestToken = 0; + private selectedText: string | null = null; + private readonly listeners = new Set(); + + state: DefinitionState = { kind: "idle" }; + + constructor(private readonly dependencies: DefinitionControllerDependencies) {} + + subscribe(listener: DefinitionStateListener): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + async open(text: string): Promise { + const token = ++this.requestToken; + this.selectedText = text; + await this.lookupSelection(text, token); + } + + async retry(): Promise { + if (!this.selectedText) return; + await this.open(this.selectedText); + } + + async download(): Promise { + if (this.state.kind !== "missing-pack" || !this.selectedText) return; + + const { descriptor, language } = this.state; + const text = this.selectedText; + const token = this.requestToken; + this.setState({ kind: "downloading", language, progress: 0 }); + + try { + await this.dependencies.install(descriptor, (progress) => { + if (this.isCurrent(token)) { + this.setState({ kind: "downloading", language, progress: clampProgress(progress) }); + } + }); + } catch (error) { + if (this.isCurrent(token)) this.setState({ kind: "error", message: messageOf(error) }); + return; + } + + if (this.isCurrent(token)) await this.lookupSelection(text, token); + } + + close(): void { + this.requestToken += 1; + this.selectedText = null; + this.setState({ kind: "idle" }); + } + + private async lookupSelection(text: string, token: number): Promise { + const selection = prepareDictionarySelection(text); + if (!selection.ok) { + if (this.isCurrent(token)) this.setState({ kind: "unsupported", reason: selection.reason }); + return; + } + + this.setState({ kind: "loading", displayText: selection.displayText }); + try { + const entries = await this.dependencies.lookup(text); + if (!this.isCurrent(token)) return; + this.setState( + entries.length > 0 + ? { kind: "result", displayText: selection.displayText, entries } + : { kind: "no-match", displayText: selection.displayText }, + ); + } catch (error) { + if (!this.isCurrent(token)) return; + const dictionaryError = error as DictionaryError; + if (dictionaryError.code === "pack-not-installed") { + const descriptor = this.dependencies.getDescriptor(selection.language); + if (descriptor) { + this.setState({ kind: "missing-pack", language: selection.language, descriptor }); + return; + } + } + if (dictionaryError.code === "unsupported-selection") { + this.setState({ kind: "unsupported", reason: "unsupported-selection" }); + return; + } + this.setState({ kind: "error", message: messageOf(error) }); + } + } + + private isCurrent(token: number): boolean { + return token === this.requestToken; + } + + private setState(state: DefinitionState): void { + this.state = state; + for (const listener of this.listeners) listener(state); + } +} + +function clampProgress(progress: number): number { + return Number.isFinite(progress) ? Math.max(0, Math.min(1, progress)) : 0; +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/app-expo/src/config/dictionary-config.test.ts b/packages/app-expo/src/config/dictionary-config.test.ts new file mode 100644 index 000000000..66f135376 --- /dev/null +++ b/packages/app-expo/src/config/dictionary-config.test.ts @@ -0,0 +1,75 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as dictionaryConfig from "./dictionary-config"; + +afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); +}); + +describe("dictionary configuration", () => { + it("allows a build-time manifest override", async () => { + vi.stubEnv("EXPO_PUBLIC_DICTIONARY_MANIFEST_URL", "https://example.com/dictionaries.json"); + vi.resetModules(); + const config = await import("./dictionary-config"); + expect(config.DICTIONARY_REMOTE_MANIFEST_URL).toBe("https://example.com/dictionaries.json"); + }); + + it("uses the official URL for an empty build-time override", async () => { + vi.stubEnv("EXPO_PUBLIC_DICTIONARY_MANIFEST_URL", " "); + vi.resetModules(); + const config = await import("./dictionary-config"); + expect(config.DICTIONARY_REMOTE_MANIFEST_URL).toBe( + "https://raw.githubusercontent.com/codedogQBY/ReadAny/main/dictionary-packs/manifest.json", + ); + }); + it("uses the official dictionary manifest URL", () => { + expect(dictionaryConfig.DICTIONARY_REMOTE_MANIFEST_URL).toBe( + "https://raw.githubusercontent.com/codedogQBY/ReadAny/main/dictionary-packs/manifest.json", + ); + }); + + it("exposes a parsed bundled manifest with the verified mixed sources", () => { + expect(dictionaryConfig).toHaveProperty("DICTIONARY_BUNDLED_MANIFEST"); + const bundled = Reflect.get(dictionaryConfig, "DICTIONARY_BUNDLED_MANIFEST"); + expect(bundled).toMatchObject({ + manifestVersion: 1, + packs: { + en: { + language: "en", + schemaVersion: 1, + sourceEdition: "wordnet-3.1", + license: "WordNet 3.1 License", + sourceArchiveUrl: "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + attributionUrl: "https://wordnet.princeton.edu/license-and-commercial-use", + }, + zh: { + language: "zh", + schemaVersion: 1, + sourceEdition: "zhwiktionary", + license: "CC BY-SA 4.0", + sourceArchiveUrl: + "https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2", + attributionUrl: + "https://zh.wiktionary.org/wiki/Wiktionary:%E7%89%88%E6%9D%83%E4%BF%A1%E6%81%AF", + }, + }, + }); + + for (const pack of Object.values(bundled.packs)) { + expect(pack.attributionUrl).not.toBe(pack.sourceArchiveUrl); + expect(pack.attributionUrl).not.toMatch(/\.(?:tar\.gz|xml\.bz2)$/u); + } + }); + + it("bundles bytes identical to the canonical release manifest", async () => { + const canonicalPath = resolve( + import.meta.dirname, + "../../../../dictionary-packs/manifest.json", + ); + const bundledPath = resolve(import.meta.dirname, "dictionary-manifest.json"); + + await expect(readFile(bundledPath)).resolves.toEqual(await readFile(canonicalPath)); + }); +}); diff --git a/packages/app-expo/src/config/dictionary-config.ts b/packages/app-expo/src/config/dictionary-config.ts new file mode 100644 index 000000000..d61592fc6 --- /dev/null +++ b/packages/app-expo/src/config/dictionary-config.ts @@ -0,0 +1,8 @@ +import { parseDictionaryManifest } from "@readany/core/dictionary"; +import bundledManifest from "./dictionary-manifest.json"; + +export const DICTIONARY_REMOTE_MANIFEST_URL = + process.env.EXPO_PUBLIC_DICTIONARY_MANIFEST_URL?.trim() || + "https://raw.githubusercontent.com/codedogQBY/ReadAny/main/dictionary-packs/manifest.json"; + +export const DICTIONARY_BUNDLED_MANIFEST = parseDictionaryManifest(bundledManifest); diff --git a/packages/app-expo/src/config/dictionary-manifest.json b/packages/app-expo/src/config/dictionary-manifest.json new file mode 100644 index 000000000..08150a75b --- /dev/null +++ b/packages/app-expo/src/config/dictionary-manifest.json @@ -0,0 +1,31 @@ +{ + "manifestVersion": 1, + "packs": { + "en": { + "language": "en", + "version": "1.0.0", + "schemaVersion": 1, + "sourceEdition": "wordnet-3.1", + "sourceDumpDate": "2011-05-26", + "sizeBytes": 29069312, + "sha256": "90adbeab5ee325b31f2e34bbfa7c5b699932c900bfd1efdfc62eabd9e004ee0e", + "url": "https://github.com/cha1latte/ReadAny/releases/download/dictionary-packs-v1/readany-dictionary-en-v1.sqlite", + "sourceArchiveUrl": "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + "attributionUrl": "https://wordnet.princeton.edu/license-and-commercial-use", + "license": "WordNet 3.1 License" + }, + "zh": { + "language": "zh", + "version": "1.0.0", + "schemaVersion": 1, + "sourceEdition": "zhwiktionary", + "sourceDumpDate": "2026-09-01", + "sizeBytes": 11866112, + "sha256": "d50218459f78a5e7bcba819fbb7db699271bdaf30757dc8b2443cb1e37bc11b1", + "url": "https://github.com/cha1latte/ReadAny/releases/download/dictionary-packs-v1/readany-dictionary-zh-v1.sqlite", + "sourceArchiveUrl": "https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2", + "attributionUrl": "https://zh.wiktionary.org/wiki/Wiktionary:%E7%89%88%E6%9D%83%E4%BF%A1%E6%81%AF", + "license": "CC BY-SA 4.0" + } + } +} diff --git a/packages/app-expo/src/lib/dictionary/dictionary-database.ts b/packages/app-expo/src/lib/dictionary/dictionary-database.ts new file mode 100644 index 000000000..692be58a9 --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-database.ts @@ -0,0 +1,81 @@ +import type { DictionaryLanguage } from "@readany/core/dictionary"; + +export interface DictionaryDatabaseConnection { + getAllAsync(sql: string, ...params: unknown[]): Promise; + closeAsync(): Promise; +} + +export interface DictionaryDatabaseAdapter { + open(language: DictionaryLanguage, absolutePath: string): Promise; +} + +export class DictionaryLookupError extends Error { + constructor( + readonly code: "unsupported-selection" | "pack-not-installed" | "pack-invalid", + message: string, + ) { + super(message); + this.name = "DictionaryLookupError"; + } +} + +interface DictionaryMetadataRow { + key: string; + value: string; +} + +function splitDatabasePath(absolutePath: string): { fileName: string; directory: string } { + const normalizedPath = absolutePath.replace(/[\\/]+$/, ""); + const separator = Math.max(normalizedPath.lastIndexOf("/"), normalizedPath.lastIndexOf("\\")); + if (separator < 0 || separator === normalizedPath.length - 1) { + throw new DictionaryLookupError("pack-invalid", "Dictionary pack path must include a filename"); + } + + return { + fileName: normalizedPath.slice(separator + 1), + directory: separator === 0 ? normalizedPath.slice(0, 1) : normalizedPath.slice(0, separator), + }; +} + +export class ExpoDictionaryDatabaseAdapter implements DictionaryDatabaseAdapter { + async open( + language: DictionaryLanguage, + absolutePath: string, + ): Promise { + const { fileName, directory } = splitDatabasePath(absolutePath); + const SQLite = await import("expo-sqlite"); + const database = await SQLite.openDatabaseAsync( + fileName, + { useNewConnection: true }, + directory, + ); + + try { + await database.execAsync("PRAGMA query_only = ON"); + const metadata = new Map( + ( + await database.getAllAsync( + "SELECT key, value FROM metadata WHERE key IN ('schema_version', 'language')", + ) + ).map((row) => [row.key, row.value]), + ); + if (metadata.get("schema_version") !== "1" || metadata.get("language") !== language) { + throw new Error("metadata did not match the expected schema version and language"); + } + } catch (error) { + try { + await database.closeAsync(); + } catch { + // The invalid handle cannot be reused; retain the original validation error. + } + const detail = error instanceof Error ? error.message : String(error); + throw new DictionaryLookupError("pack-invalid", `Dictionary pack is invalid: ${detail}`); + } + + return { + getAllAsync: async (sql: string, ...params: unknown[]) => + database.getAllAsync(sql, ...(params as string[])), + closeAsync: () => database.closeAsync(), + }; + } +} diff --git a/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.test.ts b/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.test.ts new file mode 100644 index 000000000..b74ec5e53 --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.test.ts @@ -0,0 +1,266 @@ +import type { DictionaryEntry, DictionaryLanguage } from "@readany/core/dictionary"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + type DictionaryDatabaseAdapter, + type DictionaryDatabaseConnection, + ExpoDictionaryDatabaseAdapter, +} from "./dictionary-database"; +import { DictionaryLookupService } from "./dictionary-lookup-service"; + +const expoSqlite = vi.hoisted(() => ({ + openDatabaseAsync: vi.fn(), +})); + +vi.mock("expo-sqlite", () => expoSqlite); + +interface FixtureEntry extends DictionaryEntry { + lookupKeys: string[]; +} + +function createFixtureAdapter( + fixtures: Partial>, +): DictionaryDatabaseAdapter & { + openCount: number; + closeCount: number; + openedPaths: unknown[]; + queries: Array<{ sql: string; params: unknown[] }>; +} { + let openCount = 0; + let closeCount = 0; + const queries: Array<{ sql: string; params: unknown[] }> = []; + const openedPaths: unknown[] = []; + + return { + get openCount() { + return openCount; + }, + get closeCount() { + return closeCount; + }, + openedPaths, + queries, + async open(language, absolutePath): Promise { + openCount += 1; + openedPaths.push(absolutePath); + return { + async getAllAsync(sql: string, ...params: unknown[]): Promise { + queries.push({ sql, params }); + const [lookupKey, selectedLanguage] = params as [string, DictionaryLanguage]; + return (fixtures[language] ?? []) + .filter( + (entry) => selectedLanguage === language && entry.lookupKeys.includes(lookupKey), + ) + .flatMap((entry) => { + const rank = entry.lookupKeys.indexOf(lookupKey); + return entry.senses.map((sense) => ({ + entry_id: entry.id, + language: entry.language, + headword: entry.headword, + simplified: entry.simplified ?? null, + traditional: entry.traditional ?? null, + pronunciation: entry.pronunciation ?? null, + part_of_speech: entry.partOfSpeech, + rank, + sense_order: sense.order, + definition: sense.definition, + })); + }) + .sort( + (left, right) => + left.rank - right.rank || + left.entry_id - right.entry_id || + left.sense_order - right.sense_order, + ) as T[]; + }, + async closeAsync(): Promise { + closeCount += 1; + }, + }; + }, + }; +} + +function createChineseFixtureAdapter() { + return createFixtureAdapter({ + zh: [ + { + id: 1, + language: "zh", + headword: "閱讀", + simplified: "阅读", + traditional: "閱讀", + partOfSpeech: "verb", + senses: [{ order: 0, definition: "看书。" }], + lookupKeys: ["閱讀", "阅读"], + }, + ], + }); +} + +describe("DictionaryLookupService", () => { + it("returns ordered English senses through a supplied inflection alias", async () => { + const database = createFixtureAdapter({ + en: [ + { + id: 2, + language: "en", + headword: "desire", + pronunciation: "/dɪˈzaɪəɹ/", + partOfSpeech: "noun", + senses: [ + { order: 1, definition: "A strong wish." }, + { order: 0, definition: "An object of longing." }, + ], + lookupKeys: ["desire", "desires"], + }, + ], + }); + const service = new DictionaryLookupService(database, () => "/dict/en.sqlite"); + + await expect(service.lookup("Desires")).resolves.toEqual([ + expect.objectContaining({ + headword: "desire", + senses: [ + { order: 0, definition: "An object of longing." }, + { order: 1, definition: "A strong wish." }, + ], + }), + ]); + expect(database.queries[0]).toMatchObject({ params: ["desires", "en"] }); + expect(database.queries[0]?.sql).toContain( + "ORDER BY matched.rank ASC, e.id ASC, s.sense_order ASC", + ); + expect(database.queries[0]?.sql).toContain("LIMIT 20"); + }); + + it("returns one Chinese entry through simplified and traditional aliases", async () => { + const service = new DictionaryLookupService( + createChineseFixtureAdapter(), + () => "/dict/zh.sqlite", + ); + + expect((await service.lookup("阅读"))[0]?.traditional).toBe("閱讀"); + expect((await service.lookup("閱讀"))[0]?.simplified).toBe("阅读"); + }); + + it("returns exact-match entries in rank and entry order", async () => { + const service = new DictionaryLookupService( + createFixtureAdapter({ + en: [ + { + id: 4, + language: "en", + headword: "read", + partOfSpeech: "verb", + senses: [{ order: 0, definition: "To examine writing." }], + lookupKeys: ["read", "reads"], + }, + { + id: 2, + language: "en", + headword: "read", + partOfSpeech: "verb", + senses: [{ order: 0, definition: "To interpret writing." }], + lookupKeys: ["read", "reads"], + }, + ], + }), + () => "/dict/en.sqlite", + ); + + await expect(service.lookup("reads")).resolves.toMatchObject([ + { id: 2, headword: "read" }, + { id: 4, headword: "read" }, + ]); + }); + + it("does not open a database for unsupported selection", async () => { + const adapter = createFixtureAdapter({}); + + await expect( + new DictionaryLookupService(adapter, () => null).lookup("read閱讀"), + ).rejects.toMatchObject({ code: "unsupported-selection" }); + expect(adapter.openCount).toBe(0); + }); + + it("reports a missing selected pack without opening a database", async () => { + const adapter = createFixtureAdapter({}); + + await expect( + new DictionaryLookupService(adapter, () => null).lookup("read"), + ).rejects.toMatchObject({ code: "pack-not-installed" }); + expect(adapter.openCount).toBe(0); + }); + + it("awaits an asynchronous installed-pack path resolver", async () => { + const adapter = createFixtureAdapter({ en: [] }); + const service = new DictionaryLookupService(adapter, async () => "/dict/en.sqlite"); + + await service.lookup("desire"); + + expect(adapter.openedPaths).toEqual(["/dict/en.sqlite"]); + }); + + it("caches one connection per language and releases it when closed", async () => { + const adapter = createFixtureAdapter({ en: [] }); + const service = new DictionaryLookupService(adapter, () => "/dict/en.sqlite"); + + await service.lookup("read"); + await service.lookup("reads"); + expect(adapter.openCount).toBe(1); + + await service.close("en"); + expect(adapter.closeCount).toBe(1); + await service.lookup("read"); + expect(adapter.openCount).toBe(2); + + await service.close(); + expect(adapter.closeCount).toBe(2); + }); +}); + +describe("ExpoDictionaryDatabaseAdapter", () => { + const connection = { + execAsync: vi.fn(), + getAllAsync: vi.fn(), + closeAsync: vi.fn(), + }; + + beforeEach(() => { + vi.clearAllMocks(); + expoSqlite.openDatabaseAsync.mockResolvedValue(connection); + connection.getAllAsync.mockResolvedValue([ + { key: "schema_version", value: "1" }, + { key: "language", value: "en" }, + ]); + }); + + it("opens a verified read-only Expo pack by filename and directory", async () => { + const adapter = new ExpoDictionaryDatabaseAdapter(); + + const opened = await adapter.open("en", "/dict/en.sqlite"); + + expect(expoSqlite.openDatabaseAsync).toHaveBeenCalledWith( + "en.sqlite", + { useNewConnection: true }, + "/dict", + ); + expect(connection.execAsync).toHaveBeenCalledWith("PRAGMA query_only = ON"); + expect(opened).toMatchObject({ + getAllAsync: expect.any(Function), + closeAsync: expect.any(Function), + }); + }); + + it("closes an invalid Expo pack before reporting it", async () => { + connection.getAllAsync.mockResolvedValue([ + { key: "schema_version", value: "1" }, + { key: "language", value: "zh" }, + ]); + + await expect( + new ExpoDictionaryDatabaseAdapter().open("en", "/dict/en.sqlite"), + ).rejects.toMatchObject({ code: "pack-invalid" }); + expect(connection.closeAsync).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.ts b/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.ts new file mode 100644 index 000000000..08d13d6ca --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.ts @@ -0,0 +1,163 @@ +import { + type DictionaryEntry, + type DictionaryLanguage, + prepareDictionarySelection, +} from "@readany/core/dictionary"; +import { + type DictionaryDatabaseAdapter, + type DictionaryDatabaseConnection, + DictionaryLookupError, +} from "./dictionary-database"; + +const LOOKUP_SQL = ` + WITH matched AS ( + SELECT lookup.entry_id, lookup.rank + FROM lookup + INNER JOIN entries ON entries.id = lookup.entry_id + WHERE lookup.lookup_key = ? AND entries.language = ? + ORDER BY lookup.rank ASC, lookup.entry_id ASC + LIMIT 20 + ) + SELECT + e.id AS entry_id, + e.language, + e.headword, + e.simplified, + e.traditional, + e.pronunciation, + e.part_of_speech, + matched.rank, + s.sense_order, + s.definition + FROM matched + INNER JOIN entries e ON e.id = matched.entry_id + INNER JOIN senses s ON s.entry_id = e.id + ORDER BY matched.rank ASC, e.id ASC, s.sense_order ASC +`; + +interface DictionaryLookupRow { + entry_id: number; + language: DictionaryLanguage; + headword: string; + simplified: string | null; + traditional: string | null; + pronunciation: string | null; + part_of_speech: string; + rank: number; + sense_order: number; + definition: string; +} + +export type DictionaryPackPathResolver = ( + language: DictionaryLanguage, +) => string | null | Promise; + +export type DictionaryPackInvalidator = (language: DictionaryLanguage) => Promise | void; + +export class DictionaryLookupService { + private readonly connections = new Map< + DictionaryLanguage, + Promise + >(); + + constructor( + private readonly database: DictionaryDatabaseAdapter, + private readonly resolvePackPath: DictionaryPackPathResolver, + private readonly invalidatePack: DictionaryPackInvalidator = () => {}, + ) {} + + async lookup(text: string): Promise { + const selection = prepareDictionarySelection(text); + if (!selection.ok) { + throw new DictionaryLookupError( + "unsupported-selection", + `Dictionary lookup does not support this selection: ${selection.reason}`, + ); + } + + const absolutePath = await this.resolvePackPath(selection.language); + if (!absolutePath) { + throw new DictionaryLookupError( + "pack-not-installed", + `The ${selection.language} dictionary pack is not installed`, + ); + } + + try { + const database = await this.connectionFor(selection.language, absolutePath); + const rows = await database.getAllAsync( + LOOKUP_SQL, + selection.key, + selection.language, + ); + return this.entriesFromRows(rows); + } catch (error) { + const cleanupErrors: unknown[] = []; + try { + await this.close(selection.language); + } catch (cleanupError) { + cleanupErrors.push(cleanupError); + } + try { + await this.invalidatePack(selection.language); + } catch (cleanupError) { + cleanupErrors.push(cleanupError); + } + if (error instanceof Error && cleanupErrors.length > 0) { + (error as Error & { cleanupErrors?: unknown[] }).cleanupErrors = cleanupErrors; + } + throw error; + } + } + + async close(language?: DictionaryLanguage): Promise { + const languages = language ? [language] : [...this.connections.keys()]; + await Promise.all( + languages.map(async (currentLanguage) => { + const connection = this.connections.get(currentLanguage); + if (!connection) return; + this.connections.delete(currentLanguage); + await (await connection).closeAsync(); + }), + ); + } + + private connectionFor( + language: DictionaryLanguage, + absolutePath: string, + ): Promise { + const existing = this.connections.get(language); + if (existing) return existing; + + const opening = this.database.open(language, absolutePath); + this.connections.set(language, opening); + void opening.catch(() => { + if (this.connections.get(language) === opening) { + this.connections.delete(language); + } + }); + return opening; + } + + private entriesFromRows(rows: DictionaryLookupRow[]): DictionaryEntry[] { + const entries = new Map(); + for (const row of rows) { + let entry = entries.get(row.entry_id); + if (!entry) { + entry = { + id: row.entry_id, + language: row.language, + headword: row.headword, + simplified: row.simplified ?? undefined, + traditional: row.traditional ?? undefined, + pronunciation: row.pronunciation ?? undefined, + partOfSpeech: row.part_of_speech, + senses: [], + }; + entries.set(row.entry_id, entry); + } + entry.senses.push({ order: row.sense_order, definition: row.definition }); + } + return [...entries.values()]; + } +} diff --git a/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.test.ts b/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.test.ts new file mode 100644 index 000000000..b66083ea7 --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.test.ts @@ -0,0 +1,613 @@ +import type { + ChineseDictionaryPackDescriptor, + DictionaryLanguage, + DictionaryManifest, + DictionaryPackDescriptor, + EnglishDictionaryPackDescriptor, +} from "@readany/core/dictionary"; +import { describe, expect, it, vi } from "vitest"; +import { + DictionaryPackManager, + type DictionaryPackMetadata, + type DictionaryPackPlatform, + type DictionaryPackStatus, +} from "./dictionary-pack-manager"; + +const enV1 = descriptor("en", "2026.08", "a"); +const enV2 = descriptor("en", "2026.09", "b"); +const zhV1 = descriptor("zh", "2026.09", "c"); + +function descriptor( + language: "en", + version: string, + hashCharacter: string, +): EnglishDictionaryPackDescriptor; +function descriptor( + language: "zh", + version: string, + hashCharacter: string, +): ChineseDictionaryPackDescriptor; +function descriptor( + language: DictionaryLanguage, + version: string, + hashCharacter: string, +): DictionaryPackDescriptor { + const shared = { + version, + schemaVersion: 1 as const, + sourceDumpDate: "2026-09-01", + sha256: hashCharacter.repeat(64), + license: "CC BY-SA 4.0" as const, + }; + return language === "en" + ? { + ...shared, + language, + sourceEdition: "enwiktionary", + sizeBytes: 123, + url: `https://example.test/en-${version}.sqlite`, + sourceArchiveUrl: `https://example.test/en-${version}.tar.gz`, + attributionUrl: "https://en.wiktionary.org/wiki/Wiktionary:Copyrights", + } + : { + ...shared, + language, + sourceEdition: "zhwiktionary", + sizeBytes: 234, + url: `https://example.test/zh-${version}.sqlite`, + sourceArchiveUrl: `https://example.test/zh-${version}.xml.bz2`, + attributionUrl: "https://zh.wiktionary.org/wiki/Wiktionary:Copyrights", + }; +} + +function metadataOf(pack: DictionaryPackDescriptor): DictionaryPackMetadata { + const { + language, + version, + schemaVersion, + sourceEdition, + sourceDumpDate, + sourceArchiveUrl, + url, + attributionUrl, + license, + } = pack; + return { + language, + version, + schemaVersion, + sourceEdition, + sourceDumpDate, + sourceArchiveUrl, + url, + attributionUrl, + license, + licenseNotice: `Complete ${license} notice.`, + creatorAttribution: `${sourceEdition} contributors.`, + }; +} + +interface MemoryFile { + id: string; + content: string; + size: number; + hash: string; + metadata: DictionaryPackMetadata; + validSchema: boolean; +} + +interface Deferred { + promise: Promise; + resolve(): void; +} + +function deferred(): Deferred { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +class StatefulMemoryPlatform implements DictionaryPackPlatform { + readonly root = "/docs/dictionaries"; + readonly files = new Map(); + readonly events: string[] = []; + readonly downloads = new Map([ + [enV1.url, enV1], + [enV2.url, enV2], + [zhV1.url, zhV1], + ]); + readonly downloadBarriers = new Map(); + readonly removeFaults = new Map(); + readonly metadataFaults: Array<{ path: string; id?: string; error: Error }> = []; + downloadCalls = 0; + downloadMutation?: (file: MemoryFile) => void; + downloadFailure?: Error; + + async ensureDirectory(): Promise { + this.events.push("ensure-directory"); + } + + async download(url: string, path: string, onProgress: (fraction: number) => void): Promise { + this.downloadCalls += 1; + this.events.push(`download-start:${url}`); + onProgress(0.25); + await this.downloadBarriers.get(url)?.promise; + const pack = this.downloads.get(url); + if (!pack) throw new Error(`No test pack for ${url}`); + const file = this.file(pack, `download-${this.downloadCalls}`, `content:${pack.version}`); + this.downloadMutation?.(file); + this.files.set(path, file); + if (this.downloadFailure) throw this.downloadFailure; + onProgress(1); + this.events.push(`download-complete:${url}`); + } + + async exists(path: string): Promise { + return this.files.has(path); + } + + async size(path: string): Promise { + return this.requiredFile(path).size; + } + + async sha256(path: string): Promise { + return this.requiredFile(path).hash; + } + + async readMetadata(path: string): Promise { + const file = this.requiredFile(path); + this.events.push(`metadata:${path}:${file.id}`); + const faultIndex = this.metadataFaults.findIndex( + (fault) => fault.path === path && (!fault.id || fault.id === file.id), + ); + if (faultIndex >= 0) throw this.metadataFaults.splice(faultIndex, 1)[0].error; + if (!file.validSchema) throw new Error(`invalid schema: ${file.id}`); + return { ...file.metadata }; + } + + async move(from: string, to: string): Promise { + this.events.push(`move:${from}->${to}`); + if (this.files.has(to)) throw new Error(`destination exists: ${to}`); + const file = this.requiredFile(from); + this.files.delete(from); + this.files.set(to, file); + } + + async remove(path: string): Promise { + this.events.push(`remove:${path}`); + const faults = this.removeFaults.get(path); + const fault = faults?.shift(); + if (fault) throw fault; + this.files.delete(path); + } + + put(path: string, pack: DictionaryPackDescriptor, id: string, content = `content:${id}`): void { + this.files.set(path, this.file(pack, id, content)); + } + + pauseDownload(url: string): () => void { + const barrier = deferred(); + this.downloadBarriers.set(url, barrier); + return barrier.resolve; + } + + failMetadataOnce(path: string, error: Error, id?: string): void { + this.metadataFaults.push({ path, id, error }); + } + + private file(pack: DictionaryPackDescriptor, id: string, content: string): MemoryFile { + return { + id, + content, + size: pack.sizeBytes, + hash: pack.sha256, + metadata: metadataOf(pack), + validSchema: true, + }; + } + + private requiredFile(path: string): MemoryFile { + const file = this.files.get(path); + if (!file) throw new Error(`missing ${path}`); + return file; + } +} + +function paths(platform: StatefulMemoryPlatform, language: DictionaryLanguage = "en") { + const active = `${platform.root}/readany-dictionary-${language}.sqlite`; + return { active, staged: `${active}.download`, backup: `${active}.backup` }; +} + +function closer(platform: StatefulMemoryPlatform) { + return { + close: vi.fn(async (language?: DictionaryLanguage) => { + platform.events.push(`close:${language ?? "all"}`); + }), + }; +} + +function manifest(en = enV2): DictionaryManifest { + return { manifestVersion: 1, packs: { en, zh: zhV1 } }; +} + +describe("DictionaryPackManager", () => { + it("activates the exact validated staged file", async () => { + const platform = new StatefulMemoryPlatform(); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await manager.install(enV2); + + const activeFile = platform.files.get(paths(platform).active); + expect(activeFile).toMatchObject({ id: "download-1", content: "content:2026.09" }); + expect(await manager.getInstalledDescriptor("en")).toEqual({ + ...metadataOf(enV2), + sizeBytes: enV2.sizeBytes, + sha256: enV2.sha256, + }); + }); + + it.each([ + [ + "size", + (file: MemoryFile): void => { + file.size = 1; + }, + ], + [ + "hash", + (file: MemoryFile): void => { + file.hash = "d".repeat(64); + }, + ], + [ + "schema", + (file: MemoryFile): void => { + file.validSchema = false; + }, + ], + ] as const)("retains the exact active file on staged %s mismatch", async (_kind, mutate) => { + const platform = new StatefulMemoryPlatform(); + const { active, staged } = paths(platform); + platform.put(active, enV1, "old", "trusted old bytes"); + platform.downloadMutation = mutate; + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.install(enV2)).rejects.toThrow(); + + expect(platform.files.get(active)).toMatchObject({ id: "old", content: "trusted old bytes" }); + expect(platform.files.has(staged)).toBe(false); + }); + + it("removes a partial staged file after interrupted download and permits retry", async () => { + const platform = new StatefulMemoryPlatform(); + const { active, staged } = paths(platform); + platform.put(active, enV1, "old", "trusted old bytes"); + platform.downloadMutation = (file) => { + file.id = "partial"; + file.content = "partial bytes"; + }; + platform.downloadFailure = new Error("interrupted"); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.install(enV2)).rejects.toThrow("interrupted"); + expect(platform.files.has(staged)).toBe(false); + expect(platform.files.get(active)?.content).toBe("trusted old bytes"); + + platform.downloadMutation = undefined; + platform.downloadFailure = undefined; + await manager.install(enV2); + expect(platform.files.get(active)?.content).toBe("content:2026.09"); + }); + + it("uses but does not move or delete the last-known-good backup during read-only discovery", async () => { + const platform = new StatefulMemoryPlatform(); + const { active, backup } = paths(platform); + platform.put(active, enV2, "uncommitted-new", "new bytes"); + platform.put(backup, enV1, "trusted-backup", "trusted backup bytes"); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + expect(await manager.getActivePath("en")).toBe(backup); + + expect(platform.files.get(active)).toMatchObject({ + id: "uncommitted-new", + content: "new bytes", + }); + expect(platform.files.get(backup)).toMatchObject({ + id: "trusted-backup", + content: "trusted backup bytes", + }); + expect(platform.events).not.toContain(`remove:${active}`); + expect(platform.events).not.toContain(`move:${backup}->${active}`); + }); + + it("preserves staged and corrupt backup artifacts during read-only discovery", async () => { + const platform = new StatefulMemoryPlatform(); + const { active, staged, backup } = paths(platform); + platform.put(active, enV1, "trusted-active"); + platform.put(staged, enV2, "interrupted-download"); + platform.put(backup, enV2, "corrupt-backup"); + const corruptBackup = platform.files.get(backup); + if (corruptBackup) corruptBackup.validSchema = false; + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.refresh(manifest())).resolves.toMatchObject({ + en: { state: "update-available", installedVersion: enV1.version }, + }); + + expect(platform.files.get(active)?.id).toBe("trusted-active"); + expect(platform.files.get(staged)?.id).toBe("interrupted-download"); + expect(platform.files.get(backup)?.id).toBe("corrupt-backup"); + expect(platform.events.filter((event) => event.startsWith("remove:"))).toEqual([]); + }); + + it.each([ + ["a corrupt backup with no active", false], + ["corrupt active and backup files", true], + ])("repairs %s only during an explicit install", async (_name, includeActive) => { + const platform = new StatefulMemoryPlatform(); + const { active, backup } = paths(platform); + platform.put(backup, enV1, "corrupt-backup"); + const corruptBackup = platform.files.get(backup); + if (corruptBackup) corruptBackup.validSchema = false; + if (includeActive) { + platform.put(active, enV1, "corrupt-active"); + const corruptActive = platform.files.get(active); + if (corruptActive) corruptActive.validSchema = false; + } + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.refresh(manifest())).resolves.toMatchObject({ + en: { state: "error", hasActivePack: true }, + }); + expect(platform.files.get(backup)?.id).toBe("corrupt-backup"); + if (includeActive) expect(platform.files.get(active)?.id).toBe("corrupt-active"); + + await expect(manager.install(enV2)).resolves.toBeUndefined(); + expect(platform.files.get(active)?.id).toBe("download-1"); + expect(platform.files.has(backup)).toBe(false); + }); + + it("keeps repair available when a retry fails with only a corrupt backup", async () => { + const platform = new StatefulMemoryPlatform(); + const { backup } = paths(platform); + platform.put(backup, enV1, "corrupt-backup"); + const corruptBackup = platform.files.get(backup); + if (corruptBackup) corruptBackup.validSchema = false; + platform.downloadFailure = new Error("offline"); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + const statuses: DictionaryPackStatus[] = []; + + await expect(manager.install(enV2, (status) => statuses.push(status))).rejects.toThrow( + "offline", + ); + + expect(statuses.at(-1)).toEqual({ + state: "error", + message: "offline", + hasActivePack: true, + }); + expect(platform.files.get(backup)?.id).toBe("corrupt-backup"); + }); + + it("discovers installed metadata from SQLite after restart and detects updates", async () => { + const platform = new StatefulMemoryPlatform(); + platform.put(paths(platform).active, enV1, "persisted", "persisted bytes"); + const restartedManager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + expect(await restartedManager.getInstalledDescriptor("en")).toEqual({ + ...metadataOf(enV1), + sizeBytes: enV1.sizeBytes, + sha256: enV1.sha256, + }); + await expect(restartedManager.refresh(manifest())).resolves.toMatchObject({ + en: { + state: "update-available", + installedVersion: enV1.version, + availableVersion: enV2.version, + sizeBytes: enV1.sizeBytes, + }, + }); + }); + + it("offers repair when the manifest has the same version but a different full identity", async () => { + const platform = new StatefulMemoryPlatform(); + platform.put(paths(platform).active, enV2, "old-same-version"); + const replacement = { + ...enV2, + sizeBytes: 124, + sha256: "d".repeat(64), + url: "https://example.test/en-2026.09-repacked.sqlite", + sourceArchiveUrl: "https://example.test/en-2026.09-repacked.tar.gz", + attributionUrl: "https://example.test/en-2026.09-attribution", + }; + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.refresh(manifest(replacement))).resolves.toMatchObject({ + en: { + state: "update-available", + installedVersion: enV2.version, + availableVersion: replacement.version, + }, + }); + }); + + it("keeps a valid manifest usable when one installed language is corrupt", async () => { + const platform = new StatefulMemoryPlatform(); + const enPath = paths(platform, "en").active; + const zhPath = paths(platform, "zh").active; + platform.put(enPath, enV1, "corrupt-en"); + const corruptEnglish = platform.files.get(enPath); + if (corruptEnglish) corruptEnglish.validSchema = false; + platform.put(zhPath, zhV1, "healthy-zh"); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.refresh(manifest())).resolves.toMatchObject({ + en: { state: "error", hasActivePack: true }, + zh: { state: "installed", version: zhV1.version }, + }); + expect(platform.files.get(enPath)?.id).toBe("corrupt-en"); + expect(await manager.getInstalledDescriptor("zh")).toMatchObject({ version: zhV1.version }); + }); + + it("rolls back using the backup's own metadata after final validation fails", async () => { + const platform = new StatefulMemoryPlatform(); + const { active, backup } = paths(platform); + platform.put(active, enV1, "old", "trusted old bytes"); + const activationError = new Error("post-activation validation failed"); + platform.failMetadataOnce(active, activationError, "download-1"); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.install(enV2)).rejects.toBe(activationError); + + expect(platform.events).toContain(`metadata:${backup}:old`); + expect(platform.files.get(active)).toMatchObject({ id: "old", content: "trusted old bytes" }); + expect(await manager.getInstalledDescriptor("en")).toMatchObject({ version: enV1.version }); + }); + + it("removes a broken active when final validation fails without a previous pack", async () => { + const platform = new StatefulMemoryPlatform(); + const { active, staged } = paths(platform); + platform.failMetadataOnce(active, new Error("post-activation validation failed"), "download-1"); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.install(enV2)).rejects.toThrow("post-activation validation failed"); + + expect(platform.files.has(active)).toBe(false); + expect(platform.files.has(staged)).toBe(false); + }); + + it("preserves the operation error and records cleanup failure", async () => { + const platform = new StatefulMemoryPlatform(); + const { active } = paths(platform); + const operationError = new Error("post-activation validation failed"); + const cleanupError = new Error("cannot remove broken active"); + platform.failMetadataOnce(active, operationError, "download-1"); + platform.removeFaults.set(active, [cleanupError]); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + let thrown: unknown; + try { + await manager.install(enV2); + } catch (error) { + thrown = error; + } + + expect(thrown).toBe(operationError); + expect((thrown as Error & { cleanupErrors?: unknown[] }).cleanupErrors).toContain(cleanupError); + }); + + it("coalesces concurrent installs for the same language", async () => { + const platform = new StatefulMemoryPlatform(); + const release = platform.pauseDownload(enV2.url); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + const first = manager.install(enV2); + const second = manager.install(enV2); + await vi.waitFor(() => expect(platform.downloadCalls).toBe(1)); + release(); + await Promise.all([first, second]); + + expect(platform.downloadCalls).toBe(1); + }); + + it("allows different languages to install in parallel", async () => { + const platform = new StatefulMemoryPlatform(); + const releaseEnglish = platform.pauseDownload(enV2.url); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + let englishFinished = false; + + const english = manager.install(enV2).then(() => { + englishFinished = true; + }); + await vi.waitFor(() => expect(platform.downloadCalls).toBe(1)); + + await manager.install(zhV1); + expect(englishFinished).toBe(false); + expect(platform.files.get(paths(platform, "zh").active)?.content).toBe("content:2026.09"); + + releaseEnglish(); + await english; + }); + + it("serializes refresh behind an in-flight install for the same language", async () => { + const platform = new StatefulMemoryPlatform(); + const release = platform.pauseDownload(enV2.url); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + const install = manager.install(enV2); + await vi.waitFor(() => expect(platform.downloadCalls).toBe(1)); + let refreshFinished = false; + + const refresh = manager.refresh(manifest()).then((status) => { + refreshFinished = true; + return status; + }); + await Promise.resolve(); + expect(refreshFinished).toBe(false); + + release(); + await install; + await expect(refresh).resolves.toMatchObject({ + en: { state: "installed", version: enV2.version }, + }); + }); + + it("serializes removal behind an in-flight install and leaves no pack artifacts", async () => { + const platform = new StatefulMemoryPlatform(); + const release = platform.pauseDownload(enV2.url); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + const install = manager.install(enV2); + await vi.waitFor(() => expect(platform.downloadCalls).toBe(1)); + let removeFinished = false; + + const remove = manager.remove("en").then(() => { + removeFinished = true; + }); + await Promise.resolve(); + expect(removeFinished).toBe(false); + + release(); + await install; + await remove; + expect(Object.values(paths(platform)).some((path) => platform.files.has(path))).toBe(false); + }); + + it("closes before moving an active pack or deleting one", async () => { + const platform = new StatefulMemoryPlatform(); + const { active, backup } = paths(platform); + platform.put(active, enV1, "old"); + const lookup = closer(platform); + const manager = new DictionaryPackManager(platform, platform.root, lookup); + + await manager.install(enV2); + const closeBeforeMove = platform.events.indexOf("close:en"); + expect(closeBeforeMove).toBeGreaterThanOrEqual(0); + expect(closeBeforeMove).toBeLessThan(platform.events.indexOf(`move:${active}->${backup}`)); + + platform.events.length = 0; + await manager.remove("en"); + expect(platform.events.indexOf("close:en")).toBeLessThan( + platform.events.indexOf(`remove:${active}`), + ); + }); + + it("removes only the requested language", async () => { + const platform = new StatefulMemoryPlatform(); + const enPaths = paths(platform, "en"); + const zhPaths = paths(platform, "zh"); + platform.put(enPaths.active, enV1, "english", "english bytes"); + platform.put(zhPaths.active, zhV1, "chinese", "chinese bytes"); + const lookup = closer(platform); + const manager = new DictionaryPackManager(platform, platform.root, lookup); + + await manager.remove("en"); + + expect(lookup.close).toHaveBeenCalledWith("en"); + expect(platform.files.has(enPaths.active)).toBe(false); + expect(platform.files.get(zhPaths.active)).toMatchObject({ + id: "chinese", + content: "chinese bytes", + }); + }); +}); diff --git a/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.ts b/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.ts new file mode 100644 index 000000000..575a846a8 --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.ts @@ -0,0 +1,436 @@ +import type { + DictionaryLanguage, + DictionaryManifest, + DictionaryPackDescriptor, +} from "@readany/core/dictionary"; + +const dictionaryLanguages = ["en", "zh"] as const; + +export type DictionaryPackMetadata = Pick< + DictionaryPackDescriptor, + | "language" + | "version" + | "schemaVersion" + | "sourceEdition" + | "sourceDumpDate" + | "sourceArchiveUrl" + | "url" + | "attributionUrl" + | "license" +> & { + licenseNotice: string; + creatorAttribution: string; +}; + +export type InstalledDictionaryPack = DictionaryPackMetadata & + Pick; + +export interface DictionaryPackPlatform { + ensureDirectory(path: string): Promise; + download(url: string, path: string, onProgress: (fraction: number) => void): Promise; + exists(path: string): Promise; + size(path: string): Promise; + sha256(path: string): Promise; + readMetadata(path: string): Promise; + move(from: string, to: string): Promise; + remove(path: string): Promise; +} + +export type DictionaryPackStatus = + | { state: "not-installed" } + | { state: "downloading"; progress: number } + | { state: "installed"; version: string; sizeBytes: number } + | { + state: "update-available"; + installedVersion: string; + availableVersion: string; + sizeBytes: number; + } + | { state: "error"; message: string; hasActivePack: boolean }; + +export interface DictionaryHandleCloser { + close(language?: DictionaryLanguage): Promise | void; +} + +type ErrorWithCleanup = Error & { cleanupErrors?: unknown[] }; + +interface InspectedArtifact { + path: string; + exists: boolean; + installed?: InstalledDictionaryPack; + error?: unknown; +} + +interface InspectedArtifacts { + active: InspectedArtifact; + backup: InspectedArtifact; +} + +export class DictionaryPackManager { + private readonly installs = new Map>(); + private readonly gates = new Map>(); + private readonly validatedPacks = new Map< + DictionaryLanguage, + { path: string; installed: InstalledDictionaryPack } | null + >(); + + constructor( + private readonly platform: DictionaryPackPlatform, + private readonly directory: string, + private readonly lookup: DictionaryHandleCloser, + ) {} + + async refresh( + manifest: DictionaryManifest, + ): Promise> { + const entries = await Promise.all( + dictionaryLanguages.map((language) => + this.runExclusive(language, async () => { + try { + const discovered = await this.discoverInstalledLocked(language); + const installed = discovered?.installed; + const available = manifest.packs[language]; + const status: DictionaryPackStatus = !installed + ? { state: "not-installed" } + : sameDescriptorIdentity(installed, available) + ? { state: "installed", version: installed.version, sizeBytes: installed.sizeBytes } + : { + state: "update-available", + installedVersion: installed.version, + availableVersion: available.version, + sizeBytes: installed.sizeBytes, + }; + return [language, status] as const; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const hasActivePack = await this.hasPackArtifacts(language); + return [ + language, + { state: "error", message, hasActivePack } satisfies DictionaryPackStatus, + ] as const; + } + }), + ), + ); + return Object.fromEntries(entries) as Record; + } + + install( + descriptor: DictionaryPackDescriptor, + onStatus: (status: DictionaryPackStatus) => void = () => {}, + ): Promise { + const language = descriptor.language; + const existing = this.installs.get(language); + if (existing) return existing; + + const operation = this.runExclusive(language, () => + this.installLocked(descriptor, onStatus), + ).finally(() => { + if (this.installs.get(language) === operation) this.installs.delete(language); + }); + this.installs.set(language, operation); + return operation; + } + + remove(language: DictionaryLanguage): Promise { + return this.runExclusive(language, async () => { + this.invalidate(language); + await this.platform.ensureDirectory(this.directory); + await this.lookup.close(language); + await this.removeIfPresent(this.activePath(language)); + await this.removeIfPresent(this.stagedPath(language)); + await this.removeIfPresent(this.backupPath(language)); + }); + } + + getActivePath(language: DictionaryLanguage): Promise { + return this.runExclusive(language, async () => { + const discovered = await this.discoverInstalledLocked(language); + return discovered?.path ?? null; + }); + } + + getInstalledDescriptor(language: DictionaryLanguage): Promise { + return this.runExclusive(language, async () => { + return (await this.discoverInstalledLocked(language))?.installed ?? null; + }); + } + + invalidate(language: DictionaryLanguage): void { + this.validatedPacks.delete(language); + } + + private async installLocked( + descriptor: DictionaryPackDescriptor, + onStatus: (status: DictionaryPackStatus) => void, + ): Promise { + const language = descriptor.language; + this.invalidate(language); + const staged = this.stagedPath(language); + const active = this.activePath(language); + const backup = this.backupPath(language); + try { + await this.platform.ensureDirectory(this.directory); + await this.removeIfPresent(staged); + const artifacts = await this.inspectArtifactsLocked(language); + onStatus({ state: "downloading", progress: 0 }); + await this.platform.download(descriptor.url, staged, (progress) => + onStatus({ state: "downloading", progress: clamp(progress) }), + ); + await this.verifyExpected(staged, descriptor); + await this.lookup.close(language); + + let hasRollbackBackup = Boolean(artifacts.backup.installed); + let activationStarted = false; + try { + if (artifacts.backup.installed) { + activationStarted = true; + await this.removeIfPresent(active); + } else if (artifacts.active.installed) { + await this.removeIfPresent(backup); + await this.platform.move(active, backup); + hasRollbackBackup = true; + activationStarted = true; + } else { + await this.removeIfPresent(active); + await this.removeIfPresent(backup); + activationStarted = true; + } + await this.platform.move(staged, active); + const installed = await this.verifyExpected(active, descriptor); + await this.removeIfPresent(backup); + this.validatedPacks.set(language, { path: active, installed }); + } catch (operationError) { + const cleanupErrors: unknown[] = []; + if (hasRollbackBackup) { + await this.captureCleanupFailure(cleanupErrors, () => + this.rollbackLocked(active, backup), + ); + } else if (activationStarted) { + await this.captureCleanupFailure(cleanupErrors, () => this.removeIfPresent(active)); + } + attachCleanupErrors(operationError, cleanupErrors); + throw operationError; + } + + onStatus({ + state: "installed", + version: descriptor.version, + sizeBytes: descriptor.sizeBytes, + }); + } catch (operationError) { + const cleanupErrors: unknown[] = []; + await this.captureCleanupFailure(cleanupErrors, () => this.removeIfPresent(staged)); + attachCleanupErrors(operationError, cleanupErrors); + const message = + operationError instanceof Error ? operationError.message : String(operationError); + onStatus({ + state: "error", + message, + hasActivePack: await this.hasPackArtifacts(language), + }); + throw operationError; + } + } + + private async verifyExpected( + path: string, + descriptor: DictionaryPackDescriptor, + ): Promise { + const installed = await this.inspectPack(path); + if (installed.sizeBytes !== descriptor.sizeBytes) + throw new Error("Dictionary pack size did not match manifest"); + if (installed.sha256.toLowerCase() !== descriptor.sha256.toLowerCase()) + throw new Error("Dictionary pack checksum did not match manifest"); + assertMetadataMatches(installed, descriptor); + return installed; + } + + private async rollbackLocked(active: string, backup: string): Promise { + if (!(await this.platform.exists(backup))) return; + const backupSnapshot = await this.inspectPack(backup); + await this.removeIfPresent(active); + await this.platform.move(backup, active); + const restoredSnapshot = await this.inspectPack(active); + assertSamePack(backupSnapshot, restoredSnapshot); + this.validatedPacks.set(restoredSnapshot.language, { + path: active, + installed: restoredSnapshot, + }); + } + + private async discoverInstalledLocked( + language: DictionaryLanguage, + ): Promise<{ path: string; installed: InstalledDictionaryPack } | null> { + if (this.validatedPacks.has(language)) return this.validatedPacks.get(language) ?? null; + await this.platform.ensureDirectory(this.directory); + const artifacts = await this.inspectArtifactsLocked(language); + if (artifacts.backup.installed) { + const discovered = { path: artifacts.backup.path, installed: artifacts.backup.installed }; + this.validatedPacks.set(language, discovered); + return discovered; + } + if (artifacts.active.installed) { + const discovered = { path: artifacts.active.path, installed: artifacts.active.installed }; + this.validatedPacks.set(language, discovered); + return discovered; + } + + const errors = [artifacts.backup.error, artifacts.active.error].filter( + (error): error is NonNullable => error !== undefined, + ); + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) { + throw new AggregateError(errors, `No valid ${language} dictionary recovery artifact exists`); + } + this.validatedPacks.set(language, null); + return null; + } + + private async inspectArtifactsLocked(language: DictionaryLanguage): Promise { + const [active, backup] = await Promise.all([ + this.inspectArtifact(this.activePath(language), language), + this.inspectArtifact(this.backupPath(language), language), + ]); + return { active, backup }; + } + + private async inspectArtifact( + path: string, + language: DictionaryLanguage, + ): Promise { + if (!(await this.platform.exists(path))) return { path, exists: false }; + try { + const installed = await this.inspectPack(path); + if (installed.language !== language) { + throw new Error(`Dictionary metadata language did not match ${language}`); + } + return { path, exists: true, installed }; + } catch (error) { + return { path, exists: true, error }; + } + } + + private async hasPackArtifacts(language: DictionaryLanguage): Promise { + return ( + (await this.platform.exists(this.activePath(language))) || + (await this.platform.exists(this.backupPath(language))) || + (await this.platform.exists(this.stagedPath(language))) + ); + } + + private async inspectPack(path: string): Promise { + const [metadata, sizeBytes, sha256] = await Promise.all([ + this.platform.readMetadata(path), + this.platform.size(path), + this.platform.sha256(path), + ]); + return { ...metadata, sizeBytes, sha256: sha256.toLowerCase() }; + } + + private runExclusive(language: DictionaryLanguage, action: () => Promise): Promise { + const previous = this.gates.get(language) ?? Promise.resolve(); + const operation = previous.catch(() => undefined).then(action); + const tail = operation.then( + () => undefined, + () => undefined, + ); + this.gates.set(language, tail); + void tail.then(() => { + if (this.gates.get(language) === tail) this.gates.delete(language); + }); + return operation; + } + + private async captureCleanupFailure( + errors: unknown[], + cleanup: () => Promise, + ): Promise { + try { + await cleanup(); + } catch (error) { + errors.push(error); + } + } + + private activePath(language: DictionaryLanguage): string { + return `${this.directory}/readany-dictionary-${language}.sqlite`; + } + + private stagedPath(language: DictionaryLanguage): string { + return `${this.activePath(language)}.download`; + } + + private backupPath(language: DictionaryLanguage): string { + return `${this.activePath(language)}.backup`; + } + + private async removeIfPresent(path: string): Promise { + if (await this.platform.exists(path)) await this.platform.remove(path); + } +} + +function assertMetadataMatches( + installed: DictionaryPackMetadata, + expected: DictionaryPackDescriptor, +): void { + for (const key of [ + "language", + "version", + "schemaVersion", + "sourceEdition", + "sourceDumpDate", + "sourceArchiveUrl", + "url", + "attributionUrl", + "license", + ] as const) { + if (installed[key] !== expected[key]) + throw new Error(`Dictionary metadata ${key} did not match manifest`); + } +} + +function sameDescriptorIdentity( + installed: InstalledDictionaryPack, + available: DictionaryPackDescriptor, +): boolean { + return ( + installed.sizeBytes === available.sizeBytes && + installed.sha256.toLowerCase() === available.sha256.toLowerCase() && + [ + "language", + "version", + "schemaVersion", + "sourceEdition", + "sourceDumpDate", + "sourceArchiveUrl", + "url", + "attributionUrl", + "license", + ].every( + (key) => + installed[key as keyof InstalledDictionaryPack] === + available[key as keyof DictionaryPackDescriptor], + ) + ); +} + +function assertSamePack(expected: InstalledDictionaryPack, actual: InstalledDictionaryPack): void { + if ( + expected.sizeBytes !== actual.sizeBytes || + expected.sha256 !== actual.sha256 || + JSON.stringify(expected) !== JSON.stringify(actual) + ) { + throw new Error("Restored dictionary pack did not match the validated backup"); + } +} + +function attachCleanupErrors(operationError: unknown, cleanupErrors: unknown[]): void { + if (cleanupErrors.length === 0 || !(operationError instanceof Error)) return; + const error = operationError as ErrorWithCleanup; + error.cleanupErrors = [...(error.cleanupErrors ?? []), ...cleanupErrors]; +} + +function clamp(value: number): number { + return Number.isFinite(value) ? Math.max(0, Math.min(1, value)) : 0; +} diff --git a/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.test.ts b/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.test.ts new file mode 100644 index 000000000..191c30183 --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.test.ts @@ -0,0 +1,151 @@ +import type { DictionaryPackDescriptor } from "@readany/core/dictionary"; +import { describe, expect, it, vi } from "vitest"; +import { + type DictionaryValidationDatabase, + validateDictionaryDatabase, +} from "./dictionary-pack-platform"; + +vi.mock("@dr.pogodin/react-native-fs", () => ({ hash: vi.fn() })); +vi.mock("expo-file-system", () => ({ + Directory: class {}, + File: class {}, + Paths: { document: { uri: "file:///documents" } }, +})); +vi.mock("expo-file-system/legacy", () => ({ createDownloadResumable: vi.fn() })); + +const descriptor: DictionaryPackDescriptor = { + language: "en", + version: "2026.09", + schemaVersion: 1, + sourceEdition: "wordnet-3.1", + sourceDumpDate: "2011-05-26", + sizeBytes: 123, + sha256: "a".repeat(64), + url: "https://example.test/en.sqlite", + sourceArchiveUrl: "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + attributionUrl: "https://wordnet.princeton.edu/", + license: "WordNet 3.1 License", +}; + +const expectedColumns = { + metadata: ["key", "value"], + entries: [ + "id", + "language", + "headword", + "simplified", + "traditional", + "pronunciation", + "part_of_speech", + ], + senses: ["entry_id", "sense_order", "definition"], + lookup: ["lookup_key", "entry_id", "rank"], +}; + +class SchemaDatabase implements DictionaryValidationDatabase { + integrity = "ok"; + objects = [ + { name: "metadata", type: "table", tbl_name: "metadata" }, + { name: "entries", type: "table", tbl_name: "entries" }, + { name: "senses", type: "table", tbl_name: "senses" }, + { name: "lookup", type: "table", tbl_name: "lookup" }, + { name: "lookup_key_rank_idx", type: "index", tbl_name: "lookup" }, + ]; + columns = structuredClone(expectedColumns); + indexColumns = ["lookup_key", "rank", "entry_id"]; + metadata = new Map([ + ["schema_version", "1"], + ["language", "en"], + ["version", descriptor.version], + ["source_edition", descriptor.sourceEdition], + ["source_dump_date", descriptor.sourceDumpDate], + ["source_archive_url", descriptor.sourceArchiveUrl], + ["asset_url", descriptor.url], + ["attribution_url", descriptor.attributionUrl], + ["license", descriptor.license], + ["license_notice", "Complete WordNet license notice."], + ["creator_attribution", "Princeton University."], + ]); + + async getFirstAsync(): Promise { + return { integrity_check: this.integrity } as T; + } + + async getAllAsync(sql: string): Promise { + if (sql.includes("sqlite_master")) return structuredClone(this.objects) as T[]; + const table = /table_info\('([^']+)'\)/.exec(sql)?.[1] as keyof typeof this.columns | undefined; + if (table) return this.columns[table].map((name) => ({ name })) as T[]; + if (sql.includes("index_info")) + return this.indexColumns.map((name, seqno) => ({ name, seqno })) as T[]; + if (sql.includes("FROM metadata")) + return [...this.metadata].map(([key, value]) => ({ key, value })) as T[]; + throw new Error(`Unexpected SQL: ${sql}`); + } +} + +describe("dictionary SQLite validation", () => { + it("returns intrinsic pack metadata after validating the complete schema", async () => { + await expect(validateDictionaryDatabase(new SchemaDatabase())).resolves.toEqual({ + language: descriptor.language, + version: descriptor.version, + schemaVersion: descriptor.schemaVersion, + sourceEdition: descriptor.sourceEdition, + sourceDumpDate: descriptor.sourceDumpDate, + sourceArchiveUrl: descriptor.sourceArchiveUrl, + url: descriptor.url, + attributionUrl: descriptor.attributionUrl, + license: descriptor.license, + licenseNotice: "Complete WordNet license notice.", + creatorAttribution: "Princeton University.", + }); + }); + + it("rejects an expected object with the wrong SQLite type", async () => { + const database = new SchemaDatabase(); + const index = database.objects.find((object) => object.name === "lookup_key_rank_idx"); + if (index) index.type = "table"; + + await expect(validateDictionaryDatabase(database)).rejects.toThrow("lookup_key_rank_idx index"); + }); + + it("rejects the lookup index when it belongs to another table", async () => { + const database = new SchemaDatabase(); + const index = database.objects.find((object) => object.name === "lookup_key_rank_idx"); + if (index) index.tbl_name = "entries"; + + await expect(validateDictionaryDatabase(database)).rejects.toThrow( + "lookup_key_rank_idx must belong to lookup", + ); + }); + + it("rejects a table missing a required column", async () => { + const database = new SchemaDatabase(); + database.columns.entries = database.columns.entries.filter((name) => name !== "headword"); + + await expect(validateDictionaryDatabase(database)).rejects.toThrow("entries columns"); + }); + + it("rejects a lookup index with the wrong column order", async () => { + const database = new SchemaDatabase(); + database.indexColumns = ["rank", "lookup_key", "entry_id"]; + + await expect(validateDictionaryDatabase(database)).rejects.toThrow("lookup index columns"); + }); + + it("rejects metadata whose source and license do not form a supported pair", async () => { + const database = new SchemaDatabase(); + database.metadata.set("license", "CC BY-SA 4.0"); + + await expect(validateDictionaryDatabase(database)).rejects.toThrow("source/license"); + }); + + it.each(["license_notice", "creator_attribution"])( + "rejects blank required metadata %s", + async (key) => { + const database = new SchemaDatabase(); + database.metadata.set(key, " "); + + await expect(validateDictionaryDatabase(database)).rejects.toThrow(key); + }, + ); +}); diff --git a/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.ts b/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.ts new file mode 100644 index 000000000..679e76033 --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.ts @@ -0,0 +1,219 @@ +import { hash } from "@dr.pogodin/react-native-fs"; +import { Directory, File, Paths } from "expo-file-system"; +import * as LegacyFileSystem from "expo-file-system/legacy"; +import type { DictionaryPackMetadata, DictionaryPackPlatform } from "./dictionary-pack-manager"; + +interface SqliteObjectRow { + name: string; + type: string; + tbl_name: string; +} + +interface SqliteColumnRow { + name: string; +} + +interface SqliteIndexColumnRow { + name: string; + seqno: number; +} + +interface DictionaryMetadataRow { + key: string; + value: string; +} + +export interface DictionaryValidationDatabase { + getFirstAsync(sql: string): Promise; + getAllAsync(sql: string): Promise; +} + +const requiredObjects = new Map([ + ["metadata", "table"], + ["entries", "table"], + ["senses", "table"], + ["lookup", "table"], + ["lookup_key_rank_idx", "index"], +]); + +const requiredColumns = { + metadata: ["key", "value"], + entries: [ + "id", + "language", + "headword", + "simplified", + "traditional", + "pronunciation", + "part_of_speech", + ], + senses: ["entry_id", "sense_order", "definition"], + lookup: ["lookup_key", "entry_id", "rank"], +} as const; + +const requiredMetadataKeys = [ + "schema_version", + "language", + "version", + "source_edition", + "source_dump_date", + "source_archive_url", + "asset_url", + "attribution_url", + "license", + "license_notice", + "creator_attribution", +] as const; + +function nativePath(path: string): string { + return path.replace(/^file:\/\//, ""); +} + +export function createExpoDictionaryPackPlatform(): DictionaryPackPlatform { + return { + async ensureDirectory(path) { + new Directory(path).create({ idempotent: true, intermediates: true }); + }, + async download(url, path, onProgress) { + const task = LegacyFileSystem.createDownloadResumable( + url, + path, + {}, + ({ totalBytesWritten, totalBytesExpectedToWrite }) => + onProgress( + totalBytesExpectedToWrite > 0 ? totalBytesWritten / totalBytesExpectedToWrite : 0, + ), + ); + const result = await task.downloadAsync(); + if (!result || result.status < 200 || result.status >= 300) + throw new Error("Dictionary pack download failed"); + }, + async exists(path) { + return new File(path).exists; + }, + async size(path) { + return new File(path).size ?? 0; + }, + async sha256(path) { + return hash(nativePath(path), "sha256"); + }, + async readMetadata(path) { + return readAndValidateSqlite(path); + }, + async move(from, to) { + const target = new File(to); + if (target.exists) throw new Error(`Dictionary move target already exists: ${to}`); + new File(from).move(target); + }, + async remove(path) { + const file = new File(path); + if (file.exists) file.delete(); + }, + }; +} + +export async function validateDictionaryDatabase( + database: DictionaryValidationDatabase, +): Promise { + const integrity = await database.getFirstAsync<{ integrity_check: string }>( + "PRAGMA integrity_check", + ); + if (integrity?.integrity_check !== "ok") + throw new Error("Dictionary SQLite integrity check failed"); + + const objects = await database.getAllAsync( + "SELECT name, type, tbl_name FROM sqlite_master WHERE name IN ('metadata', 'entries', 'senses', 'lookup', 'lookup_key_rank_idx')", + ); + const objectsByName = new Map(objects.map((object) => [object.name, object])); + for (const [name, type] of requiredObjects) { + if (objectsByName.get(name)?.type !== type) + throw new Error(`Dictionary SQLite schema requires ${name} ${type}`); + } + if (objectsByName.get("lookup_key_rank_idx")?.tbl_name !== "lookup") + throw new Error("Dictionary SQLite lookup_key_rank_idx must belong to lookup"); + + for (const [table, expectedColumns] of Object.entries(requiredColumns)) { + const columns = await database.getAllAsync(`PRAGMA table_info('${table}')`); + const actualColumns = columns.map((column) => column.name); + if (!sameArray(actualColumns, expectedColumns)) + throw new Error(`Dictionary SQLite ${table} columns did not match the required schema`); + } + + const indexColumns = await database.getAllAsync( + "PRAGMA index_info('lookup_key_rank_idx')", + ); + const orderedIndexColumns = [...indexColumns] + .sort((left, right) => left.seqno - right.seqno) + .map((column) => column.name); + if (!sameArray(orderedIndexColumns, ["lookup_key", "rank", "entry_id"])) + throw new Error("Dictionary SQLite lookup index columns were not in the required order"); + + const metadataRows = await database.getAllAsync( + `SELECT key, value FROM metadata WHERE key IN (${requiredMetadataKeys + .map((key) => `'${key}'`) + .join(", ")})`, + ); + const metadata = new Map(metadataRows.map((row) => [row.key, row.value])); + const value = (key: (typeof requiredMetadataKeys)[number]): string => { + const found = metadata.get(key); + if (!found?.trim()) throw new Error(`Dictionary metadata ${key} is missing`); + return found; + }; + + const schemaVersion = value("schema_version"); + if (schemaVersion !== "1") throw new Error("Dictionary metadata schema_version is unsupported"); + const language = value("language"); + if (language !== "en" && language !== "zh") + throw new Error("Dictionary metadata language is unsupported"); + const sourceEdition = value("source_edition"); + const license = value("license"); + const common = { + version: value("version"), + sourceDumpDate: value("source_dump_date"), + sourceArchiveUrl: value("source_archive_url"), + url: value("asset_url"), + attributionUrl: value("attribution_url"), + licenseNotice: value("license_notice"), + creatorAttribution: value("creator_attribution"), + }; + if (language === "en" && sourceEdition === "wordnet-3.1" && license === "WordNet 3.1 License") { + return { ...common, schemaVersion: 1, language, sourceEdition, license }; + } + if (language === "en" && sourceEdition === "enwiktionary" && license === "CC BY-SA 4.0") { + return { ...common, schemaVersion: 1, language, sourceEdition, license }; + } + if (language === "zh" && sourceEdition === "zhwiktionary" && license === "CC BY-SA 4.0") { + return { ...common, schemaVersion: 1, language, sourceEdition, license }; + } + throw new Error("Dictionary metadata source/license combination is unsupported"); +} + +async function readAndValidateSqlite(path: string): Promise { + const SQLite = await import("expo-sqlite"); + const { fileName, directory } = splitDatabasePath(path); + const database = await SQLite.openDatabaseAsync(fileName, { useNewConnection: true }, directory); + try { + return await validateDictionaryDatabase(database); + } finally { + await database.closeAsync(); + } +} + +function splitDatabasePath(path: string): { fileName: string; directory: string } { + const normalized = path.replace(/[\\/]+$/, ""); + const separator = Math.max(normalized.lastIndexOf("/"), normalized.lastIndexOf("\\")); + if (separator < 0 || separator === normalized.length - 1) + throw new Error("Dictionary path has no filename"); + return { + fileName: normalized.slice(separator + 1), + directory: separator === 0 ? normalized.slice(0, 1) : normalized.slice(0, separator), + }; +} + +function sameArray(actual: readonly string[], expected: readonly string[]): boolean { + return ( + actual.length === expected.length && actual.every((value, index) => value === expected[index]) + ); +} + +export const dictionaryPackDirectory = `${Paths.document.uri.replace(/\/$/, "")}/dictionaries`; diff --git a/packages/app-expo/src/lib/dictionary/dictionary-runtime.test.ts b/packages/app-expo/src/lib/dictionary/dictionary-runtime.test.ts new file mode 100644 index 000000000..de5a0b263 --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-runtime.test.ts @@ -0,0 +1,170 @@ +import type { DictionaryLanguage } from "@readany/core/dictionary"; +import { describe, expect, it } from "vitest"; +import type { DictionaryDatabaseAdapter } from "./dictionary-database"; +import type { DictionaryPackMetadata, DictionaryPackPlatform } from "./dictionary-pack-manager"; +import { createDictionaryRuntime } from "./dictionary-runtime"; + +const metadata: DictionaryPackMetadata = { + language: "en", + version: "1.0.0", + schemaVersion: 1, + sourceEdition: "wordnet-3.1", + sourceDumpDate: "2011-05-26", + sourceArchiveUrl: "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + url: "https://example.test/en.sqlite", + attributionUrl: "https://wordnet.princeton.edu/", + license: "WordNet 3.1 License", + licenseNotice: "Complete WordNet license notice.", + creatorAttribution: "Princeton University.", +}; + +describe("createDictionaryRuntime", () => { + it("inspects an active pack once per lifecycle and re-inspects after mutation", async () => { + const directory = "/docs/dictionaries"; + const activePath = `${directory}/readany-dictionary-en.sqlite`; + const files = new Set([activePath]); + let metadataReads = 0; + let hashReads = 0; + const platform: DictionaryPackPlatform = { + ensureDirectory: async () => {}, + download: async () => { + throw new Error("not used"); + }, + exists: async (path) => files.has(path), + size: async () => 123, + sha256: async () => { + hashReads += 1; + return "a".repeat(64); + }, + readMetadata: async () => { + metadataReads += 1; + return metadata; + }, + move: async () => { + throw new Error("not used"); + }, + remove: async (path) => { + files.delete(path); + }, + }; + let openCount = 0; + const database: DictionaryDatabaseAdapter = { + open: async () => { + openCount += 1; + return { getAllAsync: async () => [], closeAsync: async () => {} }; + }, + }; + const runtime = createDictionaryRuntime({ database, directory, platform }); + + await runtime.lookup.lookup("desire"); + await runtime.lookup.lookup("desires"); + expect({ metadataReads, hashReads, openCount }).toEqual({ + metadataReads: 1, + hashReads: 1, + openCount: 1, + }); + + await runtime.manager.remove("en"); + files.add(activePath); + await runtime.lookup.lookup("desire"); + expect({ metadataReads, hashReads, openCount }).toEqual({ + metadataReads: 2, + hashReads: 2, + openCount: 2, + }); + }); + + it("invalidates the inspected path and cached connection after a SQLite query error", async () => { + const directory = "/docs/dictionaries"; + const activePath = `${directory}/readany-dictionary-en.sqlite`; + let metadataReads = 0; + let openCount = 0; + let closeCount = 0; + const platform: DictionaryPackPlatform = { + ensureDirectory: async () => {}, + download: async () => { + throw new Error("not used"); + }, + exists: async (path) => path === activePath, + size: async () => 123, + sha256: async () => "a".repeat(64), + readMetadata: async () => { + metadataReads += 1; + return metadata; + }, + move: async () => { + throw new Error("not used"); + }, + remove: async () => {}, + }; + const database: DictionaryDatabaseAdapter = { + open: async () => { + openCount += 1; + const thisOpen = openCount; + return { + getAllAsync: async () => { + if (thisOpen === 1) throw new Error("SQLITE_CORRUPT"); + return []; + }, + closeAsync: async () => { + closeCount += 1; + }, + }; + }, + }; + const runtime = createDictionaryRuntime({ database, directory, platform }); + + await expect(runtime.lookup.lookup("desire")).rejects.toThrow("SQLITE_CORRUPT"); + await expect(runtime.lookup.lookup("desire")).resolves.toEqual([]); + + expect({ metadataReads, openCount, closeCount }).toEqual({ + metadataReads: 2, + openCount: 2, + closeCount: 1, + }); + }); + + it("shares one real lookup between installed-path resolution and manager close", async () => { + const directory = "/docs/dictionaries"; + const activePath = `${directory}/readany-dictionary-en.sqlite`; + const files = new Set([activePath]); + const events: string[] = []; + const platform: DictionaryPackPlatform = { + ensureDirectory: async () => {}, + download: async () => { + throw new Error("not used"); + }, + exists: async (path) => files.has(path), + size: async () => 123, + sha256: async () => "a".repeat(64), + readMetadata: async () => metadata, + move: async () => { + throw new Error("not used"); + }, + remove: async (path) => { + events.push(`remove:${path}`); + files.delete(path); + }, + }; + const openedPaths: string[] = []; + const database: DictionaryDatabaseAdapter = { + open: async (_language: DictionaryLanguage, path: string) => { + openedPaths.push(path); + return { + getAllAsync: async () => [], + closeAsync: async () => { + events.push("close:en"); + }, + }; + }, + }; + + const runtime = createDictionaryRuntime({ database, directory, platform }); + await expect(runtime.lookup.lookup("desire")).resolves.toEqual([]); + expect(openedPaths).toEqual([activePath]); + + await runtime.manager.remove("en"); + + expect(events).toEqual(["close:en", `remove:${activePath}`]); + }); +}); diff --git a/packages/app-expo/src/lib/dictionary/dictionary-runtime.ts b/packages/app-expo/src/lib/dictionary/dictionary-runtime.ts new file mode 100644 index 000000000..ec4bebeba --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-runtime.ts @@ -0,0 +1,29 @@ +import type { DictionaryDatabaseAdapter } from "./dictionary-database"; +import { DictionaryLookupService } from "./dictionary-lookup-service"; +import { DictionaryPackManager, type DictionaryPackPlatform } from "./dictionary-pack-manager"; + +export interface DictionaryRuntimeOptions { + database: DictionaryDatabaseAdapter; + directory: string; + platform: DictionaryPackPlatform; +} + +export function createDictionaryRuntime(options: DictionaryRuntimeOptions): { + lookup: DictionaryLookupService; + manager: DictionaryPackManager; +} { + // biome-ignore lint/style/useConst: the lookup resolves installed paths through the manager created below. + let manager: DictionaryPackManager | undefined; + const lookup = new DictionaryLookupService( + options.database, + async (language) => { + if (!manager) throw new Error("Dictionary runtime is not initialized"); + return manager.getActivePath(language); + }, + (language) => { + manager?.invalidate(language); + }, + ); + manager = new DictionaryPackManager(options.platform, options.directory, lookup); + return { lookup, manager }; +} diff --git a/packages/app-expo/src/navigation/RootNavigator.tsx b/packages/app-expo/src/navigation/RootNavigator.tsx index d720f534b..f71af724f 100644 --- a/packages/app-expo/src/navigation/RootNavigator.tsx +++ b/packages/app-expo/src/navigation/RootNavigator.tsx @@ -11,6 +11,7 @@ import { WebDavImportBrowserScreen } from "@/screens/library/WebDavImportBrowser import AISettingsScreen from "@/screens/settings/AISettingsScreen"; import AboutScreen from "@/screens/settings/AboutScreen"; import AppearanceSettingsScreen from "@/screens/settings/AppearanceSettingsScreen"; +import { DictionarySettingsScreen } from "@/screens/settings/DictionarySettingsScreen"; import FeedbackDetailScreen from "@/screens/settings/FeedbackDetailScreen"; import FeedbackScreen from "@/screens/settings/FeedbackScreen"; import FontSettingsScreen from "@/screens/settings/FontSettingsScreen"; @@ -40,6 +41,7 @@ export type RootStackParamList = { AISettings: undefined; TTSSettings: undefined; TranslationSettings: undefined; + DictionarySettings: undefined; SyncSettings: undefined; About: undefined; Feedback: undefined; @@ -105,6 +107,7 @@ export function RootNavigator() { + diff --git a/packages/app-expo/src/screens/ProfileScreen.tsx b/packages/app-expo/src/screens/ProfileScreen.tsx index bc19ebce0..6c1b2edd9 100644 --- a/packages/app-expo/src/screens/ProfileScreen.tsx +++ b/packages/app-expo/src/screens/ProfileScreen.tsx @@ -78,6 +78,7 @@ type ProfileMenuRoute = Extract< | "AISettings" | "TTSSettings" | "TranslationSettings" + | "DictionarySettings" | "Skills" | "VectorModelSettings" | "Feedback" @@ -464,6 +465,11 @@ export function ProfileScreen() { label: t("settings.translationTab", "翻译"), route: "TranslationSettings" as const, }, + { + icon: BookOpenIcon, + label: t("dictionary.dictionaries", "Dictionaries"), + route: "DictionarySettings" as const, + }, { icon: PuzzleIcon, label: t("skills.title", "技能"), route: "Skills" as const }, { icon: CpuIcon, diff --git a/packages/app-expo/src/screens/ReaderScreen.tsx b/packages/app-expo/src/screens/ReaderScreen.tsx index a87fe5edc..6a8170213 100644 --- a/packages/app-expo/src/screens/ReaderScreen.tsx +++ b/packages/app-expo/src/screens/ReaderScreen.tsx @@ -1,6 +1,7 @@ import { MarkdownRenderer } from "@/components/chat/MarkdownRenderer"; import { BookmarkRibbon } from "@/components/reader/BookmarkRibbon"; import { ChapterTranslationSheet } from "@/components/reader/ChapterTranslationSheet"; +import { DefinitionSheet } from "@/components/reader/DefinitionSheet"; import { ReadingProgressSlider } from "@/components/reader/ReadingProgressSlider"; import { SelectionPopover } from "@/components/reader/SelectionPopover"; import { TTSPage } from "@/components/reader/TTSPage"; @@ -226,6 +227,8 @@ export function ReaderScreen({ route, navigation }: Props) { const [showNotebook, setShowNotebook] = useState(false); const [showTranslation, setShowTranslation] = useState(false); const [translationText, setTranslationText] = useState(""); + const [definitionText, setDefinitionText] = useState(""); + const [showDefinition, setShowDefinition] = useState(false); const [showTTS, setShowTTS] = useState(false); const [showChapterTranslation, setShowChapterTranslation] = useState(false); const [isReimporting, setIsReimporting] = useState(false); @@ -1607,6 +1610,11 @@ export function ReaderScreen({ route, navigation }: Props) { onCopy={() => { setSelection(null); }} + onDefine={() => { + setDefinitionText(selectionPopoverSelection.text); + setShowDefinition(true); + setSelection(null); + }} onSpeak={(text, cfi) => { tts.startSelectionTTS(text, cfi); setSelection(null); @@ -2135,6 +2143,20 @@ export function ReaderScreen({ route, navigation }: Props) { /> )} + { + setShowDefinition(false); + setDefinitionText(""); + }} + onManageDictionaries={() => { + setShowDefinition(false); + setDefinitionText(""); + navigation.navigate("DictionarySettings" as never); + }} + /> + {/* ─── Chapter Translation Sheet ─── */} { + it("owns the selected definition text separately from selection and translation state", () => { + expect(readerScreen).toContain('const [definitionText, setDefinitionText] = useState("");'); + expect(readerScreen).toContain("const [showDefinition, setShowDefinition] = useState(false);"); + expect(readerScreen).toMatch( + /onDefine=\{\(\) => \{\s*setDefinitionText\(selectionPopoverSelection\.text\);\s*setShowDefinition\(true\);\s*setSelection\(null\);\s*\}\}/, + ); + expect(readerScreen).toMatch( + / { + expect(readerScreen).toMatch( + /onClose=\{\(\) => \{\s*setShowDefinition\(false\);\s*setDefinitionText\(""\);\s*\}\}/, + ); + expect(readerScreen).toMatch( + /onManageDictionaries=\{\(\) => \{\s*setShowDefinition\(false\);\s*setDefinitionText\(""\);\s*navigation\.navigate\("DictionarySettings" as never\);\s*\}\}/, + ); + }); +}); diff --git a/packages/app-expo/src/screens/settings/DictionarySettingsScreen.test.tsx b/packages/app-expo/src/screens/settings/DictionarySettingsScreen.test.tsx new file mode 100644 index 000000000..76c456a80 --- /dev/null +++ b/packages/app-expo/src/screens/settings/DictionarySettingsScreen.test.tsx @@ -0,0 +1,325 @@ +import type { DictionaryStoreState } from "@/stores/dictionary-store"; +import type { DictionaryManifest } from "@readany/core/dictionary"; +import i18n, { i18nReady } from "@readany/core/i18n"; +import React from "react"; +import TestRenderer, { act } from "react-test-renderer"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { create } from "zustand"; +import { DictionarySettingsScreen } from "./DictionarySettingsScreen"; + +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = + true; +(globalThis as typeof globalThis & { React: typeof React }).React = React; + +const { alert, openURL } = vi.hoisted(() => ({ + alert: vi.fn(), + openURL: vi.fn(async () => true), +})); + +vi.mock("react-native", async () => { + const ReactModule = await import("react"); + const host = (name: string) => + function HostComponent(props: Record) { + return ReactModule.createElement(name, props, props.children as React.ReactNode); + }; + return { + ActivityIndicator: host("ActivityIndicator"), + Alert: { alert }, + Linking: { openURL }, + ScrollView: host("ScrollView"), + StyleSheet: { create: (styles: unknown) => styles, hairlineWidth: 1 }, + Text: host("Text"), + TouchableOpacity: host("TouchableOpacity"), + View: host("View"), + }; +}); + +vi.mock("react-native-safe-area-context", () => ({ + SafeAreaView: ({ children }: { children: React.ReactNode }) => children, +})); + +vi.mock("@/stores", () => ({ + useDictionaryStore: () => { + throw new Error("DictionarySettingsScreen tests inject their dictionary store"); + }, +})); +vi.mock("@/components/ui/Icon", () => ({ ChevronLeftIcon: () => null })); +vi.mock("@/components/ui/KeyboardAwareScrollView", () => ({ + KeyboardAwareScrollView: ({ children }: { children: React.ReactNode }) => children, +})); +vi.mock("@/hooks/use-responsive-layout", () => ({ + useResponsiveLayout: () => ({ centeredContentWidth: 720 }), +})); +vi.mock("@/styles/theme", () => ({ + fontSize: { base: 16, lg: 18, sm: 14, xs: 12 }, + fontWeight: { medium: "500", semibold: "600" }, + radius: { lg: 8, xl: 12 }, + spacing: { lg: 16, xl: 20, xxl: 24 }, + useColors: () => ({ + background: "background", + border: "border", + card: "card", + destructive: "destructive", + foreground: "foreground", + mutedForeground: "mutedForeground", + primary: "primary", + primaryForeground: "primaryForeground", + }), +})); +vi.mock("./SettingsHeader", () => ({ + SettingsHeader: ({ title, subtitle }: { title: string; subtitle: string }) => + React.createElement("Text", {}, title, subtitle), +})); + +const descriptors: DictionaryManifest["packs"] = { + en: { + language: "en", + version: "2026.09", + schemaVersion: 1, + sourceEdition: "enwiktionary", + sourceDumpDate: "2026-09-01", + sizeBytes: 1_572_864, + sha256: "a".repeat(64), + url: "https://example.test/en.sqlite", + sourceArchiveUrl: "https://dumps.example.test/en.xml.bz2", + attributionUrl: "https://en.wiktionary.org", + license: "CC BY-SA 4.0", + }, + zh: { + language: "zh", + version: "2026.09", + schemaVersion: 1, + sourceEdition: "zhwiktionary", + sourceDumpDate: "2026-09-01", + sizeBytes: 2_621_440, + sha256: "b".repeat(64), + url: "https://example.test/zh.sqlite", + sourceArchiveUrl: "https://dumps.example.test/zh.xml.bz2", + attributionUrl: "https://zh.wiktionary.org", + license: "CC BY-SA 4.0", + }, +}; + +function makeStore( + overrides: Partial = {}, + actionOverrides: Partial< + Pick + > = {}, +) { + const initialize = actionOverrides.initialize ?? vi.fn(async () => {}); + const refreshManifest = actionOverrides.refreshManifest ?? vi.fn(async () => {}); + const install = actionOverrides.install ?? vi.fn(async () => {}); + const remove = actionOverrides.remove ?? vi.fn(async () => {}); + const retry = actionOverrides.retry ?? vi.fn(async () => {}); + const lookup = vi.fn(async () => []); + return { + actions: { initialize, install, refreshManifest, remove, retry }, + store: create(() => ({ + manifest: { manifestVersion: 1, packs: descriptors }, + packs: { + en: { state: "not-installed" }, + zh: { state: "not-installed" }, + ...overrides, + }, + initialize, + refreshManifest, + install, + remove, + retry, + lookup, + })), + }; +} + +function textContent(renderer: TestRenderer.ReactTestRenderer): string { + return renderer.root + .findAll((node) => String(node.type) === "Text") + .flatMap((node) => node.children) + .join(""); +} + +function press(renderer: TestRenderer.ReactTestRenderer, label: string): void { + const button = renderer.root.findByProps({ accessibilityLabel: label }); + button.props.onPress(); +} + +describe("DictionarySettingsScreen", () => { + beforeEach(async () => { + await i18nReady; + await act(async () => { + await i18n.changeLanguage("en"); + }); + }); + it.each([ + ["en", "Dictionaries", "English", "Download English dictionary"], + ["zh-TW", "字典", "英語", "英語字典下載"], + ["fr", "Dictionaries", "English", "Download English dictionary"], + ])( + "renders real resources with English fallback in %s", + async (language, title, name, action) => { + await act(async () => { + await i18n.changeLanguage(language); + }); + const { store } = makeStore(); + let renderer!: TestRenderer.ReactTestRenderer; + await act(async () => { + renderer = TestRenderer.create(); + }); + expect(textContent(renderer)).toContain(title); + expect(textContent(renderer)).toContain(name); + expect(renderer.root.findByProps({ accessibilityLabel: action })).toBeTruthy(); + expect(JSON.stringify(renderer.toJSON())).not.toMatch(/(?:reader\.)?dictionary\./); + await act(async () => renderer.unmount()); + }, + ); + + it("renders both language rows after exactly one initial remote-first refresh", async () => { + const calls: string[] = []; + let resolveInitialization!: () => void; + const initialize = vi.fn( + () => + new Promise((resolve) => { + calls.push("initialize"); + resolveInitialization = resolve; + }), + ); + const refreshManifest = vi.fn(async () => { + calls.push("refreshManifest"); + }); + const { store: orderedStore, actions: orderedActions } = makeStore( + {}, + { initialize, refreshManifest }, + ); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create(); + }); + + expect(calls).toEqual(["initialize"]); + await act(async () => { + resolveInitialization(); + await Promise.resolve(); + }); + + expect(textContent(renderer)).toContain("English"); + expect(textContent(renderer)).toContain("Chinese"); + expect(textContent(renderer)).toContain("Not downloaded"); + expect(textContent(renderer)).toContain("1.5 MB"); + expect(textContent(renderer)).toContain("2.5 MB"); + expect(orderedActions.initialize).toHaveBeenCalledTimes(1); + expect(orderedActions.refreshManifest).not.toHaveBeenCalled(); + expect(calls).toEqual(["initialize"]); + }); + + it("shows unavailable without a download action when the manifest is absent", async () => { + const { store, actions } = makeStore(); + store.setState({ manifest: null }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create(); + }); + + expect(textContent(renderer)).toContain("Unavailable"); + expect( + renderer.root.findAll( + (node) => node.props.accessibilityLabel === "Download English dictionary", + ), + ).toHaveLength(0); + expect(actions.install).not.toHaveBeenCalled(); + }); + + it("keeps download and update actions scoped to the selected language", async () => { + const { store, actions } = makeStore({ + zh: { + state: "update-available", + installedVersion: "2026.08", + availableVersion: "2026.09", + sizeBytes: 2_621_440, + }, + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create(); + }); + await act(async () => { + press(renderer, "Download English dictionary"); + press(renderer, "Update Chinese dictionary"); + }); + + expect(actions.install).toHaveBeenNthCalledWith(1, "en"); + expect(actions.install).toHaveBeenNthCalledWith(2, "zh"); + expect(textContent(renderer)).toContain("Update available"); + expect(textContent(renderer)).toContain("2026.08"); + expect(textContent(renderer)).toContain("2026.09"); + }); + + it("renders progress, installed metadata, retry, and a removal confirmation for their own rows", async () => { + const { store, actions } = makeStore({ + en: { state: "downloading", progress: 0.37 }, + zh: { state: "installed", version: "2026.09", sizeBytes: 2_621_440 }, + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create(); + }); + expect(textContent(renderer)).toContain("37%"); + expect(textContent(renderer)).toContain("Installed"); + expect(textContent(renderer)).toContain("2026.09"); + + await act(async () => press(renderer, "Remove Chinese dictionary")); + const confirm = alert.mock.calls + .at(-1)?.[2] + ?.find((button: { style?: string }) => button.style === "destructive"); + await act(async () => confirm?.onPress()); + expect(actions.remove).toHaveBeenCalledWith("zh"); + }); + + it("shows a localized per-pack error, logs its detail, retries only that language, and opens attribution", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { store, actions } = makeStore({ + en: { state: "error", message: "network unavailable", hasActivePack: false }, + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create(); + }); + expect(textContent(renderer)).toContain("English dictionary couldn't be prepared. Try again."); + expect(textContent(renderer)).not.toContain("network unavailable"); + expect(warn).toHaveBeenCalledWith("[DictionarySettings] dictionary pack error", { + error: "network unavailable", + language: "en", + }); + await act(async () => { + press(renderer, "Retry English dictionary"); + press(renderer, "English dictionary attribution"); + }); + + expect(actions.install).toHaveBeenCalledWith("en"); + expect(actions.retry).not.toHaveBeenCalled(); + expect(openURL).toHaveBeenCalledWith(descriptors.en.attributionUrl); + warn.mockRestore(); + }); + + it("offers both repair and removal when the active pack is corrupt", async () => { + const { store, actions } = makeStore({ + en: { state: "error", message: "invalid schema", hasActivePack: true } as never, + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create(); + }); + await act(async () => { + press(renderer, "Repair English dictionary"); + press(renderer, "Remove English dictionary"); + }); + + expect(actions.install).toHaveBeenCalledWith("en"); + expect(alert).toHaveBeenCalled(); + }); +}); diff --git a/packages/app-expo/src/screens/settings/DictionarySettingsScreen.tsx b/packages/app-expo/src/screens/settings/DictionarySettingsScreen.tsx new file mode 100644 index 000000000..c08e75134 --- /dev/null +++ b/packages/app-expo/src/screens/settings/DictionarySettingsScreen.tsx @@ -0,0 +1,344 @@ +import { KeyboardAwareScrollView } from "@/components/ui/KeyboardAwareScrollView"; +import { useResponsiveLayout } from "@/hooks/use-responsive-layout"; +import { useDictionaryStore } from "@/stores"; +import { type ThemeColors, fontSize, fontWeight, radius, spacing, useColors } from "@/styles/theme"; +import type { DictionaryLanguage, DictionaryPackDescriptor } from "@readany/core/dictionary"; +import { useEffect } from "react"; +import { useTranslation } from "react-i18next"; +import { + ActivityIndicator, + Alert, + Linking, + StyleSheet, + Text, + TouchableOpacity, + View, +} from "react-native"; +import { SafeAreaView } from "react-native-safe-area-context"; +import type { StoreApi, UseBoundStore } from "zustand"; +import type { DictionaryStoreState } from "../../stores/dictionary-store"; +import { SettingsHeader } from "./SettingsHeader"; + +type DictionaryStore = UseBoundStore>; + +export interface DictionarySettingsScreenProps { + /** Allows a configured store to be supplied by the application or tests. */ + dictionaryStore?: DictionaryStore; +} + +const languages: DictionaryLanguage[] = ["en", "zh"]; + +function formatBytes(sizeBytes: number): string { + if (sizeBytes < 1024 * 1024) return `${Math.round(sizeBytes / 1024)} KB`; + return `${(sizeBytes / (1024 * 1024)).toFixed(1)} MB`; +} + +function DictionaryPackRow({ + descriptor, + language, + pack, + onInstall, + onRemove, + onRetry, +}: { + descriptor?: DictionaryPackDescriptor; + language: DictionaryLanguage; + pack: DictionaryStoreState["packs"][DictionaryLanguage]; + onInstall: (language: DictionaryLanguage) => void; + onRemove: (language: DictionaryLanguage) => void; + onRetry: (language: DictionaryLanguage) => void; +}) { + const colors = useColors(); + const styles = makeStyles(colors); + const { t } = useTranslation(); + const name = t(`dictionary.${language === "en" ? "english" : "chinese"}`); + const actionLabel = (action: string) => t("dictionary.actionLabel", { action, language: name }); + const descriptorSize = descriptor ? formatBytes(descriptor.sizeBytes) : null; + const unavailable = !descriptor && pack.state === "not-installed"; + + const status = (() => { + if (unavailable) return t("dictionary.unavailable"); + + switch (pack.state) { + case "downloading": + return t("dictionary.downloading", { + progress: Math.round(pack.progress * 100), + }); + case "installed": + return t("dictionary.installedStatus", { + installed: t("dictionary.installed"), + size: formatBytes(pack.sizeBytes), + version: t("dictionary.version", { version: pack.version }), + }); + case "update-available": + return t("dictionary.updateStatus", { + availableVersion: pack.availableVersion, + installedVersion: pack.installedVersion, + updateAvailable: t("dictionary.updateAvailable"), + }); + case "error": + return t("dictionary.packError", { language: name }); + default: + return t("dictionary.notDownloaded"); + } + })(); + + return ( + + + {name} + {status} + {(pack.state === "not-installed" || pack.state === "update-available") && descriptorSize ? ( + {t("dictionary.size", { size: descriptorSize })} + ) : null} + {descriptor ? ( + + void Linking.openURL(descriptor.attributionUrl)} + > + {t("dictionary.attribution")} + + + {"· "} + {t("dictionary.licenseDetail", { + label: t("dictionary.license"), + license: descriptor.license, + })} + + + ) : null} + + + + {pack.state === "not-installed" && !unavailable ? ( + onInstall(language)} + /> + ) : null} + {pack.state === "downloading" ? ( + + ) : null} + {pack.state === "installed" ? ( + onRemove(language)} + /> + ) : null} + {pack.state === "update-available" ? ( + <> + onInstall(language)} + /> + onRemove(language)} + /> + + ) : null} + {pack.state === "error" ? ( + <> + {descriptor ? ( + onRetry(language)} + /> + ) : null} + {pack.hasActivePack ? ( + onRemove(language)} + /> + ) : null} + + ) : null} + + + ); +} + +function ActionButton({ + accessibilityLabel, + destructive = false, + label, + onPress, +}: { + accessibilityLabel: string; + destructive?: boolean; + label: string; + onPress: () => void; +}) { + const colors = useColors(); + const styles = makeStyles(colors); + return ( + + {label} + + ); +} + +export function DictionarySettingsScreen({ + dictionaryStore = useDictionaryStore, +}: DictionarySettingsScreenProps) { + const colors = useColors(); + const styles = makeStyles(colors); + const layout = useResponsiveLayout(); + const { t } = useTranslation(); + const manifest = dictionaryStore((state) => state.manifest); + const packs = dictionaryStore((state) => state.packs); + const initialize = dictionaryStore((state) => state.initialize); + const install = dictionaryStore((state) => state.install); + const remove = dictionaryStore((state) => state.remove); + + useEffect(() => { + void initialize().catch((error) => + console.warn("[DictionarySettings] failed to initialize dictionary packs", error), + ); + }, [initialize]); + + useEffect(() => { + for (const language of languages) { + const pack = packs[language]; + if (pack.state === "error") { + console.warn("[DictionarySettings] dictionary pack error", { + error: pack.message, + language, + }); + } + } + }, [packs]); + + const handleInstall = (language: DictionaryLanguage) => { + void install(language).catch((error) => + console.warn(`[DictionarySettings] failed to install ${language} dictionary`, error), + ); + }; + + const handleRemove = (language: DictionaryLanguage) => { + const languageName = t(`dictionary.${language === "en" ? "english" : "chinese"}`); + Alert.alert( + t("dictionary.removeTitle", { language: languageName }), + t("dictionary.removeMessage", { + language: languageName, + }), + [ + { text: t("dictionary.cancel"), style: "cancel" }, + { + text: t("dictionary.remove"), + style: "destructive", + onPress: () => + void remove(language).catch((error) => + console.warn(`[DictionarySettings] failed to remove ${language} dictionary`, error), + ), + }, + ], + ); + }; + + return ( + + + + + {t("dictionary.manageDictionaries")} + + {languages.map((language, index) => ( + + + + ))} + + + + + ); +} + +const makeStyles = (colors: ThemeColors) => + StyleSheet.create({ + container: { flex: 1 }, + scroll: { flex: 1 }, + scrollContent: { padding: spacing.lg, paddingBottom: 56 }, + contentColumn: {}, + sectionTitle: { + color: colors.foreground, + fontSize: fontSize.base, + fontWeight: fontWeight.semibold, + marginBottom: 10, + }, + listCard: { + backgroundColor: colors.card, + borderColor: colors.border, + borderRadius: radius.xl, + borderWidth: 1, + overflow: "hidden", + }, + rowBorder: { borderBottomColor: colors.border, borderBottomWidth: StyleSheet.hairlineWidth }, + packRow: { + alignItems: "flex-start", + flexDirection: "row", + gap: spacing.md, + justifyContent: "space-between", + padding: spacing.lg, + }, + packCopy: { flex: 1, gap: 3 }, + packName: { color: colors.foreground, fontSize: fontSize.base, fontWeight: fontWeight.medium }, + statusText: { color: colors.mutedForeground, fontSize: fontSize.sm, lineHeight: 20 }, + metaText: { color: colors.mutedForeground, fontSize: fontSize.xs, lineHeight: 18 }, + licenseLine: { alignItems: "center", flexDirection: "row", flexWrap: "wrap", marginTop: 2 }, + linkText: { color: colors.primary, fontSize: fontSize.xs, lineHeight: 18 }, + actions: { alignItems: "flex-end", gap: spacing.sm }, + actionButton: { + borderColor: colors.primary, + borderRadius: radius.lg, + borderWidth: 1, + paddingHorizontal: 10, + paddingVertical: 7, + }, + actionLabel: { color: colors.primary, fontSize: fontSize.xs, fontWeight: fontWeight.semibold }, + destructiveButton: { borderColor: colors.destructive }, + destructiveLabel: { color: colors.destructive }, + }); diff --git a/packages/app-expo/src/screens/settings/dictionary-locales.test.ts b/packages/app-expo/src/screens/settings/dictionary-locales.test.ts new file mode 100644 index 000000000..11a0de342 --- /dev/null +++ b/packages/app-expo/src/screens/settings/dictionary-locales.test.ts @@ -0,0 +1,107 @@ +import i18n, { i18nReady } from "@readany/core/i18n"; +import { describe, expect, it } from "vitest"; +import enReader from "../../../../core/src/i18n/locales/en/reader.json"; +import zhTwReader from "../../../../core/src/i18n/locales/zh-TW/reader.json"; +import zhReader from "../../../../core/src/i18n/locales/zh/reader.json"; + +const requiredDictionaryKeys = [ + "define", + "title", + "close", + "loadingDefinition", + "dictionaries", + "english", + "chinese", + "download", + "update", + "remove", + "retry", + "repair", + "retryLookup", + "manageDictionaries", + "notDownloaded", + "noDefinitionFound", + "unsupportedSelection", + "lookupError", + "downloadDefinition", + "downloadingDefinition", + "downloadAccessibility", + "downloadingAccessibility", + "offlinePrivacy", + "downloading", + "installed", + "installedStatus", + "updateAvailable", + "updateStatus", + "error", + "packError", + "unavailable", + "version", + "size", + "attribution", + "attributionLabel", + "license", + "licenseDetail", + "actionLabel", + "statusLabel", + "removeTitle", + "removeMessage", + "cancel", +] as const; + +const requiredDictionaryTemplates = { + actionLabel: ["action", "language"], + attributionLabel: ["language"], + downloading: ["progress"], + downloadingDefinition: ["language", "progress"], + downloadDefinition: ["language", "size"], + downloadAccessibility: ["language"], + downloadingAccessibility: ["language"], + installedStatus: ["installed", "version", "size"], + licenseDetail: ["label", "license"], + packError: ["language"], + removeMessage: ["language"], + removeTitle: ["language"], + size: ["size"], + statusLabel: ["language", "status"], + updateStatus: ["updateAvailable", "installedVersion", "availableVersion"], + version: ["version"], +} as const; + +describe("dictionary locale copy", () => { + it.each(["en", "zh", "zh-TW", "ja", "ko", "fr", "es"])( + "resolves every dictionary key through the app resources in %s", + async (language) => { + await i18nReady; + const t = i18n.getFixedT(language); + for (const key of requiredDictionaryKeys) { + expect(t(`dictionary.${key}`)).not.toBe(`dictionary.${key}`); + } + }, + ); + + it("falls back to English for dictionary copy missing from French", async () => { + await i18nReady; + expect(i18n.getResource("fr", "translation", "dictionary.download")).toBeUndefined(); + expect(i18n.getFixedT("fr")("dictionary.download")).toBe("Download"); + }); + + it.each([ + ["English", enReader], + ["Simplified Chinese", zhReader], + ["Traditional Chinese", zhTwReader], + ])("provides every dictionary string in %s", (_language, reader) => { + const dictionary = reader.dictionary as Record | undefined; + + for (const key of requiredDictionaryKeys) { + expect(dictionary?.[key], `${_language} dictionary.${key}`).toEqual(expect.any(String)); + expect((dictionary?.[key] as string).trim()).not.toBe(""); + } + + for (const [key, variables] of Object.entries(requiredDictionaryTemplates)) { + for (const variable of variables) { + expect(dictionary?.[key], `${_language} dictionary.${key}`).toContain(`{{${variable}}}`); + } + } + }); +}); diff --git a/packages/app-expo/src/stores/dictionary-store.test.ts b/packages/app-expo/src/stores/dictionary-store.test.ts new file mode 100644 index 000000000..0b49b4a41 --- /dev/null +++ b/packages/app-expo/src/stores/dictionary-store.test.ts @@ -0,0 +1,332 @@ +import type { DictionaryManifest } from "@readany/core/dictionary"; +import { describe, expect, it, vi } from "vitest"; +import * as dictionaryStoreModule from "./dictionary-store"; + +const { createDictionaryStore } = dictionaryStoreModule; + +const manifest: DictionaryManifest = { + manifestVersion: 1, + packs: { + en: { + language: "en", + version: "1.0.0", + schemaVersion: 1, + sourceEdition: "enwiktionary", + sourceDumpDate: "2026-09-01", + sizeBytes: 12, + sha256: "a".repeat(64), + url: "https://example.test/en", + sourceArchiveUrl: "https://example.test/en-source", + attributionUrl: "https://example.test/license", + license: "CC BY-SA 4.0", + }, + zh: { + language: "zh", + version: "1.0.0", + schemaVersion: 1, + sourceEdition: "zhwiktionary", + sourceDumpDate: "2026-09-01", + sizeBytes: 12, + sha256: "b".repeat(64), + url: "https://example.test/zh", + sourceArchiveUrl: "https://example.test/zh-source", + attributionUrl: "https://example.test/license", + license: "CC BY-SA 4.0", + }, + }, +}; + +function manager() { + return { + refresh: vi + .fn() + .mockResolvedValue({ en: { state: "not-installed" }, zh: { state: "not-installed" } }), + install: vi.fn().mockResolvedValue(undefined), + remove: vi.fn().mockResolvedValue(undefined), + }; +} + +function lookupService() { + return { lookup: vi.fn().mockResolvedValue([]) }; +} + +describe("dictionary store", () => { + it("completes lookup from bundled data without waiting for an in-flight remote refresh", async () => { + let resolveRemote!: (value: DictionaryManifest) => void; + const remote = vi.fn( + () => + new Promise((resolve) => { + resolveRemote = resolve; + }), + ); + const bundled = vi.fn().mockResolvedValue(manifest); + const m = manager(); + const lookup = lookupService(); + const store = createDictionaryStore({ + manager: m, + lookup, + fetchRemoteManifest: remote, + getBundledManifest: bundled, + }); + let initializeFinished = false; + const initialize = store + .getState() + .initialize() + .then(() => { + initializeFinished = true; + }); + + await expect(store.getState().lookup("desire")).resolves.toEqual([]); + + expect(initializeFinished).toBe(false); + expect(remote).toHaveBeenCalledOnce(); + expect(bundled).toHaveBeenCalledOnce(); + expect(lookup.lookup).toHaveBeenCalledWith("desire"); + expect(m.refresh).toHaveBeenCalledTimes(1); + + resolveRemote(manifest); + await initialize; + expect(m.refresh).toHaveBeenCalledTimes(2); + }); + + it("starts an explicit remote refresh even when bundled lookup readiness is in flight", async () => { + let resolveBundled!: (value: DictionaryManifest) => void; + const bundled = vi.fn( + () => + new Promise((resolve) => { + resolveBundled = resolve; + }), + ); + const remote = vi.fn().mockResolvedValue(manifest); + const store = createDictionaryStore({ + manager: manager(), + lookup: lookupService(), + fetchRemoteManifest: remote, + getBundledManifest: bundled, + }); + + const lookup = store.getState().lookup("desire"); + await vi.waitFor(() => expect(bundled).toHaveBeenCalledOnce()); + const initialize = store.getState().initialize(); + + await vi.waitFor(() => expect(remote).toHaveBeenCalledOnce()); + resolveBundled(manifest); + await Promise.all([lookup, initialize]); + }); + + it("initializes remote-first and parses the remote value", async () => { + const remote = vi.fn().mockResolvedValue(structuredClone(manifest)); + const bundled = vi.fn(); + const m = manager(); + const store = createDictionaryStore({ + manager: m, + lookup: lookupService(), + fetchRemoteManifest: remote, + getBundledManifest: bundled, + }); + + await store.getState().initialize(); + + expect(remote).toHaveBeenCalledOnce(); + expect(bundled).not.toHaveBeenCalled(); + expect(m.refresh).toHaveBeenCalledWith(manifest); + expect(store.getState().manifest).toEqual(manifest); + }); + + it("falls back to a parsed bundled manifest when the remote value is invalid", async () => { + const invalidRemote = { ...structuredClone(manifest), unexpected: true }; + const bundled = vi.fn().mockResolvedValue(structuredClone(manifest)); + const m = manager(); + const store = createDictionaryStore({ + manager: m, + lookup: lookupService(), + fetchRemoteManifest: vi.fn().mockResolvedValue(invalidRemote), + getBundledManifest: bundled, + }); + + await store.getState().initialize(); + + expect(bundled).toHaveBeenCalledOnce(); + expect(m.refresh).toHaveBeenCalledWith(manifest); + }); + + it("falls back to bundled when the remote fetch fails", async () => { + const bundled = vi.fn().mockResolvedValue(structuredClone(manifest)); + const m = manager(); + const store = createDictionaryStore({ + manager: m, + lookup: lookupService(), + fetchRemoteManifest: vi.fn().mockRejectedValue(new Error("offline")), + getBundledManifest: bundled, + }); + + await store.getState().refreshManifest(); + + expect(bundled).toHaveBeenCalledOnce(); + expect(store.getState().manifest).toEqual(manifest); + }); + + it("rejects when both remote and bundled values are invalid", async () => { + const store = createDictionaryStore({ + manager: manager(), + lookup: lookupService(), + fetchRemoteManifest: vi.fn().mockResolvedValue({ nope: true }), + getBundledManifest: vi.fn().mockResolvedValue({ stillNope: true }), + }); + + await expect(store.getState().initialize()).rejects.toThrow("manifest"); + }); + + it("clears a failed explicit refresh so retry performs a new remote-first attempt", async () => { + const remote = vi.fn().mockResolvedValueOnce({ nope: true }).mockResolvedValueOnce(manifest); + const bundled = vi.fn().mockResolvedValueOnce({ stillNope: true }); + const store = createDictionaryStore({ + manager: manager(), + lookup: lookupService(), + fetchRemoteManifest: remote, + getBundledManifest: bundled, + }); + + await expect(store.getState().initialize()).rejects.toThrow("manifest"); + await expect(store.getState().retry()).resolves.toBeUndefined(); + + expect(remote).toHaveBeenCalledTimes(2); + expect(bundled).toHaveBeenCalledOnce(); + }); + + it("does not hide manager refresh failures behind bundled fallback", async () => { + const refreshError = new Error("filesystem recovery failed"); + const m = manager(); + m.refresh.mockRejectedValueOnce(refreshError); + const bundled = vi.fn().mockResolvedValue(manifest); + const store = createDictionaryStore({ + manager: m, + lookup: lookupService(), + fetchRemoteManifest: vi.fn().mockResolvedValue(manifest), + getBundledManifest: bundled, + }); + + await expect(store.getState().initialize()).rejects.toBe(refreshError); + expect(bundled).not.toHaveBeenCalled(); + }); + + it("exposes the composed lookup service through the same store", async () => { + const lookup = lookupService(); + lookup.lookup.mockResolvedValueOnce([ + { + id: 1, + language: "en", + headword: "desire", + partOfSpeech: "noun", + senses: [{ order: 0, definition: "an inclination to want things" }], + }, + ]); + const store = createDictionaryStore({ + manager: manager(), + lookup, + fetchRemoteManifest: vi.fn().mockResolvedValue(manifest), + getBundledManifest: vi.fn().mockResolvedValue(manifest), + }); + + await expect(store.getState().lookup("desire")).resolves.toMatchObject([ + { headword: "desire" }, + ]); + expect(lookup.lookup).toHaveBeenCalledWith("desire"); + }); + + it("loads only the bundled manifest before the first lookup so lookup cannot reach the network", async () => { + const lookup = lookupService(); + lookup.lookup.mockRejectedValueOnce( + Object.assign(new Error("English pack is not installed"), { code: "pack-not-installed" }), + ); + const remote = vi.fn().mockResolvedValue(manifest); + const bundled = vi.fn().mockResolvedValue(manifest); + const m = manager(); + const store = createDictionaryStore({ + manager: m, + lookup, + fetchRemoteManifest: remote, + getBundledManifest: bundled, + }); + + await expect(store.getState().lookup("desire")).rejects.toMatchObject({ + code: "pack-not-installed", + }); + + expect(remote).not.toHaveBeenCalled(); + expect(bundled).toHaveBeenCalledOnce(); + expect(m.refresh).toHaveBeenCalledWith(manifest); + expect(store.getState().manifest).toEqual(manifest); + }); + + it("loads one shared runtime for manager operations and lookup", async () => { + expect(dictionaryStoreModule).toHaveProperty("createRuntimeBackedDictionaryStore"); + const createRuntimeBackedDictionaryStore = Reflect.get( + dictionaryStoreModule, + "createRuntimeBackedDictionaryStore", + ); + const m = manager(); + const lookup = lookupService(); + const loadRuntime = vi.fn().mockResolvedValue({ manager: m, lookup }); + const store = createRuntimeBackedDictionaryStore({ + loadRuntime, + fetchRemoteManifest: vi.fn().mockResolvedValue(manifest), + getBundledManifest: vi.fn().mockResolvedValue(manifest), + }); + + await store.getState().initialize(); + await store.getState().lookup("desire"); + await store.getState().remove("en"); + + expect(loadRuntime).toHaveBeenCalledOnce(); + expect(m.refresh).toHaveBeenCalledWith(manifest); + expect(lookup.lookup).toHaveBeenCalledWith("desire"); + expect(m.remove).toHaveBeenCalledWith("en"); + }); + + it("clears a rejected runtime construction so Retry can construct it again", async () => { + const m = manager(); + const lookup = lookupService(); + const loadRuntime = vi + .fn() + .mockRejectedValueOnce(new Error("native module unavailable")) + .mockResolvedValueOnce({ manager: m, lookup }); + const store = dictionaryStoreModule.createRuntimeBackedDictionaryStore({ + loadRuntime, + fetchRemoteManifest: vi.fn().mockResolvedValue(manifest), + getBundledManifest: vi.fn().mockResolvedValue(manifest), + }); + + await expect(store.getState().initialize()).rejects.toThrow("native module unavailable"); + await expect(store.getState().retry()).resolves.toBeUndefined(); + + expect(loadRuntime).toHaveBeenCalledTimes(2); + expect(store.getState().manifest).toEqual(manifest); + }); + + it("coalesces explicit remote manifest refreshes while they are in flight", async () => { + let resolveRemote!: (value: DictionaryManifest) => void; + const remote = vi.fn( + () => + new Promise((resolve) => { + resolveRemote = resolve; + }), + ); + const m = manager(); + const store = createDictionaryStore({ + manager: m, + lookup: lookupService(), + fetchRemoteManifest: remote, + getBundledManifest: vi.fn().mockResolvedValue(manifest), + }); + + const initialize = store.getState().initialize(); + const refresh = store.getState().refreshManifest(); + expect(remote).toHaveBeenCalledOnce(); + resolveRemote(manifest); + await Promise.all([initialize, refresh]); + + expect(remote).toHaveBeenCalledOnce(); + expect(m.refresh).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/app-expo/src/stores/dictionary-store.ts b/packages/app-expo/src/stores/dictionary-store.ts new file mode 100644 index 000000000..99a379308 --- /dev/null +++ b/packages/app-expo/src/stores/dictionary-store.ts @@ -0,0 +1,191 @@ +import { + type DictionaryEntry, + type DictionaryLanguage, + type DictionaryManifest, + type DictionaryPackDescriptor, + parseDictionaryManifest, +} from "@readany/core/dictionary"; +import { type StoreApi, type UseBoundStore, create } from "zustand"; +import { + DICTIONARY_BUNDLED_MANIFEST, + DICTIONARY_REMOTE_MANIFEST_URL, +} from "../config/dictionary-config"; +import type { DictionaryLookupService } from "../lib/dictionary/dictionary-lookup-service"; +import type { + DictionaryPackManager, + DictionaryPackStatus, +} from "../lib/dictionary/dictionary-pack-manager"; + +export interface DictionaryStoreDependencies { + manager: Pick; + lookup: Pick; + fetchRemoteManifest: () => Promise; + getBundledManifest: () => Promise; +} + +export interface DictionaryRuntime { + manager: Pick; + lookup: Pick; +} + +export interface RuntimeBackedDictionaryStoreDependencies { + loadRuntime: () => Promise; + fetchRemoteManifest: () => Promise; + getBundledManifest: () => Promise; +} + +export interface DictionaryStoreState { + manifest: DictionaryManifest | null; + packs: Record; + initialize(): Promise; + refreshManifest(): Promise; + install(language: DictionaryLanguage): Promise; + remove(language: DictionaryLanguage): Promise; + retry(): Promise; + lookup(text: string): Promise; +} + +const emptyPacks = (): Record => ({ + en: { state: "not-installed" }, + zh: { state: "not-installed" }, +}); + +export function createDictionaryStore( + deps: DictionaryStoreDependencies, +): UseBoundStore> { + return create()((set, get) => { + let remoteRefreshPromise: Promise | undefined; + let bundledReadinessPromise: Promise | undefined; + const applyManifest = async (manifest: DictionaryManifest) => { + const packs = await deps.manager.refresh(manifest); + set({ manifest, packs }); + }; + const loadManifest = async (): Promise => { + let remoteError: unknown; + try { + return parseDictionaryManifest(await deps.fetchRemoteManifest()); + } catch (error) { + remoteError = error; + } + try { + return parseDictionaryManifest(await deps.getBundledManifest()); + } catch (bundledError) { + throw new AggregateError( + [remoteError, bundledError], + "No valid dictionary manifest is available", + ); + } + }; + const refreshFromSources = (): Promise => { + if (remoteRefreshPromise) return remoteRefreshPromise; + const operation = loadManifest().then(applyManifest); + remoteRefreshPromise = operation; + void operation.then( + () => { + if (remoteRefreshPromise === operation) remoteRefreshPromise = undefined; + }, + () => { + if (remoteRefreshPromise === operation) remoteRefreshPromise = undefined; + }, + ); + return operation; + }; + const refreshFromBundled = (): Promise => { + if (get().manifest) return Promise.resolve(); + if (bundledReadinessPromise) return bundledReadinessPromise; + const operation = deps + .getBundledManifest() + .then(parseDictionaryManifest) + .then(async (manifest) => { + if (!get().manifest) await applyManifest(manifest); + }); + bundledReadinessPromise = operation; + void operation.then( + () => { + if (bundledReadinessPromise === operation) bundledReadinessPromise = undefined; + }, + () => { + if (bundledReadinessPromise === operation) bundledReadinessPromise = undefined; + }, + ); + return operation; + }; + + return { + manifest: null, + packs: emptyPacks(), + initialize: refreshFromSources, + refreshManifest: refreshFromSources, + install: async (language) => { + const descriptor: DictionaryPackDescriptor | undefined = get().manifest?.packs[language]; + if (!descriptor) throw new Error("Dictionary manifest is unavailable"); + await deps.manager.install(descriptor, (status) => + set((state) => ({ packs: { ...state.packs, [language]: status } })), + ); + }, + remove: async (language) => { + await deps.manager.remove(language); + set((state) => ({ packs: { ...state.packs, [language]: { state: "not-installed" } } })); + }, + retry: refreshFromSources, + lookup: async (text) => { + if (!get().manifest) await refreshFromBundled(); + return deps.lookup.lookup(text); + }, + }; + }); +} + +export function createRuntimeBackedDictionaryStore( + deps: RuntimeBackedDictionaryStoreDependencies, +): UseBoundStore> { + let runtimePromise: Promise | undefined; + const runtime = () => { + if (!runtimePromise) { + const operation = deps.loadRuntime(); + runtimePromise = operation; + void operation.catch(() => { + if (runtimePromise === operation) runtimePromise = undefined; + }); + } + return runtimePromise; + }; + return createDictionaryStore({ + manager: { + refresh: async (manifest) => (await runtime()).manager.refresh(manifest), + install: async (descriptor, onStatus) => + (await runtime()).manager.install(descriptor, onStatus), + remove: async (language) => (await runtime()).manager.remove(language), + }, + lookup: { lookup: async (text) => (await runtime()).lookup.lookup(text) }, + fetchRemoteManifest: deps.fetchRemoteManifest, + getBundledManifest: deps.getBundledManifest, + }); +} + +async function loadExpoDictionaryRuntime(): Promise { + const [databaseModule, platformModule, runtimeModule] = await Promise.all([ + import("../lib/dictionary/dictionary-database"), + import("../lib/dictionary/dictionary-pack-platform"), + import("../lib/dictionary/dictionary-runtime"), + ]); + return runtimeModule.createDictionaryRuntime({ + database: new databaseModule.ExpoDictionaryDatabaseAdapter(), + directory: platformModule.dictionaryPackDirectory, + platform: platformModule.createExpoDictionaryPackPlatform(), + }); +} + +async function fetchRemoteDictionaryManifest(): Promise { + const response = await fetch(DICTIONARY_REMOTE_MANIFEST_URL); + if (!response.ok) { + throw new Error(`Dictionary manifest request failed with HTTP ${response.status}`); + } + return response.json(); +} + +export const useDictionaryStore = createRuntimeBackedDictionaryStore({ + loadRuntime: loadExpoDictionaryRuntime, + fetchRemoteManifest: fetchRemoteDictionaryManifest, + getBundledManifest: async () => DICTIONARY_BUNDLED_MANIFEST, +}); diff --git a/packages/app-expo/src/stores/index.ts b/packages/app-expo/src/stores/index.ts index 178c88d11..bc5f900ae 100644 --- a/packages/app-expo/src/stores/index.ts +++ b/packages/app-expo/src/stores/index.ts @@ -28,4 +28,7 @@ export type { UpdateState } from "./update-store"; export { useVectorModelStore } from "./vector-model-store"; export type { BuiltinModelStatus, BuiltinModelState, VectorModelState } from "./vector-model-store"; +export { useDictionaryStore } from "./dictionary-store"; +export type { DictionaryStoreState } from "./dictionary-store"; + export { debouncedSave, loadFromFS, flushAllWrites, withPersist } from "./persist"; diff --git a/packages/cli/package.json b/packages/cli/package.json index ca553f995..616132368 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -23,6 +23,8 @@ "acceptance:validate": "node scripts/validate-acceptance.mjs", "preflight:release": "node scripts/release-preflight.mjs", "dev": "tsx src/bin/readany.ts", + "dictionary:build": "tsx scripts/build-dictionary-pack.ts", + "dictionary:convert-wordnet": "tsx scripts/convert-wordnet.ts", "test": "vitest run", "test:watch": "vitest", "check": "tsc -p tsconfig.json --noEmit" diff --git a/packages/cli/scripts/build-dictionary-pack.ts b/packages/cli/scripts/build-dictionary-pack.ts new file mode 100644 index 000000000..d7861e87b --- /dev/null +++ b/packages/cli/scripts/build-dictionary-pack.ts @@ -0,0 +1,184 @@ +import { access, readFile, stat, writeFile } from "node:fs/promises"; +import { dirname, extname, isAbsolute } from "node:path"; +import { + type DictionaryPackDescriptor, + type DictionarySource, + parseDictionaryManifest, +} from "@readany/core/dictionary"; +import { buildDictionaryPack } from "../src/dictionary/pack-builder.js"; + +const REQUIRED_FLAGS = [ + "language", + "input", + "output", + "version", + "source-edition", + "license", + "source-date", + "source-archive-url", + "attribution-url", + "license-file", + "creator-attribution", + "asset-url", + "descriptor", +] as const; + +type RequiredFlag = (typeof REQUIRED_FLAGS)[number]; +type ParsedArguments = Record; + +function usage(message: string): never { + throw new Error( + `${message}\nUsage: dictionary:build --language en|zh --input absolute-jsonl-path --output absolute-sqlite-path --version semver --source-edition wordnet-3.1|enwiktionary|zhwiktionary --license "WordNet 3.1 License"|"CC BY-SA 4.0" --source-date YYYY-MM-DD --source-archive-url https-url --attribution-url https-url --license-file absolute-txt-path --creator-attribution text --asset-url https-url --descriptor absolute-json-path`, + ); +} + +function parseArguments(argv: string[]): ParsedArguments { + const values = new Map(); + for (let index = 0; index < argv.length; index += 2) { + const flag = argv[index]; + const value = argv[index + 1]; + if (!flag?.startsWith("--")) usage(`Unexpected argument: ${flag ?? ""}`); + const key = flag.slice(2) as RequiredFlag; + if (!REQUIRED_FLAGS.includes(key) || !value || value.startsWith("--")) { + usage(`Invalid argument: ${flag}`); + } + if (values.has(key)) usage(`Duplicate argument: ${flag}`); + values.set(key, value); + } + + for (const key of REQUIRED_FLAGS) { + if (!values.has(key)) usage(`Missing required argument: --${key}`); + } + return Object.fromEntries(values) as ParsedArguments; +} + +function validateUrl(value: string, name: string): void { + try { + if (new URL(value).protocol !== "https:") usage(`${name} must be an https URL`); + } catch { + usage(`${name} must be an https URL`); + } +} + +function validateDate(value: string): void { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) usage("--source-date must use YYYY-MM-DD"); + const date = new Date(`${value}T00:00:00.000Z`); + if (Number.isNaN(date.getTime()) || date.toISOString().slice(0, 10) !== value) { + usage("--source-date must use YYYY-MM-DD"); + } +} + +function validateSemver(value: string): void { + const numericIdentifier = "(?:0|[1-9]\\d*)"; + const prereleaseIdentifier = `(?:${numericIdentifier}|\\d*[A-Za-z-][0-9A-Za-z-]*)`; + const prerelease = `(?:-${prereleaseIdentifier}(?:\\.${prereleaseIdentifier})*)?`; + const build = "(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?"; + if ( + !new RegExp( + `^${numericIdentifier}\\.${numericIdentifier}\\.${numericIdentifier}${prerelease}${build}$`, + ).test(value) + ) { + usage("--version must be a semver value"); + } +} + +async function validatePaths(args: ParsedArguments): Promise { + for (const [name, path] of [ + ["--input", args.input], + ["--license-file", args["license-file"]], + ["--output", args.output], + ["--descriptor", args.descriptor], + ] as const) { + if (!isAbsolute(path)) usage(`${name} must be an absolute path`); + } + if (extname(args.input) !== ".jsonl") usage("--input must be a .jsonl path"); + if (extname(args["license-file"]) !== ".txt") usage("--license-file must be a .txt path"); + if (extname(args.output) !== ".sqlite") usage("--output must be a .sqlite path"); + if (extname(args.descriptor) !== ".json") usage("--descriptor must be a .json path"); + await Promise.all([access(args.input), access(args["license-file"])]); + await Promise.all([stat(dirname(args.output)), stat(dirname(args.descriptor))]); +} + +function validateArguments(args: ParsedArguments): DictionarySource { + if (args.language !== "en" && args.language !== "zh") usage("--language must be en or zh"); + validateSemver(args.version); + validateDate(args["source-date"]); + validateUrl(args["source-archive-url"], "--source-archive-url"); + validateUrl(args["attribution-url"], "--attribution-url"); + validateUrl(args["asset-url"], "--asset-url"); + if (!args["creator-attribution"].trim()) usage("--creator-attribution must not be empty"); + if ( + args.language === "en" && + args["source-edition"] === "wordnet-3.1" && + args.license === "WordNet 3.1 License" + ) { + return { language: "en", sourceEdition: "wordnet-3.1", license: "WordNet 3.1 License" }; + } + if ( + args.language === "en" && + args["source-edition"] === "enwiktionary" && + args.license === "CC BY-SA 4.0" + ) { + return { language: "en", sourceEdition: "enwiktionary", license: "CC BY-SA 4.0" }; + } + if ( + args.language === "zh" && + args["source-edition"] === "zhwiktionary" && + args.license === "CC BY-SA 4.0" + ) { + return { language: "zh", sourceEdition: "zhwiktionary", license: "CC BY-SA 4.0" }; + } + usage("--language, --source-edition, and --license must be a supported combination"); +} + +function validateDescriptor(descriptor: DictionaryPackDescriptor): DictionaryPackDescriptor { + const englishPlaceholder = { + ...descriptor, + language: "en", + sourceEdition: "wordnet-3.1", + license: "WordNet 3.1 License", + } as const; + const chinesePlaceholder = { + ...descriptor, + language: "zh", + sourceEdition: "zhwiktionary", + license: "CC BY-SA 4.0", + } as const; + const manifest = parseDictionaryManifest({ + manifestVersion: 1, + packs: + descriptor.language === "en" + ? { en: descriptor, zh: chinesePlaceholder } + : { en: englishPlaceholder, zh: descriptor }, + }); + return manifest.packs[descriptor.language]; +} + +async function main(): Promise { + const args = parseArguments(process.argv.slice(2)); + const source = validateArguments(args); + await validatePaths(args); + const licenseNotice = (await readFile(args["license-file"], "utf8")).trim(); + if (!licenseNotice) usage("--license-file must contain a license notice"); + const descriptor = validateDescriptor( + await buildDictionaryPack({ + ...source, + inputPath: args.input, + outputPath: args.output, + version: args.version, + sourceDumpDate: args["source-date"], + sourceArchiveUrl: args["source-archive-url"], + attributionUrl: args["attribution-url"], + licenseNotice, + creatorAttribution: args["creator-attribution"].trim(), + assetUrl: args["asset-url"], + }), + ); + await writeFile(args.descriptor, `${JSON.stringify(descriptor, null, 2)}\n`, "utf8"); + process.stdout.write(`${JSON.stringify(descriptor)}\n`); +} + +main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; +}); diff --git a/packages/cli/scripts/convert-wordnet.ts b/packages/cli/scripts/convert-wordnet.ts new file mode 100644 index 000000000..e1d419136 --- /dev/null +++ b/packages/cli/scripts/convert-wordnet.ts @@ -0,0 +1,55 @@ +import { access, stat } from "node:fs/promises"; +import { dirname, extname, isAbsolute } from "node:path"; +import { convertWordNetDirectory } from "../src/dictionary/wordnet-converter.js"; + +interface Arguments { + inputDirectory: string; + output: string; +} + +function usage(message: string): never { + throw new Error( + `${message}\nUsage: dictionary:convert-wordnet --input-directory absolute-dict-path --output absolute-jsonl-path`, + ); +} + +function parseArguments(argv: string[]): Arguments { + const values = new Map(); + for (let index = 0; index < argv.length; index += 2) { + const flag = argv[index]; + const value = argv[index + 1]; + if (!flag?.startsWith("--") || !value || value.startsWith("--")) { + usage(`Invalid argument: ${flag ?? ""}`); + } + const key = flag.slice(2); + if (key !== "input-directory" && key !== "output") usage(`Unexpected argument: ${flag}`); + if (values.has(key)) usage(`Duplicate argument: ${flag}`); + values.set(key, value); + } + const inputDirectory = values.get("input-directory"); + const output = values.get("output"); + if (!inputDirectory) usage("Missing required argument: --input-directory"); + if (!output) usage("Missing required argument: --output"); + return { inputDirectory, output }; +} + +async function main(): Promise { + const args = parseArguments(process.argv.slice(2)); + if (!isAbsolute(args.inputDirectory)) usage("--input-directory must be an absolute path"); + if (!isAbsolute(args.output)) usage("--output must be an absolute path"); + if (extname(args.output) !== ".jsonl") usage("--output must be a .jsonl path"); + if (!(await stat(args.inputDirectory)).isDirectory()) { + usage("--input-directory must identify a directory"); + } + await Promise.all([access(args.inputDirectory), stat(dirname(args.output))]); + const stats = await convertWordNetDirectory({ + inputDirectory: args.inputDirectory, + outputPath: args.output, + }); + process.stdout.write(`${JSON.stringify(stats)}\n`); +} + +main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; +}); diff --git a/packages/cli/src/dictionary/fixtures/en-unnormalizable-canonical.jsonl b/packages/cli/src/dictionary/fixtures/en-unnormalizable-canonical.jsonl new file mode 100644 index 000000000..dc37faa05 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/en-unnormalizable-canonical.jsonl @@ -0,0 +1 @@ +{"word":"reading閱讀","lang_code":"en","pos":"noun","forms":[{"form":"readings","tags":["plural"]}],"senses":[{"glosses":["A record that must be skipped because its canonical headword is mixed-script."]}]} diff --git a/packages/cli/src/dictionary/fixtures/en-wordnet.jsonl b/packages/cli/src/dictionary/fixtures/en-wordnet.jsonl new file mode 100644 index 000000000..3c1920fb2 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/en-wordnet.jsonl @@ -0,0 +1,2 @@ +{"word":"desire","lang_code":"en","pos":"verb","senses":[{"glosses":["feel or have a desire for"]}],"forms":[{"form":"desires","tags":["present"]}]} +{"word":"child","lang_code":"en","pos":"noun","senses":[{"glosses":["a young person"]}],"forms":[{"form":"children","tags":["wordnet-exception"]}]} diff --git a/packages/cli/src/dictionary/fixtures/en.jsonl b/packages/cli/src/dictionary/fixtures/en.jsonl new file mode 100644 index 000000000..ef2f369db --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/en.jsonl @@ -0,0 +1 @@ +{"word":"desire","lang_code":"en","pos":"noun","sounds":[{"ipa":"/dɪˈzaɪəɹ/"}],"forms":[{"form":"desires","tags":["plural"]}],"senses":[{"glosses":["A strong wish."],"examples":["Her desire was strong."],"translations":["EN_TRANSLATION_SHOULD_NOT_APPEAR"],"images":["https://example.invalid/desire.png"],"audio":["https://example.invalid/desire.ogg"]}]} diff --git a/packages/cli/src/dictionary/fixtures/test-license.txt b/packages/cli/src/dictionary/fixtures/test-license.txt new file mode 100644 index 000000000..ebe6ddcf8 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/test-license.txt @@ -0,0 +1 @@ +Complete test license notice. diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/adj.exc b/packages/cli/src/dictionary/fixtures/wordnet-invalid/adj.exc new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/adj.exc @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/adv.exc b/packages/cli/src/dictionary/fixtures/wordnet-invalid/adv.exc new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/adv.exc @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adj b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adj new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adj @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adv b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adv new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adv @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.noun b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.noun new file mode 100644 index 000000000..3590cb57c --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.noun @@ -0,0 +1 @@ +not-a-wordnet-data-row diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.verb b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.verb new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.verb @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/noun.exc b/packages/cli/src/dictionary/fixtures/wordnet-invalid/noun.exc new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/noun.exc @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/verb.exc b/packages/cli/src/dictionary/fixtures/wordnet-invalid/verb.exc new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/verb.exc @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet/adj.exc b/packages/cli/src/dictionary/fixtures/wordnet/adj.exc new file mode 100644 index 000000000..404a2e4dd --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/adj.exc @@ -0,0 +1 @@ +better good diff --git a/packages/cli/src/dictionary/fixtures/wordnet/adv.exc b/packages/cli/src/dictionary/fixtures/wordnet/adv.exc new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/adv.exc @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet/data.adj b/packages/cli/src/dictionary/fixtures/wordnet/data.adj new file mode 100644 index 000000000..e439f7fbd --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/data.adj @@ -0,0 +1,2 @@ + WordNet fixture header +00000008 00 a 01 good(p) 0 000 | having desirable qualities diff --git a/packages/cli/src/dictionary/fixtures/wordnet/data.adv b/packages/cli/src/dictionary/fixtures/wordnet/data.adv new file mode 100644 index 000000000..4c54cb670 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/data.adv @@ -0,0 +1 @@ + WordNet fixture header diff --git a/packages/cli/src/dictionary/fixtures/wordnet/data.noun b/packages/cli/src/dictionary/fixtures/wordnet/data.noun new file mode 100644 index 000000000..450bcc565 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/data.noun @@ -0,0 +1,7 @@ + WordNet fixture header +00000001 00 n 02 desire 0 wish 0 000 | a strong feeling of wanting something; "his desire was obvious" +00000002 00 n 01 desire 0 000 | something that is wanted; "the house was his desire" +00000003 00 n 01 city 0 000 | a large and densely populated urban area +00000004 00 n 01 child 0 000 | a young person +00000005 00 n 01 ice_cream 0 000 | a frozen dessert +00000009 00 n 01 tattoo 0 000 | a permanent design on skin diff --git a/packages/cli/src/dictionary/fixtures/wordnet/data.verb b/packages/cli/src/dictionary/fixtures/wordnet/data.verb new file mode 100644 index 000000000..473d0f056 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/data.verb @@ -0,0 +1,7 @@ + WordNet fixture header +00000006 00 v 02 desire 0 wish 0 000 00 | feel or have a desire for; "I desire a quiet room" +00000007 00 v 01 go 0 000 00 | move from one place to another +00000010 00 v 01 do 0 000 00 | perform an action +00000011 00 v 01 have 0 000 00 | possess something +00000012 00 v 01 be 0 000 00 | exist +00000013 00 v 01 tattoo 0 000 00 | mark skin permanently diff --git a/packages/cli/src/dictionary/fixtures/wordnet/noun.exc b/packages/cli/src/dictionary/fixtures/wordnet/noun.exc new file mode 100644 index 000000000..bcd4bd668 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/noun.exc @@ -0,0 +1 @@ +children child diff --git a/packages/cli/src/dictionary/fixtures/wordnet/verb.exc b/packages/cli/src/dictionary/fixtures/wordnet/verb.exc new file mode 100644 index 000000000..cc1a01633 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/verb.exc @@ -0,0 +1,4 @@ +went go +did do +has have +is be diff --git a/packages/cli/src/dictionary/fixtures/zh-redirects.jsonl b/packages/cli/src/dictionary/fixtures/zh-redirects.jsonl new file mode 100644 index 000000000..919a31254 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/zh-redirects.jsonl @@ -0,0 +1,12 @@ +{"title":"首页","redirect":"首頁","pos":"hard-redirect"} +{"word":"首頁","lang_code":"zh","pos":"noun","senses":[{"glosses":["網站的主要頁面。"]}]} +{"word":"锂","lang_code":"zh","pos":"soft-redirect","senses":[{"tags":["no-gloss"]}],"redirects":["鋰"]} +{"word":"鋰","lang_code":"zh","pos":"noun","senses":[{"glosses":["原子序數為三的化學元素。"]}]} +{"title":"日语","redirect":"日文","pos":"hard-redirect"} +{"word":"日文","lang_code":"zh","pos":"soft-redirect","senses":[{"tags":["no-gloss"]}],"redirects":["日語"]} +{"word":"日語","lang_code":"zh","pos":"noun","senses":[{"glosses":["日本使用的語言。"]}]} +{"title":"循环甲","redirect":"循环乙","pos":"hard-redirect"} +{"title":"循环乙","redirect":"循环甲","pos":"hard-redirect"} +{"title":"失踪词","redirect":"不存在","pos":"hard-redirect"} +{"title":"unsupported-alias","redirect":"首頁","pos":"hard-redirect"} +{"word":"錯誤語言","lang_code":"en","pos":"soft-redirect","senses":[{"tags":["no-gloss"]}],"redirects":["首頁"]} diff --git a/packages/cli/src/dictionary/fixtures/zh.jsonl b/packages/cli/src/dictionary/fixtures/zh.jsonl new file mode 100644 index 000000000..282a6267d --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/zh.jsonl @@ -0,0 +1 @@ +{"word":"閱讀","lang_code":"zh","pos":"verb","sounds":[{"zh-pron":"yuèdú"}],"forms":[{"form":"阅读","tags":["Simplified-Chinese"]},{"form":"閱讀","tags":["Traditional-Chinese"]}],"senses":[{"glosses":["看並理解文字的內容。"],"examples":["閱讀是一種樂趣。"],"translations":["ZH_TRANSLATION_SHOULD_NOT_APPEAR"],"images":["https://example.invalid/read.png"],"audio":["https://example.invalid/read.ogg"]}]} diff --git a/packages/cli/src/dictionary/pack-builder.test.ts b/packages/cli/src/dictionary/pack-builder.test.ts new file mode 100644 index 000000000..3a39f4d94 --- /dev/null +++ b/packages/cli/src/dictionary/pack-builder.test.ts @@ -0,0 +1,404 @@ +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdtemp, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import Database from "better-sqlite3"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildDictionaryPack } from "./pack-builder.js"; + +const fixtureDirectory = resolve(import.meta.dirname, "fixtures"); +const temporaryDirectories: string[] = []; +const TEST_LICENSE_NOTICE = "Complete test license notice."; +const TEST_CREATOR_ATTRIBUTION = "Wiktionary contributors."; + +async function buildFixture(language: "en" | "zh") { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-pack-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, `${language}.sqlite`); + + const descriptor = await buildDictionaryPack({ + language, + inputPath: join(fixtureDirectory, `${language}.jsonl`), + outputPath, + version: "1.2.3", + sourceEdition: language === "en" ? "enwiktionary" : "zhwiktionary", + license: "CC BY-SA 4.0", + sourceDumpDate: "2026-09-03", + sourceArchiveUrl: `https://example.invalid/${language}-archive`, + attributionUrl: `https://example.invalid/${language}-attribution`, + licenseNotice: TEST_LICENSE_NOTICE, + creatorAttribution: TEST_CREATOR_ATTRIBUTION, + assetUrl: `https://example.invalid/${language}.sqlite`, + }); + + return { descriptor, outputPath }; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map(async (directory) => { + const { rm } = await import("node:fs/promises"); + await rm(directory, { recursive: true, force: true }); + }), + ); +}); + +describe("buildDictionaryPack", () => { + it("resolves Chinese hard, soft, and chained redirects without cycle or missing-target rows", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-redirects-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "zh.sqlite"); + + await buildDictionaryPack({ + language: "zh", + inputPath: join(fixtureDirectory, "zh-redirects.jsonl"), + outputPath, + version: "1.2.3", + sourceEdition: "zhwiktionary", + license: "CC BY-SA 4.0", + sourceDumpDate: "2026-09-03", + sourceArchiveUrl: "https://example.invalid/zh-archive", + attributionUrl: "https://example.invalid/zh-attribution", + licenseNotice: TEST_LICENSE_NOTICE, + creatorAttribution: TEST_CREATOR_ATTRIBUTION, + assetUrl: "https://example.invalid/zh.sqlite", + }); + + const database = new Database(outputPath, { readonly: true }); + try { + const redirected = database + .prepare(` + SELECT l.lookup_key, l.rank, e.headword + FROM lookup l + JOIN entries e ON e.id = l.entry_id + WHERE l.lookup_key IN ('首页', '锂', '日语') + ORDER BY l.lookup_key, e.id + `) + .all(); + expect(redirected).toEqual([ + { lookup_key: "日语", rank: 1, headword: "日語" }, + { lookup_key: "锂", rank: 1, headword: "鋰" }, + { lookup_key: "首页", rank: 1, headword: "首頁" }, + ]); + expect( + database + .prepare( + "SELECT COUNT(*) FROM lookup WHERE lookup_key IN ('循环甲', '循环乙', '失踪词', 'unsupported-alias', '錯誤語言')", + ) + .pluck() + .get(), + ).toBe(0); + expect(database.pragma("foreign_key_check")).toEqual([]); + } finally { + database.close(); + } + }); + + it("builds deterministic English and Chinese SQLite packs with lookup aliases", async () => { + const [english, chinese] = await Promise.all([buildFixture("en"), buildFixture("zh")]); + const englishDatabase = new Database(english.outputPath, { readonly: true }); + const chineseDatabase = new Database(chinese.outputPath, { readonly: true }); + + try { + expect( + englishDatabase + .prepare("SELECT value FROM metadata WHERE key = 'schema_version'") + .pluck() + .get(), + ).toBe("1"); + expect( + chineseDatabase + .prepare("SELECT value FROM metadata WHERE key = 'schema_version'") + .pluck() + .get(), + ).toBe("1"); + expect( + englishDatabase + .prepare("SELECT value FROM metadata WHERE key = 'source_archive_url'") + .pluck() + .get(), + ).toBe("https://example.invalid/en-archive"); + expect( + englishDatabase + .prepare("SELECT value FROM metadata WHERE key = 'attribution_url'") + .pluck() + .get(), + ).toBe("https://example.invalid/en-attribution"); + expect( + englishDatabase + .prepare("SELECT value FROM metadata WHERE key = 'license_notice'") + .pluck() + .get(), + ).toBe(TEST_LICENSE_NOTICE); + expect( + englishDatabase + .prepare("SELECT value FROM metadata WHERE key = 'creator_attribution'") + .pluck() + .get(), + ).toBe(TEST_CREATOR_ATTRIBUTION); + expect( + englishDatabase + .prepare("SELECT entry_id FROM lookup WHERE lookup_key = ?") + .pluck() + .get("desires"), + ).toBeTypeOf("number"); + expect( + englishDatabase + .prepare("SELECT rank FROM lookup WHERE lookup_key = ?") + .pluck() + .get("desire"), + ).toBe(0); + expect( + englishDatabase + .prepare("SELECT rank FROM lookup WHERE lookup_key = ?") + .pluck() + .get("desires"), + ).toBe(1); + expect( + chineseDatabase + .prepare("SELECT entry_id FROM lookup WHERE lookup_key = ?") + .pluck() + .get("阅读"), + ).toBeTypeOf("number"); + expect( + englishDatabase.prepare("SELECT definition FROM senses ORDER BY sense_order").pluck().all(), + ).toContain("A strong wish."); + + expect(englishDatabase.prepare("SELECT pronunciation FROM entries").pluck().get()).toBe( + "/dɪˈzaɪəɹ/", + ); + expect(chineseDatabase.prepare("SELECT pronunciation FROM entries").pluck().get()).toBe( + "yuèdú", + ); + expect(chineseDatabase.prepare("SELECT simplified FROM entries").pluck().get()).toBe("阅读"); + expect(chineseDatabase.prepare("SELECT traditional FROM entries").pluck().get()).toBe("閱讀"); + } finally { + englishDatabase.close(); + chineseDatabase.close(); + } + + const databaseText = await Promise.all([ + readFile(english.outputPath), + readFile(chinese.outputPath), + ]).then((files) => Buffer.concat(files).toString("utf8")); + for (const forbiddenText of [ + "Her desire was strong.", + "閱讀是一種樂趣。", + "EN_TRANSLATION_SHOULD_NOT_APPEAR", + "ZH_TRANSLATION_SHOULD_NOT_APPEAR", + "https://example.invalid/desire.png", + "https://example.invalid/read.ogg", + ]) { + expect(databaseText).not.toContain(forbiddenText); + } + + expect(english.descriptor).toMatchObject({ + language: "en", + schemaVersion: 1, + sourceEdition: "enwiktionary", + url: "https://example.invalid/en.sqlite", + sourceArchiveUrl: "https://example.invalid/en-archive", + attributionUrl: "https://example.invalid/en-attribution", + license: "CC BY-SA 4.0", + }); + expect(chinese.descriptor).toMatchObject({ language: "zh", schemaVersion: 1 }); + }); + + it("skips a record whose canonical headword cannot become an English lookup key", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-unnormalizable-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "en.sqlite"); + + await buildDictionaryPack({ + language: "en", + inputPath: join(fixtureDirectory, "en-unnormalizable-canonical.jsonl"), + outputPath, + version: "1.2.3", + sourceEdition: "enwiktionary", + license: "CC BY-SA 4.0", + sourceDumpDate: "2026-09-03", + sourceArchiveUrl: "https://example.invalid/en-archive", + attributionUrl: "https://example.invalid/en-attribution", + licenseNotice: TEST_LICENSE_NOTICE, + creatorAttribution: TEST_CREATOR_ATTRIBUTION, + assetUrl: "https://example.invalid/en.sqlite", + }); + + const database = new Database(outputPath, { readonly: true }); + try { + expect(database.prepare("SELECT COUNT(*) FROM entries").pluck().get()).toBe(0); + expect(database.prepare("SELECT COUNT(*) FROM lookup").pluck().get()).toBe(0); + } finally { + database.close(); + } + }); + + it.each([ + ["license notice", { licenseNotice: "" }], + ["creator attribution", { creatorAttribution: " " }], + ])("rejects an empty %s before creating a pack", async (_name, override) => { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-empty-notice-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "en.sqlite"); + + await expect( + buildDictionaryPack({ + language: "en", + inputPath: join(fixtureDirectory, "en.jsonl"), + outputPath, + version: "1.2.3", + sourceEdition: "enwiktionary", + license: "CC BY-SA 4.0", + sourceDumpDate: "2026-09-03", + sourceArchiveUrl: "https://example.invalid/en-archive", + attributionUrl: "https://example.invalid/en-attribution", + licenseNotice: TEST_LICENSE_NOTICE, + creatorAttribution: TEST_CREATOR_ATTRIBUTION, + assetUrl: "https://example.invalid/en.sqlite", + ...override, + }), + ).rejects.toThrow(/license notice|creator attribution/i); + expect(existsSync(outputPath)).toBe(false); + }); + + it("builds WordNet metadata and keeps deterministic exception aliases", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-wordnet-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "en.sqlite"); + + const descriptor = await buildDictionaryPack({ + language: "en", + inputPath: join(fixtureDirectory, "en-wordnet.jsonl"), + outputPath, + version: "1.0.0", + sourceEdition: "wordnet-3.1", + sourceDumpDate: "2011-05-26", + sourceArchiveUrl: "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + attributionUrl: "https://wordnet.princeton.edu/license-and-commercial-use", + licenseNotice: "Complete WordNet license notice.", + creatorAttribution: "Princeton University.", + assetUrl: "https://example.invalid/en.sqlite", + license: "WordNet 3.1 License", + }); + + const database = new Database(outputPath, { readonly: true }); + try { + expect( + database.prepare("SELECT value FROM metadata WHERE key = 'source_edition'").pluck().get(), + ).toBe("wordnet-3.1"); + expect( + database.prepare("SELECT value FROM metadata WHERE key = 'license'").pluck().get(), + ).toBe("WordNet 3.1 License"); + expect( + database.prepare("SELECT rank FROM lookup WHERE lookup_key = 'children'").pluck().get(), + ).toBe(1); + } finally { + database.close(); + } + expect(descriptor).toMatchObject({ + language: "en", + sourceEdition: "wordnet-3.1", + license: "WordNet 3.1 License", + }); + }); + + it("builds a validated, pretty-printed descriptor from the required command arguments", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-command-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "en.sqlite"); + const descriptorPath = join(directory, "en.json"); + const scriptPath = resolve(import.meta.dirname, "../../scripts/build-dictionary-pack.ts"); + const tsxPath = resolve(import.meta.dirname, "../../../../node_modules/tsx/dist/cli.mjs"); + const result = spawnSync( + process.execPath, + [ + tsxPath, + scriptPath, + "--language", + "en", + "--input", + join(fixtureDirectory, "en.jsonl"), + "--output", + outputPath, + "--version", + "1.2.3", + "--source-edition", + "enwiktionary", + "--license", + "CC BY-SA 4.0", + "--source-date", + "2026-09-03", + "--source-archive-url", + "https://example.invalid/en-archive", + "--attribution-url", + "https://example.invalid/en-attribution", + "--license-file", + join(fixtureDirectory, "test-license.txt"), + "--creator-attribution", + TEST_CREATOR_ATTRIBUTION, + "--asset-url", + "https://example.invalid/en.sqlite", + "--descriptor", + descriptorPath, + ], + { encoding: "utf8" }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(await readFile(descriptorPath, "utf8")).toBe( + `${JSON.stringify(JSON.parse(await readFile(descriptorPath, "utf8")), null, 2)}\n`, + ); + }); + + it.each(["01.2.3", "1.2.3-01"])( + "rejects non-SemVer version %s before creating output files", + async (version) => { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-invalid-semver-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "en.sqlite"); + const descriptorPath = join(directory, "en.json"); + const scriptPath = resolve(import.meta.dirname, "../../scripts/build-dictionary-pack.ts"); + const tsxPath = resolve(import.meta.dirname, "../../../../node_modules/tsx/dist/cli.mjs"); + const result = spawnSync( + process.execPath, + [ + tsxPath, + scriptPath, + "--language", + "en", + "--input", + join(fixtureDirectory, "en.jsonl"), + "--output", + outputPath, + "--version", + version, + "--source-edition", + "enwiktionary", + "--license", + "CC BY-SA 4.0", + "--source-date", + "2026-09-03", + "--source-archive-url", + "https://example.invalid/en-archive", + "--attribution-url", + "https://example.invalid/en-attribution", + "--license-file", + join(fixtureDirectory, "test-license.txt"), + "--creator-attribution", + TEST_CREATOR_ATTRIBUTION, + "--asset-url", + "https://example.invalid/en.sqlite", + "--descriptor", + descriptorPath, + ], + { encoding: "utf8" }, + ); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("--version must be a semver value"); + expect(existsSync(outputPath)).toBe(false); + expect(existsSync(descriptorPath)).toBe(false); + }, + ); +}); diff --git a/packages/cli/src/dictionary/pack-builder.ts b/packages/cli/src/dictionary/pack-builder.ts new file mode 100644 index 000000000..d173ead74 --- /dev/null +++ b/packages/cli/src/dictionary/pack-builder.ts @@ -0,0 +1,379 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { stat } from "node:fs/promises"; +import readline from "node:readline"; +import { + type DictionaryLanguage, + type DictionaryPackDescriptor, + type DictionarySource, + prepareDictionarySelection, +} from "@readany/core/dictionary"; +import Database from "better-sqlite3"; +import { DICTIONARY_SCHEMA_SQL, DICTIONARY_SCHEMA_VERSION } from "./schema.js"; + +const ACCEPTED_FORM_TAGS: Record> = { + en: new Set(["plural", "past", "present", "participle", "wordnet-exception"]), + zh: new Set(["Simplified-Chinese", "Traditional-Chinese"]), +}; + +const TRANSFORMED_BY = "ReadAny offline dictionary pack builder"; + +interface WiktionarySound { + ipa?: unknown; + "zh-pron"?: unknown; +} + +interface WiktionaryForm { + form?: unknown; + tags?: unknown; +} + +interface WiktionarySense { + glosses?: unknown; +} + +interface WiktionaryRecord { + title?: unknown; + redirect?: unknown; + redirects?: unknown; + word?: unknown; + lang_code?: unknown; + pos?: unknown; + sounds?: unknown; + forms?: unknown; + senses?: unknown; +} + +type RedirectGraph = Map>; + +interface PendingEntry { + headword: string; + simplified?: string; + traditional?: string; + pronunciation?: string; + partOfSpeech: string; + definitions: string[]; + lookupRanks: Map; +} + +interface DictionaryPackBuildOptionsBase { + inputPath: string; + outputPath: string; + version: string; + sourceDumpDate: string; + sourceArchiveUrl: string; + attributionUrl: string; + licenseNotice: string; + creatorAttribution: string; + assetUrl: string; +} + +export type DictionaryPackBuildOptions = DictionaryPackBuildOptionsBase & DictionarySource; + +function nonEmptyString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function stringArray(value: unknown): string[] { + if (!Array.isArray(value)) return []; + return value.flatMap((item) => { + const text = nonEmptyString(item); + return text ? [text] : []; + }); +} + +function firstSound(sounds: unknown, property: keyof WiktionarySound): string | undefined { + if (!Array.isArray(sounds)) return undefined; + for (const sound of sounds as WiktionarySound[]) { + const value = nonEmptyString(sound?.[property]); + if (value) return value; + } + return undefined; +} + +function normalizeLookupKey(value: string, language: DictionaryLanguage): string | undefined { + const selection = prepareDictionarySelection(value); + return selection.ok && selection.language === language ? selection.key : undefined; +} + +function normalizeAlias(value: string): string { + return value.normalize("NFKC").trim(); +} + +function collectChineseRedirect(record: WiktionaryRecord, graph: RedirectGraph): void { + const isHardRedirect = record.pos === "hard-redirect"; + const isSoftRedirect = record.pos === "soft-redirect" && record.lang_code === "zh"; + if (!isHardRedirect && !isSoftRedirect) return; + + const alias = normalizeLookupKey( + nonEmptyString(isHardRedirect ? record.title : record.word) ?? "", + "zh", + ); + const targets = isHardRedirect + ? [nonEmptyString(record.redirect)].filter((target): target is string => Boolean(target)) + : stringArray(record.redirects); + if (!alias) return; + + for (const target of targets) { + const normalizedTarget = normalizeLookupKey(target, "zh"); + if (!normalizedTarget || normalizedTarget === alias) continue; + const existingTargets = graph.get(alias) ?? new Set(); + existingTargets.add(normalizedTarget); + graph.set(alias, existingTargets); + } +} + +function resolvedRedirectTargets( + alias: string, + graph: RedirectGraph, + canonicalKeys: ReadonlySet, + memo: Map>, + visiting = new Set(), +): ReadonlySet { + if (canonicalKeys.has(alias)) return new Set([alias]); + const cached = memo.get(alias); + if (cached) return cached; + if (visiting.has(alias)) return new Set(); + + visiting.add(alias); + const resolved = new Set(); + for (const target of [...(graph.get(alias) ?? [])].sort()) { + for (const canonicalTarget of resolvedRedirectTargets( + target, + graph, + canonicalKeys, + memo, + visiting, + )) { + resolved.add(canonicalTarget); + } + } + visiting.delete(alias); + memo.set(alias, resolved); + return resolved; +} + +function prepareEntry( + record: WiktionaryRecord, + language: DictionaryLanguage, +): PendingEntry | undefined { + if (record.lang_code !== language) return undefined; + + const headword = nonEmptyString(record.word); + if (!headword) return undefined; + const definitions = Array.isArray(record.senses) + ? (record.senses as WiktionarySense[]).flatMap((sense) => stringArray(sense?.glosses)) + : []; + if (definitions.length === 0) return undefined; + + const lookupRanks = new Map(); + const canonicalKey = normalizeLookupKey(headword, language); + if (!canonicalKey) return undefined; + lookupRanks.set(canonicalKey, 0); + + let simplified: string | undefined; + let traditional: string | undefined; + if (Array.isArray(record.forms)) { + for (const form of record.forms as WiktionaryForm[]) { + const alias = nonEmptyString(form?.form); + if (!alias) continue; + const tags = new Set(stringArray(form?.tags)); + if (![...tags].some((tag) => ACCEPTED_FORM_TAGS[language].has(tag))) continue; + + const normalizedAlias = normalizeLookupKey(alias, language); + if (normalizedAlias && !lookupRanks.has(normalizedAlias)) { + lookupRanks.set(normalizedAlias, 1); + } + if (language === "zh" && tags.has("Simplified-Chinese")) { + simplified ??= normalizeAlias(alias); + } + if (language === "zh" && tags.has("Traditional-Chinese")) { + traditional ??= normalizeAlias(alias); + } + } + } + + return { + headword: normalizeAlias(headword), + simplified, + traditional, + pronunciation: firstSound(record.sounds, language === "en" ? "ipa" : "zh-pron"), + partOfSpeech: nonEmptyString(record.pos) ?? "unknown", + definitions, + lookupRanks, + }; +} + +async function sha256File(path: string): Promise { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) { + hash.update(chunk); + } + return hash.digest("hex"); +} + +function assertSqliteChecks(database: Database.Database): void { + const foreignKeyViolations = database.pragma("foreign_key_check") as unknown[]; + if (foreignKeyViolations.length > 0) { + throw new Error("Dictionary pack foreign key check failed"); + } + const integrity = database.pragma("integrity_check", { simple: true }); + if (integrity !== "ok") { + throw new Error(`Dictionary pack integrity check failed: ${String(integrity)}`); + } +} + +export async function buildDictionaryPack( + options: DictionaryPackBuildOptions, +): Promise { + if (!options.licenseNotice.trim()) throw new Error("Dictionary license notice is required"); + if (!options.creatorAttribution.trim()) + throw new Error("Dictionary creator attribution is required"); + const database = new Database(options.outputPath); + try { + database.exec(DICTIONARY_SCHEMA_SQL); + const insertMetadata = database.prepare("INSERT INTO metadata (key, value) VALUES (?, ?)"); + const insertEntry = database.prepare(` + INSERT INTO entries (language, headword, simplified, traditional, pronunciation, part_of_speech) + VALUES (?, ?, ?, ?, ?, ?) + `); + const insertSense = database.prepare( + "INSERT INTO senses (entry_id, sense_order, definition) VALUES (?, ?, ?)", + ); + const insertLookup = database.prepare( + "INSERT INTO lookup (lookup_key, entry_id, rank) VALUES (?, ?, ?)", + ); + const insertRedirectLookup = database.prepare(` + INSERT OR IGNORE INTO lookup (lookup_key, entry_id, rank) + SELECT ?, entry_id, 1 + FROM lookup + WHERE lookup_key = ? AND rank = 0 + ORDER BY entry_id + `); + + const metadata: ReadonlyArray = [ + ["schema_version", String(DICTIONARY_SCHEMA_VERSION)], + ["language", options.language], + ["version", options.version], + ["source_edition", options.sourceEdition], + ["source_dump_date", options.sourceDumpDate], + ["source_archive_url", options.sourceArchiveUrl], + ["asset_url", options.assetUrl], + ["license", options.license], + ["license_notice", options.licenseNotice], + ["creator_attribution", options.creatorAttribution], + ["attribution_url", options.attributionUrl], + ["transformed_by", TRANSFORMED_BY], + ]; + database.transaction(() => { + for (const [key, value] of metadata) insertMetadata.run(key, value); + })(); + + const insertBatch = database.transaction((entries: PendingEntry[]) => { + for (const entry of entries) { + const entryId = Number( + insertEntry.run( + options.language, + entry.headword, + entry.simplified ?? null, + entry.traditional ?? null, + entry.pronunciation ?? null, + entry.partOfSpeech, + ).lastInsertRowid, + ); + for (const [index, definition] of entry.definitions.entries()) { + insertSense.run(entryId, index, definition); + } + for (const [lookupKey, rank] of entry.lookupRanks) { + insertLookup.run(lookupKey, entryId, rank); + } + } + }); + + const lines = readline.createInterface({ + input: createReadStream(options.inputPath, { encoding: "utf8" }), + crlfDelay: Number.POSITIVE_INFINITY, + }); + let lineNumber = 0; + let batch: PendingEntry[] = []; + const redirects: RedirectGraph = new Map(); + for await (const line of lines) { + lineNumber += 1; + if (!line.trim()) continue; + let parsed: WiktionaryRecord; + try { + parsed = JSON.parse(line) as WiktionaryRecord; + } catch (error) { + throw new Error(`Invalid JSONL at line ${lineNumber}: ${String(error)}`); + } + if (options.language === "zh") collectChineseRedirect(parsed, redirects); + const entry = prepareEntry(parsed, options.language); + if (!entry) continue; + batch.push(entry); + if (batch.length === 5_000) { + insertBatch(batch); + batch = []; + } + } + if (batch.length > 0) insertBatch(batch); + + const canonicalKeys = new Set( + database + .prepare("SELECT DISTINCT lookup_key FROM lookup WHERE rank = 0 ORDER BY lookup_key") + .pluck() + .all() as string[], + ); + const resolved = new Map>(); + database.transaction(() => { + for (const alias of [...redirects.keys()].sort()) { + for (const target of [ + ...resolvedRedirectTargets(alias, redirects, canonicalKeys, resolved), + ].sort()) { + if (alias !== target) insertRedirectLookup.run(alias, target); + } + } + })(); + + assertSqliteChecks(database); + database.exec("VACUUM"); + } finally { + database.close(); + } + + const [file, sha256] = await Promise.all([ + stat(options.outputPath), + sha256File(options.outputPath), + ]); + const descriptor = { + version: options.version, + schemaVersion: DICTIONARY_SCHEMA_VERSION, + sourceDumpDate: options.sourceDumpDate, + sizeBytes: file.size, + sha256, + url: options.assetUrl, + sourceArchiveUrl: options.sourceArchiveUrl, + attributionUrl: options.attributionUrl, + }; + if (options.language === "zh") { + return { + ...descriptor, + language: "zh", + sourceEdition: options.sourceEdition, + license: options.license, + }; + } + if (options.sourceEdition === "wordnet-3.1") { + return { + ...descriptor, + language: "en", + sourceEdition: options.sourceEdition, + license: options.license, + }; + } + return { + ...descriptor, + language: "en", + sourceEdition: options.sourceEdition, + license: options.license, + }; +} diff --git a/packages/cli/src/dictionary/schema.ts b/packages/cli/src/dictionary/schema.ts new file mode 100644 index 000000000..7475c4629 --- /dev/null +++ b/packages/cli/src/dictionary/schema.ts @@ -0,0 +1,32 @@ +export const DICTIONARY_SCHEMA_VERSION = 1 as const; + +export const DICTIONARY_SCHEMA_SQL = ` +PRAGMA journal_mode = DELETE; +PRAGMA synchronous = OFF; +CREATE TABLE metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +) WITHOUT ROWID; +CREATE TABLE entries ( + id INTEGER PRIMARY KEY, + language TEXT NOT NULL CHECK (language IN ('en', 'zh')), + headword TEXT NOT NULL, + simplified TEXT, + traditional TEXT, + pronunciation TEXT, + part_of_speech TEXT NOT NULL +); +CREATE TABLE senses ( + entry_id INTEGER NOT NULL REFERENCES entries(id), + sense_order INTEGER NOT NULL, + definition TEXT NOT NULL, + PRIMARY KEY (entry_id, sense_order) +) WITHOUT ROWID; +CREATE TABLE lookup ( + lookup_key TEXT NOT NULL, + entry_id INTEGER NOT NULL REFERENCES entries(id), + rank INTEGER NOT NULL, + PRIMARY KEY (lookup_key, entry_id) +) WITHOUT ROWID; +CREATE INDEX lookup_key_rank_idx ON lookup(lookup_key, rank, entry_id); +`; diff --git a/packages/cli/src/dictionary/wordnet-converter.test.ts b/packages/cli/src/dictionary/wordnet-converter.test.ts new file mode 100644 index 000000000..6c710eaed --- /dev/null +++ b/packages/cli/src/dictionary/wordnet-converter.test.ts @@ -0,0 +1,165 @@ +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { convertWordNetDirectory } from "./wordnet-converter.js"; + +const fixtureDirectory = resolve(import.meta.dirname, "fixtures/wordnet"); +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true })), + ); +}); + +describe("convertWordNetDirectory", () => { + it("converts WordNet synsets and conservative inflections to deterministic builder JSONL", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-wordnet-converter-")); + temporaryDirectories.push(directory); + const firstOutput = join(directory, "first.jsonl"); + const secondOutput = join(directory, "second.jsonl"); + + const firstStats = await convertWordNetDirectory({ + inputDirectory: fixtureDirectory, + outputPath: firstOutput, + }); + const secondStats = await convertWordNetDirectory({ + inputDirectory: fixtureDirectory, + outputPath: secondOutput, + }); + + const firstBytes = await readFile(firstOutput, "utf8"); + expect(await readFile(secondOutput, "utf8")).toBe(firstBytes); + expect(secondStats).toEqual(firstStats); + const records = firstBytes + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + + expect(firstStats).toEqual({ records: 14, senses: 15, aliases: 15 }); + expect(records).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + word: "desire", + lang_code: "en", + pos: "noun", + senses: [ + { glosses: ["a strong feeling of wanting something"] }, + { glosses: ["something that is wanted"] }, + ], + forms: [{ form: "desires", tags: ["plural"] }], + }), + expect.objectContaining({ + word: "desire", + pos: "verb", + senses: [{ glosses: ["feel or have a desire for"] }], + forms: [{ form: "desires", tags: ["present"] }], + }), + expect.objectContaining({ + word: "child", + pos: "noun", + forms: [{ form: "children", tags: ["wordnet-exception"] }], + }), + expect.objectContaining({ word: "good", pos: "adjective" }), + expect.objectContaining({ word: "ice cream", pos: "noun", forms: [] }), + expect.objectContaining({ + word: "go", + pos: "verb", + forms: [ + { form: "went", tags: ["wordnet-exception"] }, + { form: "goes", tags: ["present"] }, + ], + }), + expect.objectContaining({ + word: "do", + pos: "verb", + forms: [ + { form: "did", tags: ["wordnet-exception"] }, + { form: "does", tags: ["present"] }, + ], + }), + expect.objectContaining({ + word: "have", + pos: "verb", + forms: [{ form: "has", tags: ["wordnet-exception"] }], + }), + expect.objectContaining({ + word: "be", + pos: "verb", + forms: [{ form: "is", tags: ["wordnet-exception"] }], + }), + expect.objectContaining({ + word: "tattoo", + pos: "noun", + forms: [{ form: "tattoos", tags: ["plural"] }], + }), + expect.objectContaining({ + word: "tattoo", + pos: "verb", + forms: [{ form: "tattoos", tags: ["present"] }], + }), + ]), + ); + expect(firstBytes).not.toContain("tattooes"); + expect(firstBytes).not.toContain("his desire was obvious"); + expect(firstBytes).not.toContain("the house was his desire"); + expect(firstBytes).not.toContain("I desire a quiet room"); + }); + + it.each([ + ["malformed lex id", "data.noun", "city 0 000", "city zz 000"], + ["missing pointer count", "data.noun", "city 0 000", "city 0"], + ["malformed pointer", "data.noun", "city 0 000", "city 0 001 @ nope n 0000"], + ["malformed verb frame", "data.verb", "go 0 000 00", "go 0 000 01 ? 01 00"], + ["unexpected trailing token", "data.noun", "city 0 000", "city 0 000 surprise"], + ])( + "rejects %s instead of accepting a partial data-row parse", + async (_name, fileName, from, to) => { + const directory = await mkdtemp(join(tmpdir(), "readany-wordnet-grammar-")); + temporaryDirectories.push(directory); + const inputDirectory = join(directory, "wordnet"); + await cp(fixtureDirectory, inputDirectory, { recursive: true }); + const path = join(inputDirectory, fileName); + const source = await readFile(path, "utf8"); + await writeFile(path, source.replace(from, to), "utf8"); + + await expect( + convertWordNetDirectory({ inputDirectory, outputPath: join(directory, "output.jsonl") }), + ).rejects.toThrow(new RegExp(`${fileName.replace(".", "\\.")}.*line`, "i")); + }, + ); + + it("rejects a malformed WordNet data row with its file and line", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-wordnet-invalid-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "wordnet.jsonl"); + + await expect( + convertWordNetDirectory({ + inputDirectory: resolve(import.meta.dirname, "fixtures/wordnet-invalid"), + outputPath, + }), + ).rejects.toThrow(/data\.noun.*line/i); + }); + + it("exposes a validated release-side conversion command", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-wordnet-command-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "wordnet.jsonl"); + const scriptPath = resolve(import.meta.dirname, "../../scripts/convert-wordnet.ts"); + const tsxPath = resolve(import.meta.dirname, "../../../../node_modules/tsx/dist/cli.mjs"); + + const result = spawnSync( + process.execPath, + [tsxPath, scriptPath, "--input-directory", fixtureDirectory, "--output", outputPath], + { encoding: "utf8" }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ records: 14, senses: 15, aliases: 15 }); + expect(existsSync(outputPath)).toBe(true); + }); +}); diff --git a/packages/cli/src/dictionary/wordnet-converter.ts b/packages/cli/src/dictionary/wordnet-converter.ts new file mode 100644 index 000000000..b9712fa52 --- /dev/null +++ b/packages/cli/src/dictionary/wordnet-converter.ts @@ -0,0 +1,296 @@ +import { once } from "node:events"; +import { createWriteStream } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { prepareDictionarySelection } from "@readany/core/dictionary"; + +const WORDNET_DATA_FILES = [ + { fileName: "data.noun", partOfSpeech: "noun", synsetTypes: new Set(["n"]) }, + { fileName: "data.verb", partOfSpeech: "verb", synsetTypes: new Set(["v"]) }, + { fileName: "data.adj", partOfSpeech: "adjective", synsetTypes: new Set(["a", "s"]) }, + { fileName: "data.adv", partOfSpeech: "adverb", synsetTypes: new Set(["r"]) }, +] as const; + +const WORDNET_EXCEPTION_FILES = [ + { fileName: "noun.exc", partOfSpeech: "noun" }, + { fileName: "verb.exc", partOfSpeech: "verb" }, + { fileName: "adj.exc", partOfSpeech: "adjective" }, + { fileName: "adv.exc", partOfSpeech: "adverb" }, +] as const; + +type PartOfSpeech = (typeof WORDNET_DATA_FILES)[number]["partOfSpeech"]; + +interface PendingWordNetEntry { + word: string; + partOfSpeech: PartOfSpeech; + definitions: string[]; + definitionSet: Set; + forms: Map; + exceptionForms: Set; +} + +const WORDNET_POINTER_SYMBOLS = new Set([ + "!", + "@", + "@i", + "~", + "~i", + "#m", + "#s", + "#p", + "%m", + "%s", + "%p", + "=", + "+", + ";c", + "-c", + ";r", + "-r", + ";u", + "-u", + "*", + ">", + "^", + "$", + "&", + "<", + "\\", +]); + +const IRREGULAR_THIRD_PERSON_VERBS = new Set(["be", "have"]); + +export interface WordNetConversionOptions { + inputDirectory: string; + outputPath: string; +} + +export interface WordNetConversionStats { + records: number; + senses: number; + aliases: number; +} + +export async function convertWordNetDirectory( + options: WordNetConversionOptions, +): Promise { + const entries = new Map(); + + for (const dataFile of WORDNET_DATA_FILES) { + const path = join(options.inputDirectory, dataFile.fileName); + const lines = (await readFile(path, "utf8")).split(/\r?\n/); + for (const [index, line] of lines.entries()) { + if (!line || /^\s/.test(line)) continue; + const parsed = parseDataRow(path, index + 1, line, dataFile.synsetTypes); + for (const word of parsed.words) { + const selection = prepareDictionarySelection(word); + if (!selection.ok || selection.language !== "en") continue; + const canonical = selection.key; + const key = entryKey(dataFile.partOfSpeech, canonical); + let entry = entries.get(key); + if (!entry) { + entry = { + word: canonical, + partOfSpeech: dataFile.partOfSpeech, + definitions: [], + definitionSet: new Set(), + forms: new Map(), + exceptionForms: new Set(), + }; + entries.set(key, entry); + } + if (!entry.definitionSet.has(parsed.definition)) { + entry.definitionSet.add(parsed.definition); + entry.definitions.push(parsed.definition); + } + } + } + } + + for (const exceptionFile of WORDNET_EXCEPTION_FILES) { + const path = join(options.inputDirectory, exceptionFile.fileName); + const lines = (await readFile(path, "utf8")).split(/\r?\n/); + for (const [index, line] of lines.entries()) { + if (!line.trim()) continue; + const tokens = line.trim().split(/\s+/); + if (tokens.length < 2) throw formatError(path, index + 1, "invalid exception row"); + const inflected = normalizeLemma(tokens[0]); + for (const baseToken of tokens.slice(1)) { + const base = normalizeLemma(baseToken); + const selection = prepareDictionarySelection(base); + const aliasSelection = prepareDictionarySelection(inflected); + if ( + !selection.ok || + selection.language !== "en" || + !aliasSelection.ok || + aliasSelection.language !== "en" + ) { + continue; + } + const entry = entries.get(entryKey(exceptionFile.partOfSpeech, selection.key)); + if (!entry) continue; + entry.exceptionForms.add(aliasSelection.key); + addAlias(entry, aliasSelection.key, "wordnet-exception"); + } + } + } + + for (const entry of entries.values()) { + if (!/^[a-z]+$/u.test(entry.word)) continue; + if (entry.partOfSpeech === "noun" && entry.exceptionForms.size === 0) { + addAlias(entry, regularNounPlural(entry.word), "plural"); + } + if (entry.partOfSpeech === "verb" && !IRREGULAR_THIRD_PERSON_VERBS.has(entry.word)) { + addAlias(entry, regularVerbThirdPerson(entry.word), "present"); + } + } + + const output = createWriteStream(options.outputPath, { encoding: "utf8", flags: "wx" }); + let outputError: unknown; + output.on("error", (error) => { + outputError = error; + }); + for (const entry of entries.values()) { + const record = { + word: entry.word, + lang_code: "en", + pos: entry.partOfSpeech, + senses: entry.definitions.map((definition) => ({ glosses: [definition] })), + forms: [...entry.forms].map(([form, tag]) => ({ form, tags: [tag] })), + }; + if (!output.write(`${JSON.stringify(record)}\n`)) await once(output, "drain"); + if (outputError) throw outputError; + } + output.end(); + await once(output, "close"); + if (outputError) throw outputError; + + return { + records: entries.size, + senses: [...entries.values()].reduce((sum, entry) => sum + entry.definitions.length, 0), + aliases: [...entries.values()].reduce((sum, entry) => sum + entry.forms.size, 0), + }; +} + +function parseDataRow( + path: string, + lineNumber: number, + line: string, + expectedSynsetTypes: ReadonlySet, +): { words: string[]; definition: string } { + const separator = line.indexOf("|"); + if (separator < 0) throw formatError(path, lineNumber, "missing gloss separator"); + const fields = line.slice(0, separator).trim().split(/\s+/); + let cursor = 0; + const take = (name: string): string => { + const value = fields[cursor]; + if (value === undefined) throw formatError(path, lineNumber, `missing ${name}`); + cursor += 1; + return value; + }; + + if (!/^\d{8}$/.test(take("synset offset"))) + throw formatError(path, lineNumber, "invalid synset offset"); + if (!/^\d{2}$/.test(take("lexicographer file number"))) + throw formatError(path, lineNumber, "invalid lexicographer file number"); + const synsetType = take("synset type"); + if (!expectedSynsetTypes.has(synsetType)) { + throw formatError(path, lineNumber, `unexpected synset type ${synsetType}`); + } + const wordCountText = take("word count"); + if (!/^[0-9a-f]{2}$/i.test(wordCountText)) { + throw formatError(path, lineNumber, "invalid hexadecimal word count"); + } + const wordCount = Number.parseInt(wordCountText, 16); + if (wordCount === 0) throw formatError(path, lineNumber, "synset has no words"); + const words: string[] = []; + for (let index = 0; index < wordCount; index += 1) { + const rawWord = take(`word ${index + 1}`); + if (!/^[\x21-\x7e]+$/u.test(rawWord)) + throw formatError(path, lineNumber, `invalid word ${index + 1}`); + const lexId = take(`lex id ${index + 1}`); + if (!/^[0-9a-f]$/iu.test(lexId)) + throw formatError(path, lineNumber, `invalid lex id ${index + 1}`); + const word = normalizeLemma(rawWord); + if (word) words.push(word); + } + if (words.length === 0) throw formatError(path, lineNumber, "synset has no words"); + + const pointerCountText = take("pointer count"); + if (!/^\d{3}$/u.test(pointerCountText)) + throw formatError(path, lineNumber, "invalid pointer count"); + const pointerCount = Number.parseInt(pointerCountText, 10); + for (let index = 0; index < pointerCount; index += 1) { + const symbol = take(`pointer ${index + 1} symbol`); + if (!WORDNET_POINTER_SYMBOLS.has(symbol)) + throw formatError(path, lineNumber, `invalid pointer ${index + 1} symbol`); + if (!/^\d{8}$/u.test(take(`pointer ${index + 1} synset offset`))) + throw formatError(path, lineNumber, `invalid pointer ${index + 1} synset offset`); + if (!/^[nvar]$/u.test(take(`pointer ${index + 1} part of speech`))) + throw formatError(path, lineNumber, `invalid pointer ${index + 1} part of speech`); + const sourceTarget = take(`pointer ${index + 1} source/target`); + if (!/^[0-9a-f]{4}$/iu.test(sourceTarget)) + throw formatError(path, lineNumber, `invalid pointer ${index + 1} source/target`); + const sourceWord = Number.parseInt(sourceTarget.slice(0, 2), 16); + if (sourceWord > wordCount) + throw formatError(path, lineNumber, `pointer ${index + 1} source word is out of range`); + } + + if (synsetType === "v") { + const frameCountText = take("verb frame count"); + if (!/^\d{2}$/u.test(frameCountText)) + throw formatError(path, lineNumber, "invalid verb frame count"); + const frameCount = Number.parseInt(frameCountText, 10); + for (let index = 0; index < frameCount; index += 1) { + if (take(`verb frame ${index + 1} marker`) !== "+") + throw formatError(path, lineNumber, `invalid verb frame ${index + 1} marker`); + if (!/^\d{2}$/u.test(take(`verb frame ${index + 1} number`))) + throw formatError(path, lineNumber, `invalid verb frame ${index + 1} number`); + const wordNumber = take(`verb frame ${index + 1} word number`); + if (!/^[0-9a-f]{2}$/iu.test(wordNumber)) + throw formatError(path, lineNumber, `invalid verb frame ${index + 1} word number`); + if (Number.parseInt(wordNumber, 16) > wordCount) + throw formatError(path, lineNumber, `verb frame ${index + 1} word is out of range`); + } + } + if (cursor !== fields.length) + throw formatError(path, lineNumber, "unexpected token before gloss"); + + const gloss = line.slice(separator + 1).trim(); + const exampleStart = gloss.search(/;\s*"/u); + const definition = (exampleStart >= 0 ? gloss.slice(0, exampleStart) : gloss).trim(); + if (!definition) throw formatError(path, lineNumber, "synset has no definition"); + return { words, definition }; +} + +function normalizeLemma(value: string): string { + return value + .replace(/\((?:a|p|ip)\)$/u, "") + .replaceAll("_", " ") + .normalize("NFKC") + .trim(); +} + +function entryKey(partOfSpeech: PartOfSpeech, word: string): string { + return `${partOfSpeech}\0${word}`; +} + +function addAlias(entry: PendingWordNetEntry, alias: string, tag: string): void { + if (alias !== entry.word && !entry.forms.has(alias)) entry.forms.set(alias, tag); +} + +function regularNounPlural(word: string): string { + if (/[^aeiou]y$/u.test(word)) return `${word.slice(0, -1)}ies`; + if (/(?:s|x|z|ch|sh)$/u.test(word)) return `${word}es`; + return `${word}s`; +} + +function regularVerbThirdPerson(word: string): string { + if (/[^aeiou]y$/u.test(word)) return `${word.slice(0, -1)}ies`; + if (word === "do" || word === "go" || /(?:s|x|z|ch|sh)$/u.test(word)) return `${word}es`; + return `${word}s`; +} + +function formatError(path: string, lineNumber: number, message: string): Error { + return new Error(`Invalid WordNet data in ${path} at line ${lineNumber}: ${message}`); +} diff --git a/packages/core/package.json b/packages/core/package.json index 7c385a87c..892eeac63 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -11,6 +11,8 @@ }, "exports": { ".": "./src/index.ts", + "./dictionary": "./src/dictionary/index.ts", + "./dictionary/*": "./src/dictionary/*.ts", "./types": "./src/types/index.ts", "./types/*": "./src/types/*.ts", "./i18n": "./src/i18n/index.ts", diff --git a/packages/core/src/dictionary/index.ts b/packages/core/src/dictionary/index.ts new file mode 100644 index 000000000..7a16ea8dd --- /dev/null +++ b/packages/core/src/dictionary/index.ts @@ -0,0 +1,15 @@ +export { parseDictionaryManifest } from "./manifest"; +export { prepareDictionarySelection } from "./selection"; +export type { + ChineseDictionaryPackDescriptor, + DictionaryEntry, + DictionaryLanguage, + DictionaryManifest, + DictionaryPackDescriptor, + DictionaryLicense, + DictionarySense, + DictionarySource, + DictionarySourceEdition, + EnglishDictionaryPackDescriptor, + PreparedDictionarySelection, +} from "./types"; diff --git a/packages/core/src/dictionary/manifest.test.ts b/packages/core/src/dictionary/manifest.test.ts new file mode 100644 index 000000000..a8f0f1d40 --- /dev/null +++ b/packages/core/src/dictionary/manifest.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from "vitest"; +import { parseDictionaryManifest } from "./manifest"; + +const validPack = { + language: "en", + version: "2026.9.3", + schemaVersion: 1, + sourceEdition: "wordnet-3.1", + sourceDumpDate: "2011-05-26", + sizeBytes: 123, + sha256: "a".repeat(64), + url: "https://cdn.example.com/dictionaries/en.sqlite3", + sourceArchiveUrl: "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + attributionUrl: "https://wordnet.princeton.edu/", + license: "WordNet 3.1 License", +} as const; + +function validManifest() { + return { + manifestVersion: 1, + packs: { + en: validPack, + zh: { + ...validPack, + language: "zh", + sourceEdition: "zhwiktionary", + sourceDumpDate: "2026-09-01", + url: "https://cdn.example.com/dictionaries/zh.sqlite3", + sourceArchiveUrl: + "https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2", + attributionUrl: + "https://zh.wiktionary.org/wiki/Wiktionary:%E7%89%88%E6%9D%83%E4%BF%A1%E6%81%AF", + license: "CC BY-SA 4.0", + }, + }, + }; +} + +describe("parseDictionaryManifest", () => { + it("parses valid English and Chinese descriptors", () => { + expect(parseDictionaryManifest(validManifest())).toEqual(validManifest()); + }); + + it("continues to accept an English Wiktionary descriptor", () => { + const manifest = validManifest(); + manifest.packs.en = { + ...manifest.packs.en, + sourceEdition: "enwiktionary", + sourceDumpDate: "2026-09-01", + attributionUrl: "https://en.wiktionary.org/wiki/Wiktionary:Copyrights", + license: "CC BY-SA 4.0", + } as typeof manifest.packs.en; + + expect(parseDictionaryManifest(manifest)).toEqual(manifest); + }); + + it.each([ + ["WordNet with the Wiktionary license", "wordnet-3.1", "CC BY-SA 4.0"], + ["English Wiktionary with the WordNet license", "enwiktionary", "WordNet 3.1 License"], + ])("rejects %s", (_description, sourceEdition, license) => { + const manifest = validManifest(); + const invalid = { + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, sourceEdition, license } }, + }; + + expect(() => parseDictionaryManifest(invalid)).toThrow(); + }); + + it.each(["1", "1.2", "01.2.3", "1.02.3", "1.2.03", "1.2.3-01", "v1.2.3"])( + "rejects non-SemVer version %s", + (version) => { + const manifest = validManifest(); + expect(() => + parseDictionaryManifest({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, version } }, + }), + ).toThrow(); + }, + ); + + it.each(["2026-02-30", "2026-13-01", "2026-9-01", "not-a-date"])( + "rejects invalid source date %s", + (sourceDumpDate) => { + const manifest = validManifest(); + expect(() => + parseDictionaryManifest({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, sourceDumpDate } }, + }), + ).toThrow(); + }, + ); + + it("accepts a pack exactly at the 150 MiB limit", () => { + const manifest = validManifest(); + manifest.packs.en = { ...manifest.packs.en, sizeBytes: 150 * 1024 * 1024 }; + expect(parseDictionaryManifest(manifest)).toEqual(manifest); + }); + + it.each([150 * 1024 * 1024 + 1, 1.5, 0])("rejects invalid pack size %s", (sizeBytes) => { + const manifest = validManifest(); + expect(() => + parseDictionaryManifest({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, sizeBytes } }, + }), + ).toThrow(); + }); + + it.each(["http://example.test/file", "file:///tmp/file", "javascript:alert(1)"])( + "rejects non-HTTPS URL %s in every URL field", + (url) => { + for (const field of ["url", "sourceArchiveUrl", "attributionUrl"] as const) { + const manifest = validManifest(); + expect(() => + parseDictionaryManifest({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, [field]: url } }, + }), + ).toThrow(); + } + }, + ); + + it.each([ + [ + "an invalid URL", + (manifest: ReturnType) => ({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, url: "not-a-url" } }, + }), + ], + [ + "a non-hex SHA-256", + (manifest: ReturnType) => ({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, sha256: "z".repeat(64) } }, + }), + ], + [ + "a non-positive byte size", + (manifest: ReturnType) => ({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, sizeBytes: 0 } }, + }), + ], + [ + "a wrong language key", + (manifest: ReturnType) => ({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, language: "zh" } }, + }), + ], + [ + "a schema version other than 1", + (manifest: ReturnType) => ({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, schemaVersion: 2 } }, + }), + ], + [ + "an unknown descriptor field", + (manifest: ReturnType) => ({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, extra: true } }, + }), + ], + ])("rejects %s", (_description, invalidManifest) => { + expect(() => parseDictionaryManifest(invalidManifest(validManifest()))).toThrow(); + }); +}); diff --git a/packages/core/src/dictionary/manifest.ts b/packages/core/src/dictionary/manifest.ts new file mode 100644 index 000000000..a3c1e2f85 --- /dev/null +++ b/packages/core/src/dictionary/manifest.ts @@ -0,0 +1,68 @@ +import { z } from "zod"; +import type { DictionaryManifest } from "./types"; + +const MAX_DICTIONARY_PACK_BYTES = 150 * 1024 * 1024; +const SEMVER = + /^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:(?:0|[1-9]\d*)|\d*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:(?:0|[1-9]\d*)|\d*[A-Za-z-][0-9A-Za-z-]*))*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/u; + +const semverSchema = z.string().regex(SEMVER); +const calendarDateSchema = z.string().refine((value) => { + if (!/^\d{4}-\d{2}-\d{2}$/u.test(value)) return false; + const date = new Date(`${value}T00:00:00.000Z`); + return !Number.isNaN(date.getTime()) && date.toISOString().slice(0, 10) === value; +}); +const httpsUrlSchema = z + .string() + .url() + .refine((value) => { + try { + return new URL(value).protocol === "https:"; + } catch { + return false; + } + }); + +const descriptorFields = { + version: semverSchema, + schemaVersion: z.literal(1), + sourceDumpDate: calendarDateSchema, + sizeBytes: z.number().int().positive().max(MAX_DICTIONARY_PACK_BYTES), + sha256: z.string().regex(/^[a-fA-F0-9]{64}$/), + url: httpsUrlSchema, + sourceArchiveUrl: httpsUrlSchema, + attributionUrl: httpsUrlSchema, +} as const; + +const englishPackDescriptorSchema = z.discriminatedUnion("sourceEdition", [ + z.strictObject({ + ...descriptorFields, + language: z.literal("en"), + sourceEdition: z.literal("wordnet-3.1"), + license: z.literal("WordNet 3.1 License"), + }), + z.strictObject({ + ...descriptorFields, + language: z.literal("en"), + sourceEdition: z.literal("enwiktionary"), + license: z.literal("CC BY-SA 4.0"), + }), +]); + +const chinesePackDescriptorSchema = z.strictObject({ + ...descriptorFields, + language: z.literal("zh"), + sourceEdition: z.literal("zhwiktionary"), + license: z.literal("CC BY-SA 4.0"), +}); + +const dictionaryManifestSchema = z.strictObject({ + manifestVersion: z.literal(1), + packs: z.strictObject({ + en: englishPackDescriptorSchema, + zh: chinesePackDescriptorSchema, + }), +}); + +export function parseDictionaryManifest(value: unknown): DictionaryManifest { + return dictionaryManifestSchema.parse(value); +} diff --git a/packages/core/src/dictionary/selection.test.ts b/packages/core/src/dictionary/selection.test.ts new file mode 100644 index 000000000..e3a7a9db4 --- /dev/null +++ b/packages/core/src/dictionary/selection.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { prepareDictionarySelection } from "./selection"; + +describe("prepareDictionarySelection", () => { + it("normalizes English without changing display text", () => { + expect(prepareDictionarySelection(" \u201cDesires,\u201d ")).toEqual({ + ok: true, + language: "en", + key: "desires", + displayText: "Desires", + }); + }); + + it("accepts a Chinese word and strips surrounding punctuation", () => { + expect(prepareDictionarySelection("\u300a\u95b1\u8b80\u300b")).toEqual({ + ok: true, + language: "zh", + key: "\u95b1\u8b80", + displayText: "\u95b1\u8b80", + }); + }); + + it.each([ + ["", "empty"], + ["reading\u95b1\u8b80", "mixed-script"], + ["\u95b1\u8b80\u304b\u306a", "unsupported-script"], + ["\u0447\u0442\u0435\u043d\u0438\u0435", "unsupported-script"], + ["a".repeat(121), "too-long"], + ] as const)("rejects %j as %s", (text, reason) => { + expect(prepareDictionarySelection(text)).toEqual({ ok: false, reason }); + }); +}); diff --git a/packages/core/src/dictionary/selection.ts b/packages/core/src/dictionary/selection.ts new file mode 100644 index 000000000..c963cc9bf --- /dev/null +++ b/packages/core/src/dictionary/selection.ts @@ -0,0 +1,29 @@ +import type { PreparedDictionarySelection } from "./types"; + +const EDGE_PUNCTUATION = /^[\p{P}\p{Z}]+|[\p{P}\p{Z}]+$/gu; +const HAN = /\p{Script=Han}/u; +const LATIN = /\p{Script=Latin}/u; +const ALLOWED_EN = /^[\p{Script=Latin}\p{M}'’\-\s]+$/u; +const ALLOWED_ZH = /^[\p{Script=Han}\p{M}\s]+$/u; + +export function prepareDictionarySelection(text: string): PreparedDictionarySelection { + const displayText = text.normalize("NFKC").replace(EDGE_PUNCTUATION, "").trim(); + if (!displayText) return { ok: false, reason: "empty" }; + if (Array.from(displayText).length > 120) return { ok: false, reason: "too-long" }; + + const hasHan = HAN.test(displayText); + const hasLatin = LATIN.test(displayText); + if (hasHan && hasLatin) return { ok: false, reason: "mixed-script" }; + if (hasHan && ALLOWED_ZH.test(displayText)) { + return { ok: true, language: "zh", key: displayText.replace(/\s+/g, ""), displayText }; + } + if (hasLatin && ALLOWED_EN.test(displayText)) { + return { + ok: true, + language: "en", + key: displayText.replace(/\s+/g, " ").toLocaleLowerCase("en-US"), + displayText, + }; + } + return { ok: false, reason: "unsupported-script" }; +} diff --git a/packages/core/src/dictionary/types.ts b/packages/core/src/dictionary/types.ts new file mode 100644 index 000000000..0ab924d99 --- /dev/null +++ b/packages/core/src/dictionary/types.ts @@ -0,0 +1,70 @@ +export type DictionaryLanguage = "en" | "zh"; + +export interface DictionarySense { + order: number; + definition: string; +} + +export interface DictionaryEntry { + id: number; + language: DictionaryLanguage; + headword: string; + simplified?: string; + traditional?: string; + pronunciation?: string; + partOfSpeech: string; + senses: DictionarySense[]; +} + +interface DictionaryPackDescriptorBase { + version: string; + schemaVersion: 1; + sourceDumpDate: string; + sizeBytes: number; + sha256: string; + url: string; + sourceArchiveUrl: string; + attributionUrl: string; +} + +export type DictionarySourceEdition = "wordnet-3.1" | "enwiktionary" | "zhwiktionary"; +export type DictionaryLicense = "WordNet 3.1 License" | "CC BY-SA 4.0"; + +export type DictionarySource = + | { + language: "en"; + sourceEdition: "wordnet-3.1"; + license: "WordNet 3.1 License"; + } + | { + language: "en"; + sourceEdition: "enwiktionary"; + license: "CC BY-SA 4.0"; + } + | { + language: "zh"; + sourceEdition: "zhwiktionary"; + license: "CC BY-SA 4.0"; + }; + +export type EnglishDictionaryPackDescriptor = DictionaryPackDescriptorBase & + Extract; + +export type ChineseDictionaryPackDescriptor = DictionaryPackDescriptorBase & + Extract; + +export type DictionaryPackDescriptor = + | EnglishDictionaryPackDescriptor + | ChineseDictionaryPackDescriptor; + +export interface DictionaryManifest { + manifestVersion: 1; + packs: { + en: EnglishDictionaryPackDescriptor; + zh: ChineseDictionaryPackDescriptor; + }; +} + +export type PreparedDictionarySelection = + | { ok: true; language: DictionaryLanguage; key: string; displayText: string } + | { ok: false; reason: "empty" | "mixed-script" | "unsupported-script" | "too-long" }; diff --git a/packages/core/src/i18n/locales/en/reader.json b/packages/core/src/i18n/locales/en/reader.json index 072c3d462..b1e5bb214 100644 --- a/packages/core/src/i18n/locales/en/reader.json +++ b/packages/core/src/i18n/locales/en/reader.json @@ -81,6 +81,50 @@ "unknownBook": "Untitled book", "viewNote": "View note" }, + "dictionary": { + "define": "Define", + "title": "Definition", + "close": "Close definition", + "loadingDefinition": "Looking up definition…", + "dictionaries": "Dictionaries", + "english": "English", + "chinese": "Chinese", + "download": "Download", + "update": "Update", + "remove": "Remove", + "retry": "Retry", + "retryLookup": "Retry dictionary lookup", + "repair": "Repair", + "manageDictionaries": "Manage Dictionaries", + "notDownloaded": "Not downloaded", + "noDefinitionFound": "No definition found. Try selecting a single word.", + "unsupportedSelection": "This selection is not supported.", + "lookupError": "Dictionary lookup failed. Try again.", + "downloadDefinition": "Download the {{language}} dictionary ({{size}}) to look up definitions offline.", + "downloadingDefinition": "Downloading the {{language}} dictionary… {{progress}}%", + "downloadAccessibility": "Download {{language}} dictionary", + "downloadingAccessibility": "Downloading {{language}} dictionary", + "offlinePrivacy": "Definitions work offline and never use AI.", + "downloading": "Downloading {{progress}}%", + "installed": "Installed", + "updateAvailable": "Update available", + "error": "Error", + "unavailable": "Unavailable", + "packError": "{{language}} dictionary couldn't be prepared. Try again.", + "version": "Version {{version}}", + "size": "Size {{size}}", + "installedStatus": "{{installed}} · {{version}} · {{size}}", + "updateStatus": "{{updateAvailable}} · {{installedVersion}} → {{availableVersion}}", + "attribution": "Attribution", + "attributionLabel": "{{language}} dictionary attribution", + "license": "License", + "licenseDetail": "{{label}}: {{license}}", + "actionLabel": "{{action}} {{language}} dictionary", + "statusLabel": "{{language}} dictionary status: {{status}}", + "removeTitle": "Remove {{language}} dictionary?", + "removeMessage": "Remove the {{language}} dictionary from this device?", + "cancel": "Cancel" + }, "bookmarks": { "title": "Bookmarks", "add": "Bookmark this page", diff --git a/packages/core/src/i18n/locales/zh-TW/reader.json b/packages/core/src/i18n/locales/zh-TW/reader.json index 0a638fc28..934d2f90a 100644 --- a/packages/core/src/i18n/locales/zh-TW/reader.json +++ b/packages/core/src/i18n/locales/zh-TW/reader.json @@ -77,6 +77,50 @@ "unknownBook": "未命名書籍", "viewNote": "查看筆記" }, + "dictionary": { + "define": "查詞", + "title": "釋義", + "close": "關閉釋義", + "loadingDefinition": "正在查詢釋義…", + "dictionaries": "字典", + "english": "英語", + "chinese": "中文", + "download": "下載", + "update": "更新", + "remove": "移除", + "retry": "重試", + "retryLookup": "重試字典查詢", + "repair": "修復", + "manageDictionaries": "管理字典", + "notDownloaded": "尚未下載", + "noDefinitionFound": "找不到釋義。請嘗試只選取一個單字。", + "unsupportedSelection": "不支援這個選取內容。", + "lookupError": "字典查詢失敗，請重試。", + "downloadDefinition": "下載{{language}}字典（{{size}}）即可離線查詢釋義。", + "downloadingDefinition": "正在下載{{language}}字典… {{progress}}%", + "downloadAccessibility": "下載{{language}}字典", + "downloadingAccessibility": "正在下載{{language}}字典", + "offlinePrivacy": "釋義完全在離線狀態下運作，不會使用 AI。", + "downloading": "正在下載 {{progress}}%", + "installed": "已安裝", + "updateAvailable": "有可用的更新", + "error": "出錯", + "unavailable": "無法使用", + "packError": "{{language}}字典暫時無法使用，請重試。", + "version": "版本 {{version}}", + "size": "大小 {{size}}", + "installedStatus": "{{installed}} · {{version}} · {{size}}", + "updateStatus": "{{updateAvailable}} · {{installedVersion}} → {{availableVersion}}", + "attribution": "來源注記", + "attributionLabel": "{{language}}字典來源注記", + "license": "授權", + "licenseDetail": "{{label}}：{{license}}", + "actionLabel": "{{language}}字典{{action}}", + "statusLabel": "{{language}}字典狀態：{{status}}", + "removeTitle": "移除{{language}}字典？", + "removeMessage": "要從這台裝置移除{{language}}字典嗎？", + "cancel": "取消" + }, "bookmarks": { "title": "書籤", "add": "新增書籤", diff --git a/packages/core/src/i18n/locales/zh/reader.json b/packages/core/src/i18n/locales/zh/reader.json index 150d9ec9f..216a3f40a 100644 --- a/packages/core/src/i18n/locales/zh/reader.json +++ b/packages/core/src/i18n/locales/zh/reader.json @@ -81,6 +81,50 @@ "unknownBook": "未命名书籍", "viewNote": "查看笔记" }, + "dictionary": { + "define": "查词", + "title": "释义", + "close": "关闭释义", + "loadingDefinition": "正在查询释义…", + "dictionaries": "词典", + "english": "英语", + "chinese": "中文", + "download": "下载", + "update": "更新", + "remove": "移除", + "retry": "重试", + "retryLookup": "重试词典查询", + "repair": "修复", + "manageDictionaries": "管理词典", + "notDownloaded": "未下载", + "noDefinitionFound": "没有找到释义。请尝试选择一个单词。", + "unsupportedSelection": "不支持此选中内容。", + "lookupError": "词典查询失败，请重试。", + "downloadDefinition": "下载{{language}}词典（{{size}}）即可离线查询释义。", + "downloadingDefinition": "正在下载{{language}}词典… {{progress}}%", + "downloadAccessibility": "下载{{language}}词典", + "downloadingAccessibility": "正在下载{{language}}词典", + "offlinePrivacy": "释义完全离线工作，不会使用 AI。", + "downloading": "正在下载 {{progress}}%", + "installed": "已安装", + "updateAvailable": "有可用更新", + "error": "出错", + "unavailable": "不可用", + "packError": "{{language}}词典暂时不可用，请重试。", + "version": "版本 {{version}}", + "size": "大小 {{size}}", + "installedStatus": "{{installed}} · {{version}} · {{size}}", + "updateStatus": "{{updateAvailable}} · {{installedVersion}} → {{availableVersion}}", + "attribution": "来源注记", + "attributionLabel": "{{language}}词典来源注记", + "license": "许可证", + "licenseDetail": "{{label}}：{{license}}", + "actionLabel": "{{language}}词典{{action}}", + "statusLabel": "{{language}}词典状态：{{status}}", + "removeTitle": "移除{{language}}词典？", + "removeMessage": "要从本设备移除{{language}}词典吗？", + "cancel": "取消" + }, "bookmarks": { "title": "书签", "add": "添加书签", diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e8af67daa..827e9485f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -5,6 +5,9 @@ // Types export * from "./types"; +// Dictionary +export * from "./dictionary"; + // Utils export { cn, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 560ef437a..2b5acbe71 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -452,6 +452,12 @@ importers: specifier: ^5.0.5 version: 5.0.11(@types/react@19.1.17)(react@19.1.0)(use-sync-external-store@1.6.0(react@19.1.0)) devDependencies: + react-test-renderer: + specifier: 19.1.0 + version: 19.1.0(react@19.1.0) + '@types/react-test-renderer': + specifier: 19.1.0 + version: 19.1.0 '@babel/core': specifier: ^7.25.0 version: 7.29.0 @@ -649,6 +655,15 @@ importers: packages: + '@types/react-test-renderer@19.1.0': + resolution: {integrity: sha512-XD0WZrHqjNrxA/MaR9O22w/RNidWR9YZmBdRGI7wcnWGrv/3dA8wKCJ8m63Sn+tLJhcjmuhOi629N66W6kgWzQ==} + + react-test-renderer@19.1.0: + resolution: {integrity: sha512-jXkSl3CpvPYEF+p/eGDLB4sPoDX8pKkYvRl9+rR8HxLY0X04vW7hCm1/0zHoUSjPZ3bDa+wXWNTDVIw/R8aDVw==} + peerDependencies: + react: 19.1.0 + + '@0no-co/graphql.web@1.2.0': resolution: {integrity: sha512-/1iHy9TTr63gE1YcR5idjx8UREz1s0kFhydf3bBLCXyqjhkIc6igAzTOx3zPifCwFR87tsh/4Pa9cNts6d2otw==} peerDependencies: @@ -9498,6 +9513,17 @@ packages: snapshots: + '@types/react-test-renderer@19.1.0': + dependencies: + '@types/react': 19.1.17 + + react-test-renderer@19.1.0(react@19.1.0): + dependencies: + react: 19.1.0 + react-is: 19.2.4 + scheduler: 0.26.0 + + '@0no-co/graphql.web@1.2.0(graphql@16.8.1)': optionalDependencies: graphql: 16.8.1 From 0b3b8ca64d61e158d2c135833b82b9a72f92d081 Mon Sep 17 00:00:00 2001 From: Decidetto <5384177+Decidetto@users.noreply.github.com> Date: Sun, 6 Sep 2026 13:58:40 +0200 Subject: [PATCH 2/3] feat: extend offline dictionaries to desktop --- dictionary-packs/README.md | 32 +- .../src/components/reader/DefinitionSheet.tsx | 6 +- .../reader/definition-controller.test.ts | 28 +- .../reader/definition-controller.ts | 137 +----- .../src/config/dictionary-config.test.ts | 5 +- .../app-expo/src/config/dictionary-config.ts | 10 +- .../src/lib/dictionary/dictionary-database.ts | 25 +- .../dictionary/dictionary-lookup-service.ts | 164 +------ .../lib/dictionary/dictionary-pack-manager.ts | 437 +---------------- .../dictionary/dictionary-pack-platform.ts | 149 +----- .../src/lib/dictionary/dictionary-runtime.ts | 30 +- .../settings/DictionarySettingsScreen.tsx | 9 +- .../settings/dictionary-locales.test.ts | 2 +- .../app-expo/src/stores/dictionary-store.ts | 164 +------ packages/app/package.json | 8 +- .../app/src-tauri/capabilities/default.json | 1 + packages/app/src-tauri/src/dictionary.rs | 314 +++++++++++++ packages/app/src-tauri/src/lib.rs | 3 + .../reader/ChapterTranslationBar.tsx | 17 +- .../DefinitionDialog.dictionary.test.tsx | 161 +++++++ .../components/reader/DefinitionDialog.tsx | 148 ++++++ .../app/src/components/reader/ReaderView.tsx | 20 + .../components/reader/SelectionPopover.tsx | 10 +- .../src/components/settings/AboutSettings.tsx | 5 +- .../DictionarySettings.dictionary.test.tsx | 88 ++++ .../settings/DictionarySettings.tsx | 200 ++++++++ .../components/settings/SettingsDialog.tsx | 6 +- .../src/components/settings/SyncSettings.tsx | 18 +- packages/app/src/components/ui/progress.tsx | 28 ++ .../src/lib/dictionary/desktop-dictionary.ts | 96 ++++ .../lib/dictionary/desktop.dictionary.test.ts | 110 +++++ packages/app/src/stores/dictionary-store.ts | 21 + packages/app/tsconfig.json | 2 +- packages/app/vitest.dictionary.config.ts | 7 + .../src/dictionary/definition-controller.ts | 144 ++++++ .../core/src/dictionary/dictionary-config.ts | 5 + .../src/dictionary/dictionary-database.ts | 20 + .../dictionary/dictionary-lookup-service.ts | 159 +++++++ .../src/dictionary}/dictionary-manifest.json | 0 .../src/dictionary/dictionary-pack-manager.ts | 438 ++++++++++++++++++ .../core/src/dictionary/dictionary-runtime.ts | 29 ++ .../core/src/dictionary/dictionary-store.ts | 156 +++++++ .../src/dictionary/dictionary-validation.ts | 144 ++++++ packages/core/src/dictionary/manifest.ts | 2 +- packages/core/src/i18n/locales/en/reader.json | 2 +- .../core/src/i18n/locales/zh-TW/reader.json | 2 +- packages/core/src/i18n/locales/zh/reader.json | 2 +- packages/core/src/stores/app-store.ts | 1 + packages/core/tsconfig.json | 2 +- 49 files changed, 2430 insertions(+), 1137 deletions(-) create mode 100644 packages/app/src-tauri/src/dictionary.rs create mode 100644 packages/app/src/components/reader/DefinitionDialog.dictionary.test.tsx create mode 100644 packages/app/src/components/reader/DefinitionDialog.tsx create mode 100644 packages/app/src/components/settings/DictionarySettings.dictionary.test.tsx create mode 100644 packages/app/src/components/settings/DictionarySettings.tsx create mode 100644 packages/app/src/components/ui/progress.tsx create mode 100644 packages/app/src/lib/dictionary/desktop-dictionary.ts create mode 100644 packages/app/src/lib/dictionary/desktop.dictionary.test.ts create mode 100644 packages/app/src/stores/dictionary-store.ts create mode 100644 packages/app/vitest.dictionary.config.ts create mode 100644 packages/core/src/dictionary/definition-controller.ts create mode 100644 packages/core/src/dictionary/dictionary-config.ts create mode 100644 packages/core/src/dictionary/dictionary-database.ts create mode 100644 packages/core/src/dictionary/dictionary-lookup-service.ts rename packages/{app-expo/src/config => core/src/dictionary}/dictionary-manifest.json (100%) create mode 100644 packages/core/src/dictionary/dictionary-pack-manager.ts create mode 100644 packages/core/src/dictionary/dictionary-runtime.ts create mode 100644 packages/core/src/dictionary/dictionary-store.ts create mode 100644 packages/core/src/dictionary/dictionary-validation.ts diff --git a/dictionary-packs/README.md b/dictionary-packs/README.md index e524c2601..e92d937f2 100644 --- a/dictionary-packs/README.md +++ b/dictionary-packs/README.md @@ -19,7 +19,8 @@ The app reads manifest updates from the official repository after this feature is merged and falls back to its bundled manifest when offline. Maintainers can move assets by updating the manifest URLs while preserving the verified hashes. An alternate manifest can be selected at build time with -`EXPO_PUBLIC_DICTIONARY_MANIFEST_URL`. No binary packs are checked into this PR. +`EXPO_PUBLIC_DICTIONARY_MANIFEST_URL` (mobile) or `VITE_DICTIONARY_MANIFEST_URL` +(desktop). The published packs are available in the [contributor release](https://github.com/cha1latte/ReadAny/releases/tag/dictionary-packs-v1) for maintainers to migrate. WordNet gives strong ordinary English vocabulary in a compact download, but has less slang, newer language, proper-name coverage, and obscure material than the @@ -103,3 +104,32 @@ pnpm --filter @readany/cli dictionary:build -- --language zh --input ./dictionar Before publication, each descriptor must exactly match an independent local file-size and SHA-256 check. `manifest.json` and the app's bundled manifest must remain byte-identical and must pass the shared strict manifest parser. + +## Desktop and mobile integration + +Both apps use the same bundled manifest in +`packages/core/src/dictionary/dictionary-manifest.json`, selection normalization, +lookup SQL, validation, pack lifecycle, and store logic. Mobile uses Expo SQLite +and filesystem adapters. Desktop stores packs under the Tauri app-data directory's +`dictionaries` folder and runs read-only SQLite queries in a blocking native task. +Each native query closes its file handle, and the desktop adapter waits for pending +queries before pack replacement or removal. + +On desktop, select an English or Chinese word and choose **Define**. A missing +pack prompts for an explicit download. **Settings > Dictionaries** supports +downloading, updating, repairing, and removing packs. Installed lookups work offline. + +Desktop checks: `pnpm --filter app test:dictionary`, `pnpm --filter app exec tsc --noEmit`, +and `cargo test --locked dictionary::tests` from `packages/app/src-tauri`. + +Desktop pack downloads stream directly from the existing native HTTP client into +a buffered file writer, with progress events limited to ten per second. The UI +shows a separate verification phase before activating a pack. Both apps refresh +the catalog and installed-pack status when the Dictionaries settings page opens. + +A local Windows debug-preview comparison on 2026-09-06 measured the same English +pack (29,069,312 bytes) at 13.67 seconds through the original JavaScript chunk loop +and 1.58 seconds through native streaming. One full checksum/schema verification +pass took 0.59 and 0.61 seconds respectively. Progress callbacks fell from 1,776 to +13. Both files matched the published SHA-256. These are single-run measurements +on the same machine and connection, not a guaranteed download time. diff --git a/packages/app-expo/src/components/reader/DefinitionSheet.tsx b/packages/app-expo/src/components/reader/DefinitionSheet.tsx index d113383b6..21fb02b5d 100644 --- a/packages/app-expo/src/components/reader/DefinitionSheet.tsx +++ b/packages/app-expo/src/components/reader/DefinitionSheet.tsx @@ -112,6 +112,8 @@ function renderState( {t("dictionary.loadingDefinition")} ); + case "verifying": + return {t("dictionary.verifying")}; case "unsupported": return {t("dictionary.unsupportedSelection")}; case "missing-pack": @@ -232,15 +234,15 @@ function PackDownload({ function defaultDependencies(): DefinitionControllerDependencies { return { lookup: (text) => useDictionaryStore.getState().lookup(text), - install: async (descriptor, onProgress) => { + install: async (descriptor, onProgress, onVerifying) => { const unsubscribe = useDictionaryStore.subscribe((state) => { const status = state.packs[descriptor.language]; if (status.state === "downloading") onProgress(status.progress); + if (status.state === "verifying") onVerifying?.(); }); onProgress(0); try { await useDictionaryStore.getState().install(descriptor.language); - onProgress(1); } finally { unsubscribe(); } diff --git a/packages/app-expo/src/components/reader/definition-controller.test.ts b/packages/app-expo/src/components/reader/definition-controller.test.ts index 7de68af7a..b287efcdc 100644 --- a/packages/app-expo/src/components/reader/definition-controller.test.ts +++ b/packages/app-expo/src/components/reader/definition-controller.test.ts @@ -48,6 +48,7 @@ function createController(options?: { install?: ( pack: DictionaryPackDescriptor, onProgress: (progress: number) => void, + onVerifying?: () => void, ) => Promise; getDescriptor?: (language: DictionaryLanguage) => DictionaryPackDescriptor | undefined; }) { @@ -88,6 +89,31 @@ describe("DefinitionController", () => { expect(controller.state).toEqual({ kind: "missing-pack", language: "en", descriptor }); }); + it("shows verification separately and discards it after closing", async () => { + let verify: (() => void) | undefined; + let finish!: () => void; + const controller = createController({ + lookup: async () => { + throw lookupError("pack-not-installed"); + }, + install: async (_pack, _progress, onVerifying) => { + verify = onVerifying; + await new Promise((resolve) => { + finish = resolve; + }); + }, + }); + await controller.open("desire"); + const downloading = controller.download(); + verify?.(); + expect(controller.state).toEqual({ kind: "verifying", language: "en" }); + controller.close(); + verify?.(); + finish(); + await downloading; + expect(controller.state).toEqual({ kind: "idle" }); + }); + it("reports install progress then automatically retries the original selection", async () => { const lookup = vi .fn<(text: string) => Promise>() @@ -106,7 +132,7 @@ describe("DefinitionController", () => { expect(controller.state).toEqual({ kind: "downloading", language: "en", progress: 0.37 }); await downloading; - expect(install).toHaveBeenCalledWith(descriptor, expect.any(Function)); + expect(install).toHaveBeenCalledWith(descriptor, expect.any(Function), expect.any(Function)); expect(lookup).toHaveBeenNthCalledWith(2, "desire"); expect(controller.state).toEqual({ kind: "result", diff --git a/packages/app-expo/src/components/reader/definition-controller.ts b/packages/app-expo/src/components/reader/definition-controller.ts index 1ea773877..b7784b686 100644 --- a/packages/app-expo/src/components/reader/definition-controller.ts +++ b/packages/app-expo/src/components/reader/definition-controller.ts @@ -1,136 +1 @@ -import { - type DictionaryEntry, - type DictionaryLanguage, - type DictionaryPackDescriptor, - prepareDictionarySelection, -} from "@readany/core/dictionary"; - -export type DefinitionState = - | { kind: "idle" } - | { kind: "loading"; displayText: string } - | { kind: "unsupported"; reason: string } - | { kind: "missing-pack"; language: DictionaryLanguage; descriptor: DictionaryPackDescriptor } - | { kind: "downloading"; language: DictionaryLanguage; progress: number } - | { kind: "result"; displayText: string; entries: DictionaryEntry[] } - | { kind: "no-match"; displayText: string } - | { kind: "error"; message: string }; - -export interface DefinitionControllerDependencies { - lookup(text: string): Promise; - install( - descriptor: DictionaryPackDescriptor, - onProgress: (progress: number) => void, - ): Promise; - getDescriptor(language: DictionaryLanguage): DictionaryPackDescriptor | undefined; -} - -type DefinitionStateListener = (state: DefinitionState) => void; - -interface DictionaryError extends Error { - code?: string; -} - -export class DefinitionController { - private requestToken = 0; - private selectedText: string | null = null; - private readonly listeners = new Set(); - - state: DefinitionState = { kind: "idle" }; - - constructor(private readonly dependencies: DefinitionControllerDependencies) {} - - subscribe(listener: DefinitionStateListener): () => void { - this.listeners.add(listener); - return () => this.listeners.delete(listener); - } - - async open(text: string): Promise { - const token = ++this.requestToken; - this.selectedText = text; - await this.lookupSelection(text, token); - } - - async retry(): Promise { - if (!this.selectedText) return; - await this.open(this.selectedText); - } - - async download(): Promise { - if (this.state.kind !== "missing-pack" || !this.selectedText) return; - - const { descriptor, language } = this.state; - const text = this.selectedText; - const token = this.requestToken; - this.setState({ kind: "downloading", language, progress: 0 }); - - try { - await this.dependencies.install(descriptor, (progress) => { - if (this.isCurrent(token)) { - this.setState({ kind: "downloading", language, progress: clampProgress(progress) }); - } - }); - } catch (error) { - if (this.isCurrent(token)) this.setState({ kind: "error", message: messageOf(error) }); - return; - } - - if (this.isCurrent(token)) await this.lookupSelection(text, token); - } - - close(): void { - this.requestToken += 1; - this.selectedText = null; - this.setState({ kind: "idle" }); - } - - private async lookupSelection(text: string, token: number): Promise { - const selection = prepareDictionarySelection(text); - if (!selection.ok) { - if (this.isCurrent(token)) this.setState({ kind: "unsupported", reason: selection.reason }); - return; - } - - this.setState({ kind: "loading", displayText: selection.displayText }); - try { - const entries = await this.dependencies.lookup(text); - if (!this.isCurrent(token)) return; - this.setState( - entries.length > 0 - ? { kind: "result", displayText: selection.displayText, entries } - : { kind: "no-match", displayText: selection.displayText }, - ); - } catch (error) { - if (!this.isCurrent(token)) return; - const dictionaryError = error as DictionaryError; - if (dictionaryError.code === "pack-not-installed") { - const descriptor = this.dependencies.getDescriptor(selection.language); - if (descriptor) { - this.setState({ kind: "missing-pack", language: selection.language, descriptor }); - return; - } - } - if (dictionaryError.code === "unsupported-selection") { - this.setState({ kind: "unsupported", reason: "unsupported-selection" }); - return; - } - this.setState({ kind: "error", message: messageOf(error) }); - } - } - - private isCurrent(token: number): boolean { - return token === this.requestToken; - } - - private setState(state: DefinitionState): void { - this.state = state; - for (const listener of this.listeners) listener(state); - } -} - -function clampProgress(progress: number): number { - return Number.isFinite(progress) ? Math.max(0, Math.min(1, progress)) : 0; -} - -function messageOf(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} +export * from "@readany/core/dictionary/definition-controller"; diff --git a/packages/app-expo/src/config/dictionary-config.test.ts b/packages/app-expo/src/config/dictionary-config.test.ts index 66f135376..dd0c627b8 100644 --- a/packages/app-expo/src/config/dictionary-config.test.ts +++ b/packages/app-expo/src/config/dictionary-config.test.ts @@ -68,7 +68,10 @@ describe("dictionary configuration", () => { import.meta.dirname, "../../../../dictionary-packs/manifest.json", ); - const bundledPath = resolve(import.meta.dirname, "dictionary-manifest.json"); + const bundledPath = resolve( + import.meta.dirname, + "../../../core/src/dictionary/dictionary-manifest.json", + ); await expect(readFile(bundledPath)).resolves.toEqual(await readFile(canonicalPath)); }); diff --git a/packages/app-expo/src/config/dictionary-config.ts b/packages/app-expo/src/config/dictionary-config.ts index d61592fc6..7fc1f8264 100644 --- a/packages/app-expo/src/config/dictionary-config.ts +++ b/packages/app-expo/src/config/dictionary-config.ts @@ -1,8 +1,4 @@ -import { parseDictionaryManifest } from "@readany/core/dictionary"; -import bundledManifest from "./dictionary-manifest.json"; - +import { DEFAULT_DICTIONARY_MANIFEST_URL } from "@readany/core/dictionary/dictionary-config"; +export { DICTIONARY_BUNDLED_MANIFEST } from "@readany/core/dictionary/dictionary-config"; export const DICTIONARY_REMOTE_MANIFEST_URL = - process.env.EXPO_PUBLIC_DICTIONARY_MANIFEST_URL?.trim() || - "https://raw.githubusercontent.com/codedogQBY/ReadAny/main/dictionary-packs/manifest.json"; - -export const DICTIONARY_BUNDLED_MANIFEST = parseDictionaryManifest(bundledManifest); + process.env.EXPO_PUBLIC_DICTIONARY_MANIFEST_URL?.trim() || DEFAULT_DICTIONARY_MANIFEST_URL; diff --git a/packages/app-expo/src/lib/dictionary/dictionary-database.ts b/packages/app-expo/src/lib/dictionary/dictionary-database.ts index 692be58a9..3c6b2edc6 100644 --- a/packages/app-expo/src/lib/dictionary/dictionary-database.ts +++ b/packages/app-expo/src/lib/dictionary/dictionary-database.ts @@ -1,23 +1,10 @@ import type { DictionaryLanguage } from "@readany/core/dictionary"; - -export interface DictionaryDatabaseConnection { - getAllAsync(sql: string, ...params: unknown[]): Promise; - closeAsync(): Promise; -} - -export interface DictionaryDatabaseAdapter { - open(language: DictionaryLanguage, absolutePath: string): Promise; -} - -export class DictionaryLookupError extends Error { - constructor( - readonly code: "unsupported-selection" | "pack-not-installed" | "pack-invalid", - message: string, - ) { - super(message); - this.name = "DictionaryLookupError"; - } -} +import { + type DictionaryDatabaseAdapter, + type DictionaryDatabaseConnection, + DictionaryLookupError, +} from "@readany/core/dictionary/dictionary-database"; +export * from "@readany/core/dictionary/dictionary-database"; interface DictionaryMetadataRow { key: string; diff --git a/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.ts b/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.ts index 08d13d6ca..c560f577d 100644 --- a/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.ts +++ b/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.ts @@ -1,163 +1 @@ -import { - type DictionaryEntry, - type DictionaryLanguage, - prepareDictionarySelection, -} from "@readany/core/dictionary"; -import { - type DictionaryDatabaseAdapter, - type DictionaryDatabaseConnection, - DictionaryLookupError, -} from "./dictionary-database"; - -const LOOKUP_SQL = ` - WITH matched AS ( - SELECT lookup.entry_id, lookup.rank - FROM lookup - INNER JOIN entries ON entries.id = lookup.entry_id - WHERE lookup.lookup_key = ? AND entries.language = ? - ORDER BY lookup.rank ASC, lookup.entry_id ASC - LIMIT 20 - ) - SELECT - e.id AS entry_id, - e.language, - e.headword, - e.simplified, - e.traditional, - e.pronunciation, - e.part_of_speech, - matched.rank, - s.sense_order, - s.definition - FROM matched - INNER JOIN entries e ON e.id = matched.entry_id - INNER JOIN senses s ON s.entry_id = e.id - ORDER BY matched.rank ASC, e.id ASC, s.sense_order ASC -`; - -interface DictionaryLookupRow { - entry_id: number; - language: DictionaryLanguage; - headword: string; - simplified: string | null; - traditional: string | null; - pronunciation: string | null; - part_of_speech: string; - rank: number; - sense_order: number; - definition: string; -} - -export type DictionaryPackPathResolver = ( - language: DictionaryLanguage, -) => string | null | Promise; - -export type DictionaryPackInvalidator = (language: DictionaryLanguage) => Promise | void; - -export class DictionaryLookupService { - private readonly connections = new Map< - DictionaryLanguage, - Promise - >(); - - constructor( - private readonly database: DictionaryDatabaseAdapter, - private readonly resolvePackPath: DictionaryPackPathResolver, - private readonly invalidatePack: DictionaryPackInvalidator = () => {}, - ) {} - - async lookup(text: string): Promise { - const selection = prepareDictionarySelection(text); - if (!selection.ok) { - throw new DictionaryLookupError( - "unsupported-selection", - `Dictionary lookup does not support this selection: ${selection.reason}`, - ); - } - - const absolutePath = await this.resolvePackPath(selection.language); - if (!absolutePath) { - throw new DictionaryLookupError( - "pack-not-installed", - `The ${selection.language} dictionary pack is not installed`, - ); - } - - try { - const database = await this.connectionFor(selection.language, absolutePath); - const rows = await database.getAllAsync( - LOOKUP_SQL, - selection.key, - selection.language, - ); - return this.entriesFromRows(rows); - } catch (error) { - const cleanupErrors: unknown[] = []; - try { - await this.close(selection.language); - } catch (cleanupError) { - cleanupErrors.push(cleanupError); - } - try { - await this.invalidatePack(selection.language); - } catch (cleanupError) { - cleanupErrors.push(cleanupError); - } - if (error instanceof Error && cleanupErrors.length > 0) { - (error as Error & { cleanupErrors?: unknown[] }).cleanupErrors = cleanupErrors; - } - throw error; - } - } - - async close(language?: DictionaryLanguage): Promise { - const languages = language ? [language] : [...this.connections.keys()]; - await Promise.all( - languages.map(async (currentLanguage) => { - const connection = this.connections.get(currentLanguage); - if (!connection) return; - this.connections.delete(currentLanguage); - await (await connection).closeAsync(); - }), - ); - } - - private connectionFor( - language: DictionaryLanguage, - absolutePath: string, - ): Promise { - const existing = this.connections.get(language); - if (existing) return existing; - - const opening = this.database.open(language, absolutePath); - this.connections.set(language, opening); - void opening.catch(() => { - if (this.connections.get(language) === opening) { - this.connections.delete(language); - } - }); - return opening; - } - - private entriesFromRows(rows: DictionaryLookupRow[]): DictionaryEntry[] { - const entries = new Map(); - for (const row of rows) { - let entry = entries.get(row.entry_id); - if (!entry) { - entry = { - id: row.entry_id, - language: row.language, - headword: row.headword, - simplified: row.simplified ?? undefined, - traditional: row.traditional ?? undefined, - pronunciation: row.pronunciation ?? undefined, - partOfSpeech: row.part_of_speech, - senses: [], - }; - entries.set(row.entry_id, entry); - } - entry.senses.push({ order: row.sense_order, definition: row.definition }); - } - return [...entries.values()]; - } -} +export * from "@readany/core/dictionary/dictionary-lookup-service"; diff --git a/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.ts b/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.ts index 575a846a8..1b1e4ecba 100644 --- a/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.ts +++ b/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.ts @@ -1,436 +1 @@ -import type { - DictionaryLanguage, - DictionaryManifest, - DictionaryPackDescriptor, -} from "@readany/core/dictionary"; - -const dictionaryLanguages = ["en", "zh"] as const; - -export type DictionaryPackMetadata = Pick< - DictionaryPackDescriptor, - | "language" - | "version" - | "schemaVersion" - | "sourceEdition" - | "sourceDumpDate" - | "sourceArchiveUrl" - | "url" - | "attributionUrl" - | "license" -> & { - licenseNotice: string; - creatorAttribution: string; -}; - -export type InstalledDictionaryPack = DictionaryPackMetadata & - Pick; - -export interface DictionaryPackPlatform { - ensureDirectory(path: string): Promise; - download(url: string, path: string, onProgress: (fraction: number) => void): Promise; - exists(path: string): Promise; - size(path: string): Promise; - sha256(path: string): Promise; - readMetadata(path: string): Promise; - move(from: string, to: string): Promise; - remove(path: string): Promise; -} - -export type DictionaryPackStatus = - | { state: "not-installed" } - | { state: "downloading"; progress: number } - | { state: "installed"; version: string; sizeBytes: number } - | { - state: "update-available"; - installedVersion: string; - availableVersion: string; - sizeBytes: number; - } - | { state: "error"; message: string; hasActivePack: boolean }; - -export interface DictionaryHandleCloser { - close(language?: DictionaryLanguage): Promise | void; -} - -type ErrorWithCleanup = Error & { cleanupErrors?: unknown[] }; - -interface InspectedArtifact { - path: string; - exists: boolean; - installed?: InstalledDictionaryPack; - error?: unknown; -} - -interface InspectedArtifacts { - active: InspectedArtifact; - backup: InspectedArtifact; -} - -export class DictionaryPackManager { - private readonly installs = new Map>(); - private readonly gates = new Map>(); - private readonly validatedPacks = new Map< - DictionaryLanguage, - { path: string; installed: InstalledDictionaryPack } | null - >(); - - constructor( - private readonly platform: DictionaryPackPlatform, - private readonly directory: string, - private readonly lookup: DictionaryHandleCloser, - ) {} - - async refresh( - manifest: DictionaryManifest, - ): Promise> { - const entries = await Promise.all( - dictionaryLanguages.map((language) => - this.runExclusive(language, async () => { - try { - const discovered = await this.discoverInstalledLocked(language); - const installed = discovered?.installed; - const available = manifest.packs[language]; - const status: DictionaryPackStatus = !installed - ? { state: "not-installed" } - : sameDescriptorIdentity(installed, available) - ? { state: "installed", version: installed.version, sizeBytes: installed.sizeBytes } - : { - state: "update-available", - installedVersion: installed.version, - availableVersion: available.version, - sizeBytes: installed.sizeBytes, - }; - return [language, status] as const; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - const hasActivePack = await this.hasPackArtifacts(language); - return [ - language, - { state: "error", message, hasActivePack } satisfies DictionaryPackStatus, - ] as const; - } - }), - ), - ); - return Object.fromEntries(entries) as Record; - } - - install( - descriptor: DictionaryPackDescriptor, - onStatus: (status: DictionaryPackStatus) => void = () => {}, - ): Promise { - const language = descriptor.language; - const existing = this.installs.get(language); - if (existing) return existing; - - const operation = this.runExclusive(language, () => - this.installLocked(descriptor, onStatus), - ).finally(() => { - if (this.installs.get(language) === operation) this.installs.delete(language); - }); - this.installs.set(language, operation); - return operation; - } - - remove(language: DictionaryLanguage): Promise { - return this.runExclusive(language, async () => { - this.invalidate(language); - await this.platform.ensureDirectory(this.directory); - await this.lookup.close(language); - await this.removeIfPresent(this.activePath(language)); - await this.removeIfPresent(this.stagedPath(language)); - await this.removeIfPresent(this.backupPath(language)); - }); - } - - getActivePath(language: DictionaryLanguage): Promise { - return this.runExclusive(language, async () => { - const discovered = await this.discoverInstalledLocked(language); - return discovered?.path ?? null; - }); - } - - getInstalledDescriptor(language: DictionaryLanguage): Promise { - return this.runExclusive(language, async () => { - return (await this.discoverInstalledLocked(language))?.installed ?? null; - }); - } - - invalidate(language: DictionaryLanguage): void { - this.validatedPacks.delete(language); - } - - private async installLocked( - descriptor: DictionaryPackDescriptor, - onStatus: (status: DictionaryPackStatus) => void, - ): Promise { - const language = descriptor.language; - this.invalidate(language); - const staged = this.stagedPath(language); - const active = this.activePath(language); - const backup = this.backupPath(language); - try { - await this.platform.ensureDirectory(this.directory); - await this.removeIfPresent(staged); - const artifacts = await this.inspectArtifactsLocked(language); - onStatus({ state: "downloading", progress: 0 }); - await this.platform.download(descriptor.url, staged, (progress) => - onStatus({ state: "downloading", progress: clamp(progress) }), - ); - await this.verifyExpected(staged, descriptor); - await this.lookup.close(language); - - let hasRollbackBackup = Boolean(artifacts.backup.installed); - let activationStarted = false; - try { - if (artifacts.backup.installed) { - activationStarted = true; - await this.removeIfPresent(active); - } else if (artifacts.active.installed) { - await this.removeIfPresent(backup); - await this.platform.move(active, backup); - hasRollbackBackup = true; - activationStarted = true; - } else { - await this.removeIfPresent(active); - await this.removeIfPresent(backup); - activationStarted = true; - } - await this.platform.move(staged, active); - const installed = await this.verifyExpected(active, descriptor); - await this.removeIfPresent(backup); - this.validatedPacks.set(language, { path: active, installed }); - } catch (operationError) { - const cleanupErrors: unknown[] = []; - if (hasRollbackBackup) { - await this.captureCleanupFailure(cleanupErrors, () => - this.rollbackLocked(active, backup), - ); - } else if (activationStarted) { - await this.captureCleanupFailure(cleanupErrors, () => this.removeIfPresent(active)); - } - attachCleanupErrors(operationError, cleanupErrors); - throw operationError; - } - - onStatus({ - state: "installed", - version: descriptor.version, - sizeBytes: descriptor.sizeBytes, - }); - } catch (operationError) { - const cleanupErrors: unknown[] = []; - await this.captureCleanupFailure(cleanupErrors, () => this.removeIfPresent(staged)); - attachCleanupErrors(operationError, cleanupErrors); - const message = - operationError instanceof Error ? operationError.message : String(operationError); - onStatus({ - state: "error", - message, - hasActivePack: await this.hasPackArtifacts(language), - }); - throw operationError; - } - } - - private async verifyExpected( - path: string, - descriptor: DictionaryPackDescriptor, - ): Promise { - const installed = await this.inspectPack(path); - if (installed.sizeBytes !== descriptor.sizeBytes) - throw new Error("Dictionary pack size did not match manifest"); - if (installed.sha256.toLowerCase() !== descriptor.sha256.toLowerCase()) - throw new Error("Dictionary pack checksum did not match manifest"); - assertMetadataMatches(installed, descriptor); - return installed; - } - - private async rollbackLocked(active: string, backup: string): Promise { - if (!(await this.platform.exists(backup))) return; - const backupSnapshot = await this.inspectPack(backup); - await this.removeIfPresent(active); - await this.platform.move(backup, active); - const restoredSnapshot = await this.inspectPack(active); - assertSamePack(backupSnapshot, restoredSnapshot); - this.validatedPacks.set(restoredSnapshot.language, { - path: active, - installed: restoredSnapshot, - }); - } - - private async discoverInstalledLocked( - language: DictionaryLanguage, - ): Promise<{ path: string; installed: InstalledDictionaryPack } | null> { - if (this.validatedPacks.has(language)) return this.validatedPacks.get(language) ?? null; - await this.platform.ensureDirectory(this.directory); - const artifacts = await this.inspectArtifactsLocked(language); - if (artifacts.backup.installed) { - const discovered = { path: artifacts.backup.path, installed: artifacts.backup.installed }; - this.validatedPacks.set(language, discovered); - return discovered; - } - if (artifacts.active.installed) { - const discovered = { path: artifacts.active.path, installed: artifacts.active.installed }; - this.validatedPacks.set(language, discovered); - return discovered; - } - - const errors = [artifacts.backup.error, artifacts.active.error].filter( - (error): error is NonNullable => error !== undefined, - ); - if (errors.length === 1) throw errors[0]; - if (errors.length > 1) { - throw new AggregateError(errors, `No valid ${language} dictionary recovery artifact exists`); - } - this.validatedPacks.set(language, null); - return null; - } - - private async inspectArtifactsLocked(language: DictionaryLanguage): Promise { - const [active, backup] = await Promise.all([ - this.inspectArtifact(this.activePath(language), language), - this.inspectArtifact(this.backupPath(language), language), - ]); - return { active, backup }; - } - - private async inspectArtifact( - path: string, - language: DictionaryLanguage, - ): Promise { - if (!(await this.platform.exists(path))) return { path, exists: false }; - try { - const installed = await this.inspectPack(path); - if (installed.language !== language) { - throw new Error(`Dictionary metadata language did not match ${language}`); - } - return { path, exists: true, installed }; - } catch (error) { - return { path, exists: true, error }; - } - } - - private async hasPackArtifacts(language: DictionaryLanguage): Promise { - return ( - (await this.platform.exists(this.activePath(language))) || - (await this.platform.exists(this.backupPath(language))) || - (await this.platform.exists(this.stagedPath(language))) - ); - } - - private async inspectPack(path: string): Promise { - const [metadata, sizeBytes, sha256] = await Promise.all([ - this.platform.readMetadata(path), - this.platform.size(path), - this.platform.sha256(path), - ]); - return { ...metadata, sizeBytes, sha256: sha256.toLowerCase() }; - } - - private runExclusive(language: DictionaryLanguage, action: () => Promise): Promise { - const previous = this.gates.get(language) ?? Promise.resolve(); - const operation = previous.catch(() => undefined).then(action); - const tail = operation.then( - () => undefined, - () => undefined, - ); - this.gates.set(language, tail); - void tail.then(() => { - if (this.gates.get(language) === tail) this.gates.delete(language); - }); - return operation; - } - - private async captureCleanupFailure( - errors: unknown[], - cleanup: () => Promise, - ): Promise { - try { - await cleanup(); - } catch (error) { - errors.push(error); - } - } - - private activePath(language: DictionaryLanguage): string { - return `${this.directory}/readany-dictionary-${language}.sqlite`; - } - - private stagedPath(language: DictionaryLanguage): string { - return `${this.activePath(language)}.download`; - } - - private backupPath(language: DictionaryLanguage): string { - return `${this.activePath(language)}.backup`; - } - - private async removeIfPresent(path: string): Promise { - if (await this.platform.exists(path)) await this.platform.remove(path); - } -} - -function assertMetadataMatches( - installed: DictionaryPackMetadata, - expected: DictionaryPackDescriptor, -): void { - for (const key of [ - "language", - "version", - "schemaVersion", - "sourceEdition", - "sourceDumpDate", - "sourceArchiveUrl", - "url", - "attributionUrl", - "license", - ] as const) { - if (installed[key] !== expected[key]) - throw new Error(`Dictionary metadata ${key} did not match manifest`); - } -} - -function sameDescriptorIdentity( - installed: InstalledDictionaryPack, - available: DictionaryPackDescriptor, -): boolean { - return ( - installed.sizeBytes === available.sizeBytes && - installed.sha256.toLowerCase() === available.sha256.toLowerCase() && - [ - "language", - "version", - "schemaVersion", - "sourceEdition", - "sourceDumpDate", - "sourceArchiveUrl", - "url", - "attributionUrl", - "license", - ].every( - (key) => - installed[key as keyof InstalledDictionaryPack] === - available[key as keyof DictionaryPackDescriptor], - ) - ); -} - -function assertSamePack(expected: InstalledDictionaryPack, actual: InstalledDictionaryPack): void { - if ( - expected.sizeBytes !== actual.sizeBytes || - expected.sha256 !== actual.sha256 || - JSON.stringify(expected) !== JSON.stringify(actual) - ) { - throw new Error("Restored dictionary pack did not match the validated backup"); - } -} - -function attachCleanupErrors(operationError: unknown, cleanupErrors: unknown[]): void { - if (cleanupErrors.length === 0 || !(operationError instanceof Error)) return; - const error = operationError as ErrorWithCleanup; - error.cleanupErrors = [...(error.cleanupErrors ?? []), ...cleanupErrors]; -} - -function clamp(value: number): number { - return Number.isFinite(value) ? Math.max(0, Math.min(1, value)) : 0; -} +export * from "@readany/core/dictionary/dictionary-pack-manager"; diff --git a/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.ts b/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.ts index 679e76033..05f443a3e 100644 --- a/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.ts +++ b/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.ts @@ -3,68 +3,11 @@ import { Directory, File, Paths } from "expo-file-system"; import * as LegacyFileSystem from "expo-file-system/legacy"; import type { DictionaryPackMetadata, DictionaryPackPlatform } from "./dictionary-pack-manager"; -interface SqliteObjectRow { - name: string; - type: string; - tbl_name: string; -} - -interface SqliteColumnRow { - name: string; -} - -interface SqliteIndexColumnRow { - name: string; - seqno: number; -} - -interface DictionaryMetadataRow { - key: string; - value: string; -} - -export interface DictionaryValidationDatabase { - getFirstAsync(sql: string): Promise; - getAllAsync(sql: string): Promise; -} - -const requiredObjects = new Map([ - ["metadata", "table"], - ["entries", "table"], - ["senses", "table"], - ["lookup", "table"], - ["lookup_key_rank_idx", "index"], -]); - -const requiredColumns = { - metadata: ["key", "value"], - entries: [ - "id", - "language", - "headword", - "simplified", - "traditional", - "pronunciation", - "part_of_speech", - ], - senses: ["entry_id", "sense_order", "definition"], - lookup: ["lookup_key", "entry_id", "rank"], -} as const; - -const requiredMetadataKeys = [ - "schema_version", - "language", - "version", - "source_edition", - "source_dump_date", - "source_archive_url", - "asset_url", - "attribution_url", - "license", - "license_notice", - "creator_attribution", -] as const; - +import { validateDictionaryDatabase } from "@readany/core/dictionary/dictionary-validation"; +export { + validateDictionaryDatabase, + type DictionaryValidationDatabase, +} from "@readany/core/dictionary/dictionary-validation"; function nativePath(path: string): string { return path.replace(/^file:\/\//, ""); } @@ -112,82 +55,6 @@ export function createExpoDictionaryPackPlatform(): DictionaryPackPlatform { }; } -export async function validateDictionaryDatabase( - database: DictionaryValidationDatabase, -): Promise { - const integrity = await database.getFirstAsync<{ integrity_check: string }>( - "PRAGMA integrity_check", - ); - if (integrity?.integrity_check !== "ok") - throw new Error("Dictionary SQLite integrity check failed"); - - const objects = await database.getAllAsync( - "SELECT name, type, tbl_name FROM sqlite_master WHERE name IN ('metadata', 'entries', 'senses', 'lookup', 'lookup_key_rank_idx')", - ); - const objectsByName = new Map(objects.map((object) => [object.name, object])); - for (const [name, type] of requiredObjects) { - if (objectsByName.get(name)?.type !== type) - throw new Error(`Dictionary SQLite schema requires ${name} ${type}`); - } - if (objectsByName.get("lookup_key_rank_idx")?.tbl_name !== "lookup") - throw new Error("Dictionary SQLite lookup_key_rank_idx must belong to lookup"); - - for (const [table, expectedColumns] of Object.entries(requiredColumns)) { - const columns = await database.getAllAsync(`PRAGMA table_info('${table}')`); - const actualColumns = columns.map((column) => column.name); - if (!sameArray(actualColumns, expectedColumns)) - throw new Error(`Dictionary SQLite ${table} columns did not match the required schema`); - } - - const indexColumns = await database.getAllAsync( - "PRAGMA index_info('lookup_key_rank_idx')", - ); - const orderedIndexColumns = [...indexColumns] - .sort((left, right) => left.seqno - right.seqno) - .map((column) => column.name); - if (!sameArray(orderedIndexColumns, ["lookup_key", "rank", "entry_id"])) - throw new Error("Dictionary SQLite lookup index columns were not in the required order"); - - const metadataRows = await database.getAllAsync( - `SELECT key, value FROM metadata WHERE key IN (${requiredMetadataKeys - .map((key) => `'${key}'`) - .join(", ")})`, - ); - const metadata = new Map(metadataRows.map((row) => [row.key, row.value])); - const value = (key: (typeof requiredMetadataKeys)[number]): string => { - const found = metadata.get(key); - if (!found?.trim()) throw new Error(`Dictionary metadata ${key} is missing`); - return found; - }; - - const schemaVersion = value("schema_version"); - if (schemaVersion !== "1") throw new Error("Dictionary metadata schema_version is unsupported"); - const language = value("language"); - if (language !== "en" && language !== "zh") - throw new Error("Dictionary metadata language is unsupported"); - const sourceEdition = value("source_edition"); - const license = value("license"); - const common = { - version: value("version"), - sourceDumpDate: value("source_dump_date"), - sourceArchiveUrl: value("source_archive_url"), - url: value("asset_url"), - attributionUrl: value("attribution_url"), - licenseNotice: value("license_notice"), - creatorAttribution: value("creator_attribution"), - }; - if (language === "en" && sourceEdition === "wordnet-3.1" && license === "WordNet 3.1 License") { - return { ...common, schemaVersion: 1, language, sourceEdition, license }; - } - if (language === "en" && sourceEdition === "enwiktionary" && license === "CC BY-SA 4.0") { - return { ...common, schemaVersion: 1, language, sourceEdition, license }; - } - if (language === "zh" && sourceEdition === "zhwiktionary" && license === "CC BY-SA 4.0") { - return { ...common, schemaVersion: 1, language, sourceEdition, license }; - } - throw new Error("Dictionary metadata source/license combination is unsupported"); -} - async function readAndValidateSqlite(path: string): Promise { const SQLite = await import("expo-sqlite"); const { fileName, directory } = splitDatabasePath(path); @@ -210,10 +77,4 @@ function splitDatabasePath(path: string): { fileName: string; directory: string }; } -function sameArray(actual: readonly string[], expected: readonly string[]): boolean { - return ( - actual.length === expected.length && actual.every((value, index) => value === expected[index]) - ); -} - export const dictionaryPackDirectory = `${Paths.document.uri.replace(/\/$/, "")}/dictionaries`; diff --git a/packages/app-expo/src/lib/dictionary/dictionary-runtime.ts b/packages/app-expo/src/lib/dictionary/dictionary-runtime.ts index ec4bebeba..aa2c2fefa 100644 --- a/packages/app-expo/src/lib/dictionary/dictionary-runtime.ts +++ b/packages/app-expo/src/lib/dictionary/dictionary-runtime.ts @@ -1,29 +1 @@ -import type { DictionaryDatabaseAdapter } from "./dictionary-database"; -import { DictionaryLookupService } from "./dictionary-lookup-service"; -import { DictionaryPackManager, type DictionaryPackPlatform } from "./dictionary-pack-manager"; - -export interface DictionaryRuntimeOptions { - database: DictionaryDatabaseAdapter; - directory: string; - platform: DictionaryPackPlatform; -} - -export function createDictionaryRuntime(options: DictionaryRuntimeOptions): { - lookup: DictionaryLookupService; - manager: DictionaryPackManager; -} { - // biome-ignore lint/style/useConst: the lookup resolves installed paths through the manager created below. - let manager: DictionaryPackManager | undefined; - const lookup = new DictionaryLookupService( - options.database, - async (language) => { - if (!manager) throw new Error("Dictionary runtime is not initialized"); - return manager.getActivePath(language); - }, - (language) => { - manager?.invalidate(language); - }, - ); - manager = new DictionaryPackManager(options.platform, options.directory, lookup); - return { lookup, manager }; -} +export * from "@readany/core/dictionary/dictionary-runtime"; diff --git a/packages/app-expo/src/screens/settings/DictionarySettingsScreen.tsx b/packages/app-expo/src/screens/settings/DictionarySettingsScreen.tsx index c08e75134..e05a6dbcb 100644 --- a/packages/app-expo/src/screens/settings/DictionarySettingsScreen.tsx +++ b/packages/app-expo/src/screens/settings/DictionarySettingsScreen.tsx @@ -60,6 +60,8 @@ function DictionaryPackRow({ if (unavailable) return t("dictionary.unavailable"); switch (pack.state) { + case "verifying": + return t("dictionary.verifying"); case "downloading": return t("dictionary.downloading", { progress: Math.round(pack.progress * 100), @@ -119,7 +121,7 @@ function DictionaryPackRow({ onPress={() => onInstall(language)} /> ) : null} - {pack.state === "downloading" ? ( + {pack.state === "downloading" || pack.state === "verifying" ? ( - + ; - lookup: Pick; - fetchRemoteManifest: () => Promise; - getBundledManifest: () => Promise; -} - -export interface DictionaryRuntime { - manager: Pick; - lookup: Pick; -} - -export interface RuntimeBackedDictionaryStoreDependencies { - loadRuntime: () => Promise; - fetchRemoteManifest: () => Promise; - getBundledManifest: () => Promise; -} - -export interface DictionaryStoreState { - manifest: DictionaryManifest | null; - packs: Record; - initialize(): Promise; - refreshManifest(): Promise; - install(language: DictionaryLanguage): Promise; - remove(language: DictionaryLanguage): Promise; - retry(): Promise; - lookup(text: string): Promise; -} - -const emptyPacks = (): Record => ({ - en: { state: "not-installed" }, - zh: { state: "not-installed" }, -}); - -export function createDictionaryStore( - deps: DictionaryStoreDependencies, -): UseBoundStore> { - return create()((set, get) => { - let remoteRefreshPromise: Promise | undefined; - let bundledReadinessPromise: Promise | undefined; - const applyManifest = async (manifest: DictionaryManifest) => { - const packs = await deps.manager.refresh(manifest); - set({ manifest, packs }); - }; - const loadManifest = async (): Promise => { - let remoteError: unknown; - try { - return parseDictionaryManifest(await deps.fetchRemoteManifest()); - } catch (error) { - remoteError = error; - } - try { - return parseDictionaryManifest(await deps.getBundledManifest()); - } catch (bundledError) { - throw new AggregateError( - [remoteError, bundledError], - "No valid dictionary manifest is available", - ); - } - }; - const refreshFromSources = (): Promise => { - if (remoteRefreshPromise) return remoteRefreshPromise; - const operation = loadManifest().then(applyManifest); - remoteRefreshPromise = operation; - void operation.then( - () => { - if (remoteRefreshPromise === operation) remoteRefreshPromise = undefined; - }, - () => { - if (remoteRefreshPromise === operation) remoteRefreshPromise = undefined; - }, - ); - return operation; - }; - const refreshFromBundled = (): Promise => { - if (get().manifest) return Promise.resolve(); - if (bundledReadinessPromise) return bundledReadinessPromise; - const operation = deps - .getBundledManifest() - .then(parseDictionaryManifest) - .then(async (manifest) => { - if (!get().manifest) await applyManifest(manifest); - }); - bundledReadinessPromise = operation; - void operation.then( - () => { - if (bundledReadinessPromise === operation) bundledReadinessPromise = undefined; - }, - () => { - if (bundledReadinessPromise === operation) bundledReadinessPromise = undefined; - }, - ); - return operation; - }; - - return { - manifest: null, - packs: emptyPacks(), - initialize: refreshFromSources, - refreshManifest: refreshFromSources, - install: async (language) => { - const descriptor: DictionaryPackDescriptor | undefined = get().manifest?.packs[language]; - if (!descriptor) throw new Error("Dictionary manifest is unavailable"); - await deps.manager.install(descriptor, (status) => - set((state) => ({ packs: { ...state.packs, [language]: status } })), - ); - }, - remove: async (language) => { - await deps.manager.remove(language); - set((state) => ({ packs: { ...state.packs, [language]: { state: "not-installed" } } })); - }, - retry: refreshFromSources, - lookup: async (text) => { - if (!get().manifest) await refreshFromBundled(); - return deps.lookup.lookup(text); - }, - }; - }); -} - -export function createRuntimeBackedDictionaryStore( - deps: RuntimeBackedDictionaryStoreDependencies, -): UseBoundStore> { - let runtimePromise: Promise | undefined; - const runtime = () => { - if (!runtimePromise) { - const operation = deps.loadRuntime(); - runtimePromise = operation; - void operation.catch(() => { - if (runtimePromise === operation) runtimePromise = undefined; - }); - } - return runtimePromise; - }; - return createDictionaryStore({ - manager: { - refresh: async (manifest) => (await runtime()).manager.refresh(manifest), - install: async (descriptor, onStatus) => - (await runtime()).manager.install(descriptor, onStatus), - remove: async (language) => (await runtime()).manager.remove(language), - }, - lookup: { lookup: async (text) => (await runtime()).lookup.lookup(text) }, - fetchRemoteManifest: deps.fetchRemoteManifest, - getBundledManifest: deps.getBundledManifest, - }); -} - async function loadExpoDictionaryRuntime(): Promise { const [databaseModule, platformModule, runtimeModule] = await Promise.all([ import("../lib/dictionary/dictionary-database"), diff --git a/packages/app/package.json b/packages/app/package.json index 65791aa10..3eb44238d 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -7,7 +7,8 @@ "dev": "vite", "build": "tsc && vite build", "preview": "vite preview", - "tauri": "tauri" + "tauri": "tauri", + "test:dictionary": "vitest run --config vitest.dictionary.config.ts" }, "dependencies": { "@huggingface/transformers": "^3.8.1", @@ -86,6 +87,9 @@ "@vitejs/plugin-react": "^4.6.0", "tailwindcss": "^4.0.0", "typescript": "~5.8.3", - "vite": "^7.0.4" + "vite": "^7.0.4", + "vitest": "^4.1.2", + "react-test-renderer": "19.1.0", + "@types/react-test-renderer": "19.1.0" } } diff --git a/packages/app/src-tauri/capabilities/default.json b/packages/app/src-tauri/capabilities/default.json index 8a5d896e2..9471c45b7 100644 --- a/packages/app/src-tauri/capabilities/default.json +++ b/packages/app/src-tauri/capabilities/default.json @@ -21,6 +21,7 @@ "fs:allow-write", "fs:allow-exists", "fs:allow-mkdir", + "fs:allow-stat", "fs:allow-read-file", "fs:allow-read-text-file", "fs:allow-write-file", diff --git a/packages/app/src-tauri/src/dictionary.rs b/packages/app/src-tauri/src/dictionary.rs new file mode 100644 index 000000000..0d1d32e5c --- /dev/null +++ b/packages/app/src-tauri/src/dictionary.rs @@ -0,0 +1,314 @@ +use rusqlite::{params_from_iter, types::ValueRef, Connection, OpenFlags}; +use serde_json::{Map, Value}; +use std::path::{Path, PathBuf}; +use tauri::Manager; + +fn pack_path(directory: &Path, path: &Path) -> Result { + let directory = directory.canonicalize().map_err(|e| e.to_string())?; + let path = path.canonicalize().map_err(|e| e.to_string())?; + if path.parent() != Some(directory.as_path()) { + return Err("Dictionary path must be inside the dictionaries directory".into()); + } + Ok(path) +} + +fn query_pack( + path: &Path, + query: &str, + values: &[String], +) -> Result>, String> { + // Opening read-only also avoids journal-mode changes invalidating the pack checksum. + let connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY) + .map_err(|e| e.to_string())?; + let mut statement = connection.prepare(query).map_err(|e| e.to_string())?; + if !statement.readonly() { + return Err("Dictionary queries must be read-only".into()); + } + let columns: Vec = statement + .column_names() + .iter() + .map(|name| name.to_string()) + .collect(); + let mut rows = statement + .query(params_from_iter(values)) + .map_err(|e| e.to_string())?; + let mut result = Vec::new(); + while let Some(row) = rows.next().map_err(|e| e.to_string())? { + let mut object = Map::new(); + for (index, name) in columns.iter().enumerate() { + let value = match row.get_ref(index).map_err(|e| e.to_string())? { + ValueRef::Null => Value::Null, + ValueRef::Integer(value) => Value::from(value), + ValueRef::Real(value) => Value::from(value), + ValueRef::Text(value) => { + Value::from(std::str::from_utf8(value).map_err(|e| e.to_string())?) + } + ValueRef::Blob(_) => return Err("Unexpected binary dictionary value".into()), + }; + object.insert(name.clone(), value); + } + result.push(object); + } + Ok(result) +} + +#[tauri::command] +pub async fn dictionary_query( + app: tauri::AppHandle, + path: String, + query: String, + values: Vec, +) -> Result>, String> { + let directory = app + .path() + .app_data_dir() + .map_err(|e| e.to_string())? + .join("dictionaries"); + tauri::async_runtime::spawn_blocking(move || { + let path = pack_path(&directory, Path::new(&path))?; + query_pack(&path, &query, &values) + }) + .await + .map_err(|e| e.to_string())? +} + +const MAX_PACK_BYTES: u64 = 150 * 1024 * 1024; + +#[derive(Clone, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DownloadProgress { + received_bytes: u64, + total_bytes: Option, +} + +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DownloadReport { + bytes: u64, + elapsed_ms: u128, +} + +fn download_path(directory: &Path, path: &Path) -> Result { + let directory = directory.canonicalize().map_err(|e| e.to_string())?; + let parent = path + .parent() + .ok_or("Dictionary download has no directory")? + .canonicalize() + .map_err(|e| e.to_string())?; + if parent != directory + || !path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".sqlite.download")) + { + return Err("Dictionary downloads must be staged inside the dictionaries directory".into()); + } + Ok(parent.join( + path.file_name() + .ok_or("Dictionary download has no filename")?, + )) +} + +async fn download_pack( + url: &str, + path: &Path, + mut on_progress: impl FnMut(DownloadProgress), +) -> Result { + use std::time::{Duration, Instant}; + use tokio::io::AsyncWriteExt; + let started = Instant::now(); + let client = tauri_plugin_http::reqwest::Client::builder() + .https_only(url.starts_with("https://")) + .connect_timeout(Duration::from_secs(15)) + .timeout(Duration::from_secs(300)) + .build() + .map_err(|e| e.to_string())?; + let mut response = client + .get(url) + .send() + .await + .map_err(|e| e.to_string())? + .error_for_status() + .map_err(|e| e.to_string())?; + let total_bytes = response.content_length(); + if total_bytes.is_some_and(|size| size > MAX_PACK_BYTES) { + return Err("Dictionary download is too large".into()); + } + // Never follow an existing staging-file symlink or overwrite another download. + let file = tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .await + .map_err(|e| e.to_string())?; + let result = async { + let mut writer = tokio::io::BufWriter::with_capacity(1024 * 1024, file); + let mut received_bytes = 0; + let mut last_progress = Instant::now(); + on_progress(DownloadProgress { + received_bytes, + total_bytes, + }); + while let Some(chunk) = response.chunk().await.map_err(|e| e.to_string())? { + received_bytes += chunk.len() as u64; + if received_bytes > MAX_PACK_BYTES { + return Err("Dictionary download is too large".to_string()); + } + writer.write_all(&chunk).await.map_err(|e| e.to_string())?; + if last_progress.elapsed() >= Duration::from_millis(100) { + on_progress(DownloadProgress { + received_bytes, + total_bytes, + }); + last_progress = Instant::now(); + } + } + writer.flush().await.map_err(|e| e.to_string())?; + on_progress(DownloadProgress { + received_bytes, + total_bytes, + }); + Ok(DownloadReport { + bytes: received_bytes, + elapsed_ms: started.elapsed().as_millis(), + }) + } + .await; + if result.is_err() { + // The writer has dropped, releasing its handle before Windows cleanup. + let _ = tokio::fs::remove_file(path).await; + } + result +} + +#[tauri::command] +pub async fn dictionary_download( + app: tauri::AppHandle, + url: String, + path: String, + on_progress: tauri::ipc::Channel, +) -> Result { + let parsed = tauri_plugin_http::reqwest::Url::parse(&url).map_err(|e| e.to_string())?; + if parsed.scheme() != "https" { + return Err("Dictionary download URL must use HTTPS".into()); + } + let directory = app + .path() + .app_data_dir() + .map_err(|e| e.to_string())? + .join("dictionaries"); + let path = download_path(&directory, Path::new(&path))?; + download_pack(&url, &path, |progress| { + // Closing a dialog should not cancel an installation already in progress. + let _ = on_progress.send(progress); + }) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_directory() -> PathBuf { + let directory = + std::env::temp_dir().join(format!("readany-download-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&directory).unwrap(); + directory + } + + fn serve_once(status: &str, length: usize, body: Vec) -> String { + use std::io::{Read, Write}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let header = + format!("HTTP/1.1 {status}\r\nContent-Length: {length}\r\nConnection: close\r\n\r\n"); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0; 4096]; + let _ = stream.read(&mut request); + if stream.write_all(header.as_bytes()).is_ok() { + let _ = stream.write_all(&body); + } + }); + format!("http://{address}/pack") + } + + #[tokio::test] + async fn streams_native_bytes_and_limits_progress_updates() { + let directory = test_directory(); + let path = directory.join("pack.sqlite.download"); + let bytes = vec![42; 2 * 1024 * 1024]; + let url = serve_once("200 OK", bytes.len(), bytes.clone()); + let mut updates = Vec::new(); + let report = download_pack(&url, &path, |progress| updates.push(progress)) + .await + .unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + assert_eq!(report.bytes, bytes.len() as u64); + assert_eq!(updates.first().unwrap().received_bytes, 0); + assert_eq!(updates.last().unwrap().received_bytes, report.bytes); + assert!(updates.len() as u128 <= report.elapsed_ms / 100 + 2); + std::fs::rename(&path, directory.join("done.sqlite")).unwrap(); + std::fs::remove_dir_all(directory).unwrap(); + } + + #[tokio::test] + async fn rejects_failed_and_oversized_responses_and_cleans_partial_downloads() { + let directory = test_directory(); + let path = directory.join("pack.sqlite.download"); + for (status, length, bytes) in [ + ("404 Not Found", 0, vec![]), + ("200 OK", MAX_PACK_BYTES as usize + 1, vec![]), + ("200 OK", 10, vec![1, 2, 3]), + ] { + let url = serve_once(status, length, bytes); + assert!(download_pack(&url, &path, |_| {}).await.is_err()); + assert!(!path.exists()); + } + std::fs::remove_dir_all(directory).unwrap(); + } + + #[tokio::test] + async fn preserves_existing_staging_files_and_restricts_destination() { + let directory = test_directory(); + let path = directory.join("pack.sqlite.download"); + assert!(download_path(&directory, &path).is_ok()); + assert!(download_path(&directory, &directory.join("active.sqlite")).is_err()); + assert!(download_path(&directory, &directory.join("../outside.sqlite.download")).is_err()); + std::fs::write(&path, b"existing").unwrap(); + let url = serve_once("200 OK", 1, vec![1]); + assert!(download_pack(&url, &path, |_| {}).await.is_err()); + assert_eq!(std::fs::read(&path).unwrap(), b"existing"); + std::fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn reads_bound_values_without_changing_pack_and_releases_handles() { + let directory = + std::env::temp_dir().join(format!("readany-dictionary-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&directory).unwrap(); + let path = directory.join("test.sqlite"); + let connection = Connection::open(&path).unwrap(); + connection + .execute_batch( + "CREATE TABLE entries (headword TEXT); INSERT INTO entries VALUES ('hello');", + ) + .unwrap(); + drop(connection); + let before = std::fs::read(&path).unwrap(); + let rows = query_pack( + &path, + "SELECT headword FROM entries WHERE headword = ?", + &["hello".into()], + ) + .unwrap(); + assert_eq!(rows[0]["headword"], "hello"); + assert!(query_pack(&path, "DELETE FROM entries", &[]).is_err()); + assert_eq!(before, std::fs::read(&path).unwrap()); + assert!(pack_path(&directory.join("missing"), &path).is_err()); + assert!(pack_path(&directory, &path).is_ok()); + assert!(pack_path(&std::env::temp_dir(), &path).is_err()); + std::fs::rename(&path, directory.join("moved.sqlite")).unwrap(); + std::fs::remove_dir_all(directory).unwrap(); + } +} diff --git a/packages/app/src-tauri/src/lib.rs b/packages/app/src-tauri/src/lib.rs index 382c6a415..47ea855d2 100644 --- a/packages/app/src-tauri/src/lib.rs +++ b/packages/app/src-tauri/src/lib.rs @@ -1,4 +1,5 @@ mod db; +mod dictionary; mod readany_cli; mod storage; mod sync; @@ -30,6 +31,8 @@ pub fn run() { db: Mutex::new(None), }) .invoke_handler(tauri::generate_handler![ + dictionary::dictionary_query, + dictionary::dictionary_download, sync::commands::sync_vacuum_into, sync::commands::sync_integrity_check, sync::commands::sync_hash_file, diff --git a/packages/app/src/components/reader/ChapterTranslationBar.tsx b/packages/app/src/components/reader/ChapterTranslationBar.tsx index fa3f5f3fd..dc15c40b0 100644 --- a/packages/app/src/components/reader/ChapterTranslationBar.tsx +++ b/packages/app/src/components/reader/ChapterTranslationBar.tsx @@ -1,3 +1,4 @@ +import { Progress } from "@/components/ui/progress"; /** * ChapterTranslationMenu — dropdown menu attached to the toolbar Languages button. * @@ -22,8 +23,8 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; -import type { ChapterTranslationState } from "@readany/core/hooks"; import { useSettingsStore } from "@/stores/settings-store"; +import type { ChapterTranslationState } from "@readany/core/hooks"; import type { TranslationTargetLang } from "@readany/core/types/translation"; import { TRANSLATOR_LANGS } from "@readany/core/types/translation"; import { Check, Eye, EyeOff, Languages, Loader2, Trash2, X } from "lucide-react"; @@ -126,12 +127,14 @@ export function ChapterTranslationMenu({ })} -
-
-
+
({ + Dialog: ({ children }: { children: React.ReactNode }) =>
{children}
, + DialogContent: ({ children }: { children: React.ReactNode }) =>
{children}
, + DialogTitle: ({ children }: { children: React.ReactNode }) =>

{children}

, + DialogDescription: ({ children }: { children: React.ReactNode }) =>

{children}

, +})); +const mounted: TestRenderer.ReactTestRenderer[] = []; +beforeAll(async () => { + await i18nReady; + await i18n.changeLanguage("en"); +}); +afterEach(async () => { + await act(async () => { + for (const view of mounted.splice(0)) view.unmount(); + }); +}); +async function render(element: React.ReactElement) { + let view!: TestRenderer.ReactTestRenderer; + await act(async () => { + view = TestRenderer.create(element); + }); + mounted.push(view); + return view; +} +function dependencies() { + return { + lookup: vi.fn().mockResolvedValue([]), + install: vi.fn().mockResolvedValue(undefined), + getDescriptor: () => DICTIONARY_BUNDLED_MANIFEST.packs.en, + }; +} + +describe("desktop definition dialog", () => { + it("shows an offline result and closes before opening dictionary management", async () => { + const deps = dependencies(); + deps.lookup.mockResolvedValue([ + { + id: 1, + language: "en", + headword: "hello", + partOfSpeech: "noun", + senses: [{ order: 1, definition: "a greeting" }], + }, + ]); + const events: string[] = []; + const view = await render( + events.push("close")} + onManageDictionaries={() => events.push("manage")} + />, + ); + expect(JSON.stringify(view.toJSON())).toContain("a greeting"); + await act(async () => { + view.root + .findAllByType("button") + .find((button) => button.children.includes("Manage Dictionaries")) + ?.props.onClick(); + }); + expect(events).toEqual(["close", "manage"]); + }); + it("offers a download for a missing pack and retries the original selection", async () => { + const deps = dependencies(); + deps.lookup + .mockRejectedValueOnce(Object.assign(new Error("missing"), { code: "pack-not-installed" })) + .mockResolvedValueOnce([]); + const view = await render( + , + ); + expect(deps.install).not.toHaveBeenCalled(); + await act(async () => { + view.root + .findAllByType("button") + .find((button) => button.children.includes("Download")) + ?.props.onClick(); + }); + expect(deps.install).toHaveBeenCalledOnce(); + expect(deps.lookup).toHaveBeenLastCalledWith("hello"); + expect(JSON.stringify(view.toJSON())).toContain("No definition found"); + }); + it("offers retry on lookup failure", async () => { + const deps = dependencies(); + deps.lookup.mockRejectedValueOnce(new Error("failed")); + const view = await render( + , + ); + expect(view.root.findByProps({ role: "alert" }).children.join("")).toContain("lookup failed"); + await act(async () => { + view.root + .findAllByType("button") + .find((button) => button.children.includes("Retry")) + ?.props.onClick(); + }); + expect(deps.lookup).toHaveBeenCalledTimes(2); + }); + it("discards pending results when unmounted", async () => { + const deps = dependencies(); + let resolve!: (rows: unknown[]) => void; + deps.lookup.mockImplementation( + () => + new Promise((done) => { + resolve = done; + }), + ); + const controller = new DefinitionController(deps); + const view = await render( + , + ); + await act(async () => view.unmount()); + await act(async () => resolve([])); + expect(controller.state).toEqual({ kind: "idle" }); + }); + it("only offers Define for supported selections, including in PDFs", async () => { + const callback = vi.fn(); + const props = { + position: { x: 0, y: 0 }, + onHighlight: callback, + onRemoveHighlight: callback, + onNote: callback, + onCopy: callback, + onTranslate: callback, + onAskAI: callback, + onSpeak: callback, + onClose: callback, + onDefine: callback, + }; + const view = await render(); + expect(view.root.findByProps({ "aria-label": "Define" })).toBeDefined(); + await act(async () => + view.update(), + ); + expect(view.root.findAllByProps({ "aria-label": "Define" })).toHaveLength(0); + }); +}); diff --git a/packages/app/src/components/reader/DefinitionDialog.tsx b/packages/app/src/components/reader/DefinitionDialog.tsx new file mode 100644 index 000000000..574e3ccc0 --- /dev/null +++ b/packages/app/src/components/reader/DefinitionDialog.tsx @@ -0,0 +1,148 @@ +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog"; +import { Progress } from "@/components/ui/progress"; +import { useDictionaryStore } from "@/stores/dictionary-store"; +import { + DefinitionController, + type DefinitionState, +} from "@readany/core/dictionary/definition-controller"; +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; + +export function DefinitionDialog({ + text, + onClose, + onManageDictionaries, + controller: suppliedController, +}: { + text: string; + onClose(): void; + onManageDictionaries(): void; + controller?: DefinitionController; +}) { + const { t } = useTranslation(); + const [controller] = useState( + () => + suppliedController ?? + new DefinitionController({ + lookup: (text) => useDictionaryStore.getState().lookup(text), + getDescriptor: (language) => useDictionaryStore.getState().manifest?.packs[language], + install: async (descriptor, onProgress, onVerifying) => { + const unsubscribe = useDictionaryStore.subscribe((state) => { + const status = state.packs[descriptor.language]; + if (status.state === "downloading") onProgress(status.progress); + if (status.state === "verifying") onVerifying?.(); + }); + try { + await useDictionaryStore.getState().install(descriptor.language); + } finally { + unsubscribe(); + } + }, + }), + ); + const [state, setState] = useState(controller.state); + useEffect(() => { + const unsubscribe = controller.subscribe(setState); + void controller.open(text); + return () => { + unsubscribe(); + controller.close(); + }; + }, [controller, text]); + const language = + state.kind === "missing-pack" || state.kind === "downloading" + ? t(`dictionary.${state.language === "en" ? "english" : "chinese"}`) + : ""; + return ( + { + if (!open) onClose(); + }} + > + + {t("dictionary.title")} +
+ {state.kind === "loading" && {t("dictionary.loadingDefinition")}} + {state.kind === "verifying" && {t("dictionary.verifying")}} + {state.kind === "unsupported" &&

{t("dictionary.unsupportedSelection")}

} + {state.kind === "no-match" &&

{t("dictionary.noDefinitionFound")}

} + {state.kind === "error" && ( +
+

{t("dictionary.lookupError")}

+ +
+ )} + {state.kind === "missing-pack" && ( +
+

+ {t("dictionary.downloadDefinition", { + language, + size: `${(state.descriptor.sizeBytes / 1024 / 1024).toFixed(1)} MB`, + })} +

+ +
+ )} + {state.kind === "downloading" && ( +
+ + {t("dictionary.downloadingDefinition", { + language, + progress: Math.round(state.progress * 100), + })} + + +
+ )} + {state.kind === "result" && ( + <> +

{state.displayText}

+ {state.entries.map((entry) => ( +
+

+ {[ + ...new Set( + [entry.headword, entry.simplified, entry.traditional].filter(Boolean), + ), + ].join(" / ")} +

+ {entry.pronunciation && ( +

{entry.pronunciation}

+ )} + {entry.partOfSpeech && ( +

{entry.partOfSpeech}

+ )} +
    + {entry.senses.map((sense) => ( +
  1. {sense.definition}
    2. + ))} +
+
+ ))} + + )} +
+
+ +
+ + + ); +} diff --git a/packages/app/src/components/reader/ReaderView.tsx b/packages/app/src/components/reader/ReaderView.tsx index ade0140b9..8822d4bb7 100644 --- a/packages/app/src/components/reader/ReaderView.tsx +++ b/packages/app/src/components/reader/ReaderView.tsx @@ -43,6 +43,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { toast } from "sonner"; import { BookmarkRibbon } from "./BookmarkRibbon"; +import { DefinitionDialog } from "./DefinitionDialog"; import type { BookSelection, FoliateViewerHandle, RelocateDetail, TOCItem } from "./FoliateViewer"; import { FoliateViewer } from "./FoliateViewer"; import { FooterBar } from "./FooterBar"; @@ -799,6 +800,10 @@ export function ReaderView({ bookId, tabId }: ReaderViewProps) { const [currentPage, setCurrentPage] = useState(0); const [showChat, setShowChat] = useState(false); const [showSettings, setShowSettings] = useState(false); + const [definition, setDefinition] = useState<{ bookId: string; text: string } | null>(null); + useEffect(() => { + setDefinition((current) => (current?.bookId === bookId ? current : null)); + }, [bookId]); const [showTTS, setShowTTS] = useState(false); const [isReimporting, setIsReimporting] = useState(false); const [ttsSourceKind, setTtsSourceKind] = useState<"page" | "selection">("page"); @@ -3035,6 +3040,10 @@ export function ReaderView({ bookId, tabId }: ReaderViewProps) { onRemoveHighlight={handleRemoveHighlight} onNote={handleNote} onCopy={handleCopy} + onDefine={() => { + setDefinition({ bookId, text: selection.text }); + handleCloseSelection(); + }} onTranslate={handleTranslate} onAskAI={handleAskAI} onSpeak={handleSpeakSelection} @@ -3042,6 +3051,17 @@ export function ReaderView({ bookId, tabId }: ReaderViewProps) { /> )} + {definition?.bookId === bookId && ( + setDefinition(null)} + onManageDictionaries={() => + useAppStore.getState().setShowSettings(true, "dictionaries") + } + /> + )} + {/* Translation popover */} {showTranslation && translationText && ( void; onCopy: () => void; onTranslate: () => void; + onDefine?: () => void; onAskAI: () => void; onSpeak: () => void; onClose: () => void; @@ -38,7 +41,7 @@ const POPOVER_MARGIN = 8; export function SelectionPopover({ position, - selectedText: _selectedText, + selectedText, annotated = false, currentColor, defaultColor = "yellow", @@ -48,6 +51,7 @@ export function SelectionPopover({ onNote, onCopy, onTranslate, + onDefine, onAskAI, onSpeak, onClose, @@ -90,6 +94,9 @@ export function SelectionPopover({ }, { icon: NotebookPen, label: t("reader.note"), onClick: onNote, disabled: isPdf }, { icon: Copy, label: t("common.copy"), onClick: onCopy }, + ...(onDefine && prepareDictionarySelection(selectedText).ok + ? [{ icon: BookOpen, label: t("dictionary.define"), onClick: onDefine }] + : []), { icon: Languages, label: t("reader.translate"), onClick: onTranslate }, { icon: Sparkles, label: t("reader.askAI"), onClick: onAskAI }, { icon: Headphones, label: t("tts.speakSelection"), onClick: onSpeak }, @@ -180,6 +187,7 @@ export function SelectionPopover({ "text-muted-foreground hover:bg-destructive/10 hover:text-destructive", )} title={btn.label} + aria-label={btn.label} onClick={btn.disabled ? undefined : btn.onClick} disabled={btn.disabled} > diff --git a/packages/app/src/components/settings/AboutSettings.tsx b/packages/app/src/components/settings/AboutSettings.tsx index 8407de6e2..fe51583c7 100644 --- a/packages/app/src/components/settings/AboutSettings.tsx +++ b/packages/app/src/components/settings/AboutSettings.tsx @@ -7,6 +7,7 @@ import { DialogHeader, DialogTitle, } from "@/components/ui/dialog"; +import { Progress } from "@/components/ui/progress"; import { checkForUpdate, downloadAndInstall, @@ -157,9 +158,7 @@ export function AboutSettings() { {t("settings.downloading")} {progress}% -
-
-
+
)} diff --git a/packages/app/src/components/settings/DictionarySettings.dictionary.test.tsx b/packages/app/src/components/settings/DictionarySettings.dictionary.test.tsx new file mode 100644 index 000000000..9fcbac51f --- /dev/null +++ b/packages/app/src/components/settings/DictionarySettings.dictionary.test.tsx @@ -0,0 +1,88 @@ +import { useDictionaryStore } from "@/stores/dictionary-store"; +import { DICTIONARY_BUNDLED_MANIFEST } from "@readany/core/dictionary/dictionary-config"; +import i18n, { i18nReady } from "@readany/core/i18n"; +import type React from "react"; +import TestRenderer, { act } from "react-test-renderer"; +import { afterEach, beforeAll, beforeEach, expect, it, vi } from "vitest"; +import { DictionarySettings } from "./DictionarySettings"; +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = + true; +vi.mock("@/components/ui/dialog", () => ({ + Dialog: ({ children, open }: { children: React.ReactNode; open: boolean }) => + open ?
{children}
: null, + DialogContent: ({ children }: { children: React.ReactNode }) =>
{children}
, + DialogTitle: ({ children }: { children: React.ReactNode }) =>

{children}

, + DialogDescription: ({ children }: { children: React.ReactNode }) =>

{children}

, +})); +const install = vi.fn(); +const remove = vi.fn(); +let view: TestRenderer.ReactTestRenderer; +beforeAll(async () => { + await i18nReady; + await i18n.changeLanguage("en"); +}); +beforeEach(() => { + vi.resetAllMocks(); + useDictionaryStore.setState({ + manifest: DICTIONARY_BUNDLED_MANIFEST, + packs: { en: { state: "not-installed" }, zh: { state: "not-installed" } }, + initialize: async () => {}, + install, + remove, + }); +}); +afterEach(async () => { + await act(async () => view?.unmount()); +}); +it("downloads the selected language and displays source attribution", async () => { + await act(async () => { + view = TestRenderer.create(); + }); + expect(JSON.stringify(view.toJSON())).toContain("WordNet 3.1 License"); + await act(async () => + view.root + .findAllByType("button") + .find((button) => button.props["aria-label"] === "Download English dictionary") + ?.props.onClick(), + ); + expect(install).toHaveBeenCalledWith("en"); +}); +it("requires confirmation before removing a dictionary", async () => { + useDictionaryStore.setState({ + packs: { + en: { state: "installed", version: "1.0.0", sizeBytes: 42 }, + zh: { state: "not-installed" }, + }, + }); + await act(async () => { + view = TestRenderer.create(); + }); + await act(async () => + view.root + .findAllByType("button") + .find((button) => button.props["aria-label"] === "Remove English dictionary") + ?.props.onClick(), + ); + expect(remove).not.toHaveBeenCalled(); + expect(JSON.stringify(view.toJSON())).toContain("Remove English dictionary?"); + await act(async () => + view.root + .findAllByType("button") + .find((button) => !button.props["aria-label"] && button.children.includes("Remove")) + ?.props.onClick(), + ); + expect(remove).toHaveBeenCalledWith("en"); +}); +it("displays failed installations without an unhandled rejection", async () => { + install.mockRejectedValue(new Error("disk full")); + await act(async () => { + view = TestRenderer.create(); + }); + await act(async () => + view.root + .findAllByType("button") + .find((button) => button.props["aria-label"] === "Download English dictionary") + ?.props.onClick(), + ); + expect(view.root.findByProps({ role: "alert" }).children).toContain("Error"); +}); diff --git a/packages/app/src/components/settings/DictionarySettings.tsx b/packages/app/src/components/settings/DictionarySettings.tsx new file mode 100644 index 000000000..d3df3970c --- /dev/null +++ b/packages/app/src/components/settings/DictionarySettings.tsx @@ -0,0 +1,200 @@ +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogDescription, DialogTitle } from "@/components/ui/dialog"; +import { Progress } from "@/components/ui/progress"; +import { useDictionaryStore } from "@/stores/dictionary-store"; +import type { DictionaryLanguage } from "@readany/core/dictionary"; +import { Download, Loader2, RefreshCw, Trash2, Wrench } from "lucide-react"; +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; + +export function DictionarySettings() { + const { t } = useTranslation(); + const { manifest, packs, initialize, install, remove } = useDictionaryStore(); + const [error, setError] = useState(false); + const [busy, setBusy] = useState(false); + const [removing, setRemoving] = useState(null); + const run = async (operation: () => Promise) => { + setBusy(true); + setError(false); + try { + await operation(); + } catch { + setError(true); + } finally { + setBusy(false); + } + }; + useEffect(() => { + let active = true; + void initialize().catch(() => { + if (active) setError(true); + }); + return () => { + active = false; + }; + }, [initialize]); + return ( +
+ {error && ( +

+ {t("dictionary.error")} +

+ )} + {(["en", "zh"] as const).map((language) => { + const descriptor = manifest?.packs[language]; + const status = packs[language]; + const name = t(`dictionary.${language === "en" ? "english" : "chinese"}`); + const installed = + status.state === "installed" || + status.state === "update-available" || + (status.state === "error" && status.hasActivePack); + const downloading = status.state === "downloading"; + const installing = downloading || status.state === "verifying"; + const action = + status.state === "update-available" + ? "update" + : status.state === "error" + ? "repair" + : "download"; + const ActionIcon = installing + ? Loader2 + : action === "update" + ? RefreshCw + : action === "repair" + ? Wrench + : Download; + const statusText = t( + `dictionary.${status.state === "not-installed" ? "notDownloaded" : status.state === "update-available" ? "updateAvailable" : status.state}`, + { progress: downloading ? Math.round(status.progress * 100) : 0 }, + ); + return ( +
+
+
+

{name}

+ {statusText} +
+
+ {status.state !== "installed" && ( + + )} + {installed && ( + + )} +
+
+ {downloading && ( + + )} + {descriptor && ( +
+

+ {t("dictionary.version", { + version: + status.state === "installed" + ? status.version + : status.state === "update-available" + ? status.installedVersion + : descriptor.version, + })}{" "} + /{" "} + {t("dictionary.size", { + size: `${((status.state === "installed" ? status.sizeBytes : descriptor.sizeBytes) / 1024 / 1024).toFixed(1)} MB`, + })} +

+

+ {descriptor.sourceEdition} / {descriptor.sourceDumpDate} +

+

+ {t("dictionary.licenseDetail", { + label: t("dictionary.license"), + license: descriptor.license, + })} +

+ { + event.preventDefault(); + void run(async () => { + const { openUrl } = await import("@tauri-apps/plugin-opener"); + await openUrl(descriptor.attributionUrl); + }); + }} + > + {t("dictionary.attribution")} + +
+ )} +
+ ); + })} + { + if (!open) setRemoving(null); + }} + > + + + {t("dictionary.removeTitle", { + language: t(`dictionary.${removing === "en" ? "english" : "chinese"}`), + })} + + + {t("dictionary.removeMessage", { + language: t(`dictionary.${removing === "en" ? "english" : "chinese"}`), + })} + +
+ + +
+ + +
+ ); +} diff --git a/packages/app/src/components/settings/SettingsDialog.tsx b/packages/app/src/components/settings/SettingsDialog.tsx index 57eb767c4..3ad02ee6c 100644 --- a/packages/app/src/components/settings/SettingsDialog.tsx +++ b/packages/app/src/components/settings/SettingsDialog.tsx @@ -9,10 +9,11 @@ import { useEffect, useState } from "react"; import { useTranslation } from "react-i18next"; import { AISettings } from "./AISettings"; import { AboutSettings } from "./AboutSettings"; +import { DictionarySettings } from "./DictionarySettings"; +import { ExternalAISettings } from "./ExternalAISettings"; import { FeedbackSettings } from "./FeedbackSettings"; import { FontSettings } from "./FontSettings"; import { GeneralSettings } from "./GeneralSettings"; -import { ExternalAISettings } from "./ExternalAISettings"; import { ReadSettingsPanel } from "./ReadSettings"; import { SyncSettings } from "./SyncSettings"; import { TTSSettings } from "./TTSSettings"; @@ -28,6 +29,7 @@ const TAB_IDS: SettingsTab[] = [ "general", "reading", "fonts", + "dictionaries", "ai", "vectorModel", "tts", @@ -41,6 +43,7 @@ const TAB_KEYS: Record = { general: "settings.general", reading: "settings.reading", fonts: "settings.fonts", + dictionaries: "dictionary.dictionaries", ai: "settings.ai", vectorModel: "settings.vectorModel", tts: "settings.tts", @@ -118,6 +121,7 @@ export function SettingsDialog({ open, onClose }: SettingsDialogProps) { {settingsTab === "general" && } {settingsTab === "reading" && } {settingsTab === "fonts" && } + {settingsTab === "dictionaries" && } {settingsTab === "ai" && } {settingsTab === "vectorModel" && } {settingsTab === "tts" && } diff --git a/packages/app/src/components/settings/SyncSettings.tsx b/packages/app/src/components/settings/SyncSettings.tsx index 67d86fb0b..aae7ac68b 100644 --- a/packages/app/src/components/settings/SyncSettings.tsx +++ b/packages/app/src/components/settings/SyncSettings.tsx @@ -1,4 +1,5 @@ import { PasswordInput } from "@/components/ui/password-input"; +import { Progress } from "@/components/ui/progress"; import { Switch } from "@/components/ui/switch"; import { useSyncStore } from "@/stores/sync-store"; import { getPlatformService } from "@readany/core/services"; @@ -755,18 +756,11 @@ export function SyncSettings() { {statusLabel() &&

{statusLabel()}

} {isBusy && progress && (
-
- {progress.phase === "database" ? ( -
- ) : ( -
- )} -
+

{progressLabel()}

)} diff --git a/packages/app/src/components/ui/progress.tsx b/packages/app/src/components/ui/progress.tsx new file mode 100644 index 000000000..77400cfc5 --- /dev/null +++ b/packages/app/src/components/ui/progress.tsx @@ -0,0 +1,28 @@ +import { cn } from "@readany/core/utils"; + +interface ProgressProps { + /** Percentage from 0 to 100; omit when progress is unknown. */ + value?: number | null; + className?: string; + "aria-label": string; +} + +export function Progress({ value, className, "aria-label": label }: ProgressProps) { + const percent = + typeof value === "number" && Number.isFinite(value) + ? Math.min(100, Math.max(0, value)) + : undefined; + return ( +
+ + + ); +} diff --git a/packages/app/src/lib/dictionary/desktop-dictionary.ts b/packages/app/src/lib/dictionary/desktop-dictionary.ts new file mode 100644 index 000000000..04d38b7b0 --- /dev/null +++ b/packages/app/src/lib/dictionary/desktop-dictionary.ts @@ -0,0 +1,96 @@ +import type { DictionaryLanguage } from "@readany/core/dictionary"; +import { + type DictionaryDatabaseAdapter, + type DictionaryDatabaseConnection, + DictionaryLookupError, +} from "@readany/core/dictionary/dictionary-database"; +import type { DictionaryPackPlatform } from "@readany/core/dictionary/dictionary-pack-manager"; +import { createDictionaryRuntime } from "@readany/core/dictionary/dictionary-runtime"; +import { validateDictionaryDatabase } from "@readany/core/dictionary/dictionary-validation"; +import { Channel, invoke } from "@tauri-apps/api/core"; +import { appDataDir, join } from "@tauri-apps/api/path"; +import { exists, mkdir, remove, rename, stat } from "@tauri-apps/plugin-fs"; + +function query(path: string, sql: string, values: unknown[] = []): Promise { + return invoke("dictionary_query", { path, query: sql, values }); +} + +export class DesktopDictionaryDatabaseAdapter implements DictionaryDatabaseAdapter { + async open(language: DictionaryLanguage, path: string): Promise { + const metadata = await query<{ key: string; value: string }>( + path, + "SELECT key, value FROM metadata WHERE key IN ('schema_version', 'language')", + ); + const values = new Map(metadata.map((row) => [row.key, row.value])); + if (values.get("schema_version") !== "1" || values.get("language") !== language) { + throw new DictionaryLookupError("pack-invalid", "Dictionary metadata does not match"); + } + let closed = false; + const pending = new Set>(); + return { + getAllAsync(sql: string, ...params: unknown[]): Promise { + if (closed) return Promise.reject(new Error("Dictionary connection is closed")); + const operation = query(path, sql, params); + pending.add(operation); + void operation.then( + () => pending.delete(operation), + () => pending.delete(operation), + ); + return operation; + }, + async closeAsync() { + closed = true; + // Native queries own their handles; wait for all of them before replacing files. + await Promise.allSettled([...pending]); + }, + }; + } +} + +export function createDesktopDictionaryPackPlatform(): DictionaryPackPlatform { + return { + ensureDirectory: (path) => mkdir(path, { recursive: true }), + exists, + size: async (path) => (await stat(path)).size, + sha256: (path) => invoke("sync_hash_file", { path }), + async download(url, path, onProgress) { + let acceptingProgress = true; + const progress = new Channel<{ receivedBytes: number; totalBytes: number | null }>(); + progress.onmessage = ({ receivedBytes, totalBytes }) => { + if (acceptingProgress) + onProgress(totalBytes && totalBytes > 0 ? Math.min(receivedBytes / totalBytes, 1) : 0); + }; + try { + const report = await invoke<{ bytes: number; elapsedMs: number }>("dictionary_download", { + url, + path, + onProgress: progress, + }); + onProgress(1); + console.info(`[Dictionary] Transferred ${report.bytes} bytes in ${report.elapsedMs} ms`); + } finally { + acceptingProgress = false; + } + }, + readMetadata: (path) => + validateDictionaryDatabase({ + getAllAsync: (sql: string) => query(path, sql), + getFirstAsync: async (sql: string) => (await query(path, sql))[0] ?? null, + }), + async move(from, to) { + if (await exists(to)) throw new Error(`Dictionary move target already exists: ${to}`); + await rename(from, to); + }, + async remove(path) { + if (await exists(path)) await remove(path); + }, + }; +} + +export async function loadDesktopDictionaryRuntime() { + return createDictionaryRuntime({ + database: new DesktopDictionaryDatabaseAdapter(), + directory: await join(await appDataDir(), "dictionaries"), + platform: createDesktopDictionaryPackPlatform(), + }); +} diff --git a/packages/app/src/lib/dictionary/desktop.dictionary.test.ts b/packages/app/src/lib/dictionary/desktop.dictionary.test.ts new file mode 100644 index 000000000..203194dd0 --- /dev/null +++ b/packages/app/src/lib/dictionary/desktop.dictionary.test.ts @@ -0,0 +1,110 @@ +import { invoke } from "@tauri-apps/api/core"; +import { exists, rename } from "@tauri-apps/plugin-fs"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + DesktopDictionaryDatabaseAdapter, + createDesktopDictionaryPackPlatform, +} from "./desktop-dictionary"; +vi.mock("@tauri-apps/api/core", () => ({ + invoke: vi.fn(), + Channel: class { + onmessage = (_message: unknown) => {}; + }, +})); +vi.mock("@tauri-apps/plugin-fs", () => ({ + create: vi.fn(), + exists: vi.fn(), + rename: vi.fn(), + mkdir: vi.fn(), + remove: vi.fn(), + stat: vi.fn(), +})); +beforeEach(() => vi.resetAllMocks()); + +describe("desktop dictionary files", () => { + it("delegates transfer to native code and forwards bounded progress", async () => { + let emit!: (value: unknown) => void; + vi.mocked(invoke).mockImplementationOnce(async (_command, args) => { + const channel = (args as { onProgress: { onmessage(value: unknown): void } }).onProgress; + emit = channel.onmessage; + channel.onmessage({ receivedBytes: 1, totalBytes: 3 }); + channel.onmessage({ receivedBytes: 3, totalBytes: 3 }); + return { bytes: 3, elapsedMs: 10 }; + }); + const progress = vi.fn(); + await createDesktopDictionaryPackPlatform().download( + "https://example.com/pack", + "pack", + progress, + ); + expect(invoke).toHaveBeenCalledOnce(); + expect(invoke).toHaveBeenCalledWith( + "dictionary_download", + expect.objectContaining({ url: "https://example.com/pack", path: "pack" }), + ); + expect(progress).toHaveBeenCalledWith(1 / 3); + expect(progress).toHaveBeenLastCalledWith(1); + const count = progress.mock.calls.length; + emit({ receivedBytes: 1, totalBytes: 3 }); + expect(progress).toHaveBeenCalledTimes(count); + }); + it("propagates native transfer failures", async () => { + vi.mocked(invoke).mockRejectedValue(new Error("disk full")); + await expect( + createDesktopDictionaryPackPlatform().download("url", "pack", vi.fn()), + ).rejects.toThrow("disk full"); + }); + it("does not overwrite an existing pack during promotion", async () => { + vi.mocked(exists).mockResolvedValue(true); + await expect(createDesktopDictionaryPackPlatform().move("staged", "active")).rejects.toThrow( + "already exists", + ); + expect(rename).not.toHaveBeenCalled(); + }); +}); + +describe("desktop dictionary queries", () => { + it("rejects a pack with the wrong language", async () => { + vi.mocked(invoke).mockResolvedValue([ + { key: "schema_version", value: "1" }, + { key: "language", value: "zh" }, + ]); + await expect(new DesktopDictionaryDatabaseAdapter().open("en", "pack")).rejects.toMatchObject({ + code: "pack-invalid", + }); + }); + it("waits for active native queries before closing and rejects later queries", async () => { + vi.mocked(invoke).mockResolvedValueOnce([ + { key: "schema_version", value: "1" }, + { key: "language", value: "en" }, + ]); + const connection = await new DesktopDictionaryDatabaseAdapter().open("en", "pack"); + let finish!: (rows: unknown[]) => void; + vi.mocked(invoke).mockImplementationOnce( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + const query = connection.getAllAsync( + "SELECT headword FROM entries WHERE headword = ?", + "hello", + ); + let closed = false; + const closing = connection.closeAsync().then(() => { + closed = true; + }); + await Promise.resolve(); + expect(closed).toBe(false); + expect(invoke).toHaveBeenLastCalledWith("dictionary_query", { + path: "pack", + query: "SELECT headword FROM entries WHERE headword = ?", + values: ["hello"], + }); + finish([{ headword: "hello" }]); + await query; + await closing; + expect(closed).toBe(true); + await expect(connection.getAllAsync("SELECT 1")).rejects.toThrow("closed"); + }); +}); diff --git a/packages/app/src/stores/dictionary-store.ts b/packages/app/src/stores/dictionary-store.ts new file mode 100644 index 000000000..4a36303bf --- /dev/null +++ b/packages/app/src/stores/dictionary-store.ts @@ -0,0 +1,21 @@ +import { + DEFAULT_DICTIONARY_MANIFEST_URL, + DICTIONARY_BUNDLED_MANIFEST, +} from "@readany/core/dictionary/dictionary-config"; +import { createRuntimeBackedDictionaryStore } from "@readany/core/dictionary/dictionary-store"; + +export const DICTIONARY_REMOTE_MANIFEST_URL = + import.meta.env.VITE_DICTIONARY_MANIFEST_URL?.trim() || DEFAULT_DICTIONARY_MANIFEST_URL; + +export const useDictionaryStore = createRuntimeBackedDictionaryStore({ + loadRuntime: async () => + (await import("@/lib/dictionary/desktop-dictionary")).loadDesktopDictionaryRuntime(), + fetchRemoteManifest: async () => { + const { fetch } = await import("@tauri-apps/plugin-http"); + const response = await fetch(DICTIONARY_REMOTE_MANIFEST_URL); + if (!response.ok) + throw new Error(`Dictionary manifest request failed: HTTP ${response.status}`); + return response.json(); + }, + getBundledManifest: async () => DICTIONARY_BUNDLED_MANIFEST, +}); diff --git a/packages/app/tsconfig.json b/packages/app/tsconfig.json index 04eb5481d..46930c95e 100644 --- a/packages/app/tsconfig.json +++ b/packages/app/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "ES2020", "useDefineForClassFields": true, - "lib": ["ES2020", "DOM", "DOM.Iterable"], + "lib": ["ES2021", "DOM", "DOM.Iterable"], "module": "ESNext", "skipLibCheck": true, diff --git a/packages/app/vitest.dictionary.config.ts b/packages/app/vitest.dictionary.config.ts new file mode 100644 index 000000000..15bf25327 --- /dev/null +++ b/packages/app/vitest.dictionary.config.ts @@ -0,0 +1,7 @@ +import { fileURLToPath } from "node:url"; +import { defineConfig } from "vitest/config"; +export default defineConfig({ + resolve: { alias: { "@": fileURLToPath(new URL("./src", import.meta.url)) } }, + test: { environment: "node", include: ["src/**/*.dictionary.test.{ts,tsx}"] }, + esbuild: { jsx: "automatic" }, +}); diff --git a/packages/core/src/dictionary/definition-controller.ts b/packages/core/src/dictionary/definition-controller.ts new file mode 100644 index 000000000..b3e9457a4 --- /dev/null +++ b/packages/core/src/dictionary/definition-controller.ts @@ -0,0 +1,144 @@ +import { + type DictionaryEntry, + type DictionaryLanguage, + type DictionaryPackDescriptor, + prepareDictionarySelection, +} from "./index"; + +export type DefinitionState = + | { kind: "idle" } + | { kind: "loading"; displayText: string } + | { kind: "unsupported"; reason: string } + | { kind: "missing-pack"; language: DictionaryLanguage; descriptor: DictionaryPackDescriptor } + | { kind: "downloading"; language: DictionaryLanguage; progress: number } + | { kind: "verifying"; language: DictionaryLanguage } + | { kind: "result"; displayText: string; entries: DictionaryEntry[] } + | { kind: "no-match"; displayText: string } + | { kind: "error"; message: string }; + +export interface DefinitionControllerDependencies { + lookup(text: string): Promise; + install( + descriptor: DictionaryPackDescriptor, + onProgress: (progress: number) => void, + onVerifying?: () => void, + ): Promise; + getDescriptor(language: DictionaryLanguage): DictionaryPackDescriptor | undefined; +} + +type DefinitionStateListener = (state: DefinitionState) => void; + +interface DictionaryError extends Error { + code?: string; +} + +export class DefinitionController { + private requestToken = 0; + private selectedText: string | null = null; + private readonly listeners = new Set(); + + state: DefinitionState = { kind: "idle" }; + + constructor(private readonly dependencies: DefinitionControllerDependencies) {} + + subscribe(listener: DefinitionStateListener): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + async open(text: string): Promise { + const token = ++this.requestToken; + this.selectedText = text; + await this.lookupSelection(text, token); + } + + async retry(): Promise { + if (!this.selectedText) return; + await this.open(this.selectedText); + } + + async download(): Promise { + if (this.state.kind !== "missing-pack" || !this.selectedText) return; + + const { descriptor, language } = this.state; + const text = this.selectedText; + const token = this.requestToken; + this.setState({ kind: "downloading", language, progress: 0 }); + + try { + await this.dependencies.install( + descriptor, + (progress) => { + if (this.isCurrent(token)) { + this.setState({ kind: "downloading", language, progress: clampProgress(progress) }); + } + }, + () => { + if (this.isCurrent(token)) this.setState({ kind: "verifying", language }); + }, + ); + } catch (error) { + if (this.isCurrent(token)) this.setState({ kind: "error", message: messageOf(error) }); + return; + } + + if (this.isCurrent(token)) await this.lookupSelection(text, token); + } + + close(): void { + this.requestToken += 1; + this.selectedText = null; + this.setState({ kind: "idle" }); + } + + private async lookupSelection(text: string, token: number): Promise { + const selection = prepareDictionarySelection(text); + if (!selection.ok) { + if (this.isCurrent(token)) this.setState({ kind: "unsupported", reason: selection.reason }); + return; + } + + this.setState({ kind: "loading", displayText: selection.displayText }); + try { + const entries = await this.dependencies.lookup(text); + if (!this.isCurrent(token)) return; + this.setState( + entries.length > 0 + ? { kind: "result", displayText: selection.displayText, entries } + : { kind: "no-match", displayText: selection.displayText }, + ); + } catch (error) { + if (!this.isCurrent(token)) return; + const dictionaryError = error as DictionaryError; + if (dictionaryError.code === "pack-not-installed") { + const descriptor = this.dependencies.getDescriptor(selection.language); + if (descriptor) { + this.setState({ kind: "missing-pack", language: selection.language, descriptor }); + return; + } + } + if (dictionaryError.code === "unsupported-selection") { + this.setState({ kind: "unsupported", reason: "unsupported-selection" }); + return; + } + this.setState({ kind: "error", message: messageOf(error) }); + } + } + + private isCurrent(token: number): boolean { + return token === this.requestToken; + } + + private setState(state: DefinitionState): void { + this.state = state; + for (const listener of this.listeners) listener(state); + } +} + +function clampProgress(progress: number): number { + return Number.isFinite(progress) ? Math.max(0, Math.min(1, progress)) : 0; +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/core/src/dictionary/dictionary-config.ts b/packages/core/src/dictionary/dictionary-config.ts new file mode 100644 index 000000000..b57eb58b2 --- /dev/null +++ b/packages/core/src/dictionary/dictionary-config.ts @@ -0,0 +1,5 @@ +import manifest from "./dictionary-manifest.json"; +import { parseDictionaryManifest } from "./manifest"; +export const DICTIONARY_BUNDLED_MANIFEST = parseDictionaryManifest(manifest); +export const DEFAULT_DICTIONARY_MANIFEST_URL = + "https://raw.githubusercontent.com/codedogQBY/ReadAny/main/dictionary-packs/manifest.json"; diff --git a/packages/core/src/dictionary/dictionary-database.ts b/packages/core/src/dictionary/dictionary-database.ts new file mode 100644 index 000000000..8483b6efb --- /dev/null +++ b/packages/core/src/dictionary/dictionary-database.ts @@ -0,0 +1,20 @@ +import type { DictionaryLanguage } from "./types"; + +export interface DictionaryDatabaseConnection { + getAllAsync(sql: string, ...params: unknown[]): Promise; + closeAsync(): Promise; +} + +export interface DictionaryDatabaseAdapter { + open(language: DictionaryLanguage, absolutePath: string): Promise; +} + +export class DictionaryLookupError extends Error { + constructor( + readonly code: "unsupported-selection" | "pack-not-installed" | "pack-invalid", + message: string, + ) { + super(message); + this.name = "DictionaryLookupError"; + } +} diff --git a/packages/core/src/dictionary/dictionary-lookup-service.ts b/packages/core/src/dictionary/dictionary-lookup-service.ts new file mode 100644 index 000000000..5a93b3774 --- /dev/null +++ b/packages/core/src/dictionary/dictionary-lookup-service.ts @@ -0,0 +1,159 @@ +import { + type DictionaryDatabaseAdapter, + type DictionaryDatabaseConnection, + DictionaryLookupError, +} from "./dictionary-database"; +import { type DictionaryEntry, type DictionaryLanguage, prepareDictionarySelection } from "./index"; + +const LOOKUP_SQL = ` + WITH matched AS ( + SELECT lookup.entry_id, lookup.rank + FROM lookup + INNER JOIN entries ON entries.id = lookup.entry_id + WHERE lookup.lookup_key = ? AND entries.language = ? + ORDER BY lookup.rank ASC, lookup.entry_id ASC + LIMIT 20 + ) + SELECT + e.id AS entry_id, + e.language, + e.headword, + e.simplified, + e.traditional, + e.pronunciation, + e.part_of_speech, + matched.rank, + s.sense_order, + s.definition + FROM matched + INNER JOIN entries e ON e.id = matched.entry_id + INNER JOIN senses s ON s.entry_id = e.id + ORDER BY matched.rank ASC, e.id ASC, s.sense_order ASC +`; + +interface DictionaryLookupRow { + entry_id: number; + language: DictionaryLanguage; + headword: string; + simplified: string | null; + traditional: string | null; + pronunciation: string | null; + part_of_speech: string; + rank: number; + sense_order: number; + definition: string; +} + +export type DictionaryPackPathResolver = ( + language: DictionaryLanguage, +) => string | null | Promise; + +export type DictionaryPackInvalidator = (language: DictionaryLanguage) => Promise | void; + +export class DictionaryLookupService { + private readonly connections = new Map< + DictionaryLanguage, + Promise + >(); + + constructor( + private readonly database: DictionaryDatabaseAdapter, + private readonly resolvePackPath: DictionaryPackPathResolver, + private readonly invalidatePack: DictionaryPackInvalidator = () => {}, + ) {} + + async lookup(text: string): Promise { + const selection = prepareDictionarySelection(text); + if (!selection.ok) { + throw new DictionaryLookupError( + "unsupported-selection", + `Dictionary lookup does not support this selection: ${selection.reason}`, + ); + } + + const absolutePath = await this.resolvePackPath(selection.language); + if (!absolutePath) { + throw new DictionaryLookupError( + "pack-not-installed", + `The ${selection.language} dictionary pack is not installed`, + ); + } + + try { + const database = await this.connectionFor(selection.language, absolutePath); + const rows = await database.getAllAsync( + LOOKUP_SQL, + selection.key, + selection.language, + ); + return this.entriesFromRows(rows); + } catch (error) { + const cleanupErrors: unknown[] = []; + try { + await this.close(selection.language); + } catch (cleanupError) { + cleanupErrors.push(cleanupError); + } + try { + await this.invalidatePack(selection.language); + } catch (cleanupError) { + cleanupErrors.push(cleanupError); + } + if (error instanceof Error && cleanupErrors.length > 0) { + (error as Error & { cleanupErrors?: unknown[] }).cleanupErrors = cleanupErrors; + } + throw error; + } + } + + async close(language?: DictionaryLanguage): Promise { + const languages = language ? [language] : [...this.connections.keys()]; + await Promise.all( + languages.map(async (currentLanguage) => { + const connection = this.connections.get(currentLanguage); + if (!connection) return; + this.connections.delete(currentLanguage); + await (await connection).closeAsync(); + }), + ); + } + + private connectionFor( + language: DictionaryLanguage, + absolutePath: string, + ): Promise { + const existing = this.connections.get(language); + if (existing) return existing; + + const opening = this.database.open(language, absolutePath); + this.connections.set(language, opening); + void opening.catch(() => { + if (this.connections.get(language) === opening) { + this.connections.delete(language); + } + }); + return opening; + } + + private entriesFromRows(rows: DictionaryLookupRow[]): DictionaryEntry[] { + const entries = new Map(); + for (const row of rows) { + let entry = entries.get(row.entry_id); + if (!entry) { + entry = { + id: row.entry_id, + language: row.language, + headword: row.headword, + simplified: row.simplified ?? undefined, + traditional: row.traditional ?? undefined, + pronunciation: row.pronunciation ?? undefined, + partOfSpeech: row.part_of_speech, + senses: [], + }; + entries.set(row.entry_id, entry); + } + entry.senses.push({ order: row.sense_order, definition: row.definition }); + } + return [...entries.values()]; + } +} diff --git a/packages/app-expo/src/config/dictionary-manifest.json b/packages/core/src/dictionary/dictionary-manifest.json similarity index 100% rename from packages/app-expo/src/config/dictionary-manifest.json rename to packages/core/src/dictionary/dictionary-manifest.json diff --git a/packages/core/src/dictionary/dictionary-pack-manager.ts b/packages/core/src/dictionary/dictionary-pack-manager.ts new file mode 100644 index 000000000..49bd45097 --- /dev/null +++ b/packages/core/src/dictionary/dictionary-pack-manager.ts @@ -0,0 +1,438 @@ +import type { DictionaryLanguage, DictionaryManifest, DictionaryPackDescriptor } from "./index"; + +const dictionaryLanguages = ["en", "zh"] as const; + +export type DictionaryPackMetadata = Pick< + DictionaryPackDescriptor, + | "language" + | "version" + | "schemaVersion" + | "sourceEdition" + | "sourceDumpDate" + | "sourceArchiveUrl" + | "url" + | "attributionUrl" + | "license" +> & { + licenseNotice: string; + creatorAttribution: string; +}; + +export type InstalledDictionaryPack = DictionaryPackMetadata & + Pick; + +export interface DictionaryPackPlatform { + ensureDirectory(path: string): Promise; + download(url: string, path: string, onProgress: (fraction: number) => void): Promise; + exists(path: string): Promise; + size(path: string): Promise; + sha256(path: string): Promise; + readMetadata(path: string): Promise; + move(from: string, to: string): Promise; + remove(path: string): Promise; +} + +export type DictionaryPackStatus = + | { state: "not-installed" } + | { state: "downloading"; progress: number } + | { state: "verifying" } + | { state: "installed"; version: string; sizeBytes: number } + | { + state: "update-available"; + installedVersion: string; + availableVersion: string; + sizeBytes: number; + } + | { state: "error"; message: string; hasActivePack: boolean }; + +export interface DictionaryHandleCloser { + close(language?: DictionaryLanguage): Promise | void; +} + +type ErrorWithCleanup = Error & { cleanupErrors?: unknown[] }; + +interface InspectedArtifact { + path: string; + exists: boolean; + installed?: InstalledDictionaryPack; + error?: unknown; +} + +interface InspectedArtifacts { + active: InspectedArtifact; + backup: InspectedArtifact; +} + +export class DictionaryPackManager { + private readonly installs = new Map>(); + private readonly gates = new Map>(); + private readonly validatedPacks = new Map< + DictionaryLanguage, + { path: string; installed: InstalledDictionaryPack } | null + >(); + + constructor( + private readonly platform: DictionaryPackPlatform, + private readonly directory: string, + private readonly lookup: DictionaryHandleCloser, + ) {} + + async refresh( + manifest: DictionaryManifest, + ): Promise> { + const entries = await Promise.all( + dictionaryLanguages.map((language) => + this.runExclusive(language, async () => { + try { + const discovered = await this.discoverInstalledLocked(language); + const installed = discovered?.installed; + const available = manifest.packs[language]; + const status: DictionaryPackStatus = !installed + ? { state: "not-installed" } + : sameDescriptorIdentity(installed, available) + ? { state: "installed", version: installed.version, sizeBytes: installed.sizeBytes } + : { + state: "update-available", + installedVersion: installed.version, + availableVersion: available.version, + sizeBytes: installed.sizeBytes, + }; + return [language, status] as const; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const hasActivePack = await this.hasPackArtifacts(language); + return [ + language, + { state: "error", message, hasActivePack } satisfies DictionaryPackStatus, + ] as const; + } + }), + ), + ); + return Object.fromEntries(entries) as Record; + } + + install( + descriptor: DictionaryPackDescriptor, + onStatus: (status: DictionaryPackStatus) => void = () => {}, + ): Promise { + const language = descriptor.language; + const existing = this.installs.get(language); + if (existing) return existing; + + const operation = this.runExclusive(language, () => + this.installLocked(descriptor, onStatus), + ).finally(() => { + if (this.installs.get(language) === operation) this.installs.delete(language); + }); + this.installs.set(language, operation); + return operation; + } + + remove(language: DictionaryLanguage): Promise { + return this.runExclusive(language, async () => { + this.invalidate(language); + await this.platform.ensureDirectory(this.directory); + await this.lookup.close(language); + await this.removeIfPresent(this.activePath(language)); + await this.removeIfPresent(this.stagedPath(language)); + await this.removeIfPresent(this.backupPath(language)); + }); + } + + getActivePath(language: DictionaryLanguage): Promise { + return this.runExclusive(language, async () => { + const discovered = await this.discoverInstalledLocked(language); + return discovered?.path ?? null; + }); + } + + getInstalledDescriptor(language: DictionaryLanguage): Promise { + return this.runExclusive(language, async () => { + return (await this.discoverInstalledLocked(language))?.installed ?? null; + }); + } + + invalidate(language: DictionaryLanguage): void { + this.validatedPacks.delete(language); + } + + private async installLocked( + descriptor: DictionaryPackDescriptor, + onStatus: (status: DictionaryPackStatus) => void, + ): Promise { + const language = descriptor.language; + this.invalidate(language); + const staged = this.stagedPath(language); + const active = this.activePath(language); + const backup = this.backupPath(language); + try { + await this.platform.ensureDirectory(this.directory); + await this.removeIfPresent(staged); + const artifacts = await this.inspectArtifactsLocked(language); + onStatus({ state: "downloading", progress: 0 }); + await this.platform.download(descriptor.url, staged, (progress) => + onStatus({ state: "downloading", progress: clamp(progress) }), + ); + onStatus({ state: "verifying" }); + const verificationStarted = Date.now(); + await this.verifyExpected(staged, descriptor); + await this.lookup.close(language); + + let hasRollbackBackup = Boolean(artifacts.backup.installed); + let activationStarted = false; + try { + if (artifacts.backup.installed) { + activationStarted = true; + await this.removeIfPresent(active); + } else if (artifacts.active.installed) { + await this.removeIfPresent(backup); + await this.platform.move(active, backup); + hasRollbackBackup = true; + activationStarted = true; + } else { + await this.removeIfPresent(active); + await this.removeIfPresent(backup); + activationStarted = true; + } + await this.platform.move(staged, active); + const installed = await this.verifyExpected(active, descriptor); + await this.removeIfPresent(backup); + this.validatedPacks.set(language, { path: active, installed }); + } catch (operationError) { + const cleanupErrors: unknown[] = []; + if (hasRollbackBackup) { + await this.captureCleanupFailure(cleanupErrors, () => + this.rollbackLocked(active, backup), + ); + } else if (activationStarted) { + await this.captureCleanupFailure(cleanupErrors, () => this.removeIfPresent(active)); + } + attachCleanupErrors(operationError, cleanupErrors); + throw operationError; + } + + console.info( + `[Dictionary] Verified and installed ${language} pack in ${Date.now() - verificationStarted} ms`, + ); + onStatus({ + state: "installed", + version: descriptor.version, + sizeBytes: descriptor.sizeBytes, + }); + } catch (operationError) { + const cleanupErrors: unknown[] = []; + await this.captureCleanupFailure(cleanupErrors, () => this.removeIfPresent(staged)); + attachCleanupErrors(operationError, cleanupErrors); + const message = + operationError instanceof Error ? operationError.message : String(operationError); + onStatus({ + state: "error", + message, + hasActivePack: await this.hasPackArtifacts(language), + }); + throw operationError; + } + } + + private async verifyExpected( + path: string, + descriptor: DictionaryPackDescriptor, + ): Promise { + const installed = await this.inspectPack(path); + if (installed.sizeBytes !== descriptor.sizeBytes) + throw new Error("Dictionary pack size did not match manifest"); + if (installed.sha256.toLowerCase() !== descriptor.sha256.toLowerCase()) + throw new Error("Dictionary pack checksum did not match manifest"); + assertMetadataMatches(installed, descriptor); + return installed; + } + + private async rollbackLocked(active: string, backup: string): Promise { + if (!(await this.platform.exists(backup))) return; + const backupSnapshot = await this.inspectPack(backup); + await this.removeIfPresent(active); + await this.platform.move(backup, active); + const restoredSnapshot = await this.inspectPack(active); + assertSamePack(backupSnapshot, restoredSnapshot); + this.validatedPacks.set(restoredSnapshot.language, { + path: active, + installed: restoredSnapshot, + }); + } + + private async discoverInstalledLocked( + language: DictionaryLanguage, + ): Promise<{ path: string; installed: InstalledDictionaryPack } | null> { + if (this.validatedPacks.has(language)) return this.validatedPacks.get(language) ?? null; + await this.platform.ensureDirectory(this.directory); + const artifacts = await this.inspectArtifactsLocked(language); + if (artifacts.backup.installed) { + const discovered = { path: artifacts.backup.path, installed: artifacts.backup.installed }; + this.validatedPacks.set(language, discovered); + return discovered; + } + if (artifacts.active.installed) { + const discovered = { path: artifacts.active.path, installed: artifacts.active.installed }; + this.validatedPacks.set(language, discovered); + return discovered; + } + + const errors = [artifacts.backup.error, artifacts.active.error].filter( + (error): error is NonNullable => error !== undefined, + ); + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) { + throw new AggregateError(errors, `No valid ${language} dictionary recovery artifact exists`); + } + this.validatedPacks.set(language, null); + return null; + } + + private async inspectArtifactsLocked(language: DictionaryLanguage): Promise { + const [active, backup] = await Promise.all([ + this.inspectArtifact(this.activePath(language), language), + this.inspectArtifact(this.backupPath(language), language), + ]); + return { active, backup }; + } + + private async inspectArtifact( + path: string, + language: DictionaryLanguage, + ): Promise { + if (!(await this.platform.exists(path))) return { path, exists: false }; + try { + const installed = await this.inspectPack(path); + if (installed.language !== language) { + throw new Error(`Dictionary metadata language did not match ${language}`); + } + return { path, exists: true, installed }; + } catch (error) { + return { path, exists: true, error }; + } + } + + private async hasPackArtifacts(language: DictionaryLanguage): Promise { + return ( + (await this.platform.exists(this.activePath(language))) || + (await this.platform.exists(this.backupPath(language))) || + (await this.platform.exists(this.stagedPath(language))) + ); + } + + private async inspectPack(path: string): Promise { + const [metadata, sizeBytes, sha256] = await Promise.all([ + this.platform.readMetadata(path), + this.platform.size(path), + this.platform.sha256(path), + ]); + return { ...metadata, sizeBytes, sha256: sha256.toLowerCase() }; + } + + private runExclusive(language: DictionaryLanguage, action: () => Promise): Promise { + const previous = this.gates.get(language) ?? Promise.resolve(); + const operation = previous.catch(() => undefined).then(action); + const tail = operation.then( + () => undefined, + () => undefined, + ); + this.gates.set(language, tail); + void tail.then(() => { + if (this.gates.get(language) === tail) this.gates.delete(language); + }); + return operation; + } + + private async captureCleanupFailure( + errors: unknown[], + cleanup: () => Promise, + ): Promise { + try { + await cleanup(); + } catch (error) { + errors.push(error); + } + } + + private activePath(language: DictionaryLanguage): string { + return `${this.directory}/readany-dictionary-${language}.sqlite`; + } + + private stagedPath(language: DictionaryLanguage): string { + return `${this.activePath(language)}.download`; + } + + private backupPath(language: DictionaryLanguage): string { + return `${this.activePath(language)}.backup`; + } + + private async removeIfPresent(path: string): Promise { + if (await this.platform.exists(path)) await this.platform.remove(path); + } +} + +function assertMetadataMatches( + installed: DictionaryPackMetadata, + expected: DictionaryPackDescriptor, +): void { + for (const key of [ + "language", + "version", + "schemaVersion", + "sourceEdition", + "sourceDumpDate", + "sourceArchiveUrl", + "url", + "attributionUrl", + "license", + ] as const) { + if (installed[key] !== expected[key]) + throw new Error(`Dictionary metadata ${key} did not match manifest`); + } +} + +function sameDescriptorIdentity( + installed: InstalledDictionaryPack, + available: DictionaryPackDescriptor, +): boolean { + return ( + installed.sizeBytes === available.sizeBytes && + installed.sha256.toLowerCase() === available.sha256.toLowerCase() && + [ + "language", + "version", + "schemaVersion", + "sourceEdition", + "sourceDumpDate", + "sourceArchiveUrl", + "url", + "attributionUrl", + "license", + ].every( + (key) => + installed[key as keyof InstalledDictionaryPack] === + available[key as keyof DictionaryPackDescriptor], + ) + ); +} + +function assertSamePack(expected: InstalledDictionaryPack, actual: InstalledDictionaryPack): void { + if ( + expected.sizeBytes !== actual.sizeBytes || + expected.sha256 !== actual.sha256 || + JSON.stringify(expected) !== JSON.stringify(actual) + ) { + throw new Error("Restored dictionary pack did not match the validated backup"); + } +} + +function attachCleanupErrors(operationError: unknown, cleanupErrors: unknown[]): void { + if (cleanupErrors.length === 0 || !(operationError instanceof Error)) return; + const error = operationError as ErrorWithCleanup; + error.cleanupErrors = [...(error.cleanupErrors ?? []), ...cleanupErrors]; +} + +function clamp(value: number): number { + return Number.isFinite(value) ? Math.max(0, Math.min(1, value)) : 0; +} diff --git a/packages/core/src/dictionary/dictionary-runtime.ts b/packages/core/src/dictionary/dictionary-runtime.ts new file mode 100644 index 000000000..ec4bebeba --- /dev/null +++ b/packages/core/src/dictionary/dictionary-runtime.ts @@ -0,0 +1,29 @@ +import type { DictionaryDatabaseAdapter } from "./dictionary-database"; +import { DictionaryLookupService } from "./dictionary-lookup-service"; +import { DictionaryPackManager, type DictionaryPackPlatform } from "./dictionary-pack-manager"; + +export interface DictionaryRuntimeOptions { + database: DictionaryDatabaseAdapter; + directory: string; + platform: DictionaryPackPlatform; +} + +export function createDictionaryRuntime(options: DictionaryRuntimeOptions): { + lookup: DictionaryLookupService; + manager: DictionaryPackManager; +} { + // biome-ignore lint/style/useConst: the lookup resolves installed paths through the manager created below. + let manager: DictionaryPackManager | undefined; + const lookup = new DictionaryLookupService( + options.database, + async (language) => { + if (!manager) throw new Error("Dictionary runtime is not initialized"); + return manager.getActivePath(language); + }, + (language) => { + manager?.invalidate(language); + }, + ); + manager = new DictionaryPackManager(options.platform, options.directory, lookup); + return { lookup, manager }; +} diff --git a/packages/core/src/dictionary/dictionary-store.ts b/packages/core/src/dictionary/dictionary-store.ts new file mode 100644 index 000000000..bd9372ea2 --- /dev/null +++ b/packages/core/src/dictionary/dictionary-store.ts @@ -0,0 +1,156 @@ +import { type StoreApi, type UseBoundStore, create } from "zustand"; +import type { DictionaryLookupService } from "./dictionary-lookup-service"; +import type { DictionaryPackManager, DictionaryPackStatus } from "./dictionary-pack-manager"; +import { + type DictionaryEntry, + type DictionaryLanguage, + type DictionaryManifest, + type DictionaryPackDescriptor, + parseDictionaryManifest, +} from "./index"; +export interface DictionaryStoreDependencies { + manager: Pick; + lookup: Pick; + fetchRemoteManifest: () => Promise; + getBundledManifest: () => Promise; +} + +export interface DictionaryRuntime { + manager: Pick; + lookup: Pick; +} + +export interface RuntimeBackedDictionaryStoreDependencies { + loadRuntime: () => Promise; + fetchRemoteManifest: () => Promise; + getBundledManifest: () => Promise; +} + +export interface DictionaryStoreState { + manifest: DictionaryManifest | null; + packs: Record; + initialize(): Promise; + refreshManifest(): Promise; + install(language: DictionaryLanguage): Promise; + remove(language: DictionaryLanguage): Promise; + retry(): Promise; + lookup(text: string): Promise; +} + +const emptyPacks = (): Record => ({ + en: { state: "not-installed" }, + zh: { state: "not-installed" }, +}); + +export function createDictionaryStore( + deps: DictionaryStoreDependencies, +): UseBoundStore> { + return create()((set, get) => { + let remoteRefreshPromise: Promise | undefined; + let bundledReadinessPromise: Promise | undefined; + const applyManifest = async (manifest: DictionaryManifest) => { + const packs = await deps.manager.refresh(manifest); + set({ manifest, packs }); + }; + const loadManifest = async (): Promise => { + let remoteError: unknown; + try { + return parseDictionaryManifest(await deps.fetchRemoteManifest()); + } catch (error) { + remoteError = error; + } + try { + return parseDictionaryManifest(await deps.getBundledManifest()); + } catch (bundledError) { + throw new AggregateError( + [remoteError, bundledError], + "No valid dictionary manifest is available", + ); + } + }; + const refreshFromSources = (): Promise => { + if (remoteRefreshPromise) return remoteRefreshPromise; + const operation = loadManifest().then(applyManifest); + remoteRefreshPromise = operation; + void operation.then( + () => { + if (remoteRefreshPromise === operation) remoteRefreshPromise = undefined; + }, + () => { + if (remoteRefreshPromise === operation) remoteRefreshPromise = undefined; + }, + ); + return operation; + }; + const refreshFromBundled = (): Promise => { + if (get().manifest) return Promise.resolve(); + if (bundledReadinessPromise) return bundledReadinessPromise; + const operation = deps + .getBundledManifest() + .then(parseDictionaryManifest) + .then(async (manifest) => { + if (!get().manifest) await applyManifest(manifest); + }); + bundledReadinessPromise = operation; + void operation.then( + () => { + if (bundledReadinessPromise === operation) bundledReadinessPromise = undefined; + }, + () => { + if (bundledReadinessPromise === operation) bundledReadinessPromise = undefined; + }, + ); + return operation; + }; + + return { + manifest: null, + packs: emptyPacks(), + initialize: refreshFromSources, + refreshManifest: refreshFromSources, + install: async (language) => { + const descriptor: DictionaryPackDescriptor | undefined = get().manifest?.packs[language]; + if (!descriptor) throw new Error("Dictionary manifest is unavailable"); + await deps.manager.install(descriptor, (status) => + set((state) => ({ packs: { ...state.packs, [language]: status } })), + ); + }, + remove: async (language) => { + await deps.manager.remove(language); + set((state) => ({ packs: { ...state.packs, [language]: { state: "not-installed" } } })); + }, + retry: refreshFromSources, + lookup: async (text) => { + if (!get().manifest) await refreshFromBundled(); + return deps.lookup.lookup(text); + }, + }; + }); +} + +export function createRuntimeBackedDictionaryStore( + deps: RuntimeBackedDictionaryStoreDependencies, +): UseBoundStore> { + let runtimePromise: Promise | undefined; + const runtime = () => { + if (!runtimePromise) { + const operation = deps.loadRuntime(); + runtimePromise = operation; + void operation.catch(() => { + if (runtimePromise === operation) runtimePromise = undefined; + }); + } + return runtimePromise; + }; + return createDictionaryStore({ + manager: { + refresh: async (manifest) => (await runtime()).manager.refresh(manifest), + install: async (descriptor, onStatus) => + (await runtime()).manager.install(descriptor, onStatus), + remove: async (language) => (await runtime()).manager.remove(language), + }, + lookup: { lookup: async (text) => (await runtime()).lookup.lookup(text) }, + fetchRemoteManifest: deps.fetchRemoteManifest, + getBundledManifest: deps.getBundledManifest, + }); +} diff --git a/packages/core/src/dictionary/dictionary-validation.ts b/packages/core/src/dictionary/dictionary-validation.ts new file mode 100644 index 000000000..41e62e4e8 --- /dev/null +++ b/packages/core/src/dictionary/dictionary-validation.ts @@ -0,0 +1,144 @@ +import type { DictionaryPackMetadata } from "./dictionary-pack-manager"; +interface SqliteObjectRow { + name: string; + type: string; + tbl_name: string; +} + +interface SqliteColumnRow { + name: string; +} + +interface SqliteIndexColumnRow { + name: string; + seqno: number; +} + +interface DictionaryMetadataRow { + key: string; + value: string; +} + +export interface DictionaryValidationDatabase { + getFirstAsync(sql: string): Promise; + getAllAsync(sql: string): Promise; +} + +const requiredObjects = new Map([ + ["metadata", "table"], + ["entries", "table"], + ["senses", "table"], + ["lookup", "table"], + ["lookup_key_rank_idx", "index"], +]); + +const requiredColumns = { + metadata: ["key", "value"], + entries: [ + "id", + "language", + "headword", + "simplified", + "traditional", + "pronunciation", + "part_of_speech", + ], + senses: ["entry_id", "sense_order", "definition"], + lookup: ["lookup_key", "entry_id", "rank"], +} as const; + +const requiredMetadataKeys = [ + "schema_version", + "language", + "version", + "source_edition", + "source_dump_date", + "source_archive_url", + "asset_url", + "attribution_url", + "license", + "license_notice", + "creator_attribution", +] as const; + +export async function validateDictionaryDatabase( + database: DictionaryValidationDatabase, +): Promise { + const integrity = await database.getFirstAsync<{ integrity_check: string }>( + "PRAGMA integrity_check", + ); + if (integrity?.integrity_check !== "ok") + throw new Error("Dictionary SQLite integrity check failed"); + + const objects = await database.getAllAsync( + "SELECT name, type, tbl_name FROM sqlite_master WHERE name IN ('metadata', 'entries', 'senses', 'lookup', 'lookup_key_rank_idx')", + ); + const objectsByName = new Map(objects.map((object) => [object.name, object])); + for (const [name, type] of requiredObjects) { + if (objectsByName.get(name)?.type !== type) + throw new Error(`Dictionary SQLite schema requires ${name} ${type}`); + } + if (objectsByName.get("lookup_key_rank_idx")?.tbl_name !== "lookup") + throw new Error("Dictionary SQLite lookup_key_rank_idx must belong to lookup"); + + for (const [table, expectedColumns] of Object.entries(requiredColumns)) { + const columns = await database.getAllAsync(`PRAGMA table_info('${table}')`); + const actualColumns = columns.map((column) => column.name); + if (!sameArray(actualColumns, expectedColumns)) + throw new Error(`Dictionary SQLite ${table} columns did not match the required schema`); + } + + const indexColumns = await database.getAllAsync( + "PRAGMA index_info('lookup_key_rank_idx')", + ); + const orderedIndexColumns = [...indexColumns] + .sort((left, right) => left.seqno - right.seqno) + .map((column) => column.name); + if (!sameArray(orderedIndexColumns, ["lookup_key", "rank", "entry_id"])) + throw new Error("Dictionary SQLite lookup index columns were not in the required order"); + + const metadataRows = await database.getAllAsync( + `SELECT key, value FROM metadata WHERE key IN (${requiredMetadataKeys + .map((key) => `'${key}'`) + .join(", ")})`, + ); + const metadata = new Map(metadataRows.map((row) => [row.key, row.value])); + const value = (key: (typeof requiredMetadataKeys)[number]): string => { + const found = metadata.get(key); + if (!found?.trim()) throw new Error(`Dictionary metadata ${key} is missing`); + return found; + }; + + const schemaVersion = value("schema_version"); + if (schemaVersion !== "1") throw new Error("Dictionary metadata schema_version is unsupported"); + const language = value("language"); + if (language !== "en" && language !== "zh") + throw new Error("Dictionary metadata language is unsupported"); + const sourceEdition = value("source_edition"); + const license = value("license"); + const common = { + version: value("version"), + sourceDumpDate: value("source_dump_date"), + sourceArchiveUrl: value("source_archive_url"), + url: value("asset_url"), + attributionUrl: value("attribution_url"), + licenseNotice: value("license_notice"), + creatorAttribution: value("creator_attribution"), + }; + if (language === "en" && sourceEdition === "wordnet-3.1" && license === "WordNet 3.1 License") { + return { ...common, schemaVersion: 1, language, sourceEdition, license }; + } + if (language === "en" && sourceEdition === "enwiktionary" && license === "CC BY-SA 4.0") { + return { ...common, schemaVersion: 1, language, sourceEdition, license }; + } + if (language === "zh" && sourceEdition === "zhwiktionary" && license === "CC BY-SA 4.0") { + return { ...common, schemaVersion: 1, language, sourceEdition, license }; + } + throw new Error("Dictionary metadata source/license combination is unsupported"); +} + +function sameArray(actual: readonly string[], expected: readonly string[]): boolean { + return ( + actual.length === expected.length && actual.every((value, index) => value === expected[index]) + ); +} diff --git a/packages/core/src/dictionary/manifest.ts b/packages/core/src/dictionary/manifest.ts index a3c1e2f85..4ab4b8c15 100644 --- a/packages/core/src/dictionary/manifest.ts +++ b/packages/core/src/dictionary/manifest.ts @@ -1,7 +1,7 @@ import { z } from "zod"; import type { DictionaryManifest } from "./types"; -const MAX_DICTIONARY_PACK_BYTES = 150 * 1024 * 1024; +export const MAX_DICTIONARY_PACK_BYTES = 150 * 1024 * 1024; const SEMVER = /^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:(?:0|[1-9]\d*)|\d*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:(?:0|[1-9]\d*)|\d*[A-Za-z-][0-9A-Za-z-]*))*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/u; diff --git a/packages/core/src/i18n/locales/en/reader.json b/packages/core/src/i18n/locales/en/reader.json index b1e5bb214..17506910e 100644 --- a/packages/core/src/i18n/locales/en/reader.json +++ b/packages/core/src/i18n/locales/en/reader.json @@ -95,6 +95,7 @@ "retry": "Retry", "retryLookup": "Retry dictionary lookup", "repair": "Repair", + "verifying": "Verifying…", "manageDictionaries": "Manage Dictionaries", "notDownloaded": "Not downloaded", "noDefinitionFound": "No definition found. Try selecting a single word.", @@ -104,7 +105,6 @@ "downloadingDefinition": "Downloading the {{language}} dictionary… {{progress}}%", "downloadAccessibility": "Download {{language}} dictionary", "downloadingAccessibility": "Downloading {{language}} dictionary", - "offlinePrivacy": "Definitions work offline and never use AI.", "downloading": "Downloading {{progress}}%", "installed": "Installed", "updateAvailable": "Update available", diff --git a/packages/core/src/i18n/locales/zh-TW/reader.json b/packages/core/src/i18n/locales/zh-TW/reader.json index 934d2f90a..cea88c460 100644 --- a/packages/core/src/i18n/locales/zh-TW/reader.json +++ b/packages/core/src/i18n/locales/zh-TW/reader.json @@ -91,6 +91,7 @@ "retry": "重試", "retryLookup": "重試字典查詢", "repair": "修復", + "verifying": "正在驗證…", "manageDictionaries": "管理字典", "notDownloaded": "尚未下載", "noDefinitionFound": "找不到釋義。請嘗試只選取一個單字。", @@ -100,7 +101,6 @@ "downloadingDefinition": "正在下載{{language}}字典… {{progress}}%", "downloadAccessibility": "下載{{language}}字典", "downloadingAccessibility": "正在下載{{language}}字典", - "offlinePrivacy": "釋義完全在離線狀態下運作，不會使用 AI。", "downloading": "正在下載 {{progress}}%", "installed": "已安裝", "updateAvailable": "有可用的更新", diff --git a/packages/core/src/i18n/locales/zh/reader.json b/packages/core/src/i18n/locales/zh/reader.json index 216a3f40a..c2fe4d67a 100644 --- a/packages/core/src/i18n/locales/zh/reader.json +++ b/packages/core/src/i18n/locales/zh/reader.json @@ -95,6 +95,7 @@ "retry": "重试", "retryLookup": "重试词典查询", "repair": "修复", + "verifying": "正在验证…", "manageDictionaries": "管理词典", "notDownloaded": "未下载", "noDefinitionFound": "没有找到释义。请尝试选择一个单词。", @@ -104,7 +105,6 @@ "downloadingDefinition": "正在下载{{language}}词典… {{progress}}%", "downloadAccessibility": "下载{{language}}词典", "downloadingAccessibility": "正在下载{{language}}词典", - "offlinePrivacy": "释义完全离线工作，不会使用 AI。", "downloading": "正在下载 {{progress}}%", "installed": "已安装", "updateAvailable": "有可用更新", diff --git a/packages/core/src/stores/app-store.ts b/packages/core/src/stores/app-store.ts index 333ca4ebc..17560708a 100644 --- a/packages/core/src/stores/app-store.ts +++ b/packages/core/src/stores/app-store.ts @@ -22,6 +22,7 @@ export type SidebarTab = "chat" | "notes" | "toc" | "highlights" | "stats"; export type SettingsTab = | "general" | "reading" + | "dictionaries" | "fonts" | "ai" | "vectorModel" diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index 8024fc855..6ad7fdcf8 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "ES2020", "useDefineForClassFields": true, - "lib": ["ES2020", "DOM", "DOM.Iterable"], + "lib": ["ES2021", "DOM", "DOM.Iterable"], "module": "ESNext", "skipLibCheck": true, "moduleResolution": "bundler", From 07273400ab14d4cca42d83fae54797741e9cb2f3 Mon Sep 17 00:00:00 2001 From: Decidetto <5384177+Decidetto@users.noreply.github.com> Date: Sun, 6 Sep 2026 13:59:03 +0200 Subject: [PATCH 3/3] test: declare existing desktop dictionary test tools in lockfile --- pnpm-lock.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2b5acbe71..406488efc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -217,6 +217,15 @@ importers: specifier: ^5.0.0 version: 5.0.11(@types/react@19.1.17)(react@19.1.0)(use-sync-external-store@1.6.0(react@19.1.0)) devDependencies: + '@types/react-test-renderer': + specifier: 19.1.0 + version: 19.1.0 + react-test-renderer: + specifier: 19.1.0 + version: 19.1.0(react@19.1.0) + vitest: + specifier: ^4.1.2 + version: 4.1.2(@types/node@25.5.2)(vite@7.3.1(@types/node@25.5.2)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.0)(tsx@4.22.4)(yaml@2.8.2)) '@tailwindcss/vite': specifier: ^4.0.0 version: 4.2.1(vite@7.3.1(@types/node@25.5.2)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.0)(tsx@4.22.4)(yaml@2.8.2))