diff --git a/dictionary-packs/LICENSE-WIKTIONARY.txt b/dictionary-packs/LICENSE-WIKTIONARY.txt new file mode 100644 index 000000000..025091d4c --- /dev/null +++ b/dictionary-packs/LICENSE-WIKTIONARY.txt @@ -0,0 +1,22 @@ +Chinese Wiktionary pack attribution + +The ReadAny Chinese offline dictionary pack contains transformed material from +the Chinese-language Wiktionary dump dated 2026-09-01: +https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2 + +Human-readable attribution page: +https://zh.wiktionary.org/wiki/Wiktionary:%E7%89%88%E6%9D%83%E4%BF%A1%E6%81%AF + +Wiktionary contributors license this material under Creative Commons +Attribution-ShareAlike 4.0 International (CC BY-SA 4.0): +https://creativecommons.org/licenses/by-sa/4.0/ + +Attribution: Wiktionary contributors. + +ReadAny used Wiktextract and its deterministic pack builder to retain headwords, +parts of speech, Chinese pronunciation when present, Simplified/Traditional +forms, and non-empty definitions. It removed examples, translations, audio, +images, and other fields not represented by the ReadAny dictionary schema. + +ReadAny's transformed SQLite pack is distributed under CC BY-SA 4.0. Neither +the Wikimedia Foundation nor Wiktionary contributors endorse ReadAny. diff --git a/dictionary-packs/LICENSE-WORDNET-3.1.txt b/dictionary-packs/LICENSE-WORDNET-3.1.txt new file mode 100644 index 000000000..38b04c622 --- /dev/null +++ b/dictionary-packs/LICENSE-WORDNET-3.1.txt @@ -0,0 +1,44 @@ +WordNet 3.1 license + +This software and database is being provided to you, the LICENSEE, by +Princeton University under the following license. By obtaining, using +and/or copying this software and database, you agree that you have +read, understood, and will comply with these terms and conditions.: + +Permission to use, copy, modify and distribute this software and +database and its documentation for any purpose and without fee or +royalty is hereby granted, provided that you agree to comply with +the following copyright notice and statements, including the disclaimer, +and that the same appear on ALL copies of the software, database and +documentation, including modifications that you make for internal +use or for distribution. + +WordNet 3.1 Copyright 2011 by Princeton University. All rights reserved. + +THIS SOFTWARE AND DATABASE IS PROVIDED "AS IS" AND PRINCETON +UNIVERSITY MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PRINCETON +UNIVERSITY MAKES NO REPRESENTATIONS OR WARRANTIES OF MERCHANT- +ABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE +OF THE LICENSED SOFTWARE, DATABASE OR DOCUMENTATION WILL NOT +INFRINGE ANY THIRD PARTY PATENTS, COPYRIGHTS, TRADEMARKS OR +OTHER RIGHTS. + +The name of Princeton University or Princeton may not be used in +advertising or publicity pertaining to distribution of the software +and/or database. Title to copyright in this software, database and +any associated documentation shall at all times remain with +Princeton University and LICENSEE agrees to preserve same. + +The license text above is reproduced from the header embedded in the official +WordNet 3.1 database files distributed at: +https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz + +Human-readable WordNet license page: +https://wordnet.princeton.edu/license-and-commercial-use + +Creator attribution: WordNet 3.1 Copyright 2011 by Princeton University. All +rights reserved. + +ReadAny transforms WordNet data into a SQLite lookup pack. Princeton University +does not endorse ReadAny or this transformation. diff --git a/dictionary-packs/README.md b/dictionary-packs/README.md new file mode 100644 index 000000000..e92d937f2 --- /dev/null +++ b/dictionary-packs/README.md @@ -0,0 +1,135 @@ +# ReadAny offline dictionary packs v1 + +These are optional, one-time downloads used for on-device definition lookup. +They are not bundled into the app and lookup never falls back to AI or a web +definition service. + +## Exact user-download sizes + +| Language | Coverage/source | Entries | Bytes | MiB | SHA-256 | +| --- | --- | ---: | ---: | ---: | --- | +| English | WordNet 3.1 | 154,404 | 29,069,312 | 27.722656 | `90adbeab5ee325b31f2e34bbfa7c5b699932c900bfd1efdfc62eabd9e004ee0e` | +| Chinese | Chinese Wiktionary 2026-09-01 | 77,256 | 11,866,112 | 11.316406 | `d50218459f78a5e7bcba819fbb7db699271bdaf30757dc8b2443cb1e37bc11b1` | + +Downloading both packs is 40,935,424 bytes (39.039062 MiB). Each real SQLite +pack is below the 150 MiB hard stop used for this release. The release URLs in +`manifest.json` point to the published `dictionary-packs-v1` assets. The initial +packs are hosted by contributor cha1latte; they are data downloads, not app updates. +The app reads manifest updates from the official repository after this feature +is merged and falls back to its bundled manifest when offline. Maintainers can +move assets by updating the manifest URLs while preserving the verified hashes. +An alternate manifest can be selected at build time with +`EXPO_PUBLIC_DICTIONARY_MANIFEST_URL` (mobile) or `VITE_DICTIONARY_MANIFEST_URL` +(desktop). The published packs are available in the [contributor release](https://github.com/cha1latte/ReadAny/releases/tag/dictionary-packs-v1) for maintainers to migrate. + +WordNet gives strong ordinary English vocabulary in a compact download, but has +less slang, newer language, proper-name coverage, and obscure material than the +full English Wiktionary source that was rejected for this initial release after +its pinned extraction projected a multi-day run. + +## Pinned sources and provenance + +### English + +- Official source: `https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz` +- Archive bytes: 16,358,468 +- Archive SHA-256: `3f7d8be8ef6ecc7167d39b10d66954ec734280b5bdcd57f7d9eafe429d11c22a` +- Source edition: WordNet 3.1 +- Source date: 2011-05-26, recorded from the archive's top-level directory and + exception-file timestamps; WordNet 3.1 remains the authoritative edition identity +- License: WordNet 3.1 License; see `LICENSE-WORDNET-3.1.txt` +- Converted JSONL: 154,404 records, 206,143 definitions, 66,893 aliases +- Converted JSONL bytes: 30,477,132 +- Converted JSONL SHA-256: `65f632bef5671e85f649f510eef13d90d752d11dcedc252de65f48a6d2f0d359` + +### Chinese + +- Official source: `https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2` +- Dump bytes: 284,272,376 +- Dump SHA-256: `516d870bb8ddaa461e013cfb0d477229501c9b8f97fd56c8a7098f42e3586a22` +- Source edition/date: Chinese Wiktionary, 2026-09-01 +- License: CC BY-SA 4.0; see `LICENSE-WIKTIONARY.txt` +- Extractor: Wiktextract commit `1939b1f8b1ae5d6989b8cbaea91c639b1b5dcbef` +- Extracted JSONL: 245,518 language-matching data records, of which 80,623 + contain at least one non-empty gloss +- Accepted SQLite entries: 77,256; senses: 105,182; lookup rows: 129,666 +- Wiktextract hard redirects and same-language soft redirects contribute 10,113 + rank-1 lookup rows across 7,801 aliases that were absent from the prior pack. + Redirect chains resolve only to existing canonical entries; cycles, missing + targets, and unsupported-language aliases do not create lookup rows. + +## Deterministic transformation + +The WordNet converter reads `data.noun`, `data.verb`, `data.adj`, and `data.adv` +using the complete official database-row grammar: fixed-width offsets and +counts, hexadecimal lexical IDs, every pointer record, verb frame counts and +records, and exact pre-gloss token consumption. It converts +underscore-separated lemmas to display spaces, removes adjective syntactic +markers, groups definitions by lemma and part of speech, and omits quoted usage +examples. It attaches aliases from the four WordNet exception files. For +single-token nouns without an official noun exception, it uses conservative +plural rules: consonant-`y` to `-ies`, sibilants to `-es`, and otherwise `-s`. +For verbs it adds only third-person singular: `do`/`go` and sibilants use +`-es`, consonant-`y` uses `-ies`, and other verbs use `-s`; the official +exceptions supply irregular `be` and `have` forms. Thus `goes`, `does`, `has`, +`is`, and `tattoos` are present while `tattooes` is absent. It does not guess +past tense, gerunds, participles, or comparative/superlative forms. + +The shared pack builder retains: + +- language, canonical headword, and part of speech; +- every non-empty definition; +- WordNet exception and conservative regular aliases for English; +- the first available `zh-pron` and Simplified/Traditional aliases for Chinese; +- Chinese Wiktextract hard/soft redirects, including chains whose final target + is an existing same-language canonical entry; +- canonical lookup rank 0 and alias rank 1; and +- source edition/date, distinct source-archive, asset, and human-readable + attribution URLs, transformation identity, creator attribution, and the + complete source-specific license/notice text in SQLite metadata. + +The v1 schema has no examples, translations, audio, images, or AI-generated +data. WordNet semantic relations and verb frames are also not represented. + +## Reproduction commands + +```powershell +pnpm --filter @readany/cli dictionary:convert-wordnet -- --input-directory ./dictionary-source/wordnet-3.1\dict --output ./dictionary-source/en-wordnet-3.1.jsonl + +pnpm --filter @readany/cli dictionary:build -- --language en --input ./dictionary-source/en-wordnet-3.1.jsonl --output ./dictionary-source/readany-dictionary-en-v1.sqlite --version 1.0.0 --source-edition wordnet-3.1 --license "WordNet 3.1 License" --source-date 2011-05-26 --source-archive-url https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz --attribution-url https://wordnet.princeton.edu/license-and-commercial-use --license-file ./dictionary-packs/LICENSE-WORDNET-3.1.txt --creator-attribution "WordNet 3.1 Copyright 2011 by Princeton University. All rights reserved." --asset-url https://github.com/cha1latte/ReadAny/releases/download/dictionary-packs-v1/readany-dictionary-en-v1.sqlite --descriptor ./dictionary-source/en-descriptor.json + +pnpm --filter @readany/cli dictionary:build -- --language zh --input ./dictionary-source/zh-20260901.jsonl --output ./dictionary-source/readany-dictionary-zh-v1.sqlite --version 1.0.0 --source-edition zhwiktionary --license "CC BY-SA 4.0" --source-date 2026-09-01 --source-archive-url https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2 --attribution-url https://zh.wiktionary.org/wiki/Wiktionary:%E7%89%88%E6%9D%83%E4%BF%A1%E6%81%AF --license-file ./dictionary-packs/LICENSE-WIKTIONARY.txt --creator-attribution "Wiktionary contributors." --asset-url https://github.com/cha1latte/ReadAny/releases/download/dictionary-packs-v1/readany-dictionary-zh-v1.sqlite --descriptor ./dictionary-source/zh-descriptor.json +``` + +Before publication, each descriptor must exactly match an independent local +file-size and SHA-256 check. `manifest.json` and the app's bundled manifest must +remain byte-identical and must pass the shared strict manifest parser. + +## Desktop and mobile integration + +Both apps use the same bundled manifest in +`packages/core/src/dictionary/dictionary-manifest.json`, selection normalization, +lookup SQL, validation, pack lifecycle, and store logic. Mobile uses Expo SQLite +and filesystem adapters. Desktop stores packs under the Tauri app-data directory's +`dictionaries` folder and runs read-only SQLite queries in a blocking native task. +Each native query closes its file handle, and the desktop adapter waits for pending +queries before pack replacement or removal. + +On desktop, select an English or Chinese word and choose **Define**. A missing +pack prompts for an explicit download. **Settings > Dictionaries** supports +downloading, updating, repairing, and removing packs. Installed lookups work offline. + +Desktop checks: `pnpm --filter app test:dictionary`, `pnpm --filter app exec tsc --noEmit`, +and `cargo test --locked dictionary::tests` from `packages/app/src-tauri`. + +Desktop pack downloads stream directly from the existing native HTTP client into +a buffered file writer, with progress events limited to ten per second. The UI +shows a separate verification phase before activating a pack. Both apps refresh +the catalog and installed-pack status when the Dictionaries settings page opens. + +A local Windows debug-preview comparison on 2026-09-06 measured the same English +pack (29,069,312 bytes) at 13.67 seconds through the original JavaScript chunk loop +and 1.58 seconds through native streaming. One full checksum/schema verification +pass took 0.59 and 0.61 seconds respectively. Progress callbacks fell from 1,776 to +13. Both files matched the published SHA-256. These are single-run measurements +on the same machine and connection, not a guaranteed download time. diff --git a/dictionary-packs/manifest.json b/dictionary-packs/manifest.json new file mode 100644 index 000000000..08150a75b --- /dev/null +++ b/dictionary-packs/manifest.json @@ -0,0 +1,31 @@ +{ + "manifestVersion": 1, + "packs": { + "en": { + "language": "en", + "version": "1.0.0", + "schemaVersion": 1, + "sourceEdition": "wordnet-3.1", + "sourceDumpDate": "2011-05-26", + "sizeBytes": 29069312, + "sha256": "90adbeab5ee325b31f2e34bbfa7c5b699932c900bfd1efdfc62eabd9e004ee0e", + "url": "https://github.com/cha1latte/ReadAny/releases/download/dictionary-packs-v1/readany-dictionary-en-v1.sqlite", + "sourceArchiveUrl": "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + "attributionUrl": "https://wordnet.princeton.edu/license-and-commercial-use", + "license": "WordNet 3.1 License" + }, + "zh": { + "language": "zh", + "version": "1.0.0", + "schemaVersion": 1, + "sourceEdition": "zhwiktionary", + "sourceDumpDate": "2026-09-01", + "sizeBytes": 11866112, + "sha256": "d50218459f78a5e7bcba819fbb7db699271bdaf30757dc8b2443cb1e37bc11b1", + "url": "https://github.com/cha1latte/ReadAny/releases/download/dictionary-packs-v1/readany-dictionary-zh-v1.sqlite", + "sourceArchiveUrl": "https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2", + "attributionUrl": "https://zh.wiktionary.org/wiki/Wiktionary:%E7%89%88%E6%9D%83%E4%BF%A1%E6%81%AF", + "license": "CC BY-SA 4.0" + } + } +} diff --git a/packages/app-expo/package.json b/packages/app-expo/package.json index 8117f1158..abacc3906 100644 --- a/packages/app-expo/package.json +++ b/packages/app-expo/package.json @@ -115,11 +115,13 @@ "@babel/core": "^7.25.0", "@types/pako": "^2.0.4", "@types/react": "^19.1.17", + "@types/react-test-renderer": "19.1.0", "babel-plugin-module-resolver": "^5.0.2", "babel-plugin-transform-import-meta": "^2.3.2", "eas-cli": "^18.11.0", "esbuild": "^0.27.3", "react-native-svg-transformer": "^1.5.3", + "react-test-renderer": "19.1.0", "typescript": "^5.9.3", "vitest": "^4.1.2" } diff --git a/packages/app-expo/src/components/reader/DefinitionSheet.test.tsx b/packages/app-expo/src/components/reader/DefinitionSheet.test.tsx new file mode 100644 index 000000000..5fa7fb416 --- /dev/null +++ b/packages/app-expo/src/components/reader/DefinitionSheet.test.tsx @@ -0,0 +1,350 @@ +import type { DictionaryEntry, DictionaryPackDescriptor } from "@readany/core/dictionary"; +import i18n, { i18nReady } from "@readany/core/i18n"; +import React from "react"; +import TestRenderer, { act } from "react-test-renderer"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { DefinitionSheet } from "./DefinitionSheet"; +import { DefinitionController, type DefinitionState } from "./definition-controller"; + +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = + true; +(globalThis as typeof globalThis & { React: typeof React }).React = React; + +vi.mock("react-native", async () => { + const ReactModule = await import("react"); + const host = (name: string) => + function HostComponent(props: Record) { + return ReactModule.createElement(name, props, props.children as React.ReactNode); + }; + return { + ActivityIndicator: host("ActivityIndicator"), + Modal: host("Modal"), + Pressable: host("Pressable"), + ScrollView: host("ScrollView"), + StyleSheet: { create: (styles: unknown) => styles }, + Text: host("Text"), + TouchableOpacity: host("TouchableOpacity"), + View: host("View"), + }; +}); + +vi.mock("@/components/ui/Icon", () => ({ + XIcon: () => null, +})); + +vi.mock("@/stores", () => ({ + useDictionaryStore: Object.assign(() => undefined, { + getState: () => ({ + install: async () => {}, + manifest: null, + }), + }), +})); + +vi.mock("@/styles/theme", () => ({ + fontSize: { base: 16, lg: 18, sm: 14, xs: 12 }, + fontWeight: { medium: "500", semibold: "600" }, + radius: { full: 999, lg: 8, xl: 12 }, + useColors: () => ({ + background: "background", + border: "border", + card: "card", + destructive: "destructive", + foreground: "foreground", + muted: "muted", + mutedForeground: "mutedForeground", + primary: "primary", + primaryForeground: "primaryForeground", + }), +})); + +vi.mock("react-native-safe-area-context", () => ({ + useSafeAreaInsets: () => ({ bottom: 24 }), +})); + +const descriptor: DictionaryPackDescriptor = { + language: "en", + version: "1.0.0", + schemaVersion: 1, + sourceEdition: "enwiktionary", + sourceDumpDate: "2026-09-01", + sizeBytes: 1_572_864, + sha256: "a".repeat(64), + url: "https://example.test/readany-dictionary-en.sqlite", + sourceArchiveUrl: "https://example.test/en-source.xml.bz2", + attributionUrl: "https://en.wiktionary.org", + license: "CC BY-SA 4.0", +}; + +const englishEntry: DictionaryEntry = { + id: 1, + language: "en", + headword: "desire", + pronunciation: "/dɪˈzaɪəɹ/", + partOfSpeech: "noun", + senses: [ + { order: 0, definition: "A strong wish." }, + { order: 1, definition: "To want strongly." }, + ], +}; + +const chineseEntry: DictionaryEntry = { + id: 2, + language: "zh", + headword: "閱讀", + simplified: "阅读", + traditional: "閱讀", + pronunciation: "yuèdú", + partOfSpeech: "动词", + senses: [{ order: 0, definition: "看书或文章。" }], +}; + +function createController(state: DefinitionState, lookup = vi.fn(async () => [])) { + const controller = new DefinitionController({ + lookup, + install: vi.fn(async () => {}), + getDescriptor: () => descriptor, + }); + controller.state = state; + vi.spyOn(controller, "open").mockResolvedValue(); + return { controller, lookup }; +} + +function isHostType(node: TestRenderer.ReactTestInstance, type: string): boolean { + return node.type === type; +} + +function textContent(renderer: TestRenderer.ReactTestRenderer): string { + return renderer.root + .findAll((node) => isHostType(node, "Text")) + .flatMap((node) => node.children) + .join(""); +} + +function pressButton(renderer: TestRenderer.ReactTestRenderer, label: string): void { + const button = renderer.root + .findAll((node) => isHostType(node, "TouchableOpacity")) + .find((node) => node.findAll((child) => child.children.includes(label)).length > 0); + if (!button) throw new Error(`Button ${label} was not found`); + button.props.onPress(); +} + +describe("DefinitionSheet", () => { + beforeEach(async () => { + await i18nReady; + await act(async () => { + await i18n.changeLanguage("en"); + }); + }); + + it("consumes Traditional Chinese sheet, download-template, and accessibility translations", async () => { + await act(async () => { + await i18n.changeLanguage("zh-TW"); + }); + const chineseDescriptor: DictionaryPackDescriptor = { + ...descriptor, + language: "zh", + sourceEdition: "zhwiktionary", + }; + const { controller } = createController({ + kind: "missing-pack", + language: "zh", + descriptor: chineseDescriptor, + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + expect(textContent(renderer)).toContain("釋義"); + expect(textContent(renderer)).toContain("下載中文字典（1.5 MB）即可離線查詢釋義。"); + expect(renderer.root.findAllByProps({ accessibilityLabel: "關閉釋義" }).length).toBeGreaterThan( + 0, + ); + expect(renderer.root.findByProps({ accessibilityLabel: "下載中文字典" })).toBeTruthy(); + }); + + it("renders English headword, IPA, part of speech, and numbered senses", async () => { + const { controller } = createController({ + kind: "result", + displayText: "desire", + entries: [englishEntry], + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + expect(textContent(renderer)).toContain("desire"); + expect(textContent(renderer)).toContain("/dɪˈzaɪəɹ/"); + expect(textContent(renderer)).toContain("noun"); + expect(textContent(renderer)).toContain("1. A strong wish."); + expect(textContent(renderer)).toContain("2. To want strongly."); + }); + + it("renders Chinese simplified and traditional forms, pinyin, and Chinese senses", async () => { + const { controller } = createController({ + kind: "result", + displayText: "閱讀", + entries: [chineseEntry], + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + expect(textContent(renderer)).toContain("阅读"); + expect(textContent(renderer)).toContain("閱讀"); + expect(textContent(renderer)).toContain("yuèdú"); + expect(textContent(renderer)).toContain("动词"); + expect(textContent(renderer)).toContain("1. 看书或文章。"); + }); + + it("renders the missing pack size and a Download action", async () => { + const { controller } = createController({ kind: "missing-pack", language: "en", descriptor }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + expect(textContent(renderer)).toContain("1.5 MB"); + expect(textContent(renderer)).toContain("Download"); + }); + + it("disables Download and renders progress while the pack is downloading", async () => { + const { controller } = createController({ + kind: "downloading", + language: "en", + progress: 0.37, + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + const download = renderer.root + .findAll((node) => isHostType(node, "TouchableOpacity")) + .find((node) => node.findAll((child) => child.children.includes("Download")).length > 0); + expect(download?.props.disabled).toBe(true); + expect(textContent(renderer)).toContain("37%"); + }); + + it("renders the exact no-match guidance", async () => { + const { controller } = createController({ kind: "no-match", displayText: "unknown" }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + expect(textContent(renderer)).toContain("No definition found. Try selecting a single word."); + }); + + it("renders Retry and Manage Dictionaries for a recoverable error", async () => { + const calls: string[] = []; + const onClose = vi.fn(() => calls.push("close")); + const onManageDictionaries = vi.fn(() => calls.push("manage")); + const { controller } = createController({ kind: "error", message: "database unavailable" }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + await act(async () => { + pressButton(renderer, "Retry"); + await Promise.resolve(); + pressButton(renderer, "Manage Dictionaries"); + }); + + expect(textContent(renderer)).toContain("Dictionary lookup failed. Try again."); + expect(calls).toEqual(["close", "manage"]); + expect(onClose).toHaveBeenCalledTimes(1); + expect(onManageDictionaries).toHaveBeenCalledTimes(1); + }); + + it("closes the modal without launching another lookup", async () => { + const onClose = vi.fn(); + const { controller, lookup } = createController({ kind: "no-match", displayText: "desire" }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + const lookupCountBeforeClose = lookup.mock.calls.length; + await act(async () => { + renderer.root.findAll((node) => isHostType(node, "Modal"))[0]?.props.onRequestClose(); + }); + + expect(onClose).toHaveBeenCalledTimes(1); + expect(lookup).toHaveBeenCalledTimes(lookupCountBeforeClose); + }); +}); diff --git a/packages/app-expo/src/components/reader/DefinitionSheet.tsx b/packages/app-expo/src/components/reader/DefinitionSheet.tsx new file mode 100644 index 000000000..21fb02b5d --- /dev/null +++ b/packages/app-expo/src/components/reader/DefinitionSheet.tsx @@ -0,0 +1,411 @@ +import { XIcon } from "@/components/ui/Icon"; +import { useDictionaryStore } from "@/stores"; +import { type ThemeColors, fontSize, fontWeight, radius, useColors } from "@/styles/theme"; +import type { DictionaryEntry } from "@readany/core/dictionary"; +import { useEffect, useRef, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { + ActivityIndicator, + Modal, + Pressable, + ScrollView, + StyleSheet, + Text, + TouchableOpacity, + View, +} from "react-native"; +import { useSafeAreaInsets } from "react-native-safe-area-context"; +import { + DefinitionController, + type DefinitionControllerDependencies, + type DefinitionState, +} from "./definition-controller"; + +interface DefinitionSheetProps { + visible: boolean; + text: string; + onClose: () => void; + onManageDictionaries: () => void; + /** Injected by tests or an application-level dictionary composition. */ + controller?: DefinitionController; +} + +export function DefinitionSheet({ + visible, + text, + onClose, + onManageDictionaries, + controller: suppliedController, +}: DefinitionSheetProps) { + const { t } = useTranslation(); + const colors = useColors(); + const insets = useSafeAreaInsets(); + const s = makeStyles(colors); + const fallbackController = useRef(null); + if (!suppliedController && !fallbackController.current) { + fallbackController.current = new DefinitionController(defaultDependencies()); + } + const controller = suppliedController ?? fallbackController.current; + if (!controller) throw new Error("Definition controller was not initialized"); + const [state, setState] = useState(controller.state); + + useEffect(() => controller.subscribe(setState), [controller]); + + useEffect(() => { + if (visible) { + void controller.open(text); + } else { + controller.close(); + } + }, [controller, text, visible]); + + const close = () => onClose(); + const manageDictionaries = () => { + onClose(); + onManageDictionaries(); + }; + + return ( + + + + + + {t("dictionary.title")} + + + + + + {renderState(state, controller, manageDictionaries, s, colors.primary, t)} + + + + ); +} + +function renderState( + state: DefinitionState, + controller: DefinitionController, + onManageDictionaries: () => void, + s: ReturnType, + primaryColor: string, + t: ReturnType["t"], +) { + switch (state.kind) { + case "idle": + return null; + case "loading": + return ( + + + {t("dictionary.loadingDefinition")} + + ); + case "verifying": + return {t("dictionary.verifying")}; + case "unsupported": + return {t("dictionary.unsupportedSelection")}; + case "missing-pack": + return ; + case "downloading": + return ; + case "no-match": + return {t("dictionary.noDefinitionFound")}; + case "error": + return ( + + {t("dictionary.lookupError")} + + void controller.retry()} + accessibilityRole="button" + accessibilityLabel={t("dictionary.retryLookup")} + > + {t("dictionary.retry")} + + + {t("dictionary.manageDictionaries")} + + + + ); + case "result": + return ( + + {state.displayText} + {state.entries.map((entry) => ( + + ))} + + ); + } +} + +function EntryView({ entry, s }: { entry: DictionaryEntry; s: ReturnType }) { + const forms = + entry.language === "zh" + ? [entry.simplified ?? entry.headword, entry.traditional] + : [entry.headword]; + const uniqueForms = [...new Set(forms.filter((form): form is string => Boolean(form)))]; + + return ( + + + {uniqueForms.map((form) => ( + + {form} + + ))} + {entry.pronunciation ? {entry.pronunciation} : null} + + {entry.partOfSpeech ? {entry.partOfSpeech} : null} + + {entry.senses.map((sense, index) => ( + + {index + 1}. {sense.definition} + + ))} + + + ); +} + +function PackDownload({ + state, + controller, + s, + t, +}: { + state: + | Extract + | Extract; + controller: DefinitionController; + s: ReturnType; + t: ReturnType["t"]; +}) { + const downloading = state.kind === "downloading"; + const descriptor = state.kind === "missing-pack" ? state.descriptor : undefined; + const language = t(`dictionary.${state.language === "en" ? "english" : "chinese"}`); + const progress = downloading ? Math.round(state.progress * 100) : null; + return ( + + + {downloading + ? t("dictionary.downloadingDefinition", { language, progress }) + : t("dictionary.downloadDefinition", { + language, + size: formatBytes(descriptor?.sizeBytes ?? 0), + })} + + void controller.download()} + accessibilityRole="button" + accessibilityLabel={ + downloading + ? t("dictionary.downloadingAccessibility", { language }) + : t("dictionary.downloadAccessibility", { language }) + } + > + {t("dictionary.download")} + + + ); +} + +function defaultDependencies(): DefinitionControllerDependencies { + return { + lookup: (text) => useDictionaryStore.getState().lookup(text), + install: async (descriptor, onProgress, onVerifying) => { + const unsubscribe = useDictionaryStore.subscribe((state) => { + const status = state.packs[descriptor.language]; + if (status.state === "downloading") onProgress(status.progress); + if (status.state === "verifying") onVerifying?.(); + }); + onProgress(0); + try { + await useDictionaryStore.getState().install(descriptor.language); + } finally { + unsubscribe(); + } + }, + getDescriptor: (language) => useDictionaryStore.getState().manifest?.packs[language], + }; +} + +function formatBytes(bytes: number): string { + if (bytes < 1024 * 1024) return `${Math.max(0, Math.round(bytes / 1024))} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +const makeStyles = (colors: ThemeColors) => + StyleSheet.create({ + backdrop: { + flex: 1, + backgroundColor: "rgba(0,0,0,0.3)", + }, + container: { + position: "absolute", + bottom: 0, + left: 0, + right: 0, + maxHeight: "60%", + backgroundColor: colors.background, + borderTopLeftRadius: radius.xl, + borderTopRightRadius: radius.xl, + }, + handle: { + width: 40, + height: 4, + marginTop: 8, + alignSelf: "center", + borderRadius: 2, + backgroundColor: colors.muted, + }, + header: { + flexDirection: "row", + alignItems: "center", + justifyContent: "space-between", + paddingHorizontal: 16, + paddingVertical: 12, + borderBottomWidth: 1, + borderBottomColor: colors.border, + }, + title: { + fontSize: fontSize.base, + fontWeight: fontWeight.semibold, + color: colors.foreground, + }, + closeButton: { + width: 32, + height: 32, + alignItems: "center", + justifyContent: "center", + borderRadius: radius.full, + }, + content: { + padding: 16, + }, + loadingWrap: { + flexDirection: "row", + alignItems: "center", + gap: 8, + paddingVertical: 16, + }, + statusText: { + fontSize: fontSize.sm, + lineHeight: 20, + color: colors.mutedForeground, + }, + downloadWrap: { + gap: 12, + paddingVertical: 16, + }, + errorWrap: { + gap: 12, + paddingVertical: 16, + }, + errorText: { + fontSize: fontSize.sm, + lineHeight: 20, + color: colors.destructive, + }, + actions: { + flexDirection: "row", + flexWrap: "wrap", + gap: 8, + }, + primaryButton: { + alignSelf: "flex-start", + paddingHorizontal: 12, + paddingVertical: 8, + borderRadius: radius.lg, + backgroundColor: colors.primary, + }, + primaryButtonText: { + fontSize: fontSize.sm, + fontWeight: fontWeight.medium, + color: colors.primaryForeground, + }, + secondaryButton: { + alignSelf: "flex-start", + paddingHorizontal: 12, + paddingVertical: 8, + borderWidth: 1, + borderColor: colors.border, + borderRadius: radius.lg, + backgroundColor: colors.card, + }, + secondaryButtonText: { + fontSize: fontSize.sm, + fontWeight: fontWeight.medium, + color: colors.foreground, + }, + buttonDisabled: { + opacity: 0.55, + }, + results: { + gap: 12, + }, + selectedText: { + fontSize: fontSize.xs, + color: colors.mutedForeground, + }, + entry: { + gap: 8, + paddingBottom: 16, + borderBottomWidth: 1, + borderBottomColor: colors.border, + }, + entryHeading: { + flexDirection: "row", + flexWrap: "wrap", + alignItems: "baseline", + gap: 8, + }, + headword: { + fontSize: fontSize.lg, + fontWeight: fontWeight.semibold, + color: colors.foreground, + }, + pronunciation: { + fontSize: fontSize.sm, + color: colors.mutedForeground, + }, + partOfSpeech: { + alignSelf: "flex-start", + paddingHorizontal: 8, + paddingVertical: 3, + borderRadius: radius.lg, + overflow: "hidden", + fontSize: fontSize.xs, + color: colors.primary, + backgroundColor: colors.muted, + }, + senses: { + gap: 6, + }, + sense: { + fontSize: fontSize.base, + lineHeight: 23, + color: colors.foreground, + }, + }); diff --git a/packages/app-expo/src/components/reader/SelectionPopover.test.tsx b/packages/app-expo/src/components/reader/SelectionPopover.test.tsx new file mode 100644 index 000000000..83efe8e26 --- /dev/null +++ b/packages/app-expo/src/components/reader/SelectionPopover.test.tsx @@ -0,0 +1,121 @@ +import i18n, { i18nReady } from "@readany/core/i18n"; +import React from "react"; +import TestRenderer, { act } from "react-test-renderer"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { SelectionPopover } from "./SelectionPopover"; + +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = + true; +(globalThis as typeof globalThis & { React: typeof React }).React = React; + +vi.mock("expo-clipboard", () => ({ setStringAsync: vi.fn() })); + +vi.mock("@/components/ui/Icon", () => ({ + BookOpenIcon: () => null, + CopyIcon: () => null, + HighlighterIcon: () => null, + LanguagesIcon: () => null, + NotebookPenIcon: () => null, + SparklesIcon: () => null, + Trash2Icon: () => null, + Volume2Icon: () => null, + XIcon: () => null, +})); + +vi.mock("@/components/ui/RichTextEditor", () => ({ RichTextEditor: () => null })); + +vi.mock("@/styles/theme", () => ({ + fontSize: { base: 16, lg: 18, sm: 14 }, + fontWeight: { medium: "500", semibold: "600" }, + radius: { lg: 8, xl: 12, xxl: 16 }, + spacing: { sm: 8, md: 12, lg: 16 }, + useColors: () => ({ + border: "border", + card: "card", + destructive: "destructive", + foreground: "foreground", + muted: "muted", + mutedForeground: "mutedForeground", + primary: "primary", + primaryForeground: "primaryForeground", + }), + withOpacity: () => "transparent", +})); + +vi.mock("@readany/core/types", () => ({ + HIGHLIGHT_COLORS: ["yellow"], + HIGHLIGHT_COLOR_HEX: { yellow: "#ffff00" }, +})); + +vi.mock("react-native", async () => { + const ReactModule = await import("react"); + const host = (name: string) => + function HostComponent(props: Record) { + return ReactModule.createElement(name, props, props.children as React.ReactNode); + }; + return { + KeyboardAvoidingView: host("KeyboardAvoidingView"), + Modal: host("Modal"), + Platform: { OS: "android" }, + StyleSheet: { absoluteFill: {}, absoluteFillObject: {}, create: (styles: unknown) => styles }, + Text: host("Text"), + TouchableOpacity: host("TouchableOpacity"), + View: host("View"), + useWindowDimensions: () => ({ height: 800, width: 400 }), + }; +}); + +const selection = { + cfi: "epubcfi(/6/2)", + text: "selected dictionary word", + position: { x: 200, y: 100, selectionTop: 90, selectionBottom: 110 }, +}; + +describe("SelectionPopover", () => { + beforeEach(async () => { + await i18nReady; + await act(async () => { + await i18n.changeLanguage("zh-TW"); + }); + }); + it("passes the selected text to Define and dismisses the selection", async () => { + const onDefine = vi.fn(); + const onDismiss = vi.fn(); + let renderer!: TestRenderer.ReactTestRenderer; + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + const define = renderer.root.findByProps({ accessibilityLabel: "查詞" }); + act(() => define.props.onPress()); + + expect(onDefine).toHaveBeenCalledWith(selection.text); + expect(onDismiss).toHaveBeenCalledTimes(1); + }); + + it("does not render Define when no definition handler is supplied", async () => { + let renderer!: TestRenderer.ReactTestRenderer; + await act(async () => { + renderer = TestRenderer.create( + , + ); + }); + + expect(renderer.root.findAllByProps({ accessibilityLabel: "查詞" })).toHaveLength(0); + }); +}); diff --git a/packages/app-expo/src/components/reader/SelectionPopover.tsx b/packages/app-expo/src/components/reader/SelectionPopover.tsx index daa748117..6fc5143ab 100644 --- a/packages/app-expo/src/components/reader/SelectionPopover.tsx +++ b/packages/app-expo/src/components/reader/SelectionPopover.tsx @@ -1,4 +1,5 @@ import { + BookOpenIcon, CopyIcon, HighlighterIcon, LanguagesIcon, @@ -55,6 +56,7 @@ interface Props { onAIChat: () => void; onSpeak?: (text: string, cfi: string) => void; onNote?: (text: string, cfi: string) => void; + onDefine?: (text: string) => void; onTranslate?: (text: string) => void; onRemoveHighlight?: () => void; existingHighlight?: { id: string; color: HighlightColor; note?: string } | null; @@ -69,6 +71,7 @@ export function SelectionPopover({ onAIChat, onSpeak, onNote, + onDefine, onTranslate, onRemoveHighlight, existingHighlight, @@ -99,10 +102,7 @@ export function SelectionPopover({ }, [selection.cfi, hasExistingHighlight]); const buttonCount = - 4 + - (onNote ? 1 : 0) + - (onTranslate ? 1 : 0) + - (onSpeak ? 1 : 0); + 4 + (onNote ? 1 : 0) + (onDefine ? 1 : 0) + (onTranslate ? 1 : 0) + (onSpeak ? 1 : 0); const colorRowItemCount = HIGHLIGHT_COLORS.length + (canRemoveHighlight ? 2 : 0); const colorRowWidth = showColors ? HIGHLIGHT_COLORS.length * COLOR_DOT_SIZE + @@ -183,6 +183,13 @@ export function SelectionPopover({ onDismiss(); }, [selection.text, onTranslate, onDismiss]); + const handleDefine = useCallback(() => { + if (onDefine) { + onDefine(selection.text); + } + onDismiss(); + }, [selection.text, onDefine, onDismiss]); + const handleRemove = useCallback(() => { if (onRemoveHighlight) { onRemoveHighlight(); @@ -255,6 +262,17 @@ export function SelectionPopover({ + {onDefine && ( + + + + )} + {onTranslate && ( @@ -270,7 +288,6 @@ export function SelectionPopover({ )} - diff --git a/packages/app-expo/src/components/reader/definition-controller.test.ts b/packages/app-expo/src/components/reader/definition-controller.test.ts new file mode 100644 index 000000000..b287efcdc --- /dev/null +++ b/packages/app-expo/src/components/reader/definition-controller.test.ts @@ -0,0 +1,215 @@ +import type { + DictionaryEntry, + DictionaryLanguage, + DictionaryPackDescriptor, +} from "@readany/core/dictionary"; +import { describe, expect, it, vi } from "vitest"; +import { DefinitionController } from "./definition-controller"; + +const descriptor: DictionaryPackDescriptor = { + language: "en", + version: "1.0.0", + schemaVersion: 1, + sourceEdition: "enwiktionary", + sourceDumpDate: "2026-09-01", + sizeBytes: 1_572_864, + sha256: "a".repeat(64), + url: "https://example.test/readany-dictionary-en.sqlite", + sourceArchiveUrl: "https://example.test/en-source.xml.bz2", + attributionUrl: "https://en.wiktionary.org", + license: "CC BY-SA 4.0", +}; + +const entry: DictionaryEntry = { + id: 1, + language: "en", + headword: "desire", + pronunciation: "/dɪˈzaɪəɹ/", + partOfSpeech: "noun", + senses: [{ order: 0, definition: "A strong wish." }], +}; + +function lookupError(code: string, message = code): Error & { code: string } { + return Object.assign(new Error(message), { code }); +} + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, reject, resolve }; +} + +function createController(options?: { + lookup?: (text: string) => Promise; + install?: ( + pack: DictionaryPackDescriptor, + onProgress: (progress: number) => void, + onVerifying?: () => void, + ) => Promise; + getDescriptor?: (language: DictionaryLanguage) => DictionaryPackDescriptor | undefined; +}) { + return new DefinitionController({ + lookup: options?.lookup ?? vi.fn(async () => [entry]), + install: options?.install ?? vi.fn(async () => {}), + getDescriptor: options?.getDescriptor ?? vi.fn(() => descriptor), + }); +} + +describe("DefinitionController", () => { + it("opens a supported selection into its local result", async () => { + const lookup = vi.fn(async () => [entry]); + const controller = createController({ lookup }); + + await controller.open(" Desire "); + + expect(lookup).toHaveBeenCalledWith(" Desire "); + expect(controller.state).toEqual({ + kind: "result", + displayText: "Desire", + entries: [entry], + }); + }); + + it("offers the matching pack when the local pack is absent", async () => { + const getDescriptor = vi.fn(() => descriptor); + const controller = createController({ + lookup: async () => { + throw lookupError("pack-not-installed"); + }, + getDescriptor, + }); + + await controller.open("desire"); + + expect(getDescriptor).toHaveBeenCalledWith("en"); + expect(controller.state).toEqual({ kind: "missing-pack", language: "en", descriptor }); + }); + + it("shows verification separately and discards it after closing", async () => { + let verify: (() => void) | undefined; + let finish!: () => void; + const controller = createController({ + lookup: async () => { + throw lookupError("pack-not-installed"); + }, + install: async (_pack, _progress, onVerifying) => { + verify = onVerifying; + await new Promise((resolve) => { + finish = resolve; + }); + }, + }); + await controller.open("desire"); + const downloading = controller.download(); + verify?.(); + expect(controller.state).toEqual({ kind: "verifying", language: "en" }); + controller.close(); + verify?.(); + finish(); + await downloading; + expect(controller.state).toEqual({ kind: "idle" }); + }); + + it("reports install progress then automatically retries the original selection", async () => { + const lookup = vi + .fn<(text: string) => Promise>() + .mockRejectedValueOnce(lookupError("pack-not-installed")) + .mockResolvedValueOnce([entry]); + const install = vi.fn( + async (_pack: DictionaryPackDescriptor, onProgress: (value: number) => void) => { + onProgress(0.37); + }, + ); + const controller = createController({ lookup, install }); + + await controller.open("desire"); + const downloading = controller.download(); + + expect(controller.state).toEqual({ kind: "downloading", language: "en", progress: 0.37 }); + await downloading; + + expect(install).toHaveBeenCalledWith(descriptor, expect.any(Function), expect.any(Function)); + expect(lookup).toHaveBeenNthCalledWith(2, "desire"); + expect(controller.state).toEqual({ + kind: "result", + displayText: "desire", + entries: [entry], + }); + }); + + it("shows no-match for a supported local lookup with no entries", async () => { + const controller = createController({ lookup: async () => [] }); + + await controller.open("unknown"); + + expect(controller.state).toEqual({ kind: "no-match", displayText: "unknown" }); + }); + + it("rejects an unsupported selection before invoking lookup", async () => { + const lookup = vi.fn(async () => [entry]); + const controller = createController({ lookup }); + + await controller.open("read閱讀"); + + expect(lookup).not.toHaveBeenCalled(); + expect(controller.state).toEqual({ kind: "unsupported", reason: "mixed-script" }); + }); + + it("retries a recoverable local lookup error", async () => { + const lookup = vi + .fn<(text: string) => Promise>() + .mockRejectedValueOnce(new Error("database temporarily unavailable")) + .mockResolvedValueOnce([entry]); + const controller = createController({ lookup }); + + await controller.open("desire"); + expect(controller.state).toEqual({ + kind: "error", + message: "database temporarily unavailable", + }); + + await controller.retry(); + + expect(lookup).toHaveBeenCalledTimes(2); + expect(controller.state).toEqual({ + kind: "result", + displayText: "desire", + entries: [entry], + }); + }); + + it("suppresses a stale lookup when the selected text changes", async () => { + const first = deferred(); + const lookup = vi.fn((text: string) => + text === "first" ? first.promise : Promise.resolve([entry]), + ); + const controller = createController({ lookup }); + + const firstOpen = controller.open("first"); + await controller.open("second"); + first.resolve([]); + await firstOpen; + + expect(controller.state).toEqual({ + kind: "result", + displayText: "second", + entries: [entry], + }); + }); + + it("closes by invalidating in-flight work and resetting to idle", async () => { + const pending = deferred(); + const controller = createController({ lookup: () => pending.promise }); + + const opening = controller.open("desire"); + controller.close(); + pending.resolve([entry]); + await opening; + + expect(controller.state).toEqual({ kind: "idle" }); + }); +}); diff --git a/packages/app-expo/src/components/reader/definition-controller.ts b/packages/app-expo/src/components/reader/definition-controller.ts new file mode 100644 index 000000000..b7784b686 --- /dev/null +++ b/packages/app-expo/src/components/reader/definition-controller.ts @@ -0,0 +1 @@ +export * from "@readany/core/dictionary/definition-controller"; diff --git a/packages/app-expo/src/config/dictionary-config.test.ts b/packages/app-expo/src/config/dictionary-config.test.ts new file mode 100644 index 000000000..dd0c627b8 --- /dev/null +++ b/packages/app-expo/src/config/dictionary-config.test.ts @@ -0,0 +1,78 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import * as dictionaryConfig from "./dictionary-config"; + +afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); +}); + +describe("dictionary configuration", () => { + it("allows a build-time manifest override", async () => { + vi.stubEnv("EXPO_PUBLIC_DICTIONARY_MANIFEST_URL", "https://example.com/dictionaries.json"); + vi.resetModules(); + const config = await import("./dictionary-config"); + expect(config.DICTIONARY_REMOTE_MANIFEST_URL).toBe("https://example.com/dictionaries.json"); + }); + + it("uses the official URL for an empty build-time override", async () => { + vi.stubEnv("EXPO_PUBLIC_DICTIONARY_MANIFEST_URL", " "); + vi.resetModules(); + const config = await import("./dictionary-config"); + expect(config.DICTIONARY_REMOTE_MANIFEST_URL).toBe( + "https://raw.githubusercontent.com/codedogQBY/ReadAny/main/dictionary-packs/manifest.json", + ); + }); + it("uses the official dictionary manifest URL", () => { + expect(dictionaryConfig.DICTIONARY_REMOTE_MANIFEST_URL).toBe( + "https://raw.githubusercontent.com/codedogQBY/ReadAny/main/dictionary-packs/manifest.json", + ); + }); + + it("exposes a parsed bundled manifest with the verified mixed sources", () => { + expect(dictionaryConfig).toHaveProperty("DICTIONARY_BUNDLED_MANIFEST"); + const bundled = Reflect.get(dictionaryConfig, "DICTIONARY_BUNDLED_MANIFEST"); + expect(bundled).toMatchObject({ + manifestVersion: 1, + packs: { + en: { + language: "en", + schemaVersion: 1, + sourceEdition: "wordnet-3.1", + license: "WordNet 3.1 License", + sourceArchiveUrl: "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + attributionUrl: "https://wordnet.princeton.edu/license-and-commercial-use", + }, + zh: { + language: "zh", + schemaVersion: 1, + sourceEdition: "zhwiktionary", + license: "CC BY-SA 4.0", + sourceArchiveUrl: + "https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2", + attributionUrl: + "https://zh.wiktionary.org/wiki/Wiktionary:%E7%89%88%E6%9D%83%E4%BF%A1%E6%81%AF", + }, + }, + }); + + for (const pack of Object.values(bundled.packs)) { + expect(pack.attributionUrl).not.toBe(pack.sourceArchiveUrl); + expect(pack.attributionUrl).not.toMatch(/\.(?:tar\.gz|xml\.bz2)$/u); + } + }); + + it("bundles bytes identical to the canonical release manifest", async () => { + const canonicalPath = resolve( + import.meta.dirname, + "../../../../dictionary-packs/manifest.json", + ); + const bundledPath = resolve( + import.meta.dirname, + "../../../core/src/dictionary/dictionary-manifest.json", + ); + + await expect(readFile(bundledPath)).resolves.toEqual(await readFile(canonicalPath)); + }); +}); diff --git a/packages/app-expo/src/config/dictionary-config.ts b/packages/app-expo/src/config/dictionary-config.ts new file mode 100644 index 000000000..7fc1f8264 --- /dev/null +++ b/packages/app-expo/src/config/dictionary-config.ts @@ -0,0 +1,4 @@ +import { DEFAULT_DICTIONARY_MANIFEST_URL } from "@readany/core/dictionary/dictionary-config"; +export { DICTIONARY_BUNDLED_MANIFEST } from "@readany/core/dictionary/dictionary-config"; +export const DICTIONARY_REMOTE_MANIFEST_URL = + process.env.EXPO_PUBLIC_DICTIONARY_MANIFEST_URL?.trim() || DEFAULT_DICTIONARY_MANIFEST_URL; diff --git a/packages/app-expo/src/lib/dictionary/dictionary-database.ts b/packages/app-expo/src/lib/dictionary/dictionary-database.ts new file mode 100644 index 000000000..3c6b2edc6 --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-database.ts @@ -0,0 +1,68 @@ +import type { DictionaryLanguage } from "@readany/core/dictionary"; +import { + type DictionaryDatabaseAdapter, + type DictionaryDatabaseConnection, + DictionaryLookupError, +} from "@readany/core/dictionary/dictionary-database"; +export * from "@readany/core/dictionary/dictionary-database"; + +interface DictionaryMetadataRow { + key: string; + value: string; +} + +function splitDatabasePath(absolutePath: string): { fileName: string; directory: string } { + const normalizedPath = absolutePath.replace(/[\\/]+$/, ""); + const separator = Math.max(normalizedPath.lastIndexOf("/"), normalizedPath.lastIndexOf("\\")); + if (separator < 0 || separator === normalizedPath.length - 1) { + throw new DictionaryLookupError("pack-invalid", "Dictionary pack path must include a filename"); + } + + return { + fileName: normalizedPath.slice(separator + 1), + directory: separator === 0 ? normalizedPath.slice(0, 1) : normalizedPath.slice(0, separator), + }; +} + +export class ExpoDictionaryDatabaseAdapter implements DictionaryDatabaseAdapter { + async open( + language: DictionaryLanguage, + absolutePath: string, + ): Promise { + const { fileName, directory } = splitDatabasePath(absolutePath); + const SQLite = await import("expo-sqlite"); + const database = await SQLite.openDatabaseAsync( + fileName, + { useNewConnection: true }, + directory, + ); + + try { + await database.execAsync("PRAGMA query_only = ON"); + const metadata = new Map( + ( + await database.getAllAsync( + "SELECT key, value FROM metadata WHERE key IN ('schema_version', 'language')", + ) + ).map((row) => [row.key, row.value]), + ); + if (metadata.get("schema_version") !== "1" || metadata.get("language") !== language) { + throw new Error("metadata did not match the expected schema version and language"); + } + } catch (error) { + try { + await database.closeAsync(); + } catch { + // The invalid handle cannot be reused; retain the original validation error. + } + const detail = error instanceof Error ? error.message : String(error); + throw new DictionaryLookupError("pack-invalid", `Dictionary pack is invalid: ${detail}`); + } + + return { + getAllAsync: async (sql: string, ...params: unknown[]) => + database.getAllAsync(sql, ...(params as string[])), + closeAsync: () => database.closeAsync(), + }; + } +} diff --git a/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.test.ts b/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.test.ts new file mode 100644 index 000000000..b74ec5e53 --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.test.ts @@ -0,0 +1,266 @@ +import type { DictionaryEntry, DictionaryLanguage } from "@readany/core/dictionary"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + type DictionaryDatabaseAdapter, + type DictionaryDatabaseConnection, + ExpoDictionaryDatabaseAdapter, +} from "./dictionary-database"; +import { DictionaryLookupService } from "./dictionary-lookup-service"; + +const expoSqlite = vi.hoisted(() => ({ + openDatabaseAsync: vi.fn(), +})); + +vi.mock("expo-sqlite", () => expoSqlite); + +interface FixtureEntry extends DictionaryEntry { + lookupKeys: string[]; +} + +function createFixtureAdapter( + fixtures: Partial>, +): DictionaryDatabaseAdapter & { + openCount: number; + closeCount: number; + openedPaths: unknown[]; + queries: Array<{ sql: string; params: unknown[] }>; +} { + let openCount = 0; + let closeCount = 0; + const queries: Array<{ sql: string; params: unknown[] }> = []; + const openedPaths: unknown[] = []; + + return { + get openCount() { + return openCount; + }, + get closeCount() { + return closeCount; + }, + openedPaths, + queries, + async open(language, absolutePath): Promise { + openCount += 1; + openedPaths.push(absolutePath); + return { + async getAllAsync(sql: string, ...params: unknown[]): Promise { + queries.push({ sql, params }); + const [lookupKey, selectedLanguage] = params as [string, DictionaryLanguage]; + return (fixtures[language] ?? []) + .filter( + (entry) => selectedLanguage === language && entry.lookupKeys.includes(lookupKey), + ) + .flatMap((entry) => { + const rank = entry.lookupKeys.indexOf(lookupKey); + return entry.senses.map((sense) => ({ + entry_id: entry.id, + language: entry.language, + headword: entry.headword, + simplified: entry.simplified ?? null, + traditional: entry.traditional ?? null, + pronunciation: entry.pronunciation ?? null, + part_of_speech: entry.partOfSpeech, + rank, + sense_order: sense.order, + definition: sense.definition, + })); + }) + .sort( + (left, right) => + left.rank - right.rank || + left.entry_id - right.entry_id || + left.sense_order - right.sense_order, + ) as T[]; + }, + async closeAsync(): Promise { + closeCount += 1; + }, + }; + }, + }; +} + +function createChineseFixtureAdapter() { + return createFixtureAdapter({ + zh: [ + { + id: 1, + language: "zh", + headword: "閱讀", + simplified: "阅读", + traditional: "閱讀", + partOfSpeech: "verb", + senses: [{ order: 0, definition: "看书。" }], + lookupKeys: ["閱讀", "阅读"], + }, + ], + }); +} + +describe("DictionaryLookupService", () => { + it("returns ordered English senses through a supplied inflection alias", async () => { + const database = createFixtureAdapter({ + en: [ + { + id: 2, + language: "en", + headword: "desire", + pronunciation: "/dɪˈzaɪəɹ/", + partOfSpeech: "noun", + senses: [ + { order: 1, definition: "A strong wish." }, + { order: 0, definition: "An object of longing." }, + ], + lookupKeys: ["desire", "desires"], + }, + ], + }); + const service = new DictionaryLookupService(database, () => "/dict/en.sqlite"); + + await expect(service.lookup("Desires")).resolves.toEqual([ + expect.objectContaining({ + headword: "desire", + senses: [ + { order: 0, definition: "An object of longing." }, + { order: 1, definition: "A strong wish." }, + ], + }), + ]); + expect(database.queries[0]).toMatchObject({ params: ["desires", "en"] }); + expect(database.queries[0]?.sql).toContain( + "ORDER BY matched.rank ASC, e.id ASC, s.sense_order ASC", + ); + expect(database.queries[0]?.sql).toContain("LIMIT 20"); + }); + + it("returns one Chinese entry through simplified and traditional aliases", async () => { + const service = new DictionaryLookupService( + createChineseFixtureAdapter(), + () => "/dict/zh.sqlite", + ); + + expect((await service.lookup("阅读"))[0]?.traditional).toBe("閱讀"); + expect((await service.lookup("閱讀"))[0]?.simplified).toBe("阅读"); + }); + + it("returns exact-match entries in rank and entry order", async () => { + const service = new DictionaryLookupService( + createFixtureAdapter({ + en: [ + { + id: 4, + language: "en", + headword: "read", + partOfSpeech: "verb", + senses: [{ order: 0, definition: "To examine writing." }], + lookupKeys: ["read", "reads"], + }, + { + id: 2, + language: "en", + headword: "read", + partOfSpeech: "verb", + senses: [{ order: 0, definition: "To interpret writing." }], + lookupKeys: ["read", "reads"], + }, + ], + }), + () => "/dict/en.sqlite", + ); + + await expect(service.lookup("reads")).resolves.toMatchObject([ + { id: 2, headword: "read" }, + { id: 4, headword: "read" }, + ]); + }); + + it("does not open a database for unsupported selection", async () => { + const adapter = createFixtureAdapter({}); + + await expect( + new DictionaryLookupService(adapter, () => null).lookup("read閱讀"), + ).rejects.toMatchObject({ code: "unsupported-selection" }); + expect(adapter.openCount).toBe(0); + }); + + it("reports a missing selected pack without opening a database", async () => { + const adapter = createFixtureAdapter({}); + + await expect( + new DictionaryLookupService(adapter, () => null).lookup("read"), + ).rejects.toMatchObject({ code: "pack-not-installed" }); + expect(adapter.openCount).toBe(0); + }); + + it("awaits an asynchronous installed-pack path resolver", async () => { + const adapter = createFixtureAdapter({ en: [] }); + const service = new DictionaryLookupService(adapter, async () => "/dict/en.sqlite"); + + await service.lookup("desire"); + + expect(adapter.openedPaths).toEqual(["/dict/en.sqlite"]); + }); + + it("caches one connection per language and releases it when closed", async () => { + const adapter = createFixtureAdapter({ en: [] }); + const service = new DictionaryLookupService(adapter, () => "/dict/en.sqlite"); + + await service.lookup("read"); + await service.lookup("reads"); + expect(adapter.openCount).toBe(1); + + await service.close("en"); + expect(adapter.closeCount).toBe(1); + await service.lookup("read"); + expect(adapter.openCount).toBe(2); + + await service.close(); + expect(adapter.closeCount).toBe(2); + }); +}); + +describe("ExpoDictionaryDatabaseAdapter", () => { + const connection = { + execAsync: vi.fn(), + getAllAsync: vi.fn(), + closeAsync: vi.fn(), + }; + + beforeEach(() => { + vi.clearAllMocks(); + expoSqlite.openDatabaseAsync.mockResolvedValue(connection); + connection.getAllAsync.mockResolvedValue([ + { key: "schema_version", value: "1" }, + { key: "language", value: "en" }, + ]); + }); + + it("opens a verified read-only Expo pack by filename and directory", async () => { + const adapter = new ExpoDictionaryDatabaseAdapter(); + + const opened = await adapter.open("en", "/dict/en.sqlite"); + + expect(expoSqlite.openDatabaseAsync).toHaveBeenCalledWith( + "en.sqlite", + { useNewConnection: true }, + "/dict", + ); + expect(connection.execAsync).toHaveBeenCalledWith("PRAGMA query_only = ON"); + expect(opened).toMatchObject({ + getAllAsync: expect.any(Function), + closeAsync: expect.any(Function), + }); + }); + + it("closes an invalid Expo pack before reporting it", async () => { + connection.getAllAsync.mockResolvedValue([ + { key: "schema_version", value: "1" }, + { key: "language", value: "zh" }, + ]); + + await expect( + new ExpoDictionaryDatabaseAdapter().open("en", "/dict/en.sqlite"), + ).rejects.toMatchObject({ code: "pack-invalid" }); + expect(connection.closeAsync).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.ts b/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.ts new file mode 100644 index 000000000..c560f577d --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-lookup-service.ts @@ -0,0 +1 @@ +export * from "@readany/core/dictionary/dictionary-lookup-service"; diff --git a/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.test.ts b/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.test.ts new file mode 100644 index 000000000..b66083ea7 --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.test.ts @@ -0,0 +1,613 @@ +import type { + ChineseDictionaryPackDescriptor, + DictionaryLanguage, + DictionaryManifest, + DictionaryPackDescriptor, + EnglishDictionaryPackDescriptor, +} from "@readany/core/dictionary"; +import { describe, expect, it, vi } from "vitest"; +import { + DictionaryPackManager, + type DictionaryPackMetadata, + type DictionaryPackPlatform, + type DictionaryPackStatus, +} from "./dictionary-pack-manager"; + +const enV1 = descriptor("en", "2026.08", "a"); +const enV2 = descriptor("en", "2026.09", "b"); +const zhV1 = descriptor("zh", "2026.09", "c"); + +function descriptor( + language: "en", + version: string, + hashCharacter: string, +): EnglishDictionaryPackDescriptor; +function descriptor( + language: "zh", + version: string, + hashCharacter: string, +): ChineseDictionaryPackDescriptor; +function descriptor( + language: DictionaryLanguage, + version: string, + hashCharacter: string, +): DictionaryPackDescriptor { + const shared = { + version, + schemaVersion: 1 as const, + sourceDumpDate: "2026-09-01", + sha256: hashCharacter.repeat(64), + license: "CC BY-SA 4.0" as const, + }; + return language === "en" + ? { + ...shared, + language, + sourceEdition: "enwiktionary", + sizeBytes: 123, + url: `https://example.test/en-${version}.sqlite`, + sourceArchiveUrl: `https://example.test/en-${version}.tar.gz`, + attributionUrl: "https://en.wiktionary.org/wiki/Wiktionary:Copyrights", + } + : { + ...shared, + language, + sourceEdition: "zhwiktionary", + sizeBytes: 234, + url: `https://example.test/zh-${version}.sqlite`, + sourceArchiveUrl: `https://example.test/zh-${version}.xml.bz2`, + attributionUrl: "https://zh.wiktionary.org/wiki/Wiktionary:Copyrights", + }; +} + +function metadataOf(pack: DictionaryPackDescriptor): DictionaryPackMetadata { + const { + language, + version, + schemaVersion, + sourceEdition, + sourceDumpDate, + sourceArchiveUrl, + url, + attributionUrl, + license, + } = pack; + return { + language, + version, + schemaVersion, + sourceEdition, + sourceDumpDate, + sourceArchiveUrl, + url, + attributionUrl, + license, + licenseNotice: `Complete ${license} notice.`, + creatorAttribution: `${sourceEdition} contributors.`, + }; +} + +interface MemoryFile { + id: string; + content: string; + size: number; + hash: string; + metadata: DictionaryPackMetadata; + validSchema: boolean; +} + +interface Deferred { + promise: Promise; + resolve(): void; +} + +function deferred(): Deferred { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +class StatefulMemoryPlatform implements DictionaryPackPlatform { + readonly root = "/docs/dictionaries"; + readonly files = new Map(); + readonly events: string[] = []; + readonly downloads = new Map([ + [enV1.url, enV1], + [enV2.url, enV2], + [zhV1.url, zhV1], + ]); + readonly downloadBarriers = new Map(); + readonly removeFaults = new Map(); + readonly metadataFaults: Array<{ path: string; id?: string; error: Error }> = []; + downloadCalls = 0; + downloadMutation?: (file: MemoryFile) => void; + downloadFailure?: Error; + + async ensureDirectory(): Promise { + this.events.push("ensure-directory"); + } + + async download(url: string, path: string, onProgress: (fraction: number) => void): Promise { + this.downloadCalls += 1; + this.events.push(`download-start:${url}`); + onProgress(0.25); + await this.downloadBarriers.get(url)?.promise; + const pack = this.downloads.get(url); + if (!pack) throw new Error(`No test pack for ${url}`); + const file = this.file(pack, `download-${this.downloadCalls}`, `content:${pack.version}`); + this.downloadMutation?.(file); + this.files.set(path, file); + if (this.downloadFailure) throw this.downloadFailure; + onProgress(1); + this.events.push(`download-complete:${url}`); + } + + async exists(path: string): Promise { + return this.files.has(path); + } + + async size(path: string): Promise { + return this.requiredFile(path).size; + } + + async sha256(path: string): Promise { + return this.requiredFile(path).hash; + } + + async readMetadata(path: string): Promise { + const file = this.requiredFile(path); + this.events.push(`metadata:${path}:${file.id}`); + const faultIndex = this.metadataFaults.findIndex( + (fault) => fault.path === path && (!fault.id || fault.id === file.id), + ); + if (faultIndex >= 0) throw this.metadataFaults.splice(faultIndex, 1)[0].error; + if (!file.validSchema) throw new Error(`invalid schema: ${file.id}`); + return { ...file.metadata }; + } + + async move(from: string, to: string): Promise { + this.events.push(`move:${from}->${to}`); + if (this.files.has(to)) throw new Error(`destination exists: ${to}`); + const file = this.requiredFile(from); + this.files.delete(from); + this.files.set(to, file); + } + + async remove(path: string): Promise { + this.events.push(`remove:${path}`); + const faults = this.removeFaults.get(path); + const fault = faults?.shift(); + if (fault) throw fault; + this.files.delete(path); + } + + put(path: string, pack: DictionaryPackDescriptor, id: string, content = `content:${id}`): void { + this.files.set(path, this.file(pack, id, content)); + } + + pauseDownload(url: string): () => void { + const barrier = deferred(); + this.downloadBarriers.set(url, barrier); + return barrier.resolve; + } + + failMetadataOnce(path: string, error: Error, id?: string): void { + this.metadataFaults.push({ path, id, error }); + } + + private file(pack: DictionaryPackDescriptor, id: string, content: string): MemoryFile { + return { + id, + content, + size: pack.sizeBytes, + hash: pack.sha256, + metadata: metadataOf(pack), + validSchema: true, + }; + } + + private requiredFile(path: string): MemoryFile { + const file = this.files.get(path); + if (!file) throw new Error(`missing ${path}`); + return file; + } +} + +function paths(platform: StatefulMemoryPlatform, language: DictionaryLanguage = "en") { + const active = `${platform.root}/readany-dictionary-${language}.sqlite`; + return { active, staged: `${active}.download`, backup: `${active}.backup` }; +} + +function closer(platform: StatefulMemoryPlatform) { + return { + close: vi.fn(async (language?: DictionaryLanguage) => { + platform.events.push(`close:${language ?? "all"}`); + }), + }; +} + +function manifest(en = enV2): DictionaryManifest { + return { manifestVersion: 1, packs: { en, zh: zhV1 } }; +} + +describe("DictionaryPackManager", () => { + it("activates the exact validated staged file", async () => { + const platform = new StatefulMemoryPlatform(); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await manager.install(enV2); + + const activeFile = platform.files.get(paths(platform).active); + expect(activeFile).toMatchObject({ id: "download-1", content: "content:2026.09" }); + expect(await manager.getInstalledDescriptor("en")).toEqual({ + ...metadataOf(enV2), + sizeBytes: enV2.sizeBytes, + sha256: enV2.sha256, + }); + }); + + it.each([ + [ + "size", + (file: MemoryFile): void => { + file.size = 1; + }, + ], + [ + "hash", + (file: MemoryFile): void => { + file.hash = "d".repeat(64); + }, + ], + [ + "schema", + (file: MemoryFile): void => { + file.validSchema = false; + }, + ], + ] as const)("retains the exact active file on staged %s mismatch", async (_kind, mutate) => { + const platform = new StatefulMemoryPlatform(); + const { active, staged } = paths(platform); + platform.put(active, enV1, "old", "trusted old bytes"); + platform.downloadMutation = mutate; + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.install(enV2)).rejects.toThrow(); + + expect(platform.files.get(active)).toMatchObject({ id: "old", content: "trusted old bytes" }); + expect(platform.files.has(staged)).toBe(false); + }); + + it("removes a partial staged file after interrupted download and permits retry", async () => { + const platform = new StatefulMemoryPlatform(); + const { active, staged } = paths(platform); + platform.put(active, enV1, "old", "trusted old bytes"); + platform.downloadMutation = (file) => { + file.id = "partial"; + file.content = "partial bytes"; + }; + platform.downloadFailure = new Error("interrupted"); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.install(enV2)).rejects.toThrow("interrupted"); + expect(platform.files.has(staged)).toBe(false); + expect(platform.files.get(active)?.content).toBe("trusted old bytes"); + + platform.downloadMutation = undefined; + platform.downloadFailure = undefined; + await manager.install(enV2); + expect(platform.files.get(active)?.content).toBe("content:2026.09"); + }); + + it("uses but does not move or delete the last-known-good backup during read-only discovery", async () => { + const platform = new StatefulMemoryPlatform(); + const { active, backup } = paths(platform); + platform.put(active, enV2, "uncommitted-new", "new bytes"); + platform.put(backup, enV1, "trusted-backup", "trusted backup bytes"); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + expect(await manager.getActivePath("en")).toBe(backup); + + expect(platform.files.get(active)).toMatchObject({ + id: "uncommitted-new", + content: "new bytes", + }); + expect(platform.files.get(backup)).toMatchObject({ + id: "trusted-backup", + content: "trusted backup bytes", + }); + expect(platform.events).not.toContain(`remove:${active}`); + expect(platform.events).not.toContain(`move:${backup}->${active}`); + }); + + it("preserves staged and corrupt backup artifacts during read-only discovery", async () => { + const platform = new StatefulMemoryPlatform(); + const { active, staged, backup } = paths(platform); + platform.put(active, enV1, "trusted-active"); + platform.put(staged, enV2, "interrupted-download"); + platform.put(backup, enV2, "corrupt-backup"); + const corruptBackup = platform.files.get(backup); + if (corruptBackup) corruptBackup.validSchema = false; + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.refresh(manifest())).resolves.toMatchObject({ + en: { state: "update-available", installedVersion: enV1.version }, + }); + + expect(platform.files.get(active)?.id).toBe("trusted-active"); + expect(platform.files.get(staged)?.id).toBe("interrupted-download"); + expect(platform.files.get(backup)?.id).toBe("corrupt-backup"); + expect(platform.events.filter((event) => event.startsWith("remove:"))).toEqual([]); + }); + + it.each([ + ["a corrupt backup with no active", false], + ["corrupt active and backup files", true], + ])("repairs %s only during an explicit install", async (_name, includeActive) => { + const platform = new StatefulMemoryPlatform(); + const { active, backup } = paths(platform); + platform.put(backup, enV1, "corrupt-backup"); + const corruptBackup = platform.files.get(backup); + if (corruptBackup) corruptBackup.validSchema = false; + if (includeActive) { + platform.put(active, enV1, "corrupt-active"); + const corruptActive = platform.files.get(active); + if (corruptActive) corruptActive.validSchema = false; + } + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.refresh(manifest())).resolves.toMatchObject({ + en: { state: "error", hasActivePack: true }, + }); + expect(platform.files.get(backup)?.id).toBe("corrupt-backup"); + if (includeActive) expect(platform.files.get(active)?.id).toBe("corrupt-active"); + + await expect(manager.install(enV2)).resolves.toBeUndefined(); + expect(platform.files.get(active)?.id).toBe("download-1"); + expect(platform.files.has(backup)).toBe(false); + }); + + it("keeps repair available when a retry fails with only a corrupt backup", async () => { + const platform = new StatefulMemoryPlatform(); + const { backup } = paths(platform); + platform.put(backup, enV1, "corrupt-backup"); + const corruptBackup = platform.files.get(backup); + if (corruptBackup) corruptBackup.validSchema = false; + platform.downloadFailure = new Error("offline"); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + const statuses: DictionaryPackStatus[] = []; + + await expect(manager.install(enV2, (status) => statuses.push(status))).rejects.toThrow( + "offline", + ); + + expect(statuses.at(-1)).toEqual({ + state: "error", + message: "offline", + hasActivePack: true, + }); + expect(platform.files.get(backup)?.id).toBe("corrupt-backup"); + }); + + it("discovers installed metadata from SQLite after restart and detects updates", async () => { + const platform = new StatefulMemoryPlatform(); + platform.put(paths(platform).active, enV1, "persisted", "persisted bytes"); + const restartedManager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + expect(await restartedManager.getInstalledDescriptor("en")).toEqual({ + ...metadataOf(enV1), + sizeBytes: enV1.sizeBytes, + sha256: enV1.sha256, + }); + await expect(restartedManager.refresh(manifest())).resolves.toMatchObject({ + en: { + state: "update-available", + installedVersion: enV1.version, + availableVersion: enV2.version, + sizeBytes: enV1.sizeBytes, + }, + }); + }); + + it("offers repair when the manifest has the same version but a different full identity", async () => { + const platform = new StatefulMemoryPlatform(); + platform.put(paths(platform).active, enV2, "old-same-version"); + const replacement = { + ...enV2, + sizeBytes: 124, + sha256: "d".repeat(64), + url: "https://example.test/en-2026.09-repacked.sqlite", + sourceArchiveUrl: "https://example.test/en-2026.09-repacked.tar.gz", + attributionUrl: "https://example.test/en-2026.09-attribution", + }; + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.refresh(manifest(replacement))).resolves.toMatchObject({ + en: { + state: "update-available", + installedVersion: enV2.version, + availableVersion: replacement.version, + }, + }); + }); + + it("keeps a valid manifest usable when one installed language is corrupt", async () => { + const platform = new StatefulMemoryPlatform(); + const enPath = paths(platform, "en").active; + const zhPath = paths(platform, "zh").active; + platform.put(enPath, enV1, "corrupt-en"); + const corruptEnglish = platform.files.get(enPath); + if (corruptEnglish) corruptEnglish.validSchema = false; + platform.put(zhPath, zhV1, "healthy-zh"); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.refresh(manifest())).resolves.toMatchObject({ + en: { state: "error", hasActivePack: true }, + zh: { state: "installed", version: zhV1.version }, + }); + expect(platform.files.get(enPath)?.id).toBe("corrupt-en"); + expect(await manager.getInstalledDescriptor("zh")).toMatchObject({ version: zhV1.version }); + }); + + it("rolls back using the backup's own metadata after final validation fails", async () => { + const platform = new StatefulMemoryPlatform(); + const { active, backup } = paths(platform); + platform.put(active, enV1, "old", "trusted old bytes"); + const activationError = new Error("post-activation validation failed"); + platform.failMetadataOnce(active, activationError, "download-1"); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.install(enV2)).rejects.toBe(activationError); + + expect(platform.events).toContain(`metadata:${backup}:old`); + expect(platform.files.get(active)).toMatchObject({ id: "old", content: "trusted old bytes" }); + expect(await manager.getInstalledDescriptor("en")).toMatchObject({ version: enV1.version }); + }); + + it("removes a broken active when final validation fails without a previous pack", async () => { + const platform = new StatefulMemoryPlatform(); + const { active, staged } = paths(platform); + platform.failMetadataOnce(active, new Error("post-activation validation failed"), "download-1"); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + await expect(manager.install(enV2)).rejects.toThrow("post-activation validation failed"); + + expect(platform.files.has(active)).toBe(false); + expect(platform.files.has(staged)).toBe(false); + }); + + it("preserves the operation error and records cleanup failure", async () => { + const platform = new StatefulMemoryPlatform(); + const { active } = paths(platform); + const operationError = new Error("post-activation validation failed"); + const cleanupError = new Error("cannot remove broken active"); + platform.failMetadataOnce(active, operationError, "download-1"); + platform.removeFaults.set(active, [cleanupError]); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + let thrown: unknown; + try { + await manager.install(enV2); + } catch (error) { + thrown = error; + } + + expect(thrown).toBe(operationError); + expect((thrown as Error & { cleanupErrors?: unknown[] }).cleanupErrors).toContain(cleanupError); + }); + + it("coalesces concurrent installs for the same language", async () => { + const platform = new StatefulMemoryPlatform(); + const release = platform.pauseDownload(enV2.url); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + + const first = manager.install(enV2); + const second = manager.install(enV2); + await vi.waitFor(() => expect(platform.downloadCalls).toBe(1)); + release(); + await Promise.all([first, second]); + + expect(platform.downloadCalls).toBe(1); + }); + + it("allows different languages to install in parallel", async () => { + const platform = new StatefulMemoryPlatform(); + const releaseEnglish = platform.pauseDownload(enV2.url); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + let englishFinished = false; + + const english = manager.install(enV2).then(() => { + englishFinished = true; + }); + await vi.waitFor(() => expect(platform.downloadCalls).toBe(1)); + + await manager.install(zhV1); + expect(englishFinished).toBe(false); + expect(platform.files.get(paths(platform, "zh").active)?.content).toBe("content:2026.09"); + + releaseEnglish(); + await english; + }); + + it("serializes refresh behind an in-flight install for the same language", async () => { + const platform = new StatefulMemoryPlatform(); + const release = platform.pauseDownload(enV2.url); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + const install = manager.install(enV2); + await vi.waitFor(() => expect(platform.downloadCalls).toBe(1)); + let refreshFinished = false; + + const refresh = manager.refresh(manifest()).then((status) => { + refreshFinished = true; + return status; + }); + await Promise.resolve(); + expect(refreshFinished).toBe(false); + + release(); + await install; + await expect(refresh).resolves.toMatchObject({ + en: { state: "installed", version: enV2.version }, + }); + }); + + it("serializes removal behind an in-flight install and leaves no pack artifacts", async () => { + const platform = new StatefulMemoryPlatform(); + const release = platform.pauseDownload(enV2.url); + const manager = new DictionaryPackManager(platform, platform.root, closer(platform)); + const install = manager.install(enV2); + await vi.waitFor(() => expect(platform.downloadCalls).toBe(1)); + let removeFinished = false; + + const remove = manager.remove("en").then(() => { + removeFinished = true; + }); + await Promise.resolve(); + expect(removeFinished).toBe(false); + + release(); + await install; + await remove; + expect(Object.values(paths(platform)).some((path) => platform.files.has(path))).toBe(false); + }); + + it("closes before moving an active pack or deleting one", async () => { + const platform = new StatefulMemoryPlatform(); + const { active, backup } = paths(platform); + platform.put(active, enV1, "old"); + const lookup = closer(platform); + const manager = new DictionaryPackManager(platform, platform.root, lookup); + + await manager.install(enV2); + const closeBeforeMove = platform.events.indexOf("close:en"); + expect(closeBeforeMove).toBeGreaterThanOrEqual(0); + expect(closeBeforeMove).toBeLessThan(platform.events.indexOf(`move:${active}->${backup}`)); + + platform.events.length = 0; + await manager.remove("en"); + expect(platform.events.indexOf("close:en")).toBeLessThan( + platform.events.indexOf(`remove:${active}`), + ); + }); + + it("removes only the requested language", async () => { + const platform = new StatefulMemoryPlatform(); + const enPaths = paths(platform, "en"); + const zhPaths = paths(platform, "zh"); + platform.put(enPaths.active, enV1, "english", "english bytes"); + platform.put(zhPaths.active, zhV1, "chinese", "chinese bytes"); + const lookup = closer(platform); + const manager = new DictionaryPackManager(platform, platform.root, lookup); + + await manager.remove("en"); + + expect(lookup.close).toHaveBeenCalledWith("en"); + expect(platform.files.has(enPaths.active)).toBe(false); + expect(platform.files.get(zhPaths.active)).toMatchObject({ + id: "chinese", + content: "chinese bytes", + }); + }); +}); diff --git a/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.ts b/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.ts new file mode 100644 index 000000000..1b1e4ecba --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-pack-manager.ts @@ -0,0 +1 @@ +export * from "@readany/core/dictionary/dictionary-pack-manager"; diff --git a/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.test.ts b/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.test.ts new file mode 100644 index 000000000..191c30183 --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.test.ts @@ -0,0 +1,151 @@ +import type { DictionaryPackDescriptor } from "@readany/core/dictionary"; +import { describe, expect, it, vi } from "vitest"; +import { + type DictionaryValidationDatabase, + validateDictionaryDatabase, +} from "./dictionary-pack-platform"; + +vi.mock("@dr.pogodin/react-native-fs", () => ({ hash: vi.fn() })); +vi.mock("expo-file-system", () => ({ + Directory: class {}, + File: class {}, + Paths: { document: { uri: "file:///documents" } }, +})); +vi.mock("expo-file-system/legacy", () => ({ createDownloadResumable: vi.fn() })); + +const descriptor: DictionaryPackDescriptor = { + language: "en", + version: "2026.09", + schemaVersion: 1, + sourceEdition: "wordnet-3.1", + sourceDumpDate: "2011-05-26", + sizeBytes: 123, + sha256: "a".repeat(64), + url: "https://example.test/en.sqlite", + sourceArchiveUrl: "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + attributionUrl: "https://wordnet.princeton.edu/", + license: "WordNet 3.1 License", +}; + +const expectedColumns = { + metadata: ["key", "value"], + entries: [ + "id", + "language", + "headword", + "simplified", + "traditional", + "pronunciation", + "part_of_speech", + ], + senses: ["entry_id", "sense_order", "definition"], + lookup: ["lookup_key", "entry_id", "rank"], +}; + +class SchemaDatabase implements DictionaryValidationDatabase { + integrity = "ok"; + objects = [ + { name: "metadata", type: "table", tbl_name: "metadata" }, + { name: "entries", type: "table", tbl_name: "entries" }, + { name: "senses", type: "table", tbl_name: "senses" }, + { name: "lookup", type: "table", tbl_name: "lookup" }, + { name: "lookup_key_rank_idx", type: "index", tbl_name: "lookup" }, + ]; + columns = structuredClone(expectedColumns); + indexColumns = ["lookup_key", "rank", "entry_id"]; + metadata = new Map([ + ["schema_version", "1"], + ["language", "en"], + ["version", descriptor.version], + ["source_edition", descriptor.sourceEdition], + ["source_dump_date", descriptor.sourceDumpDate], + ["source_archive_url", descriptor.sourceArchiveUrl], + ["asset_url", descriptor.url], + ["attribution_url", descriptor.attributionUrl], + ["license", descriptor.license], + ["license_notice", "Complete WordNet license notice."], + ["creator_attribution", "Princeton University."], + ]); + + async getFirstAsync(): Promise { + return { integrity_check: this.integrity } as T; + } + + async getAllAsync(sql: string): Promise { + if (sql.includes("sqlite_master")) return structuredClone(this.objects) as T[]; + const table = /table_info\('([^']+)'\)/.exec(sql)?.[1] as keyof typeof this.columns | undefined; + if (table) return this.columns[table].map((name) => ({ name })) as T[]; + if (sql.includes("index_info")) + return this.indexColumns.map((name, seqno) => ({ name, seqno })) as T[]; + if (sql.includes("FROM metadata")) + return [...this.metadata].map(([key, value]) => ({ key, value })) as T[]; + throw new Error(`Unexpected SQL: ${sql}`); + } +} + +describe("dictionary SQLite validation", () => { + it("returns intrinsic pack metadata after validating the complete schema", async () => { + await expect(validateDictionaryDatabase(new SchemaDatabase())).resolves.toEqual({ + language: descriptor.language, + version: descriptor.version, + schemaVersion: descriptor.schemaVersion, + sourceEdition: descriptor.sourceEdition, + sourceDumpDate: descriptor.sourceDumpDate, + sourceArchiveUrl: descriptor.sourceArchiveUrl, + url: descriptor.url, + attributionUrl: descriptor.attributionUrl, + license: descriptor.license, + licenseNotice: "Complete WordNet license notice.", + creatorAttribution: "Princeton University.", + }); + }); + + it("rejects an expected object with the wrong SQLite type", async () => { + const database = new SchemaDatabase(); + const index = database.objects.find((object) => object.name === "lookup_key_rank_idx"); + if (index) index.type = "table"; + + await expect(validateDictionaryDatabase(database)).rejects.toThrow("lookup_key_rank_idx index"); + }); + + it("rejects the lookup index when it belongs to another table", async () => { + const database = new SchemaDatabase(); + const index = database.objects.find((object) => object.name === "lookup_key_rank_idx"); + if (index) index.tbl_name = "entries"; + + await expect(validateDictionaryDatabase(database)).rejects.toThrow( + "lookup_key_rank_idx must belong to lookup", + ); + }); + + it("rejects a table missing a required column", async () => { + const database = new SchemaDatabase(); + database.columns.entries = database.columns.entries.filter((name) => name !== "headword"); + + await expect(validateDictionaryDatabase(database)).rejects.toThrow("entries columns"); + }); + + it("rejects a lookup index with the wrong column order", async () => { + const database = new SchemaDatabase(); + database.indexColumns = ["rank", "lookup_key", "entry_id"]; + + await expect(validateDictionaryDatabase(database)).rejects.toThrow("lookup index columns"); + }); + + it("rejects metadata whose source and license do not form a supported pair", async () => { + const database = new SchemaDatabase(); + database.metadata.set("license", "CC BY-SA 4.0"); + + await expect(validateDictionaryDatabase(database)).rejects.toThrow("source/license"); + }); + + it.each(["license_notice", "creator_attribution"])( + "rejects blank required metadata %s", + async (key) => { + const database = new SchemaDatabase(); + database.metadata.set(key, " "); + + await expect(validateDictionaryDatabase(database)).rejects.toThrow(key); + }, + ); +}); diff --git a/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.ts b/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.ts new file mode 100644 index 000000000..05f443a3e --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-pack-platform.ts @@ -0,0 +1,80 @@ +import { hash } from "@dr.pogodin/react-native-fs"; +import { Directory, File, Paths } from "expo-file-system"; +import * as LegacyFileSystem from "expo-file-system/legacy"; +import type { DictionaryPackMetadata, DictionaryPackPlatform } from "./dictionary-pack-manager"; + +import { validateDictionaryDatabase } from "@readany/core/dictionary/dictionary-validation"; +export { + validateDictionaryDatabase, + type DictionaryValidationDatabase, +} from "@readany/core/dictionary/dictionary-validation"; +function nativePath(path: string): string { + return path.replace(/^file:\/\//, ""); +} + +export function createExpoDictionaryPackPlatform(): DictionaryPackPlatform { + return { + async ensureDirectory(path) { + new Directory(path).create({ idempotent: true, intermediates: true }); + }, + async download(url, path, onProgress) { + const task = LegacyFileSystem.createDownloadResumable( + url, + path, + {}, + ({ totalBytesWritten, totalBytesExpectedToWrite }) => + onProgress( + totalBytesExpectedToWrite > 0 ? totalBytesWritten / totalBytesExpectedToWrite : 0, + ), + ); + const result = await task.downloadAsync(); + if (!result || result.status < 200 || result.status >= 300) + throw new Error("Dictionary pack download failed"); + }, + async exists(path) { + return new File(path).exists; + }, + async size(path) { + return new File(path).size ?? 0; + }, + async sha256(path) { + return hash(nativePath(path), "sha256"); + }, + async readMetadata(path) { + return readAndValidateSqlite(path); + }, + async move(from, to) { + const target = new File(to); + if (target.exists) throw new Error(`Dictionary move target already exists: ${to}`); + new File(from).move(target); + }, + async remove(path) { + const file = new File(path); + if (file.exists) file.delete(); + }, + }; +} + +async function readAndValidateSqlite(path: string): Promise { + const SQLite = await import("expo-sqlite"); + const { fileName, directory } = splitDatabasePath(path); + const database = await SQLite.openDatabaseAsync(fileName, { useNewConnection: true }, directory); + try { + return await validateDictionaryDatabase(database); + } finally { + await database.closeAsync(); + } +} + +function splitDatabasePath(path: string): { fileName: string; directory: string } { + const normalized = path.replace(/[\\/]+$/, ""); + const separator = Math.max(normalized.lastIndexOf("/"), normalized.lastIndexOf("\\")); + if (separator < 0 || separator === normalized.length - 1) + throw new Error("Dictionary path has no filename"); + return { + fileName: normalized.slice(separator + 1), + directory: separator === 0 ? normalized.slice(0, 1) : normalized.slice(0, separator), + }; +} + +export const dictionaryPackDirectory = `${Paths.document.uri.replace(/\/$/, "")}/dictionaries`; diff --git a/packages/app-expo/src/lib/dictionary/dictionary-runtime.test.ts b/packages/app-expo/src/lib/dictionary/dictionary-runtime.test.ts new file mode 100644 index 000000000..de5a0b263 --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-runtime.test.ts @@ -0,0 +1,170 @@ +import type { DictionaryLanguage } from "@readany/core/dictionary"; +import { describe, expect, it } from "vitest"; +import type { DictionaryDatabaseAdapter } from "./dictionary-database"; +import type { DictionaryPackMetadata, DictionaryPackPlatform } from "./dictionary-pack-manager"; +import { createDictionaryRuntime } from "./dictionary-runtime"; + +const metadata: DictionaryPackMetadata = { + language: "en", + version: "1.0.0", + schemaVersion: 1, + sourceEdition: "wordnet-3.1", + sourceDumpDate: "2011-05-26", + sourceArchiveUrl: "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + url: "https://example.test/en.sqlite", + attributionUrl: "https://wordnet.princeton.edu/", + license: "WordNet 3.1 License", + licenseNotice: "Complete WordNet license notice.", + creatorAttribution: "Princeton University.", +}; + +describe("createDictionaryRuntime", () => { + it("inspects an active pack once per lifecycle and re-inspects after mutation", async () => { + const directory = "/docs/dictionaries"; + const activePath = `${directory}/readany-dictionary-en.sqlite`; + const files = new Set([activePath]); + let metadataReads = 0; + let hashReads = 0; + const platform: DictionaryPackPlatform = { + ensureDirectory: async () => {}, + download: async () => { + throw new Error("not used"); + }, + exists: async (path) => files.has(path), + size: async () => 123, + sha256: async () => { + hashReads += 1; + return "a".repeat(64); + }, + readMetadata: async () => { + metadataReads += 1; + return metadata; + }, + move: async () => { + throw new Error("not used"); + }, + remove: async (path) => { + files.delete(path); + }, + }; + let openCount = 0; + const database: DictionaryDatabaseAdapter = { + open: async () => { + openCount += 1; + return { getAllAsync: async () => [], closeAsync: async () => {} }; + }, + }; + const runtime = createDictionaryRuntime({ database, directory, platform }); + + await runtime.lookup.lookup("desire"); + await runtime.lookup.lookup("desires"); + expect({ metadataReads, hashReads, openCount }).toEqual({ + metadataReads: 1, + hashReads: 1, + openCount: 1, + }); + + await runtime.manager.remove("en"); + files.add(activePath); + await runtime.lookup.lookup("desire"); + expect({ metadataReads, hashReads, openCount }).toEqual({ + metadataReads: 2, + hashReads: 2, + openCount: 2, + }); + }); + + it("invalidates the inspected path and cached connection after a SQLite query error", async () => { + const directory = "/docs/dictionaries"; + const activePath = `${directory}/readany-dictionary-en.sqlite`; + let metadataReads = 0; + let openCount = 0; + let closeCount = 0; + const platform: DictionaryPackPlatform = { + ensureDirectory: async () => {}, + download: async () => { + throw new Error("not used"); + }, + exists: async (path) => path === activePath, + size: async () => 123, + sha256: async () => "a".repeat(64), + readMetadata: async () => { + metadataReads += 1; + return metadata; + }, + move: async () => { + throw new Error("not used"); + }, + remove: async () => {}, + }; + const database: DictionaryDatabaseAdapter = { + open: async () => { + openCount += 1; + const thisOpen = openCount; + return { + getAllAsync: async () => { + if (thisOpen === 1) throw new Error("SQLITE_CORRUPT"); + return []; + }, + closeAsync: async () => { + closeCount += 1; + }, + }; + }, + }; + const runtime = createDictionaryRuntime({ database, directory, platform }); + + await expect(runtime.lookup.lookup("desire")).rejects.toThrow("SQLITE_CORRUPT"); + await expect(runtime.lookup.lookup("desire")).resolves.toEqual([]); + + expect({ metadataReads, openCount, closeCount }).toEqual({ + metadataReads: 2, + openCount: 2, + closeCount: 1, + }); + }); + + it("shares one real lookup between installed-path resolution and manager close", async () => { + const directory = "/docs/dictionaries"; + const activePath = `${directory}/readany-dictionary-en.sqlite`; + const files = new Set([activePath]); + const events: string[] = []; + const platform: DictionaryPackPlatform = { + ensureDirectory: async () => {}, + download: async () => { + throw new Error("not used"); + }, + exists: async (path) => files.has(path), + size: async () => 123, + sha256: async () => "a".repeat(64), + readMetadata: async () => metadata, + move: async () => { + throw new Error("not used"); + }, + remove: async (path) => { + events.push(`remove:${path}`); + files.delete(path); + }, + }; + const openedPaths: string[] = []; + const database: DictionaryDatabaseAdapter = { + open: async (_language: DictionaryLanguage, path: string) => { + openedPaths.push(path); + return { + getAllAsync: async () => [], + closeAsync: async () => { + events.push("close:en"); + }, + }; + }, + }; + + const runtime = createDictionaryRuntime({ database, directory, platform }); + await expect(runtime.lookup.lookup("desire")).resolves.toEqual([]); + expect(openedPaths).toEqual([activePath]); + + await runtime.manager.remove("en"); + + expect(events).toEqual(["close:en", `remove:${activePath}`]); + }); +}); diff --git a/packages/app-expo/src/lib/dictionary/dictionary-runtime.ts b/packages/app-expo/src/lib/dictionary/dictionary-runtime.ts new file mode 100644 index 000000000..aa2c2fefa --- /dev/null +++ b/packages/app-expo/src/lib/dictionary/dictionary-runtime.ts @@ -0,0 +1 @@ +export * from "@readany/core/dictionary/dictionary-runtime"; diff --git a/packages/app-expo/src/navigation/RootNavigator.tsx b/packages/app-expo/src/navigation/RootNavigator.tsx index d720f534b..f71af724f 100644 --- a/packages/app-expo/src/navigation/RootNavigator.tsx +++ b/packages/app-expo/src/navigation/RootNavigator.tsx @@ -11,6 +11,7 @@ import { WebDavImportBrowserScreen } from "@/screens/library/WebDavImportBrowser import AISettingsScreen from "@/screens/settings/AISettingsScreen"; import AboutScreen from "@/screens/settings/AboutScreen"; import AppearanceSettingsScreen from "@/screens/settings/AppearanceSettingsScreen"; +import { DictionarySettingsScreen } from "@/screens/settings/DictionarySettingsScreen"; import FeedbackDetailScreen from "@/screens/settings/FeedbackDetailScreen"; import FeedbackScreen from "@/screens/settings/FeedbackScreen"; import FontSettingsScreen from "@/screens/settings/FontSettingsScreen"; @@ -40,6 +41,7 @@ export type RootStackParamList = { AISettings: undefined; TTSSettings: undefined; TranslationSettings: undefined; + DictionarySettings: undefined; SyncSettings: undefined; About: undefined; Feedback: undefined; @@ -105,6 +107,7 @@ export function RootNavigator() { + diff --git a/packages/app-expo/src/screens/ProfileScreen.tsx b/packages/app-expo/src/screens/ProfileScreen.tsx index bc19ebce0..6c1b2edd9 100644 --- a/packages/app-expo/src/screens/ProfileScreen.tsx +++ b/packages/app-expo/src/screens/ProfileScreen.tsx @@ -78,6 +78,7 @@ type ProfileMenuRoute = Extract< | "AISettings" | "TTSSettings" | "TranslationSettings" + | "DictionarySettings" | "Skills" | "VectorModelSettings" | "Feedback" @@ -464,6 +465,11 @@ export function ProfileScreen() { label: t("settings.translationTab", "翻译"), route: "TranslationSettings" as const, }, + { + icon: BookOpenIcon, + label: t("dictionary.dictionaries", "Dictionaries"), + route: "DictionarySettings" as const, + }, { icon: PuzzleIcon, label: t("skills.title", "技能"), route: "Skills" as const }, { icon: CpuIcon, diff --git a/packages/app-expo/src/screens/ReaderScreen.tsx b/packages/app-expo/src/screens/ReaderScreen.tsx index a87fe5edc..6a8170213 100644 --- a/packages/app-expo/src/screens/ReaderScreen.tsx +++ b/packages/app-expo/src/screens/ReaderScreen.tsx @@ -1,6 +1,7 @@ import { MarkdownRenderer } from "@/components/chat/MarkdownRenderer"; import { BookmarkRibbon } from "@/components/reader/BookmarkRibbon"; import { ChapterTranslationSheet } from "@/components/reader/ChapterTranslationSheet"; +import { DefinitionSheet } from "@/components/reader/DefinitionSheet"; import { ReadingProgressSlider } from "@/components/reader/ReadingProgressSlider"; import { SelectionPopover } from "@/components/reader/SelectionPopover"; import { TTSPage } from "@/components/reader/TTSPage"; @@ -226,6 +227,8 @@ export function ReaderScreen({ route, navigation }: Props) { const [showNotebook, setShowNotebook] = useState(false); const [showTranslation, setShowTranslation] = useState(false); const [translationText, setTranslationText] = useState(""); + const [definitionText, setDefinitionText] = useState(""); + const [showDefinition, setShowDefinition] = useState(false); const [showTTS, setShowTTS] = useState(false); const [showChapterTranslation, setShowChapterTranslation] = useState(false); const [isReimporting, setIsReimporting] = useState(false); @@ -1607,6 +1610,11 @@ export function ReaderScreen({ route, navigation }: Props) { onCopy={() => { setSelection(null); }} + onDefine={() => { + setDefinitionText(selectionPopoverSelection.text); + setShowDefinition(true); + setSelection(null); + }} onSpeak={(text, cfi) => { tts.startSelectionTTS(text, cfi); setSelection(null); @@ -2135,6 +2143,20 @@ export function ReaderScreen({ route, navigation }: Props) { /> )} + { + setShowDefinition(false); + setDefinitionText(""); + }} + onManageDictionaries={() => { + setShowDefinition(false); + setDefinitionText(""); + navigation.navigate("DictionarySettings" as never); + }} + /> + {/* ─── Chapter Translation Sheet ─── */} { + it("owns the selected definition text separately from selection and translation state", () => { + expect(readerScreen).toContain('const [definitionText, setDefinitionText] = useState("");'); + expect(readerScreen).toContain("const [showDefinition, setShowDefinition] = useState(false);"); + expect(readerScreen).toMatch( + /onDefine=\{\(\) => \{\s*setDefinitionText\(selectionPopoverSelection\.text\);\s*setShowDefinition\(true\);\s*setSelection\(null\);\s*\}\}/, + ); + expect(readerScreen).toMatch( + / { + expect(readerScreen).toMatch( + /onClose=\{\(\) => \{\s*setShowDefinition\(false\);\s*setDefinitionText\(""\);\s*\}\}/, + ); + expect(readerScreen).toMatch( + /onManageDictionaries=\{\(\) => \{\s*setShowDefinition\(false\);\s*setDefinitionText\(""\);\s*navigation\.navigate\("DictionarySettings" as never\);\s*\}\}/, + ); + }); +}); diff --git a/packages/app-expo/src/screens/settings/DictionarySettingsScreen.test.tsx b/packages/app-expo/src/screens/settings/DictionarySettingsScreen.test.tsx new file mode 100644 index 000000000..76c456a80 --- /dev/null +++ b/packages/app-expo/src/screens/settings/DictionarySettingsScreen.test.tsx @@ -0,0 +1,325 @@ +import type { DictionaryStoreState } from "@/stores/dictionary-store"; +import type { DictionaryManifest } from "@readany/core/dictionary"; +import i18n, { i18nReady } from "@readany/core/i18n"; +import React from "react"; +import TestRenderer, { act } from "react-test-renderer"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { create } from "zustand"; +import { DictionarySettingsScreen } from "./DictionarySettingsScreen"; + +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = + true; +(globalThis as typeof globalThis & { React: typeof React }).React = React; + +const { alert, openURL } = vi.hoisted(() => ({ + alert: vi.fn(), + openURL: vi.fn(async () => true), +})); + +vi.mock("react-native", async () => { + const ReactModule = await import("react"); + const host = (name: string) => + function HostComponent(props: Record) { + return ReactModule.createElement(name, props, props.children as React.ReactNode); + }; + return { + ActivityIndicator: host("ActivityIndicator"), + Alert: { alert }, + Linking: { openURL }, + ScrollView: host("ScrollView"), + StyleSheet: { create: (styles: unknown) => styles, hairlineWidth: 1 }, + Text: host("Text"), + TouchableOpacity: host("TouchableOpacity"), + View: host("View"), + }; +}); + +vi.mock("react-native-safe-area-context", () => ({ + SafeAreaView: ({ children }: { children: React.ReactNode }) => children, +})); + +vi.mock("@/stores", () => ({ + useDictionaryStore: () => { + throw new Error("DictionarySettingsScreen tests inject their dictionary store"); + }, +})); +vi.mock("@/components/ui/Icon", () => ({ ChevronLeftIcon: () => null })); +vi.mock("@/components/ui/KeyboardAwareScrollView", () => ({ + KeyboardAwareScrollView: ({ children }: { children: React.ReactNode }) => children, +})); +vi.mock("@/hooks/use-responsive-layout", () => ({ + useResponsiveLayout: () => ({ centeredContentWidth: 720 }), +})); +vi.mock("@/styles/theme", () => ({ + fontSize: { base: 16, lg: 18, sm: 14, xs: 12 }, + fontWeight: { medium: "500", semibold: "600" }, + radius: { lg: 8, xl: 12 }, + spacing: { lg: 16, xl: 20, xxl: 24 }, + useColors: () => ({ + background: "background", + border: "border", + card: "card", + destructive: "destructive", + foreground: "foreground", + mutedForeground: "mutedForeground", + primary: "primary", + primaryForeground: "primaryForeground", + }), +})); +vi.mock("./SettingsHeader", () => ({ + SettingsHeader: ({ title, subtitle }: { title: string; subtitle: string }) => + React.createElement("Text", {}, title, subtitle), +})); + +const descriptors: DictionaryManifest["packs"] = { + en: { + language: "en", + version: "2026.09", + schemaVersion: 1, + sourceEdition: "enwiktionary", + sourceDumpDate: "2026-09-01", + sizeBytes: 1_572_864, + sha256: "a".repeat(64), + url: "https://example.test/en.sqlite", + sourceArchiveUrl: "https://dumps.example.test/en.xml.bz2", + attributionUrl: "https://en.wiktionary.org", + license: "CC BY-SA 4.0", + }, + zh: { + language: "zh", + version: "2026.09", + schemaVersion: 1, + sourceEdition: "zhwiktionary", + sourceDumpDate: "2026-09-01", + sizeBytes: 2_621_440, + sha256: "b".repeat(64), + url: "https://example.test/zh.sqlite", + sourceArchiveUrl: "https://dumps.example.test/zh.xml.bz2", + attributionUrl: "https://zh.wiktionary.org", + license: "CC BY-SA 4.0", + }, +}; + +function makeStore( + overrides: Partial = {}, + actionOverrides: Partial< + Pick + > = {}, +) { + const initialize = actionOverrides.initialize ?? vi.fn(async () => {}); + const refreshManifest = actionOverrides.refreshManifest ?? vi.fn(async () => {}); + const install = actionOverrides.install ?? vi.fn(async () => {}); + const remove = actionOverrides.remove ?? vi.fn(async () => {}); + const retry = actionOverrides.retry ?? vi.fn(async () => {}); + const lookup = vi.fn(async () => []); + return { + actions: { initialize, install, refreshManifest, remove, retry }, + store: create(() => ({ + manifest: { manifestVersion: 1, packs: descriptors }, + packs: { + en: { state: "not-installed" }, + zh: { state: "not-installed" }, + ...overrides, + }, + initialize, + refreshManifest, + install, + remove, + retry, + lookup, + })), + }; +} + +function textContent(renderer: TestRenderer.ReactTestRenderer): string { + return renderer.root + .findAll((node) => String(node.type) === "Text") + .flatMap((node) => node.children) + .join(""); +} + +function press(renderer: TestRenderer.ReactTestRenderer, label: string): void { + const button = renderer.root.findByProps({ accessibilityLabel: label }); + button.props.onPress(); +} + +describe("DictionarySettingsScreen", () => { + beforeEach(async () => { + await i18nReady; + await act(async () => { + await i18n.changeLanguage("en"); + }); + }); + it.each([ + ["en", "Dictionaries", "English", "Download English dictionary"], + ["zh-TW", "字典", "英語", "英語字典下載"], + ["fr", "Dictionaries", "English", "Download English dictionary"], + ])( + "renders real resources with English fallback in %s", + async (language, title, name, action) => { + await act(async () => { + await i18n.changeLanguage(language); + }); + const { store } = makeStore(); + let renderer!: TestRenderer.ReactTestRenderer; + await act(async () => { + renderer = TestRenderer.create(); + }); + expect(textContent(renderer)).toContain(title); + expect(textContent(renderer)).toContain(name); + expect(renderer.root.findByProps({ accessibilityLabel: action })).toBeTruthy(); + expect(JSON.stringify(renderer.toJSON())).not.toMatch(/(?:reader\.)?dictionary\./); + await act(async () => renderer.unmount()); + }, + ); + + it("renders both language rows after exactly one initial remote-first refresh", async () => { + const calls: string[] = []; + let resolveInitialization!: () => void; + const initialize = vi.fn( + () => + new Promise((resolve) => { + calls.push("initialize"); + resolveInitialization = resolve; + }), + ); + const refreshManifest = vi.fn(async () => { + calls.push("refreshManifest"); + }); + const { store: orderedStore, actions: orderedActions } = makeStore( + {}, + { initialize, refreshManifest }, + ); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create(); + }); + + expect(calls).toEqual(["initialize"]); + await act(async () => { + resolveInitialization(); + await Promise.resolve(); + }); + + expect(textContent(renderer)).toContain("English"); + expect(textContent(renderer)).toContain("Chinese"); + expect(textContent(renderer)).toContain("Not downloaded"); + expect(textContent(renderer)).toContain("1.5 MB"); + expect(textContent(renderer)).toContain("2.5 MB"); + expect(orderedActions.initialize).toHaveBeenCalledTimes(1); + expect(orderedActions.refreshManifest).not.toHaveBeenCalled(); + expect(calls).toEqual(["initialize"]); + }); + + it("shows unavailable without a download action when the manifest is absent", async () => { + const { store, actions } = makeStore(); + store.setState({ manifest: null }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create(); + }); + + expect(textContent(renderer)).toContain("Unavailable"); + expect( + renderer.root.findAll( + (node) => node.props.accessibilityLabel === "Download English dictionary", + ), + ).toHaveLength(0); + expect(actions.install).not.toHaveBeenCalled(); + }); + + it("keeps download and update actions scoped to the selected language", async () => { + const { store, actions } = makeStore({ + zh: { + state: "update-available", + installedVersion: "2026.08", + availableVersion: "2026.09", + sizeBytes: 2_621_440, + }, + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create(); + }); + await act(async () => { + press(renderer, "Download English dictionary"); + press(renderer, "Update Chinese dictionary"); + }); + + expect(actions.install).toHaveBeenNthCalledWith(1, "en"); + expect(actions.install).toHaveBeenNthCalledWith(2, "zh"); + expect(textContent(renderer)).toContain("Update available"); + expect(textContent(renderer)).toContain("2026.08"); + expect(textContent(renderer)).toContain("2026.09"); + }); + + it("renders progress, installed metadata, retry, and a removal confirmation for their own rows", async () => { + const { store, actions } = makeStore({ + en: { state: "downloading", progress: 0.37 }, + zh: { state: "installed", version: "2026.09", sizeBytes: 2_621_440 }, + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create(); + }); + expect(textContent(renderer)).toContain("37%"); + expect(textContent(renderer)).toContain("Installed"); + expect(textContent(renderer)).toContain("2026.09"); + + await act(async () => press(renderer, "Remove Chinese dictionary")); + const confirm = alert.mock.calls + .at(-1)?.[2] + ?.find((button: { style?: string }) => button.style === "destructive"); + await act(async () => confirm?.onPress()); + expect(actions.remove).toHaveBeenCalledWith("zh"); + }); + + it("shows a localized per-pack error, logs its detail, retries only that language, and opens attribution", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { store, actions } = makeStore({ + en: { state: "error", message: "network unavailable", hasActivePack: false }, + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create(); + }); + expect(textContent(renderer)).toContain("English dictionary couldn't be prepared. Try again."); + expect(textContent(renderer)).not.toContain("network unavailable"); + expect(warn).toHaveBeenCalledWith("[DictionarySettings] dictionary pack error", { + error: "network unavailable", + language: "en", + }); + await act(async () => { + press(renderer, "Retry English dictionary"); + press(renderer, "English dictionary attribution"); + }); + + expect(actions.install).toHaveBeenCalledWith("en"); + expect(actions.retry).not.toHaveBeenCalled(); + expect(openURL).toHaveBeenCalledWith(descriptors.en.attributionUrl); + warn.mockRestore(); + }); + + it("offers both repair and removal when the active pack is corrupt", async () => { + const { store, actions } = makeStore({ + en: { state: "error", message: "invalid schema", hasActivePack: true } as never, + }); + let renderer!: TestRenderer.ReactTestRenderer; + + await act(async () => { + renderer = TestRenderer.create(); + }); + await act(async () => { + press(renderer, "Repair English dictionary"); + press(renderer, "Remove English dictionary"); + }); + + expect(actions.install).toHaveBeenCalledWith("en"); + expect(alert).toHaveBeenCalled(); + }); +}); diff --git a/packages/app-expo/src/screens/settings/DictionarySettingsScreen.tsx b/packages/app-expo/src/screens/settings/DictionarySettingsScreen.tsx new file mode 100644 index 000000000..e05a6dbcb --- /dev/null +++ b/packages/app-expo/src/screens/settings/DictionarySettingsScreen.tsx @@ -0,0 +1,343 @@ +import { KeyboardAwareScrollView } from "@/components/ui/KeyboardAwareScrollView"; +import { useResponsiveLayout } from "@/hooks/use-responsive-layout"; +import { useDictionaryStore } from "@/stores"; +import { type ThemeColors, fontSize, fontWeight, radius, spacing, useColors } from "@/styles/theme"; +import type { DictionaryLanguage, DictionaryPackDescriptor } from "@readany/core/dictionary"; +import { useEffect } from "react"; +import { useTranslation } from "react-i18next"; +import { + ActivityIndicator, + Alert, + Linking, + StyleSheet, + Text, + TouchableOpacity, + View, +} from "react-native"; +import { SafeAreaView } from "react-native-safe-area-context"; +import type { StoreApi, UseBoundStore } from "zustand"; +import type { DictionaryStoreState } from "../../stores/dictionary-store"; +import { SettingsHeader } from "./SettingsHeader"; + +type DictionaryStore = UseBoundStore>; + +export interface DictionarySettingsScreenProps { + /** Allows a configured store to be supplied by the application or tests. */ + dictionaryStore?: DictionaryStore; +} + +const languages: DictionaryLanguage[] = ["en", "zh"]; + +function formatBytes(sizeBytes: number): string { + if (sizeBytes < 1024 * 1024) return `${Math.round(sizeBytes / 1024)} KB`; + return `${(sizeBytes / (1024 * 1024)).toFixed(1)} MB`; +} + +function DictionaryPackRow({ + descriptor, + language, + pack, + onInstall, + onRemove, + onRetry, +}: { + descriptor?: DictionaryPackDescriptor; + language: DictionaryLanguage; + pack: DictionaryStoreState["packs"][DictionaryLanguage]; + onInstall: (language: DictionaryLanguage) => void; + onRemove: (language: DictionaryLanguage) => void; + onRetry: (language: DictionaryLanguage) => void; +}) { + const colors = useColors(); + const styles = makeStyles(colors); + const { t } = useTranslation(); + const name = t(`dictionary.${language === "en" ? "english" : "chinese"}`); + const actionLabel = (action: string) => t("dictionary.actionLabel", { action, language: name }); + const descriptorSize = descriptor ? formatBytes(descriptor.sizeBytes) : null; + const unavailable = !descriptor && pack.state === "not-installed"; + + const status = (() => { + if (unavailable) return t("dictionary.unavailable"); + + switch (pack.state) { + case "verifying": + return t("dictionary.verifying"); + case "downloading": + return t("dictionary.downloading", { + progress: Math.round(pack.progress * 100), + }); + case "installed": + return t("dictionary.installedStatus", { + installed: t("dictionary.installed"), + size: formatBytes(pack.sizeBytes), + version: t("dictionary.version", { version: pack.version }), + }); + case "update-available": + return t("dictionary.updateStatus", { + availableVersion: pack.availableVersion, + installedVersion: pack.installedVersion, + updateAvailable: t("dictionary.updateAvailable"), + }); + case "error": + return t("dictionary.packError", { language: name }); + default: + return t("dictionary.notDownloaded"); + } + })(); + + return ( + + + {name} + {status} + {(pack.state === "not-installed" || pack.state === "update-available") && descriptorSize ? ( + {t("dictionary.size", { size: descriptorSize })} + ) : null} + {descriptor ? ( + + void Linking.openURL(descriptor.attributionUrl)} + > + {t("dictionary.attribution")} + + + {"· "} + {t("dictionary.licenseDetail", { + label: t("dictionary.license"), + license: descriptor.license, + })} + + + ) : null} + + + + {pack.state === "not-installed" && !unavailable ? ( + onInstall(language)} + /> + ) : null} + {pack.state === "downloading" || pack.state === "verifying" ? ( + + ) : null} + {pack.state === "installed" ? ( + onRemove(language)} + /> + ) : null} + {pack.state === "update-available" ? ( + <> + onInstall(language)} + /> + onRemove(language)} + /> + + ) : null} + {pack.state === "error" ? ( + <> + {descriptor ? ( + onRetry(language)} + /> + ) : null} + {pack.hasActivePack ? ( + onRemove(language)} + /> + ) : null} + + ) : null} + + + ); +} + +function ActionButton({ + accessibilityLabel, + destructive = false, + label, + onPress, +}: { + accessibilityLabel: string; + destructive?: boolean; + label: string; + onPress: () => void; +}) { + const colors = useColors(); + const styles = makeStyles(colors); + return ( + + {label} + + ); +} + +export function DictionarySettingsScreen({ + dictionaryStore = useDictionaryStore, +}: DictionarySettingsScreenProps) { + const colors = useColors(); + const styles = makeStyles(colors); + const layout = useResponsiveLayout(); + const { t } = useTranslation(); + const manifest = dictionaryStore((state) => state.manifest); + const packs = dictionaryStore((state) => state.packs); + const initialize = dictionaryStore((state) => state.initialize); + const install = dictionaryStore((state) => state.install); + const remove = dictionaryStore((state) => state.remove); + + useEffect(() => { + void initialize().catch((error) => + console.warn("[DictionarySettings] failed to initialize dictionary packs", error), + ); + }, [initialize]); + + useEffect(() => { + for (const language of languages) { + const pack = packs[language]; + if (pack.state === "error") { + console.warn("[DictionarySettings] dictionary pack error", { + error: pack.message, + language, + }); + } + } + }, [packs]); + + const handleInstall = (language: DictionaryLanguage) => { + void install(language).catch((error) => + console.warn(`[DictionarySettings] failed to install ${language} dictionary`, error), + ); + }; + + const handleRemove = (language: DictionaryLanguage) => { + const languageName = t(`dictionary.${language === "en" ? "english" : "chinese"}`); + Alert.alert( + t("dictionary.removeTitle", { language: languageName }), + t("dictionary.removeMessage", { + language: languageName, + }), + [ + { text: t("dictionary.cancel"), style: "cancel" }, + { + text: t("dictionary.remove"), + style: "destructive", + onPress: () => + void remove(language).catch((error) => + console.warn(`[DictionarySettings] failed to remove ${language} dictionary`, error), + ), + }, + ], + ); + }; + + return ( + + + + + {t("dictionary.manageDictionaries")} + + {languages.map((language, index) => ( + + + + ))} + + + + + ); +} + +const makeStyles = (colors: ThemeColors) => + StyleSheet.create({ + container: { flex: 1 }, + scroll: { flex: 1 }, + scrollContent: { padding: spacing.lg, paddingBottom: 56 }, + contentColumn: {}, + sectionTitle: { + color: colors.foreground, + fontSize: fontSize.base, + fontWeight: fontWeight.semibold, + marginBottom: 10, + }, + listCard: { + backgroundColor: colors.card, + borderColor: colors.border, + borderRadius: radius.xl, + borderWidth: 1, + overflow: "hidden", + }, + rowBorder: { borderBottomColor: colors.border, borderBottomWidth: StyleSheet.hairlineWidth }, + packRow: { + alignItems: "flex-start", + flexDirection: "row", + gap: spacing.md, + justifyContent: "space-between", + padding: spacing.lg, + }, + packCopy: { flex: 1, gap: 3 }, + packName: { color: colors.foreground, fontSize: fontSize.base, fontWeight: fontWeight.medium }, + statusText: { color: colors.mutedForeground, fontSize: fontSize.sm, lineHeight: 20 }, + metaText: { color: colors.mutedForeground, fontSize: fontSize.xs, lineHeight: 18 }, + licenseLine: { alignItems: "center", flexDirection: "row", flexWrap: "wrap", marginTop: 2 }, + linkText: { color: colors.primary, fontSize: fontSize.xs, lineHeight: 18 }, + actions: { alignItems: "flex-end", gap: spacing.sm }, + actionButton: { + borderColor: colors.primary, + borderRadius: radius.lg, + borderWidth: 1, + paddingHorizontal: 10, + paddingVertical: 7, + }, + actionLabel: { color: colors.primary, fontSize: fontSize.xs, fontWeight: fontWeight.semibold }, + destructiveButton: { borderColor: colors.destructive }, + destructiveLabel: { color: colors.destructive }, + }); diff --git a/packages/app-expo/src/screens/settings/dictionary-locales.test.ts b/packages/app-expo/src/screens/settings/dictionary-locales.test.ts new file mode 100644 index 000000000..97b3045cd --- /dev/null +++ b/packages/app-expo/src/screens/settings/dictionary-locales.test.ts @@ -0,0 +1,107 @@ +import i18n, { i18nReady } from "@readany/core/i18n"; +import { describe, expect, it } from "vitest"; +import enReader from "../../../../core/src/i18n/locales/en/reader.json"; +import zhTwReader from "../../../../core/src/i18n/locales/zh-TW/reader.json"; +import zhReader from "../../../../core/src/i18n/locales/zh/reader.json"; + +const requiredDictionaryKeys = [ + "define", + "title", + "close", + "loadingDefinition", + "dictionaries", + "verifying", + "english", + "chinese", + "download", + "update", + "remove", + "retry", + "repair", + "retryLookup", + "manageDictionaries", + "notDownloaded", + "noDefinitionFound", + "unsupportedSelection", + "lookupError", + "downloadDefinition", + "downloadingDefinition", + "downloadAccessibility", + "downloadingAccessibility", + "downloading", + "installed", + "installedStatus", + "updateAvailable", + "updateStatus", + "error", + "packError", + "unavailable", + "version", + "size", + "attribution", + "attributionLabel", + "license", + "licenseDetail", + "actionLabel", + "statusLabel", + "removeTitle", + "removeMessage", + "cancel", +] as const; + +const requiredDictionaryTemplates = { + actionLabel: ["action", "language"], + attributionLabel: ["language"], + downloading: ["progress"], + downloadingDefinition: ["language", "progress"], + downloadDefinition: ["language", "size"], + downloadAccessibility: ["language"], + downloadingAccessibility: ["language"], + installedStatus: ["installed", "version", "size"], + licenseDetail: ["label", "license"], + packError: ["language"], + removeMessage: ["language"], + removeTitle: ["language"], + size: ["size"], + statusLabel: ["language", "status"], + updateStatus: ["updateAvailable", "installedVersion", "availableVersion"], + version: ["version"], +} as const; + +describe("dictionary locale copy", () => { + it.each(["en", "zh", "zh-TW", "ja", "ko", "fr", "es"])( + "resolves every dictionary key through the app resources in %s", + async (language) => { + await i18nReady; + const t = i18n.getFixedT(language); + for (const key of requiredDictionaryKeys) { + expect(t(`dictionary.${key}`)).not.toBe(`dictionary.${key}`); + } + }, + ); + + it("falls back to English for dictionary copy missing from French", async () => { + await i18nReady; + expect(i18n.getResource("fr", "translation", "dictionary.download")).toBeUndefined(); + expect(i18n.getFixedT("fr")("dictionary.download")).toBe("Download"); + }); + + it.each([ + ["English", enReader], + ["Simplified Chinese", zhReader], + ["Traditional Chinese", zhTwReader], + ])("provides every dictionary string in %s", (_language, reader) => { + const dictionary = reader.dictionary as Record | undefined; + + for (const key of requiredDictionaryKeys) { + expect(dictionary?.[key], `${_language} dictionary.${key}`).toEqual(expect.any(String)); + expect((dictionary?.[key] as string).trim()).not.toBe(""); + } + + for (const [key, variables] of Object.entries(requiredDictionaryTemplates)) { + for (const variable of variables) { + expect(dictionary?.[key], `${_language} dictionary.${key}`).toContain(`{{${variable}}}`); + } + } + }); +}); diff --git a/packages/app-expo/src/stores/dictionary-store.test.ts b/packages/app-expo/src/stores/dictionary-store.test.ts new file mode 100644 index 000000000..0b49b4a41 --- /dev/null +++ b/packages/app-expo/src/stores/dictionary-store.test.ts @@ -0,0 +1,332 @@ +import type { DictionaryManifest } from "@readany/core/dictionary"; +import { describe, expect, it, vi } from "vitest"; +import * as dictionaryStoreModule from "./dictionary-store"; + +const { createDictionaryStore } = dictionaryStoreModule; + +const manifest: DictionaryManifest = { + manifestVersion: 1, + packs: { + en: { + language: "en", + version: "1.0.0", + schemaVersion: 1, + sourceEdition: "enwiktionary", + sourceDumpDate: "2026-09-01", + sizeBytes: 12, + sha256: "a".repeat(64), + url: "https://example.test/en", + sourceArchiveUrl: "https://example.test/en-source", + attributionUrl: "https://example.test/license", + license: "CC BY-SA 4.0", + }, + zh: { + language: "zh", + version: "1.0.0", + schemaVersion: 1, + sourceEdition: "zhwiktionary", + sourceDumpDate: "2026-09-01", + sizeBytes: 12, + sha256: "b".repeat(64), + url: "https://example.test/zh", + sourceArchiveUrl: "https://example.test/zh-source", + attributionUrl: "https://example.test/license", + license: "CC BY-SA 4.0", + }, + }, +}; + +function manager() { + return { + refresh: vi + .fn() + .mockResolvedValue({ en: { state: "not-installed" }, zh: { state: "not-installed" } }), + install: vi.fn().mockResolvedValue(undefined), + remove: vi.fn().mockResolvedValue(undefined), + }; +} + +function lookupService() { + return { lookup: vi.fn().mockResolvedValue([]) }; +} + +describe("dictionary store", () => { + it("completes lookup from bundled data without waiting for an in-flight remote refresh", async () => { + let resolveRemote!: (value: DictionaryManifest) => void; + const remote = vi.fn( + () => + new Promise((resolve) => { + resolveRemote = resolve; + }), + ); + const bundled = vi.fn().mockResolvedValue(manifest); + const m = manager(); + const lookup = lookupService(); + const store = createDictionaryStore({ + manager: m, + lookup, + fetchRemoteManifest: remote, + getBundledManifest: bundled, + }); + let initializeFinished = false; + const initialize = store + .getState() + .initialize() + .then(() => { + initializeFinished = true; + }); + + await expect(store.getState().lookup("desire")).resolves.toEqual([]); + + expect(initializeFinished).toBe(false); + expect(remote).toHaveBeenCalledOnce(); + expect(bundled).toHaveBeenCalledOnce(); + expect(lookup.lookup).toHaveBeenCalledWith("desire"); + expect(m.refresh).toHaveBeenCalledTimes(1); + + resolveRemote(manifest); + await initialize; + expect(m.refresh).toHaveBeenCalledTimes(2); + }); + + it("starts an explicit remote refresh even when bundled lookup readiness is in flight", async () => { + let resolveBundled!: (value: DictionaryManifest) => void; + const bundled = vi.fn( + () => + new Promise((resolve) => { + resolveBundled = resolve; + }), + ); + const remote = vi.fn().mockResolvedValue(manifest); + const store = createDictionaryStore({ + manager: manager(), + lookup: lookupService(), + fetchRemoteManifest: remote, + getBundledManifest: bundled, + }); + + const lookup = store.getState().lookup("desire"); + await vi.waitFor(() => expect(bundled).toHaveBeenCalledOnce()); + const initialize = store.getState().initialize(); + + await vi.waitFor(() => expect(remote).toHaveBeenCalledOnce()); + resolveBundled(manifest); + await Promise.all([lookup, initialize]); + }); + + it("initializes remote-first and parses the remote value", async () => { + const remote = vi.fn().mockResolvedValue(structuredClone(manifest)); + const bundled = vi.fn(); + const m = manager(); + const store = createDictionaryStore({ + manager: m, + lookup: lookupService(), + fetchRemoteManifest: remote, + getBundledManifest: bundled, + }); + + await store.getState().initialize(); + + expect(remote).toHaveBeenCalledOnce(); + expect(bundled).not.toHaveBeenCalled(); + expect(m.refresh).toHaveBeenCalledWith(manifest); + expect(store.getState().manifest).toEqual(manifest); + }); + + it("falls back to a parsed bundled manifest when the remote value is invalid", async () => { + const invalidRemote = { ...structuredClone(manifest), unexpected: true }; + const bundled = vi.fn().mockResolvedValue(structuredClone(manifest)); + const m = manager(); + const store = createDictionaryStore({ + manager: m, + lookup: lookupService(), + fetchRemoteManifest: vi.fn().mockResolvedValue(invalidRemote), + getBundledManifest: bundled, + }); + + await store.getState().initialize(); + + expect(bundled).toHaveBeenCalledOnce(); + expect(m.refresh).toHaveBeenCalledWith(manifest); + }); + + it("falls back to bundled when the remote fetch fails", async () => { + const bundled = vi.fn().mockResolvedValue(structuredClone(manifest)); + const m = manager(); + const store = createDictionaryStore({ + manager: m, + lookup: lookupService(), + fetchRemoteManifest: vi.fn().mockRejectedValue(new Error("offline")), + getBundledManifest: bundled, + }); + + await store.getState().refreshManifest(); + + expect(bundled).toHaveBeenCalledOnce(); + expect(store.getState().manifest).toEqual(manifest); + }); + + it("rejects when both remote and bundled values are invalid", async () => { + const store = createDictionaryStore({ + manager: manager(), + lookup: lookupService(), + fetchRemoteManifest: vi.fn().mockResolvedValue({ nope: true }), + getBundledManifest: vi.fn().mockResolvedValue({ stillNope: true }), + }); + + await expect(store.getState().initialize()).rejects.toThrow("manifest"); + }); + + it("clears a failed explicit refresh so retry performs a new remote-first attempt", async () => { + const remote = vi.fn().mockResolvedValueOnce({ nope: true }).mockResolvedValueOnce(manifest); + const bundled = vi.fn().mockResolvedValueOnce({ stillNope: true }); + const store = createDictionaryStore({ + manager: manager(), + lookup: lookupService(), + fetchRemoteManifest: remote, + getBundledManifest: bundled, + }); + + await expect(store.getState().initialize()).rejects.toThrow("manifest"); + await expect(store.getState().retry()).resolves.toBeUndefined(); + + expect(remote).toHaveBeenCalledTimes(2); + expect(bundled).toHaveBeenCalledOnce(); + }); + + it("does not hide manager refresh failures behind bundled fallback", async () => { + const refreshError = new Error("filesystem recovery failed"); + const m = manager(); + m.refresh.mockRejectedValueOnce(refreshError); + const bundled = vi.fn().mockResolvedValue(manifest); + const store = createDictionaryStore({ + manager: m, + lookup: lookupService(), + fetchRemoteManifest: vi.fn().mockResolvedValue(manifest), + getBundledManifest: bundled, + }); + + await expect(store.getState().initialize()).rejects.toBe(refreshError); + expect(bundled).not.toHaveBeenCalled(); + }); + + it("exposes the composed lookup service through the same store", async () => { + const lookup = lookupService(); + lookup.lookup.mockResolvedValueOnce([ + { + id: 1, + language: "en", + headword: "desire", + partOfSpeech: "noun", + senses: [{ order: 0, definition: "an inclination to want things" }], + }, + ]); + const store = createDictionaryStore({ + manager: manager(), + lookup, + fetchRemoteManifest: vi.fn().mockResolvedValue(manifest), + getBundledManifest: vi.fn().mockResolvedValue(manifest), + }); + + await expect(store.getState().lookup("desire")).resolves.toMatchObject([ + { headword: "desire" }, + ]); + expect(lookup.lookup).toHaveBeenCalledWith("desire"); + }); + + it("loads only the bundled manifest before the first lookup so lookup cannot reach the network", async () => { + const lookup = lookupService(); + lookup.lookup.mockRejectedValueOnce( + Object.assign(new Error("English pack is not installed"), { code: "pack-not-installed" }), + ); + const remote = vi.fn().mockResolvedValue(manifest); + const bundled = vi.fn().mockResolvedValue(manifest); + const m = manager(); + const store = createDictionaryStore({ + manager: m, + lookup, + fetchRemoteManifest: remote, + getBundledManifest: bundled, + }); + + await expect(store.getState().lookup("desire")).rejects.toMatchObject({ + code: "pack-not-installed", + }); + + expect(remote).not.toHaveBeenCalled(); + expect(bundled).toHaveBeenCalledOnce(); + expect(m.refresh).toHaveBeenCalledWith(manifest); + expect(store.getState().manifest).toEqual(manifest); + }); + + it("loads one shared runtime for manager operations and lookup", async () => { + expect(dictionaryStoreModule).toHaveProperty("createRuntimeBackedDictionaryStore"); + const createRuntimeBackedDictionaryStore = Reflect.get( + dictionaryStoreModule, + "createRuntimeBackedDictionaryStore", + ); + const m = manager(); + const lookup = lookupService(); + const loadRuntime = vi.fn().mockResolvedValue({ manager: m, lookup }); + const store = createRuntimeBackedDictionaryStore({ + loadRuntime, + fetchRemoteManifest: vi.fn().mockResolvedValue(manifest), + getBundledManifest: vi.fn().mockResolvedValue(manifest), + }); + + await store.getState().initialize(); + await store.getState().lookup("desire"); + await store.getState().remove("en"); + + expect(loadRuntime).toHaveBeenCalledOnce(); + expect(m.refresh).toHaveBeenCalledWith(manifest); + expect(lookup.lookup).toHaveBeenCalledWith("desire"); + expect(m.remove).toHaveBeenCalledWith("en"); + }); + + it("clears a rejected runtime construction so Retry can construct it again", async () => { + const m = manager(); + const lookup = lookupService(); + const loadRuntime = vi + .fn() + .mockRejectedValueOnce(new Error("native module unavailable")) + .mockResolvedValueOnce({ manager: m, lookup }); + const store = dictionaryStoreModule.createRuntimeBackedDictionaryStore({ + loadRuntime, + fetchRemoteManifest: vi.fn().mockResolvedValue(manifest), + getBundledManifest: vi.fn().mockResolvedValue(manifest), + }); + + await expect(store.getState().initialize()).rejects.toThrow("native module unavailable"); + await expect(store.getState().retry()).resolves.toBeUndefined(); + + expect(loadRuntime).toHaveBeenCalledTimes(2); + expect(store.getState().manifest).toEqual(manifest); + }); + + it("coalesces explicit remote manifest refreshes while they are in flight", async () => { + let resolveRemote!: (value: DictionaryManifest) => void; + const remote = vi.fn( + () => + new Promise((resolve) => { + resolveRemote = resolve; + }), + ); + const m = manager(); + const store = createDictionaryStore({ + manager: m, + lookup: lookupService(), + fetchRemoteManifest: remote, + getBundledManifest: vi.fn().mockResolvedValue(manifest), + }); + + const initialize = store.getState().initialize(); + const refresh = store.getState().refreshManifest(); + expect(remote).toHaveBeenCalledOnce(); + resolveRemote(manifest); + await Promise.all([initialize, refresh]); + + expect(remote).toHaveBeenCalledOnce(); + expect(m.refresh).toHaveBeenCalledOnce(); + }); +}); diff --git a/packages/app-expo/src/stores/dictionary-store.ts b/packages/app-expo/src/stores/dictionary-store.ts new file mode 100644 index 000000000..c08ab3a32 --- /dev/null +++ b/packages/app-expo/src/stores/dictionary-store.ts @@ -0,0 +1,35 @@ +import { + type DictionaryRuntime, + createRuntimeBackedDictionaryStore, +} from "@readany/core/dictionary/dictionary-store"; +export * from "@readany/core/dictionary/dictionary-store"; +import { + DICTIONARY_BUNDLED_MANIFEST, + DICTIONARY_REMOTE_MANIFEST_URL, +} from "../config/dictionary-config"; +async function loadExpoDictionaryRuntime(): Promise { + const [databaseModule, platformModule, runtimeModule] = await Promise.all([ + import("../lib/dictionary/dictionary-database"), + import("../lib/dictionary/dictionary-pack-platform"), + import("../lib/dictionary/dictionary-runtime"), + ]); + return runtimeModule.createDictionaryRuntime({ + database: new databaseModule.ExpoDictionaryDatabaseAdapter(), + directory: platformModule.dictionaryPackDirectory, + platform: platformModule.createExpoDictionaryPackPlatform(), + }); +} + +async function fetchRemoteDictionaryManifest(): Promise { + const response = await fetch(DICTIONARY_REMOTE_MANIFEST_URL); + if (!response.ok) { + throw new Error(`Dictionary manifest request failed with HTTP ${response.status}`); + } + return response.json(); +} + +export const useDictionaryStore = createRuntimeBackedDictionaryStore({ + loadRuntime: loadExpoDictionaryRuntime, + fetchRemoteManifest: fetchRemoteDictionaryManifest, + getBundledManifest: async () => DICTIONARY_BUNDLED_MANIFEST, +}); diff --git a/packages/app-expo/src/stores/index.ts b/packages/app-expo/src/stores/index.ts index 178c88d11..bc5f900ae 100644 --- a/packages/app-expo/src/stores/index.ts +++ b/packages/app-expo/src/stores/index.ts @@ -28,4 +28,7 @@ export type { UpdateState } from "./update-store"; export { useVectorModelStore } from "./vector-model-store"; export type { BuiltinModelStatus, BuiltinModelState, VectorModelState } from "./vector-model-store"; +export { useDictionaryStore } from "./dictionary-store"; +export type { DictionaryStoreState } from "./dictionary-store"; + export { debouncedSave, loadFromFS, flushAllWrites, withPersist } from "./persist"; diff --git a/packages/app/package.json b/packages/app/package.json index 65791aa10..3eb44238d 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -7,7 +7,8 @@ "dev": "vite", "build": "tsc && vite build", "preview": "vite preview", - "tauri": "tauri" + "tauri": "tauri", + "test:dictionary": "vitest run --config vitest.dictionary.config.ts" }, "dependencies": { "@huggingface/transformers": "^3.8.1", @@ -86,6 +87,9 @@ "@vitejs/plugin-react": "^4.6.0", "tailwindcss": "^4.0.0", "typescript": "~5.8.3", - "vite": "^7.0.4" + "vite": "^7.0.4", + "vitest": "^4.1.2", + "react-test-renderer": "19.1.0", + "@types/react-test-renderer": "19.1.0" } } diff --git a/packages/app/src-tauri/capabilities/default.json b/packages/app/src-tauri/capabilities/default.json index 8a5d896e2..9471c45b7 100644 --- a/packages/app/src-tauri/capabilities/default.json +++ b/packages/app/src-tauri/capabilities/default.json @@ -21,6 +21,7 @@ "fs:allow-write", "fs:allow-exists", "fs:allow-mkdir", + "fs:allow-stat", "fs:allow-read-file", "fs:allow-read-text-file", "fs:allow-write-file", diff --git a/packages/app/src-tauri/src/dictionary.rs b/packages/app/src-tauri/src/dictionary.rs new file mode 100644 index 000000000..0d1d32e5c --- /dev/null +++ b/packages/app/src-tauri/src/dictionary.rs @@ -0,0 +1,314 @@ +use rusqlite::{params_from_iter, types::ValueRef, Connection, OpenFlags}; +use serde_json::{Map, Value}; +use std::path::{Path, PathBuf}; +use tauri::Manager; + +fn pack_path(directory: &Path, path: &Path) -> Result { + let directory = directory.canonicalize().map_err(|e| e.to_string())?; + let path = path.canonicalize().map_err(|e| e.to_string())?; + if path.parent() != Some(directory.as_path()) { + return Err("Dictionary path must be inside the dictionaries directory".into()); + } + Ok(path) +} + +fn query_pack( + path: &Path, + query: &str, + values: &[String], +) -> Result>, String> { + // Opening read-only also avoids journal-mode changes invalidating the pack checksum. + let connection = Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY) + .map_err(|e| e.to_string())?; + let mut statement = connection.prepare(query).map_err(|e| e.to_string())?; + if !statement.readonly() { + return Err("Dictionary queries must be read-only".into()); + } + let columns: Vec = statement + .column_names() + .iter() + .map(|name| name.to_string()) + .collect(); + let mut rows = statement + .query(params_from_iter(values)) + .map_err(|e| e.to_string())?; + let mut result = Vec::new(); + while let Some(row) = rows.next().map_err(|e| e.to_string())? { + let mut object = Map::new(); + for (index, name) in columns.iter().enumerate() { + let value = match row.get_ref(index).map_err(|e| e.to_string())? { + ValueRef::Null => Value::Null, + ValueRef::Integer(value) => Value::from(value), + ValueRef::Real(value) => Value::from(value), + ValueRef::Text(value) => { + Value::from(std::str::from_utf8(value).map_err(|e| e.to_string())?) + } + ValueRef::Blob(_) => return Err("Unexpected binary dictionary value".into()), + }; + object.insert(name.clone(), value); + } + result.push(object); + } + Ok(result) +} + +#[tauri::command] +pub async fn dictionary_query( + app: tauri::AppHandle, + path: String, + query: String, + values: Vec, +) -> Result>, String> { + let directory = app + .path() + .app_data_dir() + .map_err(|e| e.to_string())? + .join("dictionaries"); + tauri::async_runtime::spawn_blocking(move || { + let path = pack_path(&directory, Path::new(&path))?; + query_pack(&path, &query, &values) + }) + .await + .map_err(|e| e.to_string())? +} + +const MAX_PACK_BYTES: u64 = 150 * 1024 * 1024; + +#[derive(Clone, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DownloadProgress { + received_bytes: u64, + total_bytes: Option, +} + +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DownloadReport { + bytes: u64, + elapsed_ms: u128, +} + +fn download_path(directory: &Path, path: &Path) -> Result { + let directory = directory.canonicalize().map_err(|e| e.to_string())?; + let parent = path + .parent() + .ok_or("Dictionary download has no directory")? + .canonicalize() + .map_err(|e| e.to_string())?; + if parent != directory + || !path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.ends_with(".sqlite.download")) + { + return Err("Dictionary downloads must be staged inside the dictionaries directory".into()); + } + Ok(parent.join( + path.file_name() + .ok_or("Dictionary download has no filename")?, + )) +} + +async fn download_pack( + url: &str, + path: &Path, + mut on_progress: impl FnMut(DownloadProgress), +) -> Result { + use std::time::{Duration, Instant}; + use tokio::io::AsyncWriteExt; + let started = Instant::now(); + let client = tauri_plugin_http::reqwest::Client::builder() + .https_only(url.starts_with("https://")) + .connect_timeout(Duration::from_secs(15)) + .timeout(Duration::from_secs(300)) + .build() + .map_err(|e| e.to_string())?; + let mut response = client + .get(url) + .send() + .await + .map_err(|e| e.to_string())? + .error_for_status() + .map_err(|e| e.to_string())?; + let total_bytes = response.content_length(); + if total_bytes.is_some_and(|size| size > MAX_PACK_BYTES) { + return Err("Dictionary download is too large".into()); + } + // Never follow an existing staging-file symlink or overwrite another download. + let file = tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .await + .map_err(|e| e.to_string())?; + let result = async { + let mut writer = tokio::io::BufWriter::with_capacity(1024 * 1024, file); + let mut received_bytes = 0; + let mut last_progress = Instant::now(); + on_progress(DownloadProgress { + received_bytes, + total_bytes, + }); + while let Some(chunk) = response.chunk().await.map_err(|e| e.to_string())? { + received_bytes += chunk.len() as u64; + if received_bytes > MAX_PACK_BYTES { + return Err("Dictionary download is too large".to_string()); + } + writer.write_all(&chunk).await.map_err(|e| e.to_string())?; + if last_progress.elapsed() >= Duration::from_millis(100) { + on_progress(DownloadProgress { + received_bytes, + total_bytes, + }); + last_progress = Instant::now(); + } + } + writer.flush().await.map_err(|e| e.to_string())?; + on_progress(DownloadProgress { + received_bytes, + total_bytes, + }); + Ok(DownloadReport { + bytes: received_bytes, + elapsed_ms: started.elapsed().as_millis(), + }) + } + .await; + if result.is_err() { + // The writer has dropped, releasing its handle before Windows cleanup. + let _ = tokio::fs::remove_file(path).await; + } + result +} + +#[tauri::command] +pub async fn dictionary_download( + app: tauri::AppHandle, + url: String, + path: String, + on_progress: tauri::ipc::Channel, +) -> Result { + let parsed = tauri_plugin_http::reqwest::Url::parse(&url).map_err(|e| e.to_string())?; + if parsed.scheme() != "https" { + return Err("Dictionary download URL must use HTTPS".into()); + } + let directory = app + .path() + .app_data_dir() + .map_err(|e| e.to_string())? + .join("dictionaries"); + let path = download_path(&directory, Path::new(&path))?; + download_pack(&url, &path, |progress| { + // Closing a dialog should not cancel an installation already in progress. + let _ = on_progress.send(progress); + }) + .await +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_directory() -> PathBuf { + let directory = + std::env::temp_dir().join(format!("readany-download-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&directory).unwrap(); + directory + } + + fn serve_once(status: &str, length: usize, body: Vec) -> String { + use std::io::{Read, Write}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + let header = + format!("HTTP/1.1 {status}\r\nContent-Length: {length}\r\nConnection: close\r\n\r\n"); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0; 4096]; + let _ = stream.read(&mut request); + if stream.write_all(header.as_bytes()).is_ok() { + let _ = stream.write_all(&body); + } + }); + format!("http://{address}/pack") + } + + #[tokio::test] + async fn streams_native_bytes_and_limits_progress_updates() { + let directory = test_directory(); + let path = directory.join("pack.sqlite.download"); + let bytes = vec![42; 2 * 1024 * 1024]; + let url = serve_once("200 OK", bytes.len(), bytes.clone()); + let mut updates = Vec::new(); + let report = download_pack(&url, &path, |progress| updates.push(progress)) + .await + .unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + assert_eq!(report.bytes, bytes.len() as u64); + assert_eq!(updates.first().unwrap().received_bytes, 0); + assert_eq!(updates.last().unwrap().received_bytes, report.bytes); + assert!(updates.len() as u128 <= report.elapsed_ms / 100 + 2); + std::fs::rename(&path, directory.join("done.sqlite")).unwrap(); + std::fs::remove_dir_all(directory).unwrap(); + } + + #[tokio::test] + async fn rejects_failed_and_oversized_responses_and_cleans_partial_downloads() { + let directory = test_directory(); + let path = directory.join("pack.sqlite.download"); + for (status, length, bytes) in [ + ("404 Not Found", 0, vec![]), + ("200 OK", MAX_PACK_BYTES as usize + 1, vec![]), + ("200 OK", 10, vec![1, 2, 3]), + ] { + let url = serve_once(status, length, bytes); + assert!(download_pack(&url, &path, |_| {}).await.is_err()); + assert!(!path.exists()); + } + std::fs::remove_dir_all(directory).unwrap(); + } + + #[tokio::test] + async fn preserves_existing_staging_files_and_restricts_destination() { + let directory = test_directory(); + let path = directory.join("pack.sqlite.download"); + assert!(download_path(&directory, &path).is_ok()); + assert!(download_path(&directory, &directory.join("active.sqlite")).is_err()); + assert!(download_path(&directory, &directory.join("../outside.sqlite.download")).is_err()); + std::fs::write(&path, b"existing").unwrap(); + let url = serve_once("200 OK", 1, vec![1]); + assert!(download_pack(&url, &path, |_| {}).await.is_err()); + assert_eq!(std::fs::read(&path).unwrap(), b"existing"); + std::fs::remove_dir_all(directory).unwrap(); + } + + #[test] + fn reads_bound_values_without_changing_pack_and_releases_handles() { + let directory = + std::env::temp_dir().join(format!("readany-dictionary-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&directory).unwrap(); + let path = directory.join("test.sqlite"); + let connection = Connection::open(&path).unwrap(); + connection + .execute_batch( + "CREATE TABLE entries (headword TEXT); INSERT INTO entries VALUES ('hello');", + ) + .unwrap(); + drop(connection); + let before = std::fs::read(&path).unwrap(); + let rows = query_pack( + &path, + "SELECT headword FROM entries WHERE headword = ?", + &["hello".into()], + ) + .unwrap(); + assert_eq!(rows[0]["headword"], "hello"); + assert!(query_pack(&path, "DELETE FROM entries", &[]).is_err()); + assert_eq!(before, std::fs::read(&path).unwrap()); + assert!(pack_path(&directory.join("missing"), &path).is_err()); + assert!(pack_path(&directory, &path).is_ok()); + assert!(pack_path(&std::env::temp_dir(), &path).is_err()); + std::fs::rename(&path, directory.join("moved.sqlite")).unwrap(); + std::fs::remove_dir_all(directory).unwrap(); + } +} diff --git a/packages/app/src-tauri/src/lib.rs b/packages/app/src-tauri/src/lib.rs index 382c6a415..47ea855d2 100644 --- a/packages/app/src-tauri/src/lib.rs +++ b/packages/app/src-tauri/src/lib.rs @@ -1,4 +1,5 @@ mod db; +mod dictionary; mod readany_cli; mod storage; mod sync; @@ -30,6 +31,8 @@ pub fn run() { db: Mutex::new(None), }) .invoke_handler(tauri::generate_handler![ + dictionary::dictionary_query, + dictionary::dictionary_download, sync::commands::sync_vacuum_into, sync::commands::sync_integrity_check, sync::commands::sync_hash_file, diff --git a/packages/app/src/components/reader/ChapterTranslationBar.tsx b/packages/app/src/components/reader/ChapterTranslationBar.tsx index fa3f5f3fd..dc15c40b0 100644 --- a/packages/app/src/components/reader/ChapterTranslationBar.tsx +++ b/packages/app/src/components/reader/ChapterTranslationBar.tsx @@ -1,3 +1,4 @@ +import { Progress } from "@/components/ui/progress"; /** * ChapterTranslationMenu — dropdown menu attached to the toolbar Languages button. * @@ -22,8 +23,8 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; -import type { ChapterTranslationState } from "@readany/core/hooks"; import { useSettingsStore } from "@/stores/settings-store"; +import type { ChapterTranslationState } from "@readany/core/hooks"; import type { TranslationTargetLang } from "@readany/core/types/translation"; import { TRANSLATOR_LANGS } from "@readany/core/types/translation"; import { Check, Eye, EyeOff, Languages, Loader2, Trash2, X } from "lucide-react"; @@ -126,12 +127,14 @@ export function ChapterTranslationMenu({ })} -
-
-
+
({ + Dialog: ({ children }: { children: React.ReactNode }) =>
{children}
, + DialogContent: ({ children }: { children: React.ReactNode }) =>
{children}
, + DialogTitle: ({ children }: { children: React.ReactNode }) =>

{children}

, + DialogDescription: ({ children }: { children: React.ReactNode }) =>

{children}

, +})); +const mounted: TestRenderer.ReactTestRenderer[] = []; +beforeAll(async () => { + await i18nReady; + await i18n.changeLanguage("en"); +}); +afterEach(async () => { + await act(async () => { + for (const view of mounted.splice(0)) view.unmount(); + }); +}); +async function render(element: React.ReactElement) { + let view!: TestRenderer.ReactTestRenderer; + await act(async () => { + view = TestRenderer.create(element); + }); + mounted.push(view); + return view; +} +function dependencies() { + return { + lookup: vi.fn().mockResolvedValue([]), + install: vi.fn().mockResolvedValue(undefined), + getDescriptor: () => DICTIONARY_BUNDLED_MANIFEST.packs.en, + }; +} + +describe("desktop definition dialog", () => { + it("shows an offline result and closes before opening dictionary management", async () => { + const deps = dependencies(); + deps.lookup.mockResolvedValue([ + { + id: 1, + language: "en", + headword: "hello", + partOfSpeech: "noun", + senses: [{ order: 1, definition: "a greeting" }], + }, + ]); + const events: string[] = []; + const view = await render( + events.push("close")} + onManageDictionaries={() => events.push("manage")} + />, + ); + expect(JSON.stringify(view.toJSON())).toContain("a greeting"); + await act(async () => { + view.root + .findAllByType("button") + .find((button) => button.children.includes("Manage Dictionaries")) + ?.props.onClick(); + }); + expect(events).toEqual(["close", "manage"]); + }); + it("offers a download for a missing pack and retries the original selection", async () => { + const deps = dependencies(); + deps.lookup + .mockRejectedValueOnce(Object.assign(new Error("missing"), { code: "pack-not-installed" })) + .mockResolvedValueOnce([]); + const view = await render( + , + ); + expect(deps.install).not.toHaveBeenCalled(); + await act(async () => { + view.root + .findAllByType("button") + .find((button) => button.children.includes("Download")) + ?.props.onClick(); + }); + expect(deps.install).toHaveBeenCalledOnce(); + expect(deps.lookup).toHaveBeenLastCalledWith("hello"); + expect(JSON.stringify(view.toJSON())).toContain("No definition found"); + }); + it("offers retry on lookup failure", async () => { + const deps = dependencies(); + deps.lookup.mockRejectedValueOnce(new Error("failed")); + const view = await render( + , + ); + expect(view.root.findByProps({ role: "alert" }).children.join("")).toContain("lookup failed"); + await act(async () => { + view.root + .findAllByType("button") + .find((button) => button.children.includes("Retry")) + ?.props.onClick(); + }); + expect(deps.lookup).toHaveBeenCalledTimes(2); + }); + it("discards pending results when unmounted", async () => { + const deps = dependencies(); + let resolve!: (rows: unknown[]) => void; + deps.lookup.mockImplementation( + () => + new Promise((done) => { + resolve = done; + }), + ); + const controller = new DefinitionController(deps); + const view = await render( + , + ); + await act(async () => view.unmount()); + await act(async () => resolve([])); + expect(controller.state).toEqual({ kind: "idle" }); + }); + it("only offers Define for supported selections, including in PDFs", async () => { + const callback = vi.fn(); + const props = { + position: { x: 0, y: 0 }, + onHighlight: callback, + onRemoveHighlight: callback, + onNote: callback, + onCopy: callback, + onTranslate: callback, + onAskAI: callback, + onSpeak: callback, + onClose: callback, + onDefine: callback, + }; + const view = await render(); + expect(view.root.findByProps({ "aria-label": "Define" })).toBeDefined(); + await act(async () => + view.update(), + ); + expect(view.root.findAllByProps({ "aria-label": "Define" })).toHaveLength(0); + }); +}); diff --git a/packages/app/src/components/reader/DefinitionDialog.tsx b/packages/app/src/components/reader/DefinitionDialog.tsx new file mode 100644 index 000000000..574e3ccc0 --- /dev/null +++ b/packages/app/src/components/reader/DefinitionDialog.tsx @@ -0,0 +1,148 @@ +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog"; +import { Progress } from "@/components/ui/progress"; +import { useDictionaryStore } from "@/stores/dictionary-store"; +import { + DefinitionController, + type DefinitionState, +} from "@readany/core/dictionary/definition-controller"; +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; + +export function DefinitionDialog({ + text, + onClose, + onManageDictionaries, + controller: suppliedController, +}: { + text: string; + onClose(): void; + onManageDictionaries(): void; + controller?: DefinitionController; +}) { + const { t } = useTranslation(); + const [controller] = useState( + () => + suppliedController ?? + new DefinitionController({ + lookup: (text) => useDictionaryStore.getState().lookup(text), + getDescriptor: (language) => useDictionaryStore.getState().manifest?.packs[language], + install: async (descriptor, onProgress, onVerifying) => { + const unsubscribe = useDictionaryStore.subscribe((state) => { + const status = state.packs[descriptor.language]; + if (status.state === "downloading") onProgress(status.progress); + if (status.state === "verifying") onVerifying?.(); + }); + try { + await useDictionaryStore.getState().install(descriptor.language); + } finally { + unsubscribe(); + } + }, + }), + ); + const [state, setState] = useState(controller.state); + useEffect(() => { + const unsubscribe = controller.subscribe(setState); + void controller.open(text); + return () => { + unsubscribe(); + controller.close(); + }; + }, [controller, text]); + const language = + state.kind === "missing-pack" || state.kind === "downloading" + ? t(`dictionary.${state.language === "en" ? "english" : "chinese"}`) + : ""; + return ( + { + if (!open) onClose(); + }} + > + + {t("dictionary.title")} +
+ {state.kind === "loading" && {t("dictionary.loadingDefinition")}} + {state.kind === "verifying" && {t("dictionary.verifying")}} + {state.kind === "unsupported" &&

{t("dictionary.unsupportedSelection")}

} + {state.kind === "no-match" &&

{t("dictionary.noDefinitionFound")}

} + {state.kind === "error" && ( +
+

{t("dictionary.lookupError")}

+ +
+ )} + {state.kind === "missing-pack" && ( +
+

+ {t("dictionary.downloadDefinition", { + language, + size: `${(state.descriptor.sizeBytes / 1024 / 1024).toFixed(1)} MB`, + })} +

+ +
+ )} + {state.kind === "downloading" && ( +
+ + {t("dictionary.downloadingDefinition", { + language, + progress: Math.round(state.progress * 100), + })} + + +
+ )} + {state.kind === "result" && ( + <> +

{state.displayText}

+ {state.entries.map((entry) => ( +
+

+ {[ + ...new Set( + [entry.headword, entry.simplified, entry.traditional].filter(Boolean), + ), + ].join(" / ")} +

+ {entry.pronunciation && ( +

{entry.pronunciation}

+ )} + {entry.partOfSpeech && ( +

{entry.partOfSpeech}

+ )} +
    + {entry.senses.map((sense) => ( +
  1. {sense.definition}
    2. + ))} +
+
+ ))} + + )} +
+
+ +
+ + + ); +} diff --git a/packages/app/src/components/reader/ReaderView.tsx b/packages/app/src/components/reader/ReaderView.tsx index ade0140b9..8822d4bb7 100644 --- a/packages/app/src/components/reader/ReaderView.tsx +++ b/packages/app/src/components/reader/ReaderView.tsx @@ -43,6 +43,7 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { toast } from "sonner"; import { BookmarkRibbon } from "./BookmarkRibbon"; +import { DefinitionDialog } from "./DefinitionDialog"; import type { BookSelection, FoliateViewerHandle, RelocateDetail, TOCItem } from "./FoliateViewer"; import { FoliateViewer } from "./FoliateViewer"; import { FooterBar } from "./FooterBar"; @@ -799,6 +800,10 @@ export function ReaderView({ bookId, tabId }: ReaderViewProps) { const [currentPage, setCurrentPage] = useState(0); const [showChat, setShowChat] = useState(false); const [showSettings, setShowSettings] = useState(false); + const [definition, setDefinition] = useState<{ bookId: string; text: string } | null>(null); + useEffect(() => { + setDefinition((current) => (current?.bookId === bookId ? current : null)); + }, [bookId]); const [showTTS, setShowTTS] = useState(false); const [isReimporting, setIsReimporting] = useState(false); const [ttsSourceKind, setTtsSourceKind] = useState<"page" | "selection">("page"); @@ -3035,6 +3040,10 @@ export function ReaderView({ bookId, tabId }: ReaderViewProps) { onRemoveHighlight={handleRemoveHighlight} onNote={handleNote} onCopy={handleCopy} + onDefine={() => { + setDefinition({ bookId, text: selection.text }); + handleCloseSelection(); + }} onTranslate={handleTranslate} onAskAI={handleAskAI} onSpeak={handleSpeakSelection} @@ -3042,6 +3051,17 @@ export function ReaderView({ bookId, tabId }: ReaderViewProps) { /> )} + {definition?.bookId === bookId && ( + setDefinition(null)} + onManageDictionaries={() => + useAppStore.getState().setShowSettings(true, "dictionaries") + } + /> + )} + {/* Translation popover */} {showTranslation && translationText && ( void; onCopy: () => void; onTranslate: () => void; + onDefine?: () => void; onAskAI: () => void; onSpeak: () => void; onClose: () => void; @@ -38,7 +41,7 @@ const POPOVER_MARGIN = 8; export function SelectionPopover({ position, - selectedText: _selectedText, + selectedText, annotated = false, currentColor, defaultColor = "yellow", @@ -48,6 +51,7 @@ export function SelectionPopover({ onNote, onCopy, onTranslate, + onDefine, onAskAI, onSpeak, onClose, @@ -90,6 +94,9 @@ export function SelectionPopover({ }, { icon: NotebookPen, label: t("reader.note"), onClick: onNote, disabled: isPdf }, { icon: Copy, label: t("common.copy"), onClick: onCopy }, + ...(onDefine && prepareDictionarySelection(selectedText).ok + ? [{ icon: BookOpen, label: t("dictionary.define"), onClick: onDefine }] + : []), { icon: Languages, label: t("reader.translate"), onClick: onTranslate }, { icon: Sparkles, label: t("reader.askAI"), onClick: onAskAI }, { icon: Headphones, label: t("tts.speakSelection"), onClick: onSpeak }, @@ -180,6 +187,7 @@ export function SelectionPopover({ "text-muted-foreground hover:bg-destructive/10 hover:text-destructive", )} title={btn.label} + aria-label={btn.label} onClick={btn.disabled ? undefined : btn.onClick} disabled={btn.disabled} > diff --git a/packages/app/src/components/settings/AboutSettings.tsx b/packages/app/src/components/settings/AboutSettings.tsx index 8407de6e2..fe51583c7 100644 --- a/packages/app/src/components/settings/AboutSettings.tsx +++ b/packages/app/src/components/settings/AboutSettings.tsx @@ -7,6 +7,7 @@ import { DialogHeader, DialogTitle, } from "@/components/ui/dialog"; +import { Progress } from "@/components/ui/progress"; import { checkForUpdate, downloadAndInstall, @@ -157,9 +158,7 @@ export function AboutSettings() { {t("settings.downloading")} {progress}% -
-
-
+
)} diff --git a/packages/app/src/components/settings/DictionarySettings.dictionary.test.tsx b/packages/app/src/components/settings/DictionarySettings.dictionary.test.tsx new file mode 100644 index 000000000..9fcbac51f --- /dev/null +++ b/packages/app/src/components/settings/DictionarySettings.dictionary.test.tsx @@ -0,0 +1,88 @@ +import { useDictionaryStore } from "@/stores/dictionary-store"; +import { DICTIONARY_BUNDLED_MANIFEST } from "@readany/core/dictionary/dictionary-config"; +import i18n, { i18nReady } from "@readany/core/i18n"; +import type React from "react"; +import TestRenderer, { act } from "react-test-renderer"; +import { afterEach, beforeAll, beforeEach, expect, it, vi } from "vitest"; +import { DictionarySettings } from "./DictionarySettings"; +(globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = + true; +vi.mock("@/components/ui/dialog", () => ({ + Dialog: ({ children, open }: { children: React.ReactNode; open: boolean }) => + open ?
{children}
: null, + DialogContent: ({ children }: { children: React.ReactNode }) =>
{children}
, + DialogTitle: ({ children }: { children: React.ReactNode }) =>

{children}

, + DialogDescription: ({ children }: { children: React.ReactNode }) =>

{children}

, +})); +const install = vi.fn(); +const remove = vi.fn(); +let view: TestRenderer.ReactTestRenderer; +beforeAll(async () => { + await i18nReady; + await i18n.changeLanguage("en"); +}); +beforeEach(() => { + vi.resetAllMocks(); + useDictionaryStore.setState({ + manifest: DICTIONARY_BUNDLED_MANIFEST, + packs: { en: { state: "not-installed" }, zh: { state: "not-installed" } }, + initialize: async () => {}, + install, + remove, + }); +}); +afterEach(async () => { + await act(async () => view?.unmount()); +}); +it("downloads the selected language and displays source attribution", async () => { + await act(async () => { + view = TestRenderer.create(); + }); + expect(JSON.stringify(view.toJSON())).toContain("WordNet 3.1 License"); + await act(async () => + view.root + .findAllByType("button") + .find((button) => button.props["aria-label"] === "Download English dictionary") + ?.props.onClick(), + ); + expect(install).toHaveBeenCalledWith("en"); +}); +it("requires confirmation before removing a dictionary", async () => { + useDictionaryStore.setState({ + packs: { + en: { state: "installed", version: "1.0.0", sizeBytes: 42 }, + zh: { state: "not-installed" }, + }, + }); + await act(async () => { + view = TestRenderer.create(); + }); + await act(async () => + view.root + .findAllByType("button") + .find((button) => button.props["aria-label"] === "Remove English dictionary") + ?.props.onClick(), + ); + expect(remove).not.toHaveBeenCalled(); + expect(JSON.stringify(view.toJSON())).toContain("Remove English dictionary?"); + await act(async () => + view.root + .findAllByType("button") + .find((button) => !button.props["aria-label"] && button.children.includes("Remove")) + ?.props.onClick(), + ); + expect(remove).toHaveBeenCalledWith("en"); +}); +it("displays failed installations without an unhandled rejection", async () => { + install.mockRejectedValue(new Error("disk full")); + await act(async () => { + view = TestRenderer.create(); + }); + await act(async () => + view.root + .findAllByType("button") + .find((button) => button.props["aria-label"] === "Download English dictionary") + ?.props.onClick(), + ); + expect(view.root.findByProps({ role: "alert" }).children).toContain("Error"); +}); diff --git a/packages/app/src/components/settings/DictionarySettings.tsx b/packages/app/src/components/settings/DictionarySettings.tsx new file mode 100644 index 000000000..d3df3970c --- /dev/null +++ b/packages/app/src/components/settings/DictionarySettings.tsx @@ -0,0 +1,200 @@ +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogDescription, DialogTitle } from "@/components/ui/dialog"; +import { Progress } from "@/components/ui/progress"; +import { useDictionaryStore } from "@/stores/dictionary-store"; +import type { DictionaryLanguage } from "@readany/core/dictionary"; +import { Download, Loader2, RefreshCw, Trash2, Wrench } from "lucide-react"; +import { useEffect, useState } from "react"; +import { useTranslation } from "react-i18next"; + +export function DictionarySettings() { + const { t } = useTranslation(); + const { manifest, packs, initialize, install, remove } = useDictionaryStore(); + const [error, setError] = useState(false); + const [busy, setBusy] = useState(false); + const [removing, setRemoving] = useState(null); + const run = async (operation: () => Promise) => { + setBusy(true); + setError(false); + try { + await operation(); + } catch { + setError(true); + } finally { + setBusy(false); + } + }; + useEffect(() => { + let active = true; + void initialize().catch(() => { + if (active) setError(true); + }); + return () => { + active = false; + }; + }, [initialize]); + return ( +
+ {error && ( +

+ {t("dictionary.error")} +

+ )} + {(["en", "zh"] as const).map((language) => { + const descriptor = manifest?.packs[language]; + const status = packs[language]; + const name = t(`dictionary.${language === "en" ? "english" : "chinese"}`); + const installed = + status.state === "installed" || + status.state === "update-available" || + (status.state === "error" && status.hasActivePack); + const downloading = status.state === "downloading"; + const installing = downloading || status.state === "verifying"; + const action = + status.state === "update-available" + ? "update" + : status.state === "error" + ? "repair" + : "download"; + const ActionIcon = installing + ? Loader2 + : action === "update" + ? RefreshCw + : action === "repair" + ? Wrench + : Download; + const statusText = t( + `dictionary.${status.state === "not-installed" ? "notDownloaded" : status.state === "update-available" ? "updateAvailable" : status.state}`, + { progress: downloading ? Math.round(status.progress * 100) : 0 }, + ); + return ( +
+
+
+

{name}

+ {statusText} +
+
+ {status.state !== "installed" && ( + + )} + {installed && ( + + )} +
+
+ {downloading && ( + + )} + {descriptor && ( +
+

+ {t("dictionary.version", { + version: + status.state === "installed" + ? status.version + : status.state === "update-available" + ? status.installedVersion + : descriptor.version, + })}{" "} + /{" "} + {t("dictionary.size", { + size: `${((status.state === "installed" ? status.sizeBytes : descriptor.sizeBytes) / 1024 / 1024).toFixed(1)} MB`, + })} +

+

+ {descriptor.sourceEdition} / {descriptor.sourceDumpDate} +

+

+ {t("dictionary.licenseDetail", { + label: t("dictionary.license"), + license: descriptor.license, + })} +

+ { + event.preventDefault(); + void run(async () => { + const { openUrl } = await import("@tauri-apps/plugin-opener"); + await openUrl(descriptor.attributionUrl); + }); + }} + > + {t("dictionary.attribution")} + +
+ )} +
+ ); + })} + { + if (!open) setRemoving(null); + }} + > + + + {t("dictionary.removeTitle", { + language: t(`dictionary.${removing === "en" ? "english" : "chinese"}`), + })} + + + {t("dictionary.removeMessage", { + language: t(`dictionary.${removing === "en" ? "english" : "chinese"}`), + })} + +
+ + +
+ + +
+ ); +} diff --git a/packages/app/src/components/settings/SettingsDialog.tsx b/packages/app/src/components/settings/SettingsDialog.tsx index 57eb767c4..3ad02ee6c 100644 --- a/packages/app/src/components/settings/SettingsDialog.tsx +++ b/packages/app/src/components/settings/SettingsDialog.tsx @@ -9,10 +9,11 @@ import { useEffect, useState } from "react"; import { useTranslation } from "react-i18next"; import { AISettings } from "./AISettings"; import { AboutSettings } from "./AboutSettings"; +import { DictionarySettings } from "./DictionarySettings"; +import { ExternalAISettings } from "./ExternalAISettings"; import { FeedbackSettings } from "./FeedbackSettings"; import { FontSettings } from "./FontSettings"; import { GeneralSettings } from "./GeneralSettings"; -import { ExternalAISettings } from "./ExternalAISettings"; import { ReadSettingsPanel } from "./ReadSettings"; import { SyncSettings } from "./SyncSettings"; import { TTSSettings } from "./TTSSettings"; @@ -28,6 +29,7 @@ const TAB_IDS: SettingsTab[] = [ "general", "reading", "fonts", + "dictionaries", "ai", "vectorModel", "tts", @@ -41,6 +43,7 @@ const TAB_KEYS: Record = { general: "settings.general", reading: "settings.reading", fonts: "settings.fonts", + dictionaries: "dictionary.dictionaries", ai: "settings.ai", vectorModel: "settings.vectorModel", tts: "settings.tts", @@ -118,6 +121,7 @@ export function SettingsDialog({ open, onClose }: SettingsDialogProps) { {settingsTab === "general" && } {settingsTab === "reading" && } {settingsTab === "fonts" && } + {settingsTab === "dictionaries" && } {settingsTab === "ai" && } {settingsTab === "vectorModel" && } {settingsTab === "tts" && } diff --git a/packages/app/src/components/settings/SyncSettings.tsx b/packages/app/src/components/settings/SyncSettings.tsx index 67d86fb0b..aae7ac68b 100644 --- a/packages/app/src/components/settings/SyncSettings.tsx +++ b/packages/app/src/components/settings/SyncSettings.tsx @@ -1,4 +1,5 @@ import { PasswordInput } from "@/components/ui/password-input"; +import { Progress } from "@/components/ui/progress"; import { Switch } from "@/components/ui/switch"; import { useSyncStore } from "@/stores/sync-store"; import { getPlatformService } from "@readany/core/services"; @@ -755,18 +756,11 @@ export function SyncSettings() { {statusLabel() &&

{statusLabel()}

} {isBusy && progress && (
-
- {progress.phase === "database" ? ( -
- ) : ( -
- )} -
+

{progressLabel()}

)} diff --git a/packages/app/src/components/ui/progress.tsx b/packages/app/src/components/ui/progress.tsx new file mode 100644 index 000000000..77400cfc5 --- /dev/null +++ b/packages/app/src/components/ui/progress.tsx @@ -0,0 +1,28 @@ +import { cn } from "@readany/core/utils"; + +interface ProgressProps { + /** Percentage from 0 to 100; omit when progress is unknown. */ + value?: number | null; + className?: string; + "aria-label": string; +} + +export function Progress({ value, className, "aria-label": label }: ProgressProps) { + const percent = + typeof value === "number" && Number.isFinite(value) + ? Math.min(100, Math.max(0, value)) + : undefined; + return ( +
+ + + ); +} diff --git a/packages/app/src/lib/dictionary/desktop-dictionary.ts b/packages/app/src/lib/dictionary/desktop-dictionary.ts new file mode 100644 index 000000000..04d38b7b0 --- /dev/null +++ b/packages/app/src/lib/dictionary/desktop-dictionary.ts @@ -0,0 +1,96 @@ +import type { DictionaryLanguage } from "@readany/core/dictionary"; +import { + type DictionaryDatabaseAdapter, + type DictionaryDatabaseConnection, + DictionaryLookupError, +} from "@readany/core/dictionary/dictionary-database"; +import type { DictionaryPackPlatform } from "@readany/core/dictionary/dictionary-pack-manager"; +import { createDictionaryRuntime } from "@readany/core/dictionary/dictionary-runtime"; +import { validateDictionaryDatabase } from "@readany/core/dictionary/dictionary-validation"; +import { Channel, invoke } from "@tauri-apps/api/core"; +import { appDataDir, join } from "@tauri-apps/api/path"; +import { exists, mkdir, remove, rename, stat } from "@tauri-apps/plugin-fs"; + +function query(path: string, sql: string, values: unknown[] = []): Promise { + return invoke("dictionary_query", { path, query: sql, values }); +} + +export class DesktopDictionaryDatabaseAdapter implements DictionaryDatabaseAdapter { + async open(language: DictionaryLanguage, path: string): Promise { + const metadata = await query<{ key: string; value: string }>( + path, + "SELECT key, value FROM metadata WHERE key IN ('schema_version', 'language')", + ); + const values = new Map(metadata.map((row) => [row.key, row.value])); + if (values.get("schema_version") !== "1" || values.get("language") !== language) { + throw new DictionaryLookupError("pack-invalid", "Dictionary metadata does not match"); + } + let closed = false; + const pending = new Set>(); + return { + getAllAsync(sql: string, ...params: unknown[]): Promise { + if (closed) return Promise.reject(new Error("Dictionary connection is closed")); + const operation = query(path, sql, params); + pending.add(operation); + void operation.then( + () => pending.delete(operation), + () => pending.delete(operation), + ); + return operation; + }, + async closeAsync() { + closed = true; + // Native queries own their handles; wait for all of them before replacing files. + await Promise.allSettled([...pending]); + }, + }; + } +} + +export function createDesktopDictionaryPackPlatform(): DictionaryPackPlatform { + return { + ensureDirectory: (path) => mkdir(path, { recursive: true }), + exists, + size: async (path) => (await stat(path)).size, + sha256: (path) => invoke("sync_hash_file", { path }), + async download(url, path, onProgress) { + let acceptingProgress = true; + const progress = new Channel<{ receivedBytes: number; totalBytes: number | null }>(); + progress.onmessage = ({ receivedBytes, totalBytes }) => { + if (acceptingProgress) + onProgress(totalBytes && totalBytes > 0 ? Math.min(receivedBytes / totalBytes, 1) : 0); + }; + try { + const report = await invoke<{ bytes: number; elapsedMs: number }>("dictionary_download", { + url, + path, + onProgress: progress, + }); + onProgress(1); + console.info(`[Dictionary] Transferred ${report.bytes} bytes in ${report.elapsedMs} ms`); + } finally { + acceptingProgress = false; + } + }, + readMetadata: (path) => + validateDictionaryDatabase({ + getAllAsync: (sql: string) => query(path, sql), + getFirstAsync: async (sql: string) => (await query(path, sql))[0] ?? null, + }), + async move(from, to) { + if (await exists(to)) throw new Error(`Dictionary move target already exists: ${to}`); + await rename(from, to); + }, + async remove(path) { + if (await exists(path)) await remove(path); + }, + }; +} + +export async function loadDesktopDictionaryRuntime() { + return createDictionaryRuntime({ + database: new DesktopDictionaryDatabaseAdapter(), + directory: await join(await appDataDir(), "dictionaries"), + platform: createDesktopDictionaryPackPlatform(), + }); +} diff --git a/packages/app/src/lib/dictionary/desktop.dictionary.test.ts b/packages/app/src/lib/dictionary/desktop.dictionary.test.ts new file mode 100644 index 000000000..203194dd0 --- /dev/null +++ b/packages/app/src/lib/dictionary/desktop.dictionary.test.ts @@ -0,0 +1,110 @@ +import { invoke } from "@tauri-apps/api/core"; +import { exists, rename } from "@tauri-apps/plugin-fs"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + DesktopDictionaryDatabaseAdapter, + createDesktopDictionaryPackPlatform, +} from "./desktop-dictionary"; +vi.mock("@tauri-apps/api/core", () => ({ + invoke: vi.fn(), + Channel: class { + onmessage = (_message: unknown) => {}; + }, +})); +vi.mock("@tauri-apps/plugin-fs", () => ({ + create: vi.fn(), + exists: vi.fn(), + rename: vi.fn(), + mkdir: vi.fn(), + remove: vi.fn(), + stat: vi.fn(), +})); +beforeEach(() => vi.resetAllMocks()); + +describe("desktop dictionary files", () => { + it("delegates transfer to native code and forwards bounded progress", async () => { + let emit!: (value: unknown) => void; + vi.mocked(invoke).mockImplementationOnce(async (_command, args) => { + const channel = (args as { onProgress: { onmessage(value: unknown): void } }).onProgress; + emit = channel.onmessage; + channel.onmessage({ receivedBytes: 1, totalBytes: 3 }); + channel.onmessage({ receivedBytes: 3, totalBytes: 3 }); + return { bytes: 3, elapsedMs: 10 }; + }); + const progress = vi.fn(); + await createDesktopDictionaryPackPlatform().download( + "https://example.com/pack", + "pack", + progress, + ); + expect(invoke).toHaveBeenCalledOnce(); + expect(invoke).toHaveBeenCalledWith( + "dictionary_download", + expect.objectContaining({ url: "https://example.com/pack", path: "pack" }), + ); + expect(progress).toHaveBeenCalledWith(1 / 3); + expect(progress).toHaveBeenLastCalledWith(1); + const count = progress.mock.calls.length; + emit({ receivedBytes: 1, totalBytes: 3 }); + expect(progress).toHaveBeenCalledTimes(count); + }); + it("propagates native transfer failures", async () => { + vi.mocked(invoke).mockRejectedValue(new Error("disk full")); + await expect( + createDesktopDictionaryPackPlatform().download("url", "pack", vi.fn()), + ).rejects.toThrow("disk full"); + }); + it("does not overwrite an existing pack during promotion", async () => { + vi.mocked(exists).mockResolvedValue(true); + await expect(createDesktopDictionaryPackPlatform().move("staged", "active")).rejects.toThrow( + "already exists", + ); + expect(rename).not.toHaveBeenCalled(); + }); +}); + +describe("desktop dictionary queries", () => { + it("rejects a pack with the wrong language", async () => { + vi.mocked(invoke).mockResolvedValue([ + { key: "schema_version", value: "1" }, + { key: "language", value: "zh" }, + ]); + await expect(new DesktopDictionaryDatabaseAdapter().open("en", "pack")).rejects.toMatchObject({ + code: "pack-invalid", + }); + }); + it("waits for active native queries before closing and rejects later queries", async () => { + vi.mocked(invoke).mockResolvedValueOnce([ + { key: "schema_version", value: "1" }, + { key: "language", value: "en" }, + ]); + const connection = await new DesktopDictionaryDatabaseAdapter().open("en", "pack"); + let finish!: (rows: unknown[]) => void; + vi.mocked(invoke).mockImplementationOnce( + () => + new Promise((resolve) => { + finish = resolve; + }), + ); + const query = connection.getAllAsync( + "SELECT headword FROM entries WHERE headword = ?", + "hello", + ); + let closed = false; + const closing = connection.closeAsync().then(() => { + closed = true; + }); + await Promise.resolve(); + expect(closed).toBe(false); + expect(invoke).toHaveBeenLastCalledWith("dictionary_query", { + path: "pack", + query: "SELECT headword FROM entries WHERE headword = ?", + values: ["hello"], + }); + finish([{ headword: "hello" }]); + await query; + await closing; + expect(closed).toBe(true); + await expect(connection.getAllAsync("SELECT 1")).rejects.toThrow("closed"); + }); +}); diff --git a/packages/app/src/stores/dictionary-store.ts b/packages/app/src/stores/dictionary-store.ts new file mode 100644 index 000000000..4a36303bf --- /dev/null +++ b/packages/app/src/stores/dictionary-store.ts @@ -0,0 +1,21 @@ +import { + DEFAULT_DICTIONARY_MANIFEST_URL, + DICTIONARY_BUNDLED_MANIFEST, +} from "@readany/core/dictionary/dictionary-config"; +import { createRuntimeBackedDictionaryStore } from "@readany/core/dictionary/dictionary-store"; + +export const DICTIONARY_REMOTE_MANIFEST_URL = + import.meta.env.VITE_DICTIONARY_MANIFEST_URL?.trim() || DEFAULT_DICTIONARY_MANIFEST_URL; + +export const useDictionaryStore = createRuntimeBackedDictionaryStore({ + loadRuntime: async () => + (await import("@/lib/dictionary/desktop-dictionary")).loadDesktopDictionaryRuntime(), + fetchRemoteManifest: async () => { + const { fetch } = await import("@tauri-apps/plugin-http"); + const response = await fetch(DICTIONARY_REMOTE_MANIFEST_URL); + if (!response.ok) + throw new Error(`Dictionary manifest request failed: HTTP ${response.status}`); + return response.json(); + }, + getBundledManifest: async () => DICTIONARY_BUNDLED_MANIFEST, +}); diff --git a/packages/app/tsconfig.json b/packages/app/tsconfig.json index 04eb5481d..46930c95e 100644 --- a/packages/app/tsconfig.json +++ b/packages/app/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "ES2020", "useDefineForClassFields": true, - "lib": ["ES2020", "DOM", "DOM.Iterable"], + "lib": ["ES2021", "DOM", "DOM.Iterable"], "module": "ESNext", "skipLibCheck": true, diff --git a/packages/app/vitest.dictionary.config.ts b/packages/app/vitest.dictionary.config.ts new file mode 100644 index 000000000..15bf25327 --- /dev/null +++ b/packages/app/vitest.dictionary.config.ts @@ -0,0 +1,7 @@ +import { fileURLToPath } from "node:url"; +import { defineConfig } from "vitest/config"; +export default defineConfig({ + resolve: { alias: { "@": fileURLToPath(new URL("./src", import.meta.url)) } }, + test: { environment: "node", include: ["src/**/*.dictionary.test.{ts,tsx}"] }, + esbuild: { jsx: "automatic" }, +}); diff --git a/packages/cli/package.json b/packages/cli/package.json index ca553f995..616132368 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -23,6 +23,8 @@ "acceptance:validate": "node scripts/validate-acceptance.mjs", "preflight:release": "node scripts/release-preflight.mjs", "dev": "tsx src/bin/readany.ts", + "dictionary:build": "tsx scripts/build-dictionary-pack.ts", + "dictionary:convert-wordnet": "tsx scripts/convert-wordnet.ts", "test": "vitest run", "test:watch": "vitest", "check": "tsc -p tsconfig.json --noEmit" diff --git a/packages/cli/scripts/build-dictionary-pack.ts b/packages/cli/scripts/build-dictionary-pack.ts new file mode 100644 index 000000000..d7861e87b --- /dev/null +++ b/packages/cli/scripts/build-dictionary-pack.ts @@ -0,0 +1,184 @@ +import { access, readFile, stat, writeFile } from "node:fs/promises"; +import { dirname, extname, isAbsolute } from "node:path"; +import { + type DictionaryPackDescriptor, + type DictionarySource, + parseDictionaryManifest, +} from "@readany/core/dictionary"; +import { buildDictionaryPack } from "../src/dictionary/pack-builder.js"; + +const REQUIRED_FLAGS = [ + "language", + "input", + "output", + "version", + "source-edition", + "license", + "source-date", + "source-archive-url", + "attribution-url", + "license-file", + "creator-attribution", + "asset-url", + "descriptor", +] as const; + +type RequiredFlag = (typeof REQUIRED_FLAGS)[number]; +type ParsedArguments = Record; + +function usage(message: string): never { + throw new Error( + `${message}\nUsage: dictionary:build --language en|zh --input absolute-jsonl-path --output absolute-sqlite-path --version semver --source-edition wordnet-3.1|enwiktionary|zhwiktionary --license "WordNet 3.1 License"|"CC BY-SA 4.0" --source-date YYYY-MM-DD --source-archive-url https-url --attribution-url https-url --license-file absolute-txt-path --creator-attribution text --asset-url https-url --descriptor absolute-json-path`, + ); +} + +function parseArguments(argv: string[]): ParsedArguments { + const values = new Map(); + for (let index = 0; index < argv.length; index += 2) { + const flag = argv[index]; + const value = argv[index + 1]; + if (!flag?.startsWith("--")) usage(`Unexpected argument: ${flag ?? ""}`); + const key = flag.slice(2) as RequiredFlag; + if (!REQUIRED_FLAGS.includes(key) || !value || value.startsWith("--")) { + usage(`Invalid argument: ${flag}`); + } + if (values.has(key)) usage(`Duplicate argument: ${flag}`); + values.set(key, value); + } + + for (const key of REQUIRED_FLAGS) { + if (!values.has(key)) usage(`Missing required argument: --${key}`); + } + return Object.fromEntries(values) as ParsedArguments; +} + +function validateUrl(value: string, name: string): void { + try { + if (new URL(value).protocol !== "https:") usage(`${name} must be an https URL`); + } catch { + usage(`${name} must be an https URL`); + } +} + +function validateDate(value: string): void { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) usage("--source-date must use YYYY-MM-DD"); + const date = new Date(`${value}T00:00:00.000Z`); + if (Number.isNaN(date.getTime()) || date.toISOString().slice(0, 10) !== value) { + usage("--source-date must use YYYY-MM-DD"); + } +} + +function validateSemver(value: string): void { + const numericIdentifier = "(?:0|[1-9]\\d*)"; + const prereleaseIdentifier = `(?:${numericIdentifier}|\\d*[A-Za-z-][0-9A-Za-z-]*)`; + const prerelease = `(?:-${prereleaseIdentifier}(?:\\.${prereleaseIdentifier})*)?`; + const build = "(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?"; + if ( + !new RegExp( + `^${numericIdentifier}\\.${numericIdentifier}\\.${numericIdentifier}${prerelease}${build}$`, + ).test(value) + ) { + usage("--version must be a semver value"); + } +} + +async function validatePaths(args: ParsedArguments): Promise { + for (const [name, path] of [ + ["--input", args.input], + ["--license-file", args["license-file"]], + ["--output", args.output], + ["--descriptor", args.descriptor], + ] as const) { + if (!isAbsolute(path)) usage(`${name} must be an absolute path`); + } + if (extname(args.input) !== ".jsonl") usage("--input must be a .jsonl path"); + if (extname(args["license-file"]) !== ".txt") usage("--license-file must be a .txt path"); + if (extname(args.output) !== ".sqlite") usage("--output must be a .sqlite path"); + if (extname(args.descriptor) !== ".json") usage("--descriptor must be a .json path"); + await Promise.all([access(args.input), access(args["license-file"])]); + await Promise.all([stat(dirname(args.output)), stat(dirname(args.descriptor))]); +} + +function validateArguments(args: ParsedArguments): DictionarySource { + if (args.language !== "en" && args.language !== "zh") usage("--language must be en or zh"); + validateSemver(args.version); + validateDate(args["source-date"]); + validateUrl(args["source-archive-url"], "--source-archive-url"); + validateUrl(args["attribution-url"], "--attribution-url"); + validateUrl(args["asset-url"], "--asset-url"); + if (!args["creator-attribution"].trim()) usage("--creator-attribution must not be empty"); + if ( + args.language === "en" && + args["source-edition"] === "wordnet-3.1" && + args.license === "WordNet 3.1 License" + ) { + return { language: "en", sourceEdition: "wordnet-3.1", license: "WordNet 3.1 License" }; + } + if ( + args.language === "en" && + args["source-edition"] === "enwiktionary" && + args.license === "CC BY-SA 4.0" + ) { + return { language: "en", sourceEdition: "enwiktionary", license: "CC BY-SA 4.0" }; + } + if ( + args.language === "zh" && + args["source-edition"] === "zhwiktionary" && + args.license === "CC BY-SA 4.0" + ) { + return { language: "zh", sourceEdition: "zhwiktionary", license: "CC BY-SA 4.0" }; + } + usage("--language, --source-edition, and --license must be a supported combination"); +} + +function validateDescriptor(descriptor: DictionaryPackDescriptor): DictionaryPackDescriptor { + const englishPlaceholder = { + ...descriptor, + language: "en", + sourceEdition: "wordnet-3.1", + license: "WordNet 3.1 License", + } as const; + const chinesePlaceholder = { + ...descriptor, + language: "zh", + sourceEdition: "zhwiktionary", + license: "CC BY-SA 4.0", + } as const; + const manifest = parseDictionaryManifest({ + manifestVersion: 1, + packs: + descriptor.language === "en" + ? { en: descriptor, zh: chinesePlaceholder } + : { en: englishPlaceholder, zh: descriptor }, + }); + return manifest.packs[descriptor.language]; +} + +async function main(): Promise { + const args = parseArguments(process.argv.slice(2)); + const source = validateArguments(args); + await validatePaths(args); + const licenseNotice = (await readFile(args["license-file"], "utf8")).trim(); + if (!licenseNotice) usage("--license-file must contain a license notice"); + const descriptor = validateDescriptor( + await buildDictionaryPack({ + ...source, + inputPath: args.input, + outputPath: args.output, + version: args.version, + sourceDumpDate: args["source-date"], + sourceArchiveUrl: args["source-archive-url"], + attributionUrl: args["attribution-url"], + licenseNotice, + creatorAttribution: args["creator-attribution"].trim(), + assetUrl: args["asset-url"], + }), + ); + await writeFile(args.descriptor, `${JSON.stringify(descriptor, null, 2)}\n`, "utf8"); + process.stdout.write(`${JSON.stringify(descriptor)}\n`); +} + +main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; +}); diff --git a/packages/cli/scripts/convert-wordnet.ts b/packages/cli/scripts/convert-wordnet.ts new file mode 100644 index 000000000..e1d419136 --- /dev/null +++ b/packages/cli/scripts/convert-wordnet.ts @@ -0,0 +1,55 @@ +import { access, stat } from "node:fs/promises"; +import { dirname, extname, isAbsolute } from "node:path"; +import { convertWordNetDirectory } from "../src/dictionary/wordnet-converter.js"; + +interface Arguments { + inputDirectory: string; + output: string; +} + +function usage(message: string): never { + throw new Error( + `${message}\nUsage: dictionary:convert-wordnet --input-directory absolute-dict-path --output absolute-jsonl-path`, + ); +} + +function parseArguments(argv: string[]): Arguments { + const values = new Map(); + for (let index = 0; index < argv.length; index += 2) { + const flag = argv[index]; + const value = argv[index + 1]; + if (!flag?.startsWith("--") || !value || value.startsWith("--")) { + usage(`Invalid argument: ${flag ?? ""}`); + } + const key = flag.slice(2); + if (key !== "input-directory" && key !== "output") usage(`Unexpected argument: ${flag}`); + if (values.has(key)) usage(`Duplicate argument: ${flag}`); + values.set(key, value); + } + const inputDirectory = values.get("input-directory"); + const output = values.get("output"); + if (!inputDirectory) usage("Missing required argument: --input-directory"); + if (!output) usage("Missing required argument: --output"); + return { inputDirectory, output }; +} + +async function main(): Promise { + const args = parseArguments(process.argv.slice(2)); + if (!isAbsolute(args.inputDirectory)) usage("--input-directory must be an absolute path"); + if (!isAbsolute(args.output)) usage("--output must be an absolute path"); + if (extname(args.output) !== ".jsonl") usage("--output must be a .jsonl path"); + if (!(await stat(args.inputDirectory)).isDirectory()) { + usage("--input-directory must identify a directory"); + } + await Promise.all([access(args.inputDirectory), stat(dirname(args.output))]); + const stats = await convertWordNetDirectory({ + inputDirectory: args.inputDirectory, + outputPath: args.output, + }); + process.stdout.write(`${JSON.stringify(stats)}\n`); +} + +main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; +}); diff --git a/packages/cli/src/dictionary/fixtures/en-unnormalizable-canonical.jsonl b/packages/cli/src/dictionary/fixtures/en-unnormalizable-canonical.jsonl new file mode 100644 index 000000000..dc37faa05 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/en-unnormalizable-canonical.jsonl @@ -0,0 +1 @@ +{"word":"reading閱讀","lang_code":"en","pos":"noun","forms":[{"form":"readings","tags":["plural"]}],"senses":[{"glosses":["A record that must be skipped because its canonical headword is mixed-script."]}]} diff --git a/packages/cli/src/dictionary/fixtures/en-wordnet.jsonl b/packages/cli/src/dictionary/fixtures/en-wordnet.jsonl new file mode 100644 index 000000000..3c1920fb2 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/en-wordnet.jsonl @@ -0,0 +1,2 @@ +{"word":"desire","lang_code":"en","pos":"verb","senses":[{"glosses":["feel or have a desire for"]}],"forms":[{"form":"desires","tags":["present"]}]} +{"word":"child","lang_code":"en","pos":"noun","senses":[{"glosses":["a young person"]}],"forms":[{"form":"children","tags":["wordnet-exception"]}]} diff --git a/packages/cli/src/dictionary/fixtures/en.jsonl b/packages/cli/src/dictionary/fixtures/en.jsonl new file mode 100644 index 000000000..ef2f369db --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/en.jsonl @@ -0,0 +1 @@ +{"word":"desire","lang_code":"en","pos":"noun","sounds":[{"ipa":"/dɪˈzaɪəɹ/"}],"forms":[{"form":"desires","tags":["plural"]}],"senses":[{"glosses":["A strong wish."],"examples":["Her desire was strong."],"translations":["EN_TRANSLATION_SHOULD_NOT_APPEAR"],"images":["https://example.invalid/desire.png"],"audio":["https://example.invalid/desire.ogg"]}]} diff --git a/packages/cli/src/dictionary/fixtures/test-license.txt b/packages/cli/src/dictionary/fixtures/test-license.txt new file mode 100644 index 000000000..ebe6ddcf8 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/test-license.txt @@ -0,0 +1 @@ +Complete test license notice. diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/adj.exc b/packages/cli/src/dictionary/fixtures/wordnet-invalid/adj.exc new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/adj.exc @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/adv.exc b/packages/cli/src/dictionary/fixtures/wordnet-invalid/adv.exc new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/adv.exc @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adj b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adj new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adj @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adv b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adv new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.adv @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.noun b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.noun new file mode 100644 index 000000000..3590cb57c --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.noun @@ -0,0 +1 @@ +not-a-wordnet-data-row diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.verb b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.verb new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/data.verb @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/noun.exc b/packages/cli/src/dictionary/fixtures/wordnet-invalid/noun.exc new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/noun.exc @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet-invalid/verb.exc b/packages/cli/src/dictionary/fixtures/wordnet-invalid/verb.exc new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet-invalid/verb.exc @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet/adj.exc b/packages/cli/src/dictionary/fixtures/wordnet/adj.exc new file mode 100644 index 000000000..404a2e4dd --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/adj.exc @@ -0,0 +1 @@ +better good diff --git a/packages/cli/src/dictionary/fixtures/wordnet/adv.exc b/packages/cli/src/dictionary/fixtures/wordnet/adv.exc new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/adv.exc @@ -0,0 +1 @@ + diff --git a/packages/cli/src/dictionary/fixtures/wordnet/data.adj b/packages/cli/src/dictionary/fixtures/wordnet/data.adj new file mode 100644 index 000000000..e439f7fbd --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/data.adj @@ -0,0 +1,2 @@ + WordNet fixture header +00000008 00 a 01 good(p) 0 000 | having desirable qualities diff --git a/packages/cli/src/dictionary/fixtures/wordnet/data.adv b/packages/cli/src/dictionary/fixtures/wordnet/data.adv new file mode 100644 index 000000000..4c54cb670 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/data.adv @@ -0,0 +1 @@ + WordNet fixture header diff --git a/packages/cli/src/dictionary/fixtures/wordnet/data.noun b/packages/cli/src/dictionary/fixtures/wordnet/data.noun new file mode 100644 index 000000000..450bcc565 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/data.noun @@ -0,0 +1,7 @@ + WordNet fixture header +00000001 00 n 02 desire 0 wish 0 000 | a strong feeling of wanting something; "his desire was obvious" +00000002 00 n 01 desire 0 000 | something that is wanted; "the house was his desire" +00000003 00 n 01 city 0 000 | a large and densely populated urban area +00000004 00 n 01 child 0 000 | a young person +00000005 00 n 01 ice_cream 0 000 | a frozen dessert +00000009 00 n 01 tattoo 0 000 | a permanent design on skin diff --git a/packages/cli/src/dictionary/fixtures/wordnet/data.verb b/packages/cli/src/dictionary/fixtures/wordnet/data.verb new file mode 100644 index 000000000..473d0f056 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/data.verb @@ -0,0 +1,7 @@ + WordNet fixture header +00000006 00 v 02 desire 0 wish 0 000 00 | feel or have a desire for; "I desire a quiet room" +00000007 00 v 01 go 0 000 00 | move from one place to another +00000010 00 v 01 do 0 000 00 | perform an action +00000011 00 v 01 have 0 000 00 | possess something +00000012 00 v 01 be 0 000 00 | exist +00000013 00 v 01 tattoo 0 000 00 | mark skin permanently diff --git a/packages/cli/src/dictionary/fixtures/wordnet/noun.exc b/packages/cli/src/dictionary/fixtures/wordnet/noun.exc new file mode 100644 index 000000000..bcd4bd668 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/noun.exc @@ -0,0 +1 @@ +children child diff --git a/packages/cli/src/dictionary/fixtures/wordnet/verb.exc b/packages/cli/src/dictionary/fixtures/wordnet/verb.exc new file mode 100644 index 000000000..cc1a01633 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/wordnet/verb.exc @@ -0,0 +1,4 @@ +went go +did do +has have +is be diff --git a/packages/cli/src/dictionary/fixtures/zh-redirects.jsonl b/packages/cli/src/dictionary/fixtures/zh-redirects.jsonl new file mode 100644 index 000000000..919a31254 --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/zh-redirects.jsonl @@ -0,0 +1,12 @@ +{"title":"首页","redirect":"首頁","pos":"hard-redirect"} +{"word":"首頁","lang_code":"zh","pos":"noun","senses":[{"glosses":["網站的主要頁面。"]}]} +{"word":"锂","lang_code":"zh","pos":"soft-redirect","senses":[{"tags":["no-gloss"]}],"redirects":["鋰"]} +{"word":"鋰","lang_code":"zh","pos":"noun","senses":[{"glosses":["原子序數為三的化學元素。"]}]} +{"title":"日语","redirect":"日文","pos":"hard-redirect"} +{"word":"日文","lang_code":"zh","pos":"soft-redirect","senses":[{"tags":["no-gloss"]}],"redirects":["日語"]} +{"word":"日語","lang_code":"zh","pos":"noun","senses":[{"glosses":["日本使用的語言。"]}]} +{"title":"循环甲","redirect":"循环乙","pos":"hard-redirect"} +{"title":"循环乙","redirect":"循环甲","pos":"hard-redirect"} +{"title":"失踪词","redirect":"不存在","pos":"hard-redirect"} +{"title":"unsupported-alias","redirect":"首頁","pos":"hard-redirect"} +{"word":"錯誤語言","lang_code":"en","pos":"soft-redirect","senses":[{"tags":["no-gloss"]}],"redirects":["首頁"]} diff --git a/packages/cli/src/dictionary/fixtures/zh.jsonl b/packages/cli/src/dictionary/fixtures/zh.jsonl new file mode 100644 index 000000000..282a6267d --- /dev/null +++ b/packages/cli/src/dictionary/fixtures/zh.jsonl @@ -0,0 +1 @@ +{"word":"閱讀","lang_code":"zh","pos":"verb","sounds":[{"zh-pron":"yuèdú"}],"forms":[{"form":"阅读","tags":["Simplified-Chinese"]},{"form":"閱讀","tags":["Traditional-Chinese"]}],"senses":[{"glosses":["看並理解文字的內容。"],"examples":["閱讀是一種樂趣。"],"translations":["ZH_TRANSLATION_SHOULD_NOT_APPEAR"],"images":["https://example.invalid/read.png"],"audio":["https://example.invalid/read.ogg"]}]} diff --git a/packages/cli/src/dictionary/pack-builder.test.ts b/packages/cli/src/dictionary/pack-builder.test.ts new file mode 100644 index 000000000..3a39f4d94 --- /dev/null +++ b/packages/cli/src/dictionary/pack-builder.test.ts @@ -0,0 +1,404 @@ +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { mkdtemp, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import Database from "better-sqlite3"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildDictionaryPack } from "./pack-builder.js"; + +const fixtureDirectory = resolve(import.meta.dirname, "fixtures"); +const temporaryDirectories: string[] = []; +const TEST_LICENSE_NOTICE = "Complete test license notice."; +const TEST_CREATOR_ATTRIBUTION = "Wiktionary contributors."; + +async function buildFixture(language: "en" | "zh") { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-pack-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, `${language}.sqlite`); + + const descriptor = await buildDictionaryPack({ + language, + inputPath: join(fixtureDirectory, `${language}.jsonl`), + outputPath, + version: "1.2.3", + sourceEdition: language === "en" ? "enwiktionary" : "zhwiktionary", + license: "CC BY-SA 4.0", + sourceDumpDate: "2026-09-03", + sourceArchiveUrl: `https://example.invalid/${language}-archive`, + attributionUrl: `https://example.invalid/${language}-attribution`, + licenseNotice: TEST_LICENSE_NOTICE, + creatorAttribution: TEST_CREATOR_ATTRIBUTION, + assetUrl: `https://example.invalid/${language}.sqlite`, + }); + + return { descriptor, outputPath }; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map(async (directory) => { + const { rm } = await import("node:fs/promises"); + await rm(directory, { recursive: true, force: true }); + }), + ); +}); + +describe("buildDictionaryPack", () => { + it("resolves Chinese hard, soft, and chained redirects without cycle or missing-target rows", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-redirects-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "zh.sqlite"); + + await buildDictionaryPack({ + language: "zh", + inputPath: join(fixtureDirectory, "zh-redirects.jsonl"), + outputPath, + version: "1.2.3", + sourceEdition: "zhwiktionary", + license: "CC BY-SA 4.0", + sourceDumpDate: "2026-09-03", + sourceArchiveUrl: "https://example.invalid/zh-archive", + attributionUrl: "https://example.invalid/zh-attribution", + licenseNotice: TEST_LICENSE_NOTICE, + creatorAttribution: TEST_CREATOR_ATTRIBUTION, + assetUrl: "https://example.invalid/zh.sqlite", + }); + + const database = new Database(outputPath, { readonly: true }); + try { + const redirected = database + .prepare(` + SELECT l.lookup_key, l.rank, e.headword + FROM lookup l + JOIN entries e ON e.id = l.entry_id + WHERE l.lookup_key IN ('首页', '锂', '日语') + ORDER BY l.lookup_key, e.id + `) + .all(); + expect(redirected).toEqual([ + { lookup_key: "日语", rank: 1, headword: "日語" }, + { lookup_key: "锂", rank: 1, headword: "鋰" }, + { lookup_key: "首页", rank: 1, headword: "首頁" }, + ]); + expect( + database + .prepare( + "SELECT COUNT(*) FROM lookup WHERE lookup_key IN ('循环甲', '循环乙', '失踪词', 'unsupported-alias', '錯誤語言')", + ) + .pluck() + .get(), + ).toBe(0); + expect(database.pragma("foreign_key_check")).toEqual([]); + } finally { + database.close(); + } + }); + + it("builds deterministic English and Chinese SQLite packs with lookup aliases", async () => { + const [english, chinese] = await Promise.all([buildFixture("en"), buildFixture("zh")]); + const englishDatabase = new Database(english.outputPath, { readonly: true }); + const chineseDatabase = new Database(chinese.outputPath, { readonly: true }); + + try { + expect( + englishDatabase + .prepare("SELECT value FROM metadata WHERE key = 'schema_version'") + .pluck() + .get(), + ).toBe("1"); + expect( + chineseDatabase + .prepare("SELECT value FROM metadata WHERE key = 'schema_version'") + .pluck() + .get(), + ).toBe("1"); + expect( + englishDatabase + .prepare("SELECT value FROM metadata WHERE key = 'source_archive_url'") + .pluck() + .get(), + ).toBe("https://example.invalid/en-archive"); + expect( + englishDatabase + .prepare("SELECT value FROM metadata WHERE key = 'attribution_url'") + .pluck() + .get(), + ).toBe("https://example.invalid/en-attribution"); + expect( + englishDatabase + .prepare("SELECT value FROM metadata WHERE key = 'license_notice'") + .pluck() + .get(), + ).toBe(TEST_LICENSE_NOTICE); + expect( + englishDatabase + .prepare("SELECT value FROM metadata WHERE key = 'creator_attribution'") + .pluck() + .get(), + ).toBe(TEST_CREATOR_ATTRIBUTION); + expect( + englishDatabase + .prepare("SELECT entry_id FROM lookup WHERE lookup_key = ?") + .pluck() + .get("desires"), + ).toBeTypeOf("number"); + expect( + englishDatabase + .prepare("SELECT rank FROM lookup WHERE lookup_key = ?") + .pluck() + .get("desire"), + ).toBe(0); + expect( + englishDatabase + .prepare("SELECT rank FROM lookup WHERE lookup_key = ?") + .pluck() + .get("desires"), + ).toBe(1); + expect( + chineseDatabase + .prepare("SELECT entry_id FROM lookup WHERE lookup_key = ?") + .pluck() + .get("阅读"), + ).toBeTypeOf("number"); + expect( + englishDatabase.prepare("SELECT definition FROM senses ORDER BY sense_order").pluck().all(), + ).toContain("A strong wish."); + + expect(englishDatabase.prepare("SELECT pronunciation FROM entries").pluck().get()).toBe( + "/dɪˈzaɪəɹ/", + ); + expect(chineseDatabase.prepare("SELECT pronunciation FROM entries").pluck().get()).toBe( + "yuèdú", + ); + expect(chineseDatabase.prepare("SELECT simplified FROM entries").pluck().get()).toBe("阅读"); + expect(chineseDatabase.prepare("SELECT traditional FROM entries").pluck().get()).toBe("閱讀"); + } finally { + englishDatabase.close(); + chineseDatabase.close(); + } + + const databaseText = await Promise.all([ + readFile(english.outputPath), + readFile(chinese.outputPath), + ]).then((files) => Buffer.concat(files).toString("utf8")); + for (const forbiddenText of [ + "Her desire was strong.", + "閱讀是一種樂趣。", + "EN_TRANSLATION_SHOULD_NOT_APPEAR", + "ZH_TRANSLATION_SHOULD_NOT_APPEAR", + "https://example.invalid/desire.png", + "https://example.invalid/read.ogg", + ]) { + expect(databaseText).not.toContain(forbiddenText); + } + + expect(english.descriptor).toMatchObject({ + language: "en", + schemaVersion: 1, + sourceEdition: "enwiktionary", + url: "https://example.invalid/en.sqlite", + sourceArchiveUrl: "https://example.invalid/en-archive", + attributionUrl: "https://example.invalid/en-attribution", + license: "CC BY-SA 4.0", + }); + expect(chinese.descriptor).toMatchObject({ language: "zh", schemaVersion: 1 }); + }); + + it("skips a record whose canonical headword cannot become an English lookup key", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-unnormalizable-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "en.sqlite"); + + await buildDictionaryPack({ + language: "en", + inputPath: join(fixtureDirectory, "en-unnormalizable-canonical.jsonl"), + outputPath, + version: "1.2.3", + sourceEdition: "enwiktionary", + license: "CC BY-SA 4.0", + sourceDumpDate: "2026-09-03", + sourceArchiveUrl: "https://example.invalid/en-archive", + attributionUrl: "https://example.invalid/en-attribution", + licenseNotice: TEST_LICENSE_NOTICE, + creatorAttribution: TEST_CREATOR_ATTRIBUTION, + assetUrl: "https://example.invalid/en.sqlite", + }); + + const database = new Database(outputPath, { readonly: true }); + try { + expect(database.prepare("SELECT COUNT(*) FROM entries").pluck().get()).toBe(0); + expect(database.prepare("SELECT COUNT(*) FROM lookup").pluck().get()).toBe(0); + } finally { + database.close(); + } + }); + + it.each([ + ["license notice", { licenseNotice: "" }], + ["creator attribution", { creatorAttribution: " " }], + ])("rejects an empty %s before creating a pack", async (_name, override) => { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-empty-notice-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "en.sqlite"); + + await expect( + buildDictionaryPack({ + language: "en", + inputPath: join(fixtureDirectory, "en.jsonl"), + outputPath, + version: "1.2.3", + sourceEdition: "enwiktionary", + license: "CC BY-SA 4.0", + sourceDumpDate: "2026-09-03", + sourceArchiveUrl: "https://example.invalid/en-archive", + attributionUrl: "https://example.invalid/en-attribution", + licenseNotice: TEST_LICENSE_NOTICE, + creatorAttribution: TEST_CREATOR_ATTRIBUTION, + assetUrl: "https://example.invalid/en.sqlite", + ...override, + }), + ).rejects.toThrow(/license notice|creator attribution/i); + expect(existsSync(outputPath)).toBe(false); + }); + + it("builds WordNet metadata and keeps deterministic exception aliases", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-wordnet-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "en.sqlite"); + + const descriptor = await buildDictionaryPack({ + language: "en", + inputPath: join(fixtureDirectory, "en-wordnet.jsonl"), + outputPath, + version: "1.0.0", + sourceEdition: "wordnet-3.1", + sourceDumpDate: "2011-05-26", + sourceArchiveUrl: "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + attributionUrl: "https://wordnet.princeton.edu/license-and-commercial-use", + licenseNotice: "Complete WordNet license notice.", + creatorAttribution: "Princeton University.", + assetUrl: "https://example.invalid/en.sqlite", + license: "WordNet 3.1 License", + }); + + const database = new Database(outputPath, { readonly: true }); + try { + expect( + database.prepare("SELECT value FROM metadata WHERE key = 'source_edition'").pluck().get(), + ).toBe("wordnet-3.1"); + expect( + database.prepare("SELECT value FROM metadata WHERE key = 'license'").pluck().get(), + ).toBe("WordNet 3.1 License"); + expect( + database.prepare("SELECT rank FROM lookup WHERE lookup_key = 'children'").pluck().get(), + ).toBe(1); + } finally { + database.close(); + } + expect(descriptor).toMatchObject({ + language: "en", + sourceEdition: "wordnet-3.1", + license: "WordNet 3.1 License", + }); + }); + + it("builds a validated, pretty-printed descriptor from the required command arguments", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-command-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "en.sqlite"); + const descriptorPath = join(directory, "en.json"); + const scriptPath = resolve(import.meta.dirname, "../../scripts/build-dictionary-pack.ts"); + const tsxPath = resolve(import.meta.dirname, "../../../../node_modules/tsx/dist/cli.mjs"); + const result = spawnSync( + process.execPath, + [ + tsxPath, + scriptPath, + "--language", + "en", + "--input", + join(fixtureDirectory, "en.jsonl"), + "--output", + outputPath, + "--version", + "1.2.3", + "--source-edition", + "enwiktionary", + "--license", + "CC BY-SA 4.0", + "--source-date", + "2026-09-03", + "--source-archive-url", + "https://example.invalid/en-archive", + "--attribution-url", + "https://example.invalid/en-attribution", + "--license-file", + join(fixtureDirectory, "test-license.txt"), + "--creator-attribution", + TEST_CREATOR_ATTRIBUTION, + "--asset-url", + "https://example.invalid/en.sqlite", + "--descriptor", + descriptorPath, + ], + { encoding: "utf8" }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(await readFile(descriptorPath, "utf8")).toBe( + `${JSON.stringify(JSON.parse(await readFile(descriptorPath, "utf8")), null, 2)}\n`, + ); + }); + + it.each(["01.2.3", "1.2.3-01"])( + "rejects non-SemVer version %s before creating output files", + async (version) => { + const directory = await mkdtemp(join(tmpdir(), "readany-dictionary-invalid-semver-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "en.sqlite"); + const descriptorPath = join(directory, "en.json"); + const scriptPath = resolve(import.meta.dirname, "../../scripts/build-dictionary-pack.ts"); + const tsxPath = resolve(import.meta.dirname, "../../../../node_modules/tsx/dist/cli.mjs"); + const result = spawnSync( + process.execPath, + [ + tsxPath, + scriptPath, + "--language", + "en", + "--input", + join(fixtureDirectory, "en.jsonl"), + "--output", + outputPath, + "--version", + version, + "--source-edition", + "enwiktionary", + "--license", + "CC BY-SA 4.0", + "--source-date", + "2026-09-03", + "--source-archive-url", + "https://example.invalid/en-archive", + "--attribution-url", + "https://example.invalid/en-attribution", + "--license-file", + join(fixtureDirectory, "test-license.txt"), + "--creator-attribution", + TEST_CREATOR_ATTRIBUTION, + "--asset-url", + "https://example.invalid/en.sqlite", + "--descriptor", + descriptorPath, + ], + { encoding: "utf8" }, + ); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("--version must be a semver value"); + expect(existsSync(outputPath)).toBe(false); + expect(existsSync(descriptorPath)).toBe(false); + }, + ); +}); diff --git a/packages/cli/src/dictionary/pack-builder.ts b/packages/cli/src/dictionary/pack-builder.ts new file mode 100644 index 000000000..d173ead74 --- /dev/null +++ b/packages/cli/src/dictionary/pack-builder.ts @@ -0,0 +1,379 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { stat } from "node:fs/promises"; +import readline from "node:readline"; +import { + type DictionaryLanguage, + type DictionaryPackDescriptor, + type DictionarySource, + prepareDictionarySelection, +} from "@readany/core/dictionary"; +import Database from "better-sqlite3"; +import { DICTIONARY_SCHEMA_SQL, DICTIONARY_SCHEMA_VERSION } from "./schema.js"; + +const ACCEPTED_FORM_TAGS: Record> = { + en: new Set(["plural", "past", "present", "participle", "wordnet-exception"]), + zh: new Set(["Simplified-Chinese", "Traditional-Chinese"]), +}; + +const TRANSFORMED_BY = "ReadAny offline dictionary pack builder"; + +interface WiktionarySound { + ipa?: unknown; + "zh-pron"?: unknown; +} + +interface WiktionaryForm { + form?: unknown; + tags?: unknown; +} + +interface WiktionarySense { + glosses?: unknown; +} + +interface WiktionaryRecord { + title?: unknown; + redirect?: unknown; + redirects?: unknown; + word?: unknown; + lang_code?: unknown; + pos?: unknown; + sounds?: unknown; + forms?: unknown; + senses?: unknown; +} + +type RedirectGraph = Map>; + +interface PendingEntry { + headword: string; + simplified?: string; + traditional?: string; + pronunciation?: string; + partOfSpeech: string; + definitions: string[]; + lookupRanks: Map; +} + +interface DictionaryPackBuildOptionsBase { + inputPath: string; + outputPath: string; + version: string; + sourceDumpDate: string; + sourceArchiveUrl: string; + attributionUrl: string; + licenseNotice: string; + creatorAttribution: string; + assetUrl: string; +} + +export type DictionaryPackBuildOptions = DictionaryPackBuildOptionsBase & DictionarySource; + +function nonEmptyString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function stringArray(value: unknown): string[] { + if (!Array.isArray(value)) return []; + return value.flatMap((item) => { + const text = nonEmptyString(item); + return text ? [text] : []; + }); +} + +function firstSound(sounds: unknown, property: keyof WiktionarySound): string | undefined { + if (!Array.isArray(sounds)) return undefined; + for (const sound of sounds as WiktionarySound[]) { + const value = nonEmptyString(sound?.[property]); + if (value) return value; + } + return undefined; +} + +function normalizeLookupKey(value: string, language: DictionaryLanguage): string | undefined { + const selection = prepareDictionarySelection(value); + return selection.ok && selection.language === language ? selection.key : undefined; +} + +function normalizeAlias(value: string): string { + return value.normalize("NFKC").trim(); +} + +function collectChineseRedirect(record: WiktionaryRecord, graph: RedirectGraph): void { + const isHardRedirect = record.pos === "hard-redirect"; + const isSoftRedirect = record.pos === "soft-redirect" && record.lang_code === "zh"; + if (!isHardRedirect && !isSoftRedirect) return; + + const alias = normalizeLookupKey( + nonEmptyString(isHardRedirect ? record.title : record.word) ?? "", + "zh", + ); + const targets = isHardRedirect + ? [nonEmptyString(record.redirect)].filter((target): target is string => Boolean(target)) + : stringArray(record.redirects); + if (!alias) return; + + for (const target of targets) { + const normalizedTarget = normalizeLookupKey(target, "zh"); + if (!normalizedTarget || normalizedTarget === alias) continue; + const existingTargets = graph.get(alias) ?? new Set(); + existingTargets.add(normalizedTarget); + graph.set(alias, existingTargets); + } +} + +function resolvedRedirectTargets( + alias: string, + graph: RedirectGraph, + canonicalKeys: ReadonlySet, + memo: Map>, + visiting = new Set(), +): ReadonlySet { + if (canonicalKeys.has(alias)) return new Set([alias]); + const cached = memo.get(alias); + if (cached) return cached; + if (visiting.has(alias)) return new Set(); + + visiting.add(alias); + const resolved = new Set(); + for (const target of [...(graph.get(alias) ?? [])].sort()) { + for (const canonicalTarget of resolvedRedirectTargets( + target, + graph, + canonicalKeys, + memo, + visiting, + )) { + resolved.add(canonicalTarget); + } + } + visiting.delete(alias); + memo.set(alias, resolved); + return resolved; +} + +function prepareEntry( + record: WiktionaryRecord, + language: DictionaryLanguage, +): PendingEntry | undefined { + if (record.lang_code !== language) return undefined; + + const headword = nonEmptyString(record.word); + if (!headword) return undefined; + const definitions = Array.isArray(record.senses) + ? (record.senses as WiktionarySense[]).flatMap((sense) => stringArray(sense?.glosses)) + : []; + if (definitions.length === 0) return undefined; + + const lookupRanks = new Map(); + const canonicalKey = normalizeLookupKey(headword, language); + if (!canonicalKey) return undefined; + lookupRanks.set(canonicalKey, 0); + + let simplified: string | undefined; + let traditional: string | undefined; + if (Array.isArray(record.forms)) { + for (const form of record.forms as WiktionaryForm[]) { + const alias = nonEmptyString(form?.form); + if (!alias) continue; + const tags = new Set(stringArray(form?.tags)); + if (![...tags].some((tag) => ACCEPTED_FORM_TAGS[language].has(tag))) continue; + + const normalizedAlias = normalizeLookupKey(alias, language); + if (normalizedAlias && !lookupRanks.has(normalizedAlias)) { + lookupRanks.set(normalizedAlias, 1); + } + if (language === "zh" && tags.has("Simplified-Chinese")) { + simplified ??= normalizeAlias(alias); + } + if (language === "zh" && tags.has("Traditional-Chinese")) { + traditional ??= normalizeAlias(alias); + } + } + } + + return { + headword: normalizeAlias(headword), + simplified, + traditional, + pronunciation: firstSound(record.sounds, language === "en" ? "ipa" : "zh-pron"), + partOfSpeech: nonEmptyString(record.pos) ?? "unknown", + definitions, + lookupRanks, + }; +} + +async function sha256File(path: string): Promise { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) { + hash.update(chunk); + } + return hash.digest("hex"); +} + +function assertSqliteChecks(database: Database.Database): void { + const foreignKeyViolations = database.pragma("foreign_key_check") as unknown[]; + if (foreignKeyViolations.length > 0) { + throw new Error("Dictionary pack foreign key check failed"); + } + const integrity = database.pragma("integrity_check", { simple: true }); + if (integrity !== "ok") { + throw new Error(`Dictionary pack integrity check failed: ${String(integrity)}`); + } +} + +export async function buildDictionaryPack( + options: DictionaryPackBuildOptions, +): Promise { + if (!options.licenseNotice.trim()) throw new Error("Dictionary license notice is required"); + if (!options.creatorAttribution.trim()) + throw new Error("Dictionary creator attribution is required"); + const database = new Database(options.outputPath); + try { + database.exec(DICTIONARY_SCHEMA_SQL); + const insertMetadata = database.prepare("INSERT INTO metadata (key, value) VALUES (?, ?)"); + const insertEntry = database.prepare(` + INSERT INTO entries (language, headword, simplified, traditional, pronunciation, part_of_speech) + VALUES (?, ?, ?, ?, ?, ?) + `); + const insertSense = database.prepare( + "INSERT INTO senses (entry_id, sense_order, definition) VALUES (?, ?, ?)", + ); + const insertLookup = database.prepare( + "INSERT INTO lookup (lookup_key, entry_id, rank) VALUES (?, ?, ?)", + ); + const insertRedirectLookup = database.prepare(` + INSERT OR IGNORE INTO lookup (lookup_key, entry_id, rank) + SELECT ?, entry_id, 1 + FROM lookup + WHERE lookup_key = ? AND rank = 0 + ORDER BY entry_id + `); + + const metadata: ReadonlyArray = [ + ["schema_version", String(DICTIONARY_SCHEMA_VERSION)], + ["language", options.language], + ["version", options.version], + ["source_edition", options.sourceEdition], + ["source_dump_date", options.sourceDumpDate], + ["source_archive_url", options.sourceArchiveUrl], + ["asset_url", options.assetUrl], + ["license", options.license], + ["license_notice", options.licenseNotice], + ["creator_attribution", options.creatorAttribution], + ["attribution_url", options.attributionUrl], + ["transformed_by", TRANSFORMED_BY], + ]; + database.transaction(() => { + for (const [key, value] of metadata) insertMetadata.run(key, value); + })(); + + const insertBatch = database.transaction((entries: PendingEntry[]) => { + for (const entry of entries) { + const entryId = Number( + insertEntry.run( + options.language, + entry.headword, + entry.simplified ?? null, + entry.traditional ?? null, + entry.pronunciation ?? null, + entry.partOfSpeech, + ).lastInsertRowid, + ); + for (const [index, definition] of entry.definitions.entries()) { + insertSense.run(entryId, index, definition); + } + for (const [lookupKey, rank] of entry.lookupRanks) { + insertLookup.run(lookupKey, entryId, rank); + } + } + }); + + const lines = readline.createInterface({ + input: createReadStream(options.inputPath, { encoding: "utf8" }), + crlfDelay: Number.POSITIVE_INFINITY, + }); + let lineNumber = 0; + let batch: PendingEntry[] = []; + const redirects: RedirectGraph = new Map(); + for await (const line of lines) { + lineNumber += 1; + if (!line.trim()) continue; + let parsed: WiktionaryRecord; + try { + parsed = JSON.parse(line) as WiktionaryRecord; + } catch (error) { + throw new Error(`Invalid JSONL at line ${lineNumber}: ${String(error)}`); + } + if (options.language === "zh") collectChineseRedirect(parsed, redirects); + const entry = prepareEntry(parsed, options.language); + if (!entry) continue; + batch.push(entry); + if (batch.length === 5_000) { + insertBatch(batch); + batch = []; + } + } + if (batch.length > 0) insertBatch(batch); + + const canonicalKeys = new Set( + database + .prepare("SELECT DISTINCT lookup_key FROM lookup WHERE rank = 0 ORDER BY lookup_key") + .pluck() + .all() as string[], + ); + const resolved = new Map>(); + database.transaction(() => { + for (const alias of [...redirects.keys()].sort()) { + for (const target of [ + ...resolvedRedirectTargets(alias, redirects, canonicalKeys, resolved), + ].sort()) { + if (alias !== target) insertRedirectLookup.run(alias, target); + } + } + })(); + + assertSqliteChecks(database); + database.exec("VACUUM"); + } finally { + database.close(); + } + + const [file, sha256] = await Promise.all([ + stat(options.outputPath), + sha256File(options.outputPath), + ]); + const descriptor = { + version: options.version, + schemaVersion: DICTIONARY_SCHEMA_VERSION, + sourceDumpDate: options.sourceDumpDate, + sizeBytes: file.size, + sha256, + url: options.assetUrl, + sourceArchiveUrl: options.sourceArchiveUrl, + attributionUrl: options.attributionUrl, + }; + if (options.language === "zh") { + return { + ...descriptor, + language: "zh", + sourceEdition: options.sourceEdition, + license: options.license, + }; + } + if (options.sourceEdition === "wordnet-3.1") { + return { + ...descriptor, + language: "en", + sourceEdition: options.sourceEdition, + license: options.license, + }; + } + return { + ...descriptor, + language: "en", + sourceEdition: options.sourceEdition, + license: options.license, + }; +} diff --git a/packages/cli/src/dictionary/schema.ts b/packages/cli/src/dictionary/schema.ts new file mode 100644 index 000000000..7475c4629 --- /dev/null +++ b/packages/cli/src/dictionary/schema.ts @@ -0,0 +1,32 @@ +export const DICTIONARY_SCHEMA_VERSION = 1 as const; + +export const DICTIONARY_SCHEMA_SQL = ` +PRAGMA journal_mode = DELETE; +PRAGMA synchronous = OFF; +CREATE TABLE metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +) WITHOUT ROWID; +CREATE TABLE entries ( + id INTEGER PRIMARY KEY, + language TEXT NOT NULL CHECK (language IN ('en', 'zh')), + headword TEXT NOT NULL, + simplified TEXT, + traditional TEXT, + pronunciation TEXT, + part_of_speech TEXT NOT NULL +); +CREATE TABLE senses ( + entry_id INTEGER NOT NULL REFERENCES entries(id), + sense_order INTEGER NOT NULL, + definition TEXT NOT NULL, + PRIMARY KEY (entry_id, sense_order) +) WITHOUT ROWID; +CREATE TABLE lookup ( + lookup_key TEXT NOT NULL, + entry_id INTEGER NOT NULL REFERENCES entries(id), + rank INTEGER NOT NULL, + PRIMARY KEY (lookup_key, entry_id) +) WITHOUT ROWID; +CREATE INDEX lookup_key_rank_idx ON lookup(lookup_key, rank, entry_id); +`; diff --git a/packages/cli/src/dictionary/wordnet-converter.test.ts b/packages/cli/src/dictionary/wordnet-converter.test.ts new file mode 100644 index 000000000..6c710eaed --- /dev/null +++ b/packages/cli/src/dictionary/wordnet-converter.test.ts @@ -0,0 +1,165 @@ +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { convertWordNetDirectory } from "./wordnet-converter.js"; + +const fixtureDirectory = resolve(import.meta.dirname, "fixtures/wordnet"); +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all( + temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true })), + ); +}); + +describe("convertWordNetDirectory", () => { + it("converts WordNet synsets and conservative inflections to deterministic builder JSONL", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-wordnet-converter-")); + temporaryDirectories.push(directory); + const firstOutput = join(directory, "first.jsonl"); + const secondOutput = join(directory, "second.jsonl"); + + const firstStats = await convertWordNetDirectory({ + inputDirectory: fixtureDirectory, + outputPath: firstOutput, + }); + const secondStats = await convertWordNetDirectory({ + inputDirectory: fixtureDirectory, + outputPath: secondOutput, + }); + + const firstBytes = await readFile(firstOutput, "utf8"); + expect(await readFile(secondOutput, "utf8")).toBe(firstBytes); + expect(secondStats).toEqual(firstStats); + const records = firstBytes + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + + expect(firstStats).toEqual({ records: 14, senses: 15, aliases: 15 }); + expect(records).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + word: "desire", + lang_code: "en", + pos: "noun", + senses: [ + { glosses: ["a strong feeling of wanting something"] }, + { glosses: ["something that is wanted"] }, + ], + forms: [{ form: "desires", tags: ["plural"] }], + }), + expect.objectContaining({ + word: "desire", + pos: "verb", + senses: [{ glosses: ["feel or have a desire for"] }], + forms: [{ form: "desires", tags: ["present"] }], + }), + expect.objectContaining({ + word: "child", + pos: "noun", + forms: [{ form: "children", tags: ["wordnet-exception"] }], + }), + expect.objectContaining({ word: "good", pos: "adjective" }), + expect.objectContaining({ word: "ice cream", pos: "noun", forms: [] }), + expect.objectContaining({ + word: "go", + pos: "verb", + forms: [ + { form: "went", tags: ["wordnet-exception"] }, + { form: "goes", tags: ["present"] }, + ], + }), + expect.objectContaining({ + word: "do", + pos: "verb", + forms: [ + { form: "did", tags: ["wordnet-exception"] }, + { form: "does", tags: ["present"] }, + ], + }), + expect.objectContaining({ + word: "have", + pos: "verb", + forms: [{ form: "has", tags: ["wordnet-exception"] }], + }), + expect.objectContaining({ + word: "be", + pos: "verb", + forms: [{ form: "is", tags: ["wordnet-exception"] }], + }), + expect.objectContaining({ + word: "tattoo", + pos: "noun", + forms: [{ form: "tattoos", tags: ["plural"] }], + }), + expect.objectContaining({ + word: "tattoo", + pos: "verb", + forms: [{ form: "tattoos", tags: ["present"] }], + }), + ]), + ); + expect(firstBytes).not.toContain("tattooes"); + expect(firstBytes).not.toContain("his desire was obvious"); + expect(firstBytes).not.toContain("the house was his desire"); + expect(firstBytes).not.toContain("I desire a quiet room"); + }); + + it.each([ + ["malformed lex id", "data.noun", "city 0 000", "city zz 000"], + ["missing pointer count", "data.noun", "city 0 000", "city 0"], + ["malformed pointer", "data.noun", "city 0 000", "city 0 001 @ nope n 0000"], + ["malformed verb frame", "data.verb", "go 0 000 00", "go 0 000 01 ? 01 00"], + ["unexpected trailing token", "data.noun", "city 0 000", "city 0 000 surprise"], + ])( + "rejects %s instead of accepting a partial data-row parse", + async (_name, fileName, from, to) => { + const directory = await mkdtemp(join(tmpdir(), "readany-wordnet-grammar-")); + temporaryDirectories.push(directory); + const inputDirectory = join(directory, "wordnet"); + await cp(fixtureDirectory, inputDirectory, { recursive: true }); + const path = join(inputDirectory, fileName); + const source = await readFile(path, "utf8"); + await writeFile(path, source.replace(from, to), "utf8"); + + await expect( + convertWordNetDirectory({ inputDirectory, outputPath: join(directory, "output.jsonl") }), + ).rejects.toThrow(new RegExp(`${fileName.replace(".", "\\.")}.*line`, "i")); + }, + ); + + it("rejects a malformed WordNet data row with its file and line", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-wordnet-invalid-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "wordnet.jsonl"); + + await expect( + convertWordNetDirectory({ + inputDirectory: resolve(import.meta.dirname, "fixtures/wordnet-invalid"), + outputPath, + }), + ).rejects.toThrow(/data\.noun.*line/i); + }); + + it("exposes a validated release-side conversion command", async () => { + const directory = await mkdtemp(join(tmpdir(), "readany-wordnet-command-")); + temporaryDirectories.push(directory); + const outputPath = join(directory, "wordnet.jsonl"); + const scriptPath = resolve(import.meta.dirname, "../../scripts/convert-wordnet.ts"); + const tsxPath = resolve(import.meta.dirname, "../../../../node_modules/tsx/dist/cli.mjs"); + + const result = spawnSync( + process.execPath, + [tsxPath, scriptPath, "--input-directory", fixtureDirectory, "--output", outputPath], + { encoding: "utf8" }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ records: 14, senses: 15, aliases: 15 }); + expect(existsSync(outputPath)).toBe(true); + }); +}); diff --git a/packages/cli/src/dictionary/wordnet-converter.ts b/packages/cli/src/dictionary/wordnet-converter.ts new file mode 100644 index 000000000..b9712fa52 --- /dev/null +++ b/packages/cli/src/dictionary/wordnet-converter.ts @@ -0,0 +1,296 @@ +import { once } from "node:events"; +import { createWriteStream } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { prepareDictionarySelection } from "@readany/core/dictionary"; + +const WORDNET_DATA_FILES = [ + { fileName: "data.noun", partOfSpeech: "noun", synsetTypes: new Set(["n"]) }, + { fileName: "data.verb", partOfSpeech: "verb", synsetTypes: new Set(["v"]) }, + { fileName: "data.adj", partOfSpeech: "adjective", synsetTypes: new Set(["a", "s"]) }, + { fileName: "data.adv", partOfSpeech: "adverb", synsetTypes: new Set(["r"]) }, +] as const; + +const WORDNET_EXCEPTION_FILES = [ + { fileName: "noun.exc", partOfSpeech: "noun" }, + { fileName: "verb.exc", partOfSpeech: "verb" }, + { fileName: "adj.exc", partOfSpeech: "adjective" }, + { fileName: "adv.exc", partOfSpeech: "adverb" }, +] as const; + +type PartOfSpeech = (typeof WORDNET_DATA_FILES)[number]["partOfSpeech"]; + +interface PendingWordNetEntry { + word: string; + partOfSpeech: PartOfSpeech; + definitions: string[]; + definitionSet: Set; + forms: Map; + exceptionForms: Set; +} + +const WORDNET_POINTER_SYMBOLS = new Set([ + "!", + "@", + "@i", + "~", + "~i", + "#m", + "#s", + "#p", + "%m", + "%s", + "%p", + "=", + "+", + ";c", + "-c", + ";r", + "-r", + ";u", + "-u", + "*", + ">", + "^", + "$", + "&", + "<", + "\\", +]); + +const IRREGULAR_THIRD_PERSON_VERBS = new Set(["be", "have"]); + +export interface WordNetConversionOptions { + inputDirectory: string; + outputPath: string; +} + +export interface WordNetConversionStats { + records: number; + senses: number; + aliases: number; +} + +export async function convertWordNetDirectory( + options: WordNetConversionOptions, +): Promise { + const entries = new Map(); + + for (const dataFile of WORDNET_DATA_FILES) { + const path = join(options.inputDirectory, dataFile.fileName); + const lines = (await readFile(path, "utf8")).split(/\r?\n/); + for (const [index, line] of lines.entries()) { + if (!line || /^\s/.test(line)) continue; + const parsed = parseDataRow(path, index + 1, line, dataFile.synsetTypes); + for (const word of parsed.words) { + const selection = prepareDictionarySelection(word); + if (!selection.ok || selection.language !== "en") continue; + const canonical = selection.key; + const key = entryKey(dataFile.partOfSpeech, canonical); + let entry = entries.get(key); + if (!entry) { + entry = { + word: canonical, + partOfSpeech: dataFile.partOfSpeech, + definitions: [], + definitionSet: new Set(), + forms: new Map(), + exceptionForms: new Set(), + }; + entries.set(key, entry); + } + if (!entry.definitionSet.has(parsed.definition)) { + entry.definitionSet.add(parsed.definition); + entry.definitions.push(parsed.definition); + } + } + } + } + + for (const exceptionFile of WORDNET_EXCEPTION_FILES) { + const path = join(options.inputDirectory, exceptionFile.fileName); + const lines = (await readFile(path, "utf8")).split(/\r?\n/); + for (const [index, line] of lines.entries()) { + if (!line.trim()) continue; + const tokens = line.trim().split(/\s+/); + if (tokens.length < 2) throw formatError(path, index + 1, "invalid exception row"); + const inflected = normalizeLemma(tokens[0]); + for (const baseToken of tokens.slice(1)) { + const base = normalizeLemma(baseToken); + const selection = prepareDictionarySelection(base); + const aliasSelection = prepareDictionarySelection(inflected); + if ( + !selection.ok || + selection.language !== "en" || + !aliasSelection.ok || + aliasSelection.language !== "en" + ) { + continue; + } + const entry = entries.get(entryKey(exceptionFile.partOfSpeech, selection.key)); + if (!entry) continue; + entry.exceptionForms.add(aliasSelection.key); + addAlias(entry, aliasSelection.key, "wordnet-exception"); + } + } + } + + for (const entry of entries.values()) { + if (!/^[a-z]+$/u.test(entry.word)) continue; + if (entry.partOfSpeech === "noun" && entry.exceptionForms.size === 0) { + addAlias(entry, regularNounPlural(entry.word), "plural"); + } + if (entry.partOfSpeech === "verb" && !IRREGULAR_THIRD_PERSON_VERBS.has(entry.word)) { + addAlias(entry, regularVerbThirdPerson(entry.word), "present"); + } + } + + const output = createWriteStream(options.outputPath, { encoding: "utf8", flags: "wx" }); + let outputError: unknown; + output.on("error", (error) => { + outputError = error; + }); + for (const entry of entries.values()) { + const record = { + word: entry.word, + lang_code: "en", + pos: entry.partOfSpeech, + senses: entry.definitions.map((definition) => ({ glosses: [definition] })), + forms: [...entry.forms].map(([form, tag]) => ({ form, tags: [tag] })), + }; + if (!output.write(`${JSON.stringify(record)}\n`)) await once(output, "drain"); + if (outputError) throw outputError; + } + output.end(); + await once(output, "close"); + if (outputError) throw outputError; + + return { + records: entries.size, + senses: [...entries.values()].reduce((sum, entry) => sum + entry.definitions.length, 0), + aliases: [...entries.values()].reduce((sum, entry) => sum + entry.forms.size, 0), + }; +} + +function parseDataRow( + path: string, + lineNumber: number, + line: string, + expectedSynsetTypes: ReadonlySet, +): { words: string[]; definition: string } { + const separator = line.indexOf("|"); + if (separator < 0) throw formatError(path, lineNumber, "missing gloss separator"); + const fields = line.slice(0, separator).trim().split(/\s+/); + let cursor = 0; + const take = (name: string): string => { + const value = fields[cursor]; + if (value === undefined) throw formatError(path, lineNumber, `missing ${name}`); + cursor += 1; + return value; + }; + + if (!/^\d{8}$/.test(take("synset offset"))) + throw formatError(path, lineNumber, "invalid synset offset"); + if (!/^\d{2}$/.test(take("lexicographer file number"))) + throw formatError(path, lineNumber, "invalid lexicographer file number"); + const synsetType = take("synset type"); + if (!expectedSynsetTypes.has(synsetType)) { + throw formatError(path, lineNumber, `unexpected synset type ${synsetType}`); + } + const wordCountText = take("word count"); + if (!/^[0-9a-f]{2}$/i.test(wordCountText)) { + throw formatError(path, lineNumber, "invalid hexadecimal word count"); + } + const wordCount = Number.parseInt(wordCountText, 16); + if (wordCount === 0) throw formatError(path, lineNumber, "synset has no words"); + const words: string[] = []; + for (let index = 0; index < wordCount; index += 1) { + const rawWord = take(`word ${index + 1}`); + if (!/^[\x21-\x7e]+$/u.test(rawWord)) + throw formatError(path, lineNumber, `invalid word ${index + 1}`); + const lexId = take(`lex id ${index + 1}`); + if (!/^[0-9a-f]$/iu.test(lexId)) + throw formatError(path, lineNumber, `invalid lex id ${index + 1}`); + const word = normalizeLemma(rawWord); + if (word) words.push(word); + } + if (words.length === 0) throw formatError(path, lineNumber, "synset has no words"); + + const pointerCountText = take("pointer count"); + if (!/^\d{3}$/u.test(pointerCountText)) + throw formatError(path, lineNumber, "invalid pointer count"); + const pointerCount = Number.parseInt(pointerCountText, 10); + for (let index = 0; index < pointerCount; index += 1) { + const symbol = take(`pointer ${index + 1} symbol`); + if (!WORDNET_POINTER_SYMBOLS.has(symbol)) + throw formatError(path, lineNumber, `invalid pointer ${index + 1} symbol`); + if (!/^\d{8}$/u.test(take(`pointer ${index + 1} synset offset`))) + throw formatError(path, lineNumber, `invalid pointer ${index + 1} synset offset`); + if (!/^[nvar]$/u.test(take(`pointer ${index + 1} part of speech`))) + throw formatError(path, lineNumber, `invalid pointer ${index + 1} part of speech`); + const sourceTarget = take(`pointer ${index + 1} source/target`); + if (!/^[0-9a-f]{4}$/iu.test(sourceTarget)) + throw formatError(path, lineNumber, `invalid pointer ${index + 1} source/target`); + const sourceWord = Number.parseInt(sourceTarget.slice(0, 2), 16); + if (sourceWord > wordCount) + throw formatError(path, lineNumber, `pointer ${index + 1} source word is out of range`); + } + + if (synsetType === "v") { + const frameCountText = take("verb frame count"); + if (!/^\d{2}$/u.test(frameCountText)) + throw formatError(path, lineNumber, "invalid verb frame count"); + const frameCount = Number.parseInt(frameCountText, 10); + for (let index = 0; index < frameCount; index += 1) { + if (take(`verb frame ${index + 1} marker`) !== "+") + throw formatError(path, lineNumber, `invalid verb frame ${index + 1} marker`); + if (!/^\d{2}$/u.test(take(`verb frame ${index + 1} number`))) + throw formatError(path, lineNumber, `invalid verb frame ${index + 1} number`); + const wordNumber = take(`verb frame ${index + 1} word number`); + if (!/^[0-9a-f]{2}$/iu.test(wordNumber)) + throw formatError(path, lineNumber, `invalid verb frame ${index + 1} word number`); + if (Number.parseInt(wordNumber, 16) > wordCount) + throw formatError(path, lineNumber, `verb frame ${index + 1} word is out of range`); + } + } + if (cursor !== fields.length) + throw formatError(path, lineNumber, "unexpected token before gloss"); + + const gloss = line.slice(separator + 1).trim(); + const exampleStart = gloss.search(/;\s*"/u); + const definition = (exampleStart >= 0 ? gloss.slice(0, exampleStart) : gloss).trim(); + if (!definition) throw formatError(path, lineNumber, "synset has no definition"); + return { words, definition }; +} + +function normalizeLemma(value: string): string { + return value + .replace(/\((?:a|p|ip)\)$/u, "") + .replaceAll("_", " ") + .normalize("NFKC") + .trim(); +} + +function entryKey(partOfSpeech: PartOfSpeech, word: string): string { + return `${partOfSpeech}\0${word}`; +} + +function addAlias(entry: PendingWordNetEntry, alias: string, tag: string): void { + if (alias !== entry.word && !entry.forms.has(alias)) entry.forms.set(alias, tag); +} + +function regularNounPlural(word: string): string { + if (/[^aeiou]y$/u.test(word)) return `${word.slice(0, -1)}ies`; + if (/(?:s|x|z|ch|sh)$/u.test(word)) return `${word}es`; + return `${word}s`; +} + +function regularVerbThirdPerson(word: string): string { + if (/[^aeiou]y$/u.test(word)) return `${word.slice(0, -1)}ies`; + if (word === "do" || word === "go" || /(?:s|x|z|ch|sh)$/u.test(word)) return `${word}es`; + return `${word}s`; +} + +function formatError(path: string, lineNumber: number, message: string): Error { + return new Error(`Invalid WordNet data in ${path} at line ${lineNumber}: ${message}`); +} diff --git a/packages/core/package.json b/packages/core/package.json index 7c385a87c..892eeac63 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -11,6 +11,8 @@ }, "exports": { ".": "./src/index.ts", + "./dictionary": "./src/dictionary/index.ts", + "./dictionary/*": "./src/dictionary/*.ts", "./types": "./src/types/index.ts", "./types/*": "./src/types/*.ts", "./i18n": "./src/i18n/index.ts", diff --git a/packages/core/src/dictionary/definition-controller.ts b/packages/core/src/dictionary/definition-controller.ts new file mode 100644 index 000000000..b3e9457a4 --- /dev/null +++ b/packages/core/src/dictionary/definition-controller.ts @@ -0,0 +1,144 @@ +import { + type DictionaryEntry, + type DictionaryLanguage, + type DictionaryPackDescriptor, + prepareDictionarySelection, +} from "./index"; + +export type DefinitionState = + | { kind: "idle" } + | { kind: "loading"; displayText: string } + | { kind: "unsupported"; reason: string } + | { kind: "missing-pack"; language: DictionaryLanguage; descriptor: DictionaryPackDescriptor } + | { kind: "downloading"; language: DictionaryLanguage; progress: number } + | { kind: "verifying"; language: DictionaryLanguage } + | { kind: "result"; displayText: string; entries: DictionaryEntry[] } + | { kind: "no-match"; displayText: string } + | { kind: "error"; message: string }; + +export interface DefinitionControllerDependencies { + lookup(text: string): Promise; + install( + descriptor: DictionaryPackDescriptor, + onProgress: (progress: number) => void, + onVerifying?: () => void, + ): Promise; + getDescriptor(language: DictionaryLanguage): DictionaryPackDescriptor | undefined; +} + +type DefinitionStateListener = (state: DefinitionState) => void; + +interface DictionaryError extends Error { + code?: string; +} + +export class DefinitionController { + private requestToken = 0; + private selectedText: string | null = null; + private readonly listeners = new Set(); + + state: DefinitionState = { kind: "idle" }; + + constructor(private readonly dependencies: DefinitionControllerDependencies) {} + + subscribe(listener: DefinitionStateListener): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + async open(text: string): Promise { + const token = ++this.requestToken; + this.selectedText = text; + await this.lookupSelection(text, token); + } + + async retry(): Promise { + if (!this.selectedText) return; + await this.open(this.selectedText); + } + + async download(): Promise { + if (this.state.kind !== "missing-pack" || !this.selectedText) return; + + const { descriptor, language } = this.state; + const text = this.selectedText; + const token = this.requestToken; + this.setState({ kind: "downloading", language, progress: 0 }); + + try { + await this.dependencies.install( + descriptor, + (progress) => { + if (this.isCurrent(token)) { + this.setState({ kind: "downloading", language, progress: clampProgress(progress) }); + } + }, + () => { + if (this.isCurrent(token)) this.setState({ kind: "verifying", language }); + }, + ); + } catch (error) { + if (this.isCurrent(token)) this.setState({ kind: "error", message: messageOf(error) }); + return; + } + + if (this.isCurrent(token)) await this.lookupSelection(text, token); + } + + close(): void { + this.requestToken += 1; + this.selectedText = null; + this.setState({ kind: "idle" }); + } + + private async lookupSelection(text: string, token: number): Promise { + const selection = prepareDictionarySelection(text); + if (!selection.ok) { + if (this.isCurrent(token)) this.setState({ kind: "unsupported", reason: selection.reason }); + return; + } + + this.setState({ kind: "loading", displayText: selection.displayText }); + try { + const entries = await this.dependencies.lookup(text); + if (!this.isCurrent(token)) return; + this.setState( + entries.length > 0 + ? { kind: "result", displayText: selection.displayText, entries } + : { kind: "no-match", displayText: selection.displayText }, + ); + } catch (error) { + if (!this.isCurrent(token)) return; + const dictionaryError = error as DictionaryError; + if (dictionaryError.code === "pack-not-installed") { + const descriptor = this.dependencies.getDescriptor(selection.language); + if (descriptor) { + this.setState({ kind: "missing-pack", language: selection.language, descriptor }); + return; + } + } + if (dictionaryError.code === "unsupported-selection") { + this.setState({ kind: "unsupported", reason: "unsupported-selection" }); + return; + } + this.setState({ kind: "error", message: messageOf(error) }); + } + } + + private isCurrent(token: number): boolean { + return token === this.requestToken; + } + + private setState(state: DefinitionState): void { + this.state = state; + for (const listener of this.listeners) listener(state); + } +} + +function clampProgress(progress: number): number { + return Number.isFinite(progress) ? Math.max(0, Math.min(1, progress)) : 0; +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/core/src/dictionary/dictionary-config.ts b/packages/core/src/dictionary/dictionary-config.ts new file mode 100644 index 000000000..b57eb58b2 --- /dev/null +++ b/packages/core/src/dictionary/dictionary-config.ts @@ -0,0 +1,5 @@ +import manifest from "./dictionary-manifest.json"; +import { parseDictionaryManifest } from "./manifest"; +export const DICTIONARY_BUNDLED_MANIFEST = parseDictionaryManifest(manifest); +export const DEFAULT_DICTIONARY_MANIFEST_URL = + "https://raw.githubusercontent.com/codedogQBY/ReadAny/main/dictionary-packs/manifest.json"; diff --git a/packages/core/src/dictionary/dictionary-database.ts b/packages/core/src/dictionary/dictionary-database.ts new file mode 100644 index 000000000..8483b6efb --- /dev/null +++ b/packages/core/src/dictionary/dictionary-database.ts @@ -0,0 +1,20 @@ +import type { DictionaryLanguage } from "./types"; + +export interface DictionaryDatabaseConnection { + getAllAsync(sql: string, ...params: unknown[]): Promise; + closeAsync(): Promise; +} + +export interface DictionaryDatabaseAdapter { + open(language: DictionaryLanguage, absolutePath: string): Promise; +} + +export class DictionaryLookupError extends Error { + constructor( + readonly code: "unsupported-selection" | "pack-not-installed" | "pack-invalid", + message: string, + ) { + super(message); + this.name = "DictionaryLookupError"; + } +} diff --git a/packages/core/src/dictionary/dictionary-lookup-service.ts b/packages/core/src/dictionary/dictionary-lookup-service.ts new file mode 100644 index 000000000..5a93b3774 --- /dev/null +++ b/packages/core/src/dictionary/dictionary-lookup-service.ts @@ -0,0 +1,159 @@ +import { + type DictionaryDatabaseAdapter, + type DictionaryDatabaseConnection, + DictionaryLookupError, +} from "./dictionary-database"; +import { type DictionaryEntry, type DictionaryLanguage, prepareDictionarySelection } from "./index"; + +const LOOKUP_SQL = ` + WITH matched AS ( + SELECT lookup.entry_id, lookup.rank + FROM lookup + INNER JOIN entries ON entries.id = lookup.entry_id + WHERE lookup.lookup_key = ? AND entries.language = ? + ORDER BY lookup.rank ASC, lookup.entry_id ASC + LIMIT 20 + ) + SELECT + e.id AS entry_id, + e.language, + e.headword, + e.simplified, + e.traditional, + e.pronunciation, + e.part_of_speech, + matched.rank, + s.sense_order, + s.definition + FROM matched + INNER JOIN entries e ON e.id = matched.entry_id + INNER JOIN senses s ON s.entry_id = e.id + ORDER BY matched.rank ASC, e.id ASC, s.sense_order ASC +`; + +interface DictionaryLookupRow { + entry_id: number; + language: DictionaryLanguage; + headword: string; + simplified: string | null; + traditional: string | null; + pronunciation: string | null; + part_of_speech: string; + rank: number; + sense_order: number; + definition: string; +} + +export type DictionaryPackPathResolver = ( + language: DictionaryLanguage, +) => string | null | Promise; + +export type DictionaryPackInvalidator = (language: DictionaryLanguage) => Promise | void; + +export class DictionaryLookupService { + private readonly connections = new Map< + DictionaryLanguage, + Promise + >(); + + constructor( + private readonly database: DictionaryDatabaseAdapter, + private readonly resolvePackPath: DictionaryPackPathResolver, + private readonly invalidatePack: DictionaryPackInvalidator = () => {}, + ) {} + + async lookup(text: string): Promise { + const selection = prepareDictionarySelection(text); + if (!selection.ok) { + throw new DictionaryLookupError( + "unsupported-selection", + `Dictionary lookup does not support this selection: ${selection.reason}`, + ); + } + + const absolutePath = await this.resolvePackPath(selection.language); + if (!absolutePath) { + throw new DictionaryLookupError( + "pack-not-installed", + `The ${selection.language} dictionary pack is not installed`, + ); + } + + try { + const database = await this.connectionFor(selection.language, absolutePath); + const rows = await database.getAllAsync( + LOOKUP_SQL, + selection.key, + selection.language, + ); + return this.entriesFromRows(rows); + } catch (error) { + const cleanupErrors: unknown[] = []; + try { + await this.close(selection.language); + } catch (cleanupError) { + cleanupErrors.push(cleanupError); + } + try { + await this.invalidatePack(selection.language); + } catch (cleanupError) { + cleanupErrors.push(cleanupError); + } + if (error instanceof Error && cleanupErrors.length > 0) { + (error as Error & { cleanupErrors?: unknown[] }).cleanupErrors = cleanupErrors; + } + throw error; + } + } + + async close(language?: DictionaryLanguage): Promise { + const languages = language ? [language] : [...this.connections.keys()]; + await Promise.all( + languages.map(async (currentLanguage) => { + const connection = this.connections.get(currentLanguage); + if (!connection) return; + this.connections.delete(currentLanguage); + await (await connection).closeAsync(); + }), + ); + } + + private connectionFor( + language: DictionaryLanguage, + absolutePath: string, + ): Promise { + const existing = this.connections.get(language); + if (existing) return existing; + + const opening = this.database.open(language, absolutePath); + this.connections.set(language, opening); + void opening.catch(() => { + if (this.connections.get(language) === opening) { + this.connections.delete(language); + } + }); + return opening; + } + + private entriesFromRows(rows: DictionaryLookupRow[]): DictionaryEntry[] { + const entries = new Map(); + for (const row of rows) { + let entry = entries.get(row.entry_id); + if (!entry) { + entry = { + id: row.entry_id, + language: row.language, + headword: row.headword, + simplified: row.simplified ?? undefined, + traditional: row.traditional ?? undefined, + pronunciation: row.pronunciation ?? undefined, + partOfSpeech: row.part_of_speech, + senses: [], + }; + entries.set(row.entry_id, entry); + } + entry.senses.push({ order: row.sense_order, definition: row.definition }); + } + return [...entries.values()]; + } +} diff --git a/packages/core/src/dictionary/dictionary-manifest.json b/packages/core/src/dictionary/dictionary-manifest.json new file mode 100644 index 000000000..08150a75b --- /dev/null +++ b/packages/core/src/dictionary/dictionary-manifest.json @@ -0,0 +1,31 @@ +{ + "manifestVersion": 1, + "packs": { + "en": { + "language": "en", + "version": "1.0.0", + "schemaVersion": 1, + "sourceEdition": "wordnet-3.1", + "sourceDumpDate": "2011-05-26", + "sizeBytes": 29069312, + "sha256": "90adbeab5ee325b31f2e34bbfa7c5b699932c900bfd1efdfc62eabd9e004ee0e", + "url": "https://github.com/cha1latte/ReadAny/releases/download/dictionary-packs-v1/readany-dictionary-en-v1.sqlite", + "sourceArchiveUrl": "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + "attributionUrl": "https://wordnet.princeton.edu/license-and-commercial-use", + "license": "WordNet 3.1 License" + }, + "zh": { + "language": "zh", + "version": "1.0.0", + "schemaVersion": 1, + "sourceEdition": "zhwiktionary", + "sourceDumpDate": "2026-09-01", + "sizeBytes": 11866112, + "sha256": "d50218459f78a5e7bcba819fbb7db699271bdaf30757dc8b2443cb1e37bc11b1", + "url": "https://github.com/cha1latte/ReadAny/releases/download/dictionary-packs-v1/readany-dictionary-zh-v1.sqlite", + "sourceArchiveUrl": "https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2", + "attributionUrl": "https://zh.wiktionary.org/wiki/Wiktionary:%E7%89%88%E6%9D%83%E4%BF%A1%E6%81%AF", + "license": "CC BY-SA 4.0" + } + } +} diff --git a/packages/core/src/dictionary/dictionary-pack-manager.ts b/packages/core/src/dictionary/dictionary-pack-manager.ts new file mode 100644 index 000000000..49bd45097 --- /dev/null +++ b/packages/core/src/dictionary/dictionary-pack-manager.ts @@ -0,0 +1,438 @@ +import type { DictionaryLanguage, DictionaryManifest, DictionaryPackDescriptor } from "./index"; + +const dictionaryLanguages = ["en", "zh"] as const; + +export type DictionaryPackMetadata = Pick< + DictionaryPackDescriptor, + | "language" + | "version" + | "schemaVersion" + | "sourceEdition" + | "sourceDumpDate" + | "sourceArchiveUrl" + | "url" + | "attributionUrl" + | "license" +> & { + licenseNotice: string; + creatorAttribution: string; +}; + +export type InstalledDictionaryPack = DictionaryPackMetadata & + Pick; + +export interface DictionaryPackPlatform { + ensureDirectory(path: string): Promise; + download(url: string, path: string, onProgress: (fraction: number) => void): Promise; + exists(path: string): Promise; + size(path: string): Promise; + sha256(path: string): Promise; + readMetadata(path: string): Promise; + move(from: string, to: string): Promise; + remove(path: string): Promise; +} + +export type DictionaryPackStatus = + | { state: "not-installed" } + | { state: "downloading"; progress: number } + | { state: "verifying" } + | { state: "installed"; version: string; sizeBytes: number } + | { + state: "update-available"; + installedVersion: string; + availableVersion: string; + sizeBytes: number; + } + | { state: "error"; message: string; hasActivePack: boolean }; + +export interface DictionaryHandleCloser { + close(language?: DictionaryLanguage): Promise | void; +} + +type ErrorWithCleanup = Error & { cleanupErrors?: unknown[] }; + +interface InspectedArtifact { + path: string; + exists: boolean; + installed?: InstalledDictionaryPack; + error?: unknown; +} + +interface InspectedArtifacts { + active: InspectedArtifact; + backup: InspectedArtifact; +} + +export class DictionaryPackManager { + private readonly installs = new Map>(); + private readonly gates = new Map>(); + private readonly validatedPacks = new Map< + DictionaryLanguage, + { path: string; installed: InstalledDictionaryPack } | null + >(); + + constructor( + private readonly platform: DictionaryPackPlatform, + private readonly directory: string, + private readonly lookup: DictionaryHandleCloser, + ) {} + + async refresh( + manifest: DictionaryManifest, + ): Promise> { + const entries = await Promise.all( + dictionaryLanguages.map((language) => + this.runExclusive(language, async () => { + try { + const discovered = await this.discoverInstalledLocked(language); + const installed = discovered?.installed; + const available = manifest.packs[language]; + const status: DictionaryPackStatus = !installed + ? { state: "not-installed" } + : sameDescriptorIdentity(installed, available) + ? { state: "installed", version: installed.version, sizeBytes: installed.sizeBytes } + : { + state: "update-available", + installedVersion: installed.version, + availableVersion: available.version, + sizeBytes: installed.sizeBytes, + }; + return [language, status] as const; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const hasActivePack = await this.hasPackArtifacts(language); + return [ + language, + { state: "error", message, hasActivePack } satisfies DictionaryPackStatus, + ] as const; + } + }), + ), + ); + return Object.fromEntries(entries) as Record; + } + + install( + descriptor: DictionaryPackDescriptor, + onStatus: (status: DictionaryPackStatus) => void = () => {}, + ): Promise { + const language = descriptor.language; + const existing = this.installs.get(language); + if (existing) return existing; + + const operation = this.runExclusive(language, () => + this.installLocked(descriptor, onStatus), + ).finally(() => { + if (this.installs.get(language) === operation) this.installs.delete(language); + }); + this.installs.set(language, operation); + return operation; + } + + remove(language: DictionaryLanguage): Promise { + return this.runExclusive(language, async () => { + this.invalidate(language); + await this.platform.ensureDirectory(this.directory); + await this.lookup.close(language); + await this.removeIfPresent(this.activePath(language)); + await this.removeIfPresent(this.stagedPath(language)); + await this.removeIfPresent(this.backupPath(language)); + }); + } + + getActivePath(language: DictionaryLanguage): Promise { + return this.runExclusive(language, async () => { + const discovered = await this.discoverInstalledLocked(language); + return discovered?.path ?? null; + }); + } + + getInstalledDescriptor(language: DictionaryLanguage): Promise { + return this.runExclusive(language, async () => { + return (await this.discoverInstalledLocked(language))?.installed ?? null; + }); + } + + invalidate(language: DictionaryLanguage): void { + this.validatedPacks.delete(language); + } + + private async installLocked( + descriptor: DictionaryPackDescriptor, + onStatus: (status: DictionaryPackStatus) => void, + ): Promise { + const language = descriptor.language; + this.invalidate(language); + const staged = this.stagedPath(language); + const active = this.activePath(language); + const backup = this.backupPath(language); + try { + await this.platform.ensureDirectory(this.directory); + await this.removeIfPresent(staged); + const artifacts = await this.inspectArtifactsLocked(language); + onStatus({ state: "downloading", progress: 0 }); + await this.platform.download(descriptor.url, staged, (progress) => + onStatus({ state: "downloading", progress: clamp(progress) }), + ); + onStatus({ state: "verifying" }); + const verificationStarted = Date.now(); + await this.verifyExpected(staged, descriptor); + await this.lookup.close(language); + + let hasRollbackBackup = Boolean(artifacts.backup.installed); + let activationStarted = false; + try { + if (artifacts.backup.installed) { + activationStarted = true; + await this.removeIfPresent(active); + } else if (artifacts.active.installed) { + await this.removeIfPresent(backup); + await this.platform.move(active, backup); + hasRollbackBackup = true; + activationStarted = true; + } else { + await this.removeIfPresent(active); + await this.removeIfPresent(backup); + activationStarted = true; + } + await this.platform.move(staged, active); + const installed = await this.verifyExpected(active, descriptor); + await this.removeIfPresent(backup); + this.validatedPacks.set(language, { path: active, installed }); + } catch (operationError) { + const cleanupErrors: unknown[] = []; + if (hasRollbackBackup) { + await this.captureCleanupFailure(cleanupErrors, () => + this.rollbackLocked(active, backup), + ); + } else if (activationStarted) { + await this.captureCleanupFailure(cleanupErrors, () => this.removeIfPresent(active)); + } + attachCleanupErrors(operationError, cleanupErrors); + throw operationError; + } + + console.info( + `[Dictionary] Verified and installed ${language} pack in ${Date.now() - verificationStarted} ms`, + ); + onStatus({ + state: "installed", + version: descriptor.version, + sizeBytes: descriptor.sizeBytes, + }); + } catch (operationError) { + const cleanupErrors: unknown[] = []; + await this.captureCleanupFailure(cleanupErrors, () => this.removeIfPresent(staged)); + attachCleanupErrors(operationError, cleanupErrors); + const message = + operationError instanceof Error ? operationError.message : String(operationError); + onStatus({ + state: "error", + message, + hasActivePack: await this.hasPackArtifacts(language), + }); + throw operationError; + } + } + + private async verifyExpected( + path: string, + descriptor: DictionaryPackDescriptor, + ): Promise { + const installed = await this.inspectPack(path); + if (installed.sizeBytes !== descriptor.sizeBytes) + throw new Error("Dictionary pack size did not match manifest"); + if (installed.sha256.toLowerCase() !== descriptor.sha256.toLowerCase()) + throw new Error("Dictionary pack checksum did not match manifest"); + assertMetadataMatches(installed, descriptor); + return installed; + } + + private async rollbackLocked(active: string, backup: string): Promise { + if (!(await this.platform.exists(backup))) return; + const backupSnapshot = await this.inspectPack(backup); + await this.removeIfPresent(active); + await this.platform.move(backup, active); + const restoredSnapshot = await this.inspectPack(active); + assertSamePack(backupSnapshot, restoredSnapshot); + this.validatedPacks.set(restoredSnapshot.language, { + path: active, + installed: restoredSnapshot, + }); + } + + private async discoverInstalledLocked( + language: DictionaryLanguage, + ): Promise<{ path: string; installed: InstalledDictionaryPack } | null> { + if (this.validatedPacks.has(language)) return this.validatedPacks.get(language) ?? null; + await this.platform.ensureDirectory(this.directory); + const artifacts = await this.inspectArtifactsLocked(language); + if (artifacts.backup.installed) { + const discovered = { path: artifacts.backup.path, installed: artifacts.backup.installed }; + this.validatedPacks.set(language, discovered); + return discovered; + } + if (artifacts.active.installed) { + const discovered = { path: artifacts.active.path, installed: artifacts.active.installed }; + this.validatedPacks.set(language, discovered); + return discovered; + } + + const errors = [artifacts.backup.error, artifacts.active.error].filter( + (error): error is NonNullable => error !== undefined, + ); + if (errors.length === 1) throw errors[0]; + if (errors.length > 1) { + throw new AggregateError(errors, `No valid ${language} dictionary recovery artifact exists`); + } + this.validatedPacks.set(language, null); + return null; + } + + private async inspectArtifactsLocked(language: DictionaryLanguage): Promise { + const [active, backup] = await Promise.all([ + this.inspectArtifact(this.activePath(language), language), + this.inspectArtifact(this.backupPath(language), language), + ]); + return { active, backup }; + } + + private async inspectArtifact( + path: string, + language: DictionaryLanguage, + ): Promise { + if (!(await this.platform.exists(path))) return { path, exists: false }; + try { + const installed = await this.inspectPack(path); + if (installed.language !== language) { + throw new Error(`Dictionary metadata language did not match ${language}`); + } + return { path, exists: true, installed }; + } catch (error) { + return { path, exists: true, error }; + } + } + + private async hasPackArtifacts(language: DictionaryLanguage): Promise { + return ( + (await this.platform.exists(this.activePath(language))) || + (await this.platform.exists(this.backupPath(language))) || + (await this.platform.exists(this.stagedPath(language))) + ); + } + + private async inspectPack(path: string): Promise { + const [metadata, sizeBytes, sha256] = await Promise.all([ + this.platform.readMetadata(path), + this.platform.size(path), + this.platform.sha256(path), + ]); + return { ...metadata, sizeBytes, sha256: sha256.toLowerCase() }; + } + + private runExclusive(language: DictionaryLanguage, action: () => Promise): Promise { + const previous = this.gates.get(language) ?? Promise.resolve(); + const operation = previous.catch(() => undefined).then(action); + const tail = operation.then( + () => undefined, + () => undefined, + ); + this.gates.set(language, tail); + void tail.then(() => { + if (this.gates.get(language) === tail) this.gates.delete(language); + }); + return operation; + } + + private async captureCleanupFailure( + errors: unknown[], + cleanup: () => Promise, + ): Promise { + try { + await cleanup(); + } catch (error) { + errors.push(error); + } + } + + private activePath(language: DictionaryLanguage): string { + return `${this.directory}/readany-dictionary-${language}.sqlite`; + } + + private stagedPath(language: DictionaryLanguage): string { + return `${this.activePath(language)}.download`; + } + + private backupPath(language: DictionaryLanguage): string { + return `${this.activePath(language)}.backup`; + } + + private async removeIfPresent(path: string): Promise { + if (await this.platform.exists(path)) await this.platform.remove(path); + } +} + +function assertMetadataMatches( + installed: DictionaryPackMetadata, + expected: DictionaryPackDescriptor, +): void { + for (const key of [ + "language", + "version", + "schemaVersion", + "sourceEdition", + "sourceDumpDate", + "sourceArchiveUrl", + "url", + "attributionUrl", + "license", + ] as const) { + if (installed[key] !== expected[key]) + throw new Error(`Dictionary metadata ${key} did not match manifest`); + } +} + +function sameDescriptorIdentity( + installed: InstalledDictionaryPack, + available: DictionaryPackDescriptor, +): boolean { + return ( + installed.sizeBytes === available.sizeBytes && + installed.sha256.toLowerCase() === available.sha256.toLowerCase() && + [ + "language", + "version", + "schemaVersion", + "sourceEdition", + "sourceDumpDate", + "sourceArchiveUrl", + "url", + "attributionUrl", + "license", + ].every( + (key) => + installed[key as keyof InstalledDictionaryPack] === + available[key as keyof DictionaryPackDescriptor], + ) + ); +} + +function assertSamePack(expected: InstalledDictionaryPack, actual: InstalledDictionaryPack): void { + if ( + expected.sizeBytes !== actual.sizeBytes || + expected.sha256 !== actual.sha256 || + JSON.stringify(expected) !== JSON.stringify(actual) + ) { + throw new Error("Restored dictionary pack did not match the validated backup"); + } +} + +function attachCleanupErrors(operationError: unknown, cleanupErrors: unknown[]): void { + if (cleanupErrors.length === 0 || !(operationError instanceof Error)) return; + const error = operationError as ErrorWithCleanup; + error.cleanupErrors = [...(error.cleanupErrors ?? []), ...cleanupErrors]; +} + +function clamp(value: number): number { + return Number.isFinite(value) ? Math.max(0, Math.min(1, value)) : 0; +} diff --git a/packages/core/src/dictionary/dictionary-runtime.ts b/packages/core/src/dictionary/dictionary-runtime.ts new file mode 100644 index 000000000..ec4bebeba --- /dev/null +++ b/packages/core/src/dictionary/dictionary-runtime.ts @@ -0,0 +1,29 @@ +import type { DictionaryDatabaseAdapter } from "./dictionary-database"; +import { DictionaryLookupService } from "./dictionary-lookup-service"; +import { DictionaryPackManager, type DictionaryPackPlatform } from "./dictionary-pack-manager"; + +export interface DictionaryRuntimeOptions { + database: DictionaryDatabaseAdapter; + directory: string; + platform: DictionaryPackPlatform; +} + +export function createDictionaryRuntime(options: DictionaryRuntimeOptions): { + lookup: DictionaryLookupService; + manager: DictionaryPackManager; +} { + // biome-ignore lint/style/useConst: the lookup resolves installed paths through the manager created below. + let manager: DictionaryPackManager | undefined; + const lookup = new DictionaryLookupService( + options.database, + async (language) => { + if (!manager) throw new Error("Dictionary runtime is not initialized"); + return manager.getActivePath(language); + }, + (language) => { + manager?.invalidate(language); + }, + ); + manager = new DictionaryPackManager(options.platform, options.directory, lookup); + return { lookup, manager }; +} diff --git a/packages/core/src/dictionary/dictionary-store.ts b/packages/core/src/dictionary/dictionary-store.ts new file mode 100644 index 000000000..bd9372ea2 --- /dev/null +++ b/packages/core/src/dictionary/dictionary-store.ts @@ -0,0 +1,156 @@ +import { type StoreApi, type UseBoundStore, create } from "zustand"; +import type { DictionaryLookupService } from "./dictionary-lookup-service"; +import type { DictionaryPackManager, DictionaryPackStatus } from "./dictionary-pack-manager"; +import { + type DictionaryEntry, + type DictionaryLanguage, + type DictionaryManifest, + type DictionaryPackDescriptor, + parseDictionaryManifest, +} from "./index"; +export interface DictionaryStoreDependencies { + manager: Pick; + lookup: Pick; + fetchRemoteManifest: () => Promise; + getBundledManifest: () => Promise; +} + +export interface DictionaryRuntime { + manager: Pick; + lookup: Pick; +} + +export interface RuntimeBackedDictionaryStoreDependencies { + loadRuntime: () => Promise; + fetchRemoteManifest: () => Promise; + getBundledManifest: () => Promise; +} + +export interface DictionaryStoreState { + manifest: DictionaryManifest | null; + packs: Record; + initialize(): Promise; + refreshManifest(): Promise; + install(language: DictionaryLanguage): Promise; + remove(language: DictionaryLanguage): Promise; + retry(): Promise; + lookup(text: string): Promise; +} + +const emptyPacks = (): Record => ({ + en: { state: "not-installed" }, + zh: { state: "not-installed" }, +}); + +export function createDictionaryStore( + deps: DictionaryStoreDependencies, +): UseBoundStore> { + return create()((set, get) => { + let remoteRefreshPromise: Promise | undefined; + let bundledReadinessPromise: Promise | undefined; + const applyManifest = async (manifest: DictionaryManifest) => { + const packs = await deps.manager.refresh(manifest); + set({ manifest, packs }); + }; + const loadManifest = async (): Promise => { + let remoteError: unknown; + try { + return parseDictionaryManifest(await deps.fetchRemoteManifest()); + } catch (error) { + remoteError = error; + } + try { + return parseDictionaryManifest(await deps.getBundledManifest()); + } catch (bundledError) { + throw new AggregateError( + [remoteError, bundledError], + "No valid dictionary manifest is available", + ); + } + }; + const refreshFromSources = (): Promise => { + if (remoteRefreshPromise) return remoteRefreshPromise; + const operation = loadManifest().then(applyManifest); + remoteRefreshPromise = operation; + void operation.then( + () => { + if (remoteRefreshPromise === operation) remoteRefreshPromise = undefined; + }, + () => { + if (remoteRefreshPromise === operation) remoteRefreshPromise = undefined; + }, + ); + return operation; + }; + const refreshFromBundled = (): Promise => { + if (get().manifest) return Promise.resolve(); + if (bundledReadinessPromise) return bundledReadinessPromise; + const operation = deps + .getBundledManifest() + .then(parseDictionaryManifest) + .then(async (manifest) => { + if (!get().manifest) await applyManifest(manifest); + }); + bundledReadinessPromise = operation; + void operation.then( + () => { + if (bundledReadinessPromise === operation) bundledReadinessPromise = undefined; + }, + () => { + if (bundledReadinessPromise === operation) bundledReadinessPromise = undefined; + }, + ); + return operation; + }; + + return { + manifest: null, + packs: emptyPacks(), + initialize: refreshFromSources, + refreshManifest: refreshFromSources, + install: async (language) => { + const descriptor: DictionaryPackDescriptor | undefined = get().manifest?.packs[language]; + if (!descriptor) throw new Error("Dictionary manifest is unavailable"); + await deps.manager.install(descriptor, (status) => + set((state) => ({ packs: { ...state.packs, [language]: status } })), + ); + }, + remove: async (language) => { + await deps.manager.remove(language); + set((state) => ({ packs: { ...state.packs, [language]: { state: "not-installed" } } })); + }, + retry: refreshFromSources, + lookup: async (text) => { + if (!get().manifest) await refreshFromBundled(); + return deps.lookup.lookup(text); + }, + }; + }); +} + +export function createRuntimeBackedDictionaryStore( + deps: RuntimeBackedDictionaryStoreDependencies, +): UseBoundStore> { + let runtimePromise: Promise | undefined; + const runtime = () => { + if (!runtimePromise) { + const operation = deps.loadRuntime(); + runtimePromise = operation; + void operation.catch(() => { + if (runtimePromise === operation) runtimePromise = undefined; + }); + } + return runtimePromise; + }; + return createDictionaryStore({ + manager: { + refresh: async (manifest) => (await runtime()).manager.refresh(manifest), + install: async (descriptor, onStatus) => + (await runtime()).manager.install(descriptor, onStatus), + remove: async (language) => (await runtime()).manager.remove(language), + }, + lookup: { lookup: async (text) => (await runtime()).lookup.lookup(text) }, + fetchRemoteManifest: deps.fetchRemoteManifest, + getBundledManifest: deps.getBundledManifest, + }); +} diff --git a/packages/core/src/dictionary/dictionary-validation.ts b/packages/core/src/dictionary/dictionary-validation.ts new file mode 100644 index 000000000..41e62e4e8 --- /dev/null +++ b/packages/core/src/dictionary/dictionary-validation.ts @@ -0,0 +1,144 @@ +import type { DictionaryPackMetadata } from "./dictionary-pack-manager"; +interface SqliteObjectRow { + name: string; + type: string; + tbl_name: string; +} + +interface SqliteColumnRow { + name: string; +} + +interface SqliteIndexColumnRow { + name: string; + seqno: number; +} + +interface DictionaryMetadataRow { + key: string; + value: string; +} + +export interface DictionaryValidationDatabase { + getFirstAsync(sql: string): Promise; + getAllAsync(sql: string): Promise; +} + +const requiredObjects = new Map([ + ["metadata", "table"], + ["entries", "table"], + ["senses", "table"], + ["lookup", "table"], + ["lookup_key_rank_idx", "index"], +]); + +const requiredColumns = { + metadata: ["key", "value"], + entries: [ + "id", + "language", + "headword", + "simplified", + "traditional", + "pronunciation", + "part_of_speech", + ], + senses: ["entry_id", "sense_order", "definition"], + lookup: ["lookup_key", "entry_id", "rank"], +} as const; + +const requiredMetadataKeys = [ + "schema_version", + "language", + "version", + "source_edition", + "source_dump_date", + "source_archive_url", + "asset_url", + "attribution_url", + "license", + "license_notice", + "creator_attribution", +] as const; + +export async function validateDictionaryDatabase( + database: DictionaryValidationDatabase, +): Promise { + const integrity = await database.getFirstAsync<{ integrity_check: string }>( + "PRAGMA integrity_check", + ); + if (integrity?.integrity_check !== "ok") + throw new Error("Dictionary SQLite integrity check failed"); + + const objects = await database.getAllAsync( + "SELECT name, type, tbl_name FROM sqlite_master WHERE name IN ('metadata', 'entries', 'senses', 'lookup', 'lookup_key_rank_idx')", + ); + const objectsByName = new Map(objects.map((object) => [object.name, object])); + for (const [name, type] of requiredObjects) { + if (objectsByName.get(name)?.type !== type) + throw new Error(`Dictionary SQLite schema requires ${name} ${type}`); + } + if (objectsByName.get("lookup_key_rank_idx")?.tbl_name !== "lookup") + throw new Error("Dictionary SQLite lookup_key_rank_idx must belong to lookup"); + + for (const [table, expectedColumns] of Object.entries(requiredColumns)) { + const columns = await database.getAllAsync(`PRAGMA table_info('${table}')`); + const actualColumns = columns.map((column) => column.name); + if (!sameArray(actualColumns, expectedColumns)) + throw new Error(`Dictionary SQLite ${table} columns did not match the required schema`); + } + + const indexColumns = await database.getAllAsync( + "PRAGMA index_info('lookup_key_rank_idx')", + ); + const orderedIndexColumns = [...indexColumns] + .sort((left, right) => left.seqno - right.seqno) + .map((column) => column.name); + if (!sameArray(orderedIndexColumns, ["lookup_key", "rank", "entry_id"])) + throw new Error("Dictionary SQLite lookup index columns were not in the required order"); + + const metadataRows = await database.getAllAsync( + `SELECT key, value FROM metadata WHERE key IN (${requiredMetadataKeys + .map((key) => `'${key}'`) + .join(", ")})`, + ); + const metadata = new Map(metadataRows.map((row) => [row.key, row.value])); + const value = (key: (typeof requiredMetadataKeys)[number]): string => { + const found = metadata.get(key); + if (!found?.trim()) throw new Error(`Dictionary metadata ${key} is missing`); + return found; + }; + + const schemaVersion = value("schema_version"); + if (schemaVersion !== "1") throw new Error("Dictionary metadata schema_version is unsupported"); + const language = value("language"); + if (language !== "en" && language !== "zh") + throw new Error("Dictionary metadata language is unsupported"); + const sourceEdition = value("source_edition"); + const license = value("license"); + const common = { + version: value("version"), + sourceDumpDate: value("source_dump_date"), + sourceArchiveUrl: value("source_archive_url"), + url: value("asset_url"), + attributionUrl: value("attribution_url"), + licenseNotice: value("license_notice"), + creatorAttribution: value("creator_attribution"), + }; + if (language === "en" && sourceEdition === "wordnet-3.1" && license === "WordNet 3.1 License") { + return { ...common, schemaVersion: 1, language, sourceEdition, license }; + } + if (language === "en" && sourceEdition === "enwiktionary" && license === "CC BY-SA 4.0") { + return { ...common, schemaVersion: 1, language, sourceEdition, license }; + } + if (language === "zh" && sourceEdition === "zhwiktionary" && license === "CC BY-SA 4.0") { + return { ...common, schemaVersion: 1, language, sourceEdition, license }; + } + throw new Error("Dictionary metadata source/license combination is unsupported"); +} + +function sameArray(actual: readonly string[], expected: readonly string[]): boolean { + return ( + actual.length === expected.length && actual.every((value, index) => value === expected[index]) + ); +} diff --git a/packages/core/src/dictionary/index.ts b/packages/core/src/dictionary/index.ts new file mode 100644 index 000000000..7a16ea8dd --- /dev/null +++ b/packages/core/src/dictionary/index.ts @@ -0,0 +1,15 @@ +export { parseDictionaryManifest } from "./manifest"; +export { prepareDictionarySelection } from "./selection"; +export type { + ChineseDictionaryPackDescriptor, + DictionaryEntry, + DictionaryLanguage, + DictionaryManifest, + DictionaryPackDescriptor, + DictionaryLicense, + DictionarySense, + DictionarySource, + DictionarySourceEdition, + EnglishDictionaryPackDescriptor, + PreparedDictionarySelection, +} from "./types"; diff --git a/packages/core/src/dictionary/manifest.test.ts b/packages/core/src/dictionary/manifest.test.ts new file mode 100644 index 000000000..a8f0f1d40 --- /dev/null +++ b/packages/core/src/dictionary/manifest.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from "vitest"; +import { parseDictionaryManifest } from "./manifest"; + +const validPack = { + language: "en", + version: "2026.9.3", + schemaVersion: 1, + sourceEdition: "wordnet-3.1", + sourceDumpDate: "2011-05-26", + sizeBytes: 123, + sha256: "a".repeat(64), + url: "https://cdn.example.com/dictionaries/en.sqlite3", + sourceArchiveUrl: "https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz", + attributionUrl: "https://wordnet.princeton.edu/", + license: "WordNet 3.1 License", +} as const; + +function validManifest() { + return { + manifestVersion: 1, + packs: { + en: validPack, + zh: { + ...validPack, + language: "zh", + sourceEdition: "zhwiktionary", + sourceDumpDate: "2026-09-01", + url: "https://cdn.example.com/dictionaries/zh.sqlite3", + sourceArchiveUrl: + "https://dumps.wikimedia.org/zhwiktionary/20260901/zhwiktionary-20260901-pages-articles.xml.bz2", + attributionUrl: + "https://zh.wiktionary.org/wiki/Wiktionary:%E7%89%88%E6%9D%83%E4%BF%A1%E6%81%AF", + license: "CC BY-SA 4.0", + }, + }, + }; +} + +describe("parseDictionaryManifest", () => { + it("parses valid English and Chinese descriptors", () => { + expect(parseDictionaryManifest(validManifest())).toEqual(validManifest()); + }); + + it("continues to accept an English Wiktionary descriptor", () => { + const manifest = validManifest(); + manifest.packs.en = { + ...manifest.packs.en, + sourceEdition: "enwiktionary", + sourceDumpDate: "2026-09-01", + attributionUrl: "https://en.wiktionary.org/wiki/Wiktionary:Copyrights", + license: "CC BY-SA 4.0", + } as typeof manifest.packs.en; + + expect(parseDictionaryManifest(manifest)).toEqual(manifest); + }); + + it.each([ + ["WordNet with the Wiktionary license", "wordnet-3.1", "CC BY-SA 4.0"], + ["English Wiktionary with the WordNet license", "enwiktionary", "WordNet 3.1 License"], + ])("rejects %s", (_description, sourceEdition, license) => { + const manifest = validManifest(); + const invalid = { + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, sourceEdition, license } }, + }; + + expect(() => parseDictionaryManifest(invalid)).toThrow(); + }); + + it.each(["1", "1.2", "01.2.3", "1.02.3", "1.2.03", "1.2.3-01", "v1.2.3"])( + "rejects non-SemVer version %s", + (version) => { + const manifest = validManifest(); + expect(() => + parseDictionaryManifest({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, version } }, + }), + ).toThrow(); + }, + ); + + it.each(["2026-02-30", "2026-13-01", "2026-9-01", "not-a-date"])( + "rejects invalid source date %s", + (sourceDumpDate) => { + const manifest = validManifest(); + expect(() => + parseDictionaryManifest({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, sourceDumpDate } }, + }), + ).toThrow(); + }, + ); + + it("accepts a pack exactly at the 150 MiB limit", () => { + const manifest = validManifest(); + manifest.packs.en = { ...manifest.packs.en, sizeBytes: 150 * 1024 * 1024 }; + expect(parseDictionaryManifest(manifest)).toEqual(manifest); + }); + + it.each([150 * 1024 * 1024 + 1, 1.5, 0])("rejects invalid pack size %s", (sizeBytes) => { + const manifest = validManifest(); + expect(() => + parseDictionaryManifest({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, sizeBytes } }, + }), + ).toThrow(); + }); + + it.each(["http://example.test/file", "file:///tmp/file", "javascript:alert(1)"])( + "rejects non-HTTPS URL %s in every URL field", + (url) => { + for (const field of ["url", "sourceArchiveUrl", "attributionUrl"] as const) { + const manifest = validManifest(); + expect(() => + parseDictionaryManifest({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, [field]: url } }, + }), + ).toThrow(); + } + }, + ); + + it.each([ + [ + "an invalid URL", + (manifest: ReturnType) => ({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, url: "not-a-url" } }, + }), + ], + [ + "a non-hex SHA-256", + (manifest: ReturnType) => ({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, sha256: "z".repeat(64) } }, + }), + ], + [ + "a non-positive byte size", + (manifest: ReturnType) => ({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, sizeBytes: 0 } }, + }), + ], + [ + "a wrong language key", + (manifest: ReturnType) => ({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, language: "zh" } }, + }), + ], + [ + "a schema version other than 1", + (manifest: ReturnType) => ({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, schemaVersion: 2 } }, + }), + ], + [ + "an unknown descriptor field", + (manifest: ReturnType) => ({ + ...manifest, + packs: { ...manifest.packs, en: { ...manifest.packs.en, extra: true } }, + }), + ], + ])("rejects %s", (_description, invalidManifest) => { + expect(() => parseDictionaryManifest(invalidManifest(validManifest()))).toThrow(); + }); +}); diff --git a/packages/core/src/dictionary/manifest.ts b/packages/core/src/dictionary/manifest.ts new file mode 100644 index 000000000..4ab4b8c15 --- /dev/null +++ b/packages/core/src/dictionary/manifest.ts @@ -0,0 +1,68 @@ +import { z } from "zod"; +import type { DictionaryManifest } from "./types"; + +export const MAX_DICTIONARY_PACK_BYTES = 150 * 1024 * 1024; +const SEMVER = + /^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:(?:0|[1-9]\d*)|\d*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:(?:0|[1-9]\d*)|\d*[A-Za-z-][0-9A-Za-z-]*))*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/u; + +const semverSchema = z.string().regex(SEMVER); +const calendarDateSchema = z.string().refine((value) => { + if (!/^\d{4}-\d{2}-\d{2}$/u.test(value)) return false; + const date = new Date(`${value}T00:00:00.000Z`); + return !Number.isNaN(date.getTime()) && date.toISOString().slice(0, 10) === value; +}); +const httpsUrlSchema = z + .string() + .url() + .refine((value) => { + try { + return new URL(value).protocol === "https:"; + } catch { + return false; + } + }); + +const descriptorFields = { + version: semverSchema, + schemaVersion: z.literal(1), + sourceDumpDate: calendarDateSchema, + sizeBytes: z.number().int().positive().max(MAX_DICTIONARY_PACK_BYTES), + sha256: z.string().regex(/^[a-fA-F0-9]{64}$/), + url: httpsUrlSchema, + sourceArchiveUrl: httpsUrlSchema, + attributionUrl: httpsUrlSchema, +} as const; + +const englishPackDescriptorSchema = z.discriminatedUnion("sourceEdition", [ + z.strictObject({ + ...descriptorFields, + language: z.literal("en"), + sourceEdition: z.literal("wordnet-3.1"), + license: z.literal("WordNet 3.1 License"), + }), + z.strictObject({ + ...descriptorFields, + language: z.literal("en"), + sourceEdition: z.literal("enwiktionary"), + license: z.literal("CC BY-SA 4.0"), + }), +]); + +const chinesePackDescriptorSchema = z.strictObject({ + ...descriptorFields, + language: z.literal("zh"), + sourceEdition: z.literal("zhwiktionary"), + license: z.literal("CC BY-SA 4.0"), +}); + +const dictionaryManifestSchema = z.strictObject({ + manifestVersion: z.literal(1), + packs: z.strictObject({ + en: englishPackDescriptorSchema, + zh: chinesePackDescriptorSchema, + }), +}); + +export function parseDictionaryManifest(value: unknown): DictionaryManifest { + return dictionaryManifestSchema.parse(value); +} diff --git a/packages/core/src/dictionary/selection.test.ts b/packages/core/src/dictionary/selection.test.ts new file mode 100644 index 000000000..e3a7a9db4 --- /dev/null +++ b/packages/core/src/dictionary/selection.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { prepareDictionarySelection } from "./selection"; + +describe("prepareDictionarySelection", () => { + it("normalizes English without changing display text", () => { + expect(prepareDictionarySelection(" \u201cDesires,\u201d ")).toEqual({ + ok: true, + language: "en", + key: "desires", + displayText: "Desires", + }); + }); + + it("accepts a Chinese word and strips surrounding punctuation", () => { + expect(prepareDictionarySelection("\u300a\u95b1\u8b80\u300b")).toEqual({ + ok: true, + language: "zh", + key: "\u95b1\u8b80", + displayText: "\u95b1\u8b80", + }); + }); + + it.each([ + ["", "empty"], + ["reading\u95b1\u8b80", "mixed-script"], + ["\u95b1\u8b80\u304b\u306a", "unsupported-script"], + ["\u0447\u0442\u0435\u043d\u0438\u0435", "unsupported-script"], + ["a".repeat(121), "too-long"], + ] as const)("rejects %j as %s", (text, reason) => { + expect(prepareDictionarySelection(text)).toEqual({ ok: false, reason }); + }); +}); diff --git a/packages/core/src/dictionary/selection.ts b/packages/core/src/dictionary/selection.ts new file mode 100644 index 000000000..c963cc9bf --- /dev/null +++ b/packages/core/src/dictionary/selection.ts @@ -0,0 +1,29 @@ +import type { PreparedDictionarySelection } from "./types"; + +const EDGE_PUNCTUATION = /^[\p{P}\p{Z}]+|[\p{P}\p{Z}]+$/gu; +const HAN = /\p{Script=Han}/u; +const LATIN = /\p{Script=Latin}/u; +const ALLOWED_EN = /^[\p{Script=Latin}\p{M}'’\-\s]+$/u; +const ALLOWED_ZH = /^[\p{Script=Han}\p{M}\s]+$/u; + +export function prepareDictionarySelection(text: string): PreparedDictionarySelection { + const displayText = text.normalize("NFKC").replace(EDGE_PUNCTUATION, "").trim(); + if (!displayText) return { ok: false, reason: "empty" }; + if (Array.from(displayText).length > 120) return { ok: false, reason: "too-long" }; + + const hasHan = HAN.test(displayText); + const hasLatin = LATIN.test(displayText); + if (hasHan && hasLatin) return { ok: false, reason: "mixed-script" }; + if (hasHan && ALLOWED_ZH.test(displayText)) { + return { ok: true, language: "zh", key: displayText.replace(/\s+/g, ""), displayText }; + } + if (hasLatin && ALLOWED_EN.test(displayText)) { + return { + ok: true, + language: "en", + key: displayText.replace(/\s+/g, " ").toLocaleLowerCase("en-US"), + displayText, + }; + } + return { ok: false, reason: "unsupported-script" }; +} diff --git a/packages/core/src/dictionary/types.ts b/packages/core/src/dictionary/types.ts new file mode 100644 index 000000000..0ab924d99 --- /dev/null +++ b/packages/core/src/dictionary/types.ts @@ -0,0 +1,70 @@ +export type DictionaryLanguage = "en" | "zh"; + +export interface DictionarySense { + order: number; + definition: string; +} + +export interface DictionaryEntry { + id: number; + language: DictionaryLanguage; + headword: string; + simplified?: string; + traditional?: string; + pronunciation?: string; + partOfSpeech: string; + senses: DictionarySense[]; +} + +interface DictionaryPackDescriptorBase { + version: string; + schemaVersion: 1; + sourceDumpDate: string; + sizeBytes: number; + sha256: string; + url: string; + sourceArchiveUrl: string; + attributionUrl: string; +} + +export type DictionarySourceEdition = "wordnet-3.1" | "enwiktionary" | "zhwiktionary"; +export type DictionaryLicense = "WordNet 3.1 License" | "CC BY-SA 4.0"; + +export type DictionarySource = + | { + language: "en"; + sourceEdition: "wordnet-3.1"; + license: "WordNet 3.1 License"; + } + | { + language: "en"; + sourceEdition: "enwiktionary"; + license: "CC BY-SA 4.0"; + } + | { + language: "zh"; + sourceEdition: "zhwiktionary"; + license: "CC BY-SA 4.0"; + }; + +export type EnglishDictionaryPackDescriptor = DictionaryPackDescriptorBase & + Extract; + +export type ChineseDictionaryPackDescriptor = DictionaryPackDescriptorBase & + Extract; + +export type DictionaryPackDescriptor = + | EnglishDictionaryPackDescriptor + | ChineseDictionaryPackDescriptor; + +export interface DictionaryManifest { + manifestVersion: 1; + packs: { + en: EnglishDictionaryPackDescriptor; + zh: ChineseDictionaryPackDescriptor; + }; +} + +export type PreparedDictionarySelection = + | { ok: true; language: DictionaryLanguage; key: string; displayText: string } + | { ok: false; reason: "empty" | "mixed-script" | "unsupported-script" | "too-long" }; diff --git a/packages/core/src/i18n/locales/en/reader.json b/packages/core/src/i18n/locales/en/reader.json index 072c3d462..17506910e 100644 --- a/packages/core/src/i18n/locales/en/reader.json +++ b/packages/core/src/i18n/locales/en/reader.json @@ -81,6 +81,50 @@ "unknownBook": "Untitled book", "viewNote": "View note" }, + "dictionary": { + "define": "Define", + "title": "Definition", + "close": "Close definition", + "loadingDefinition": "Looking up definition…", + "dictionaries": "Dictionaries", + "english": "English", + "chinese": "Chinese", + "download": "Download", + "update": "Update", + "remove": "Remove", + "retry": "Retry", + "retryLookup": "Retry dictionary lookup", + "repair": "Repair", + "verifying": "Verifying…", + "manageDictionaries": "Manage Dictionaries", + "notDownloaded": "Not downloaded", + "noDefinitionFound": "No definition found. Try selecting a single word.", + "unsupportedSelection": "This selection is not supported.", + "lookupError": "Dictionary lookup failed. Try again.", + "downloadDefinition": "Download the {{language}} dictionary ({{size}}) to look up definitions offline.", + "downloadingDefinition": "Downloading the {{language}} dictionary… {{progress}}%", + "downloadAccessibility": "Download {{language}} dictionary", + "downloadingAccessibility": "Downloading {{language}} dictionary", + "downloading": "Downloading {{progress}}%", + "installed": "Installed", + "updateAvailable": "Update available", + "error": "Error", + "unavailable": "Unavailable", + "packError": "{{language}} dictionary couldn't be prepared. Try again.", + "version": "Version {{version}}", + "size": "Size {{size}}", + "installedStatus": "{{installed}} · {{version}} · {{size}}", + "updateStatus": "{{updateAvailable}} · {{installedVersion}} → {{availableVersion}}", + "attribution": "Attribution", + "attributionLabel": "{{language}} dictionary attribution", + "license": "License", + "licenseDetail": "{{label}}: {{license}}", + "actionLabel": "{{action}} {{language}} dictionary", + "statusLabel": "{{language}} dictionary status: {{status}}", + "removeTitle": "Remove {{language}} dictionary?", + "removeMessage": "Remove the {{language}} dictionary from this device?", + "cancel": "Cancel" + }, "bookmarks": { "title": "Bookmarks", "add": "Bookmark this page", diff --git a/packages/core/src/i18n/locales/zh-TW/reader.json b/packages/core/src/i18n/locales/zh-TW/reader.json index 0a638fc28..cea88c460 100644 --- a/packages/core/src/i18n/locales/zh-TW/reader.json +++ b/packages/core/src/i18n/locales/zh-TW/reader.json @@ -77,6 +77,50 @@ "unknownBook": "未命名書籍", "viewNote": "查看筆記" }, + "dictionary": { + "define": "查詞", + "title": "釋義", + "close": "關閉釋義", + "loadingDefinition": "正在查詢釋義…", + "dictionaries": "字典", + "english": "英語", + "chinese": "中文", + "download": "下載", + "update": "更新", + "remove": "移除", + "retry": "重試", + "retryLookup": "重試字典查詢", + "repair": "修復", + "verifying": "正在驗證…", + "manageDictionaries": "管理字典", + "notDownloaded": "尚未下載", + "noDefinitionFound": "找不到釋義。請嘗試只選取一個單字。", + "unsupportedSelection": "不支援這個選取內容。", + "lookupError": "字典查詢失敗，請重試。", + "downloadDefinition": "下載{{language}}字典（{{size}}）即可離線查詢釋義。", + "downloadingDefinition": "正在下載{{language}}字典… {{progress}}%", + "downloadAccessibility": "下載{{language}}字典", + "downloadingAccessibility": "正在下載{{language}}字典", + "downloading": "正在下載 {{progress}}%", + "installed": "已安裝", + "updateAvailable": "有可用的更新", + "error": "出錯", + "unavailable": "無法使用", + "packError": "{{language}}字典暫時無法使用，請重試。", + "version": "版本 {{version}}", + "size": "大小 {{size}}", + "installedStatus": "{{installed}} · {{version}} · {{size}}", + "updateStatus": "{{updateAvailable}} · {{installedVersion}} → {{availableVersion}}", + "attribution": "來源注記", + "attributionLabel": "{{language}}字典來源注記", + "license": "授權", + "licenseDetail": "{{label}}：{{license}}", + "actionLabel": "{{language}}字典{{action}}", + "statusLabel": "{{language}}字典狀態：{{status}}", + "removeTitle": "移除{{language}}字典？", + "removeMessage": "要從這台裝置移除{{language}}字典嗎？", + "cancel": "取消" + }, "bookmarks": { "title": "書籤", "add": "新增書籤", diff --git a/packages/core/src/i18n/locales/zh/reader.json b/packages/core/src/i18n/locales/zh/reader.json index 150d9ec9f..c2fe4d67a 100644 --- a/packages/core/src/i18n/locales/zh/reader.json +++ b/packages/core/src/i18n/locales/zh/reader.json @@ -81,6 +81,50 @@ "unknownBook": "未命名书籍", "viewNote": "查看笔记" }, + "dictionary": { + "define": "查词", + "title": "释义", + "close": "关闭释义", + "loadingDefinition": "正在查询释义…", + "dictionaries": "词典", + "english": "英语", + "chinese": "中文", + "download": "下载", + "update": "更新", + "remove": "移除", + "retry": "重试", + "retryLookup": "重试词典查询", + "repair": "修复", + "verifying": "正在验证…", + "manageDictionaries": "管理词典", + "notDownloaded": "未下载", + "noDefinitionFound": "没有找到释义。请尝试选择一个单词。", + "unsupportedSelection": "不支持此选中内容。", + "lookupError": "词典查询失败，请重试。", + "downloadDefinition": "下载{{language}}词典（{{size}}）即可离线查询释义。", + "downloadingDefinition": "正在下载{{language}}词典… {{progress}}%", + "downloadAccessibility": "下载{{language}}词典", + "downloadingAccessibility": "正在下载{{language}}词典", + "downloading": "正在下载 {{progress}}%", + "installed": "已安装", + "updateAvailable": "有可用更新", + "error": "出错", + "unavailable": "不可用", + "packError": "{{language}}词典暂时不可用，请重试。", + "version": "版本 {{version}}", + "size": "大小 {{size}}", + "installedStatus": "{{installed}} · {{version}} · {{size}}", + "updateStatus": "{{updateAvailable}} · {{installedVersion}} → {{availableVersion}}", + "attribution": "来源注记", + "attributionLabel": "{{language}}词典来源注记", + "license": "许可证", + "licenseDetail": "{{label}}：{{license}}", + "actionLabel": "{{language}}词典{{action}}", + "statusLabel": "{{language}}词典状态：{{status}}", + "removeTitle": "移除{{language}}词典？", + "removeMessage": "要从本设备移除{{language}}词典吗？", + "cancel": "取消" + }, "bookmarks": { "title": "书签", "add": "添加书签", diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e8af67daa..827e9485f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -5,6 +5,9 @@ // Types export * from "./types"; +// Dictionary +export * from "./dictionary"; + // Utils export { cn, diff --git a/packages/core/src/stores/app-store.ts b/packages/core/src/stores/app-store.ts index 333ca4ebc..17560708a 100644 --- a/packages/core/src/stores/app-store.ts +++ b/packages/core/src/stores/app-store.ts @@ -22,6 +22,7 @@ export type SidebarTab = "chat" | "notes" | "toc" | "highlights" | "stats"; export type SettingsTab = | "general" | "reading" + | "dictionaries" | "fonts" | "ai" | "vectorModel" diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index 8024fc855..6ad7fdcf8 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "ES2020", "useDefineForClassFields": true, - "lib": ["ES2020", "DOM", "DOM.Iterable"], + "lib": ["ES2021", "DOM", "DOM.Iterable"], "module": "ESNext", "skipLibCheck": true, "moduleResolution": "bundler", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 560ef437a..406488efc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -217,6 +217,15 @@ importers: specifier: ^5.0.0 version: 5.0.11(@types/react@19.1.17)(react@19.1.0)(use-sync-external-store@1.6.0(react@19.1.0)) devDependencies: + '@types/react-test-renderer': + specifier: 19.1.0 + version: 19.1.0 + react-test-renderer: + specifier: 19.1.0 + version: 19.1.0(react@19.1.0) + vitest: + specifier: ^4.1.2 + version: 4.1.2(@types/node@25.5.2)(vite@7.3.1(@types/node@25.5.2)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.0)(tsx@4.22.4)(yaml@2.8.2)) '@tailwindcss/vite': specifier: ^4.0.0 version: 4.2.1(vite@7.3.1(@types/node@25.5.2)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.0)(tsx@4.22.4)(yaml@2.8.2)) @@ -452,6 +461,12 @@ importers: specifier: ^5.0.5 version: 5.0.11(@types/react@19.1.17)(react@19.1.0)(use-sync-external-store@1.6.0(react@19.1.0)) devDependencies: + react-test-renderer: + specifier: 19.1.0 + version: 19.1.0(react@19.1.0) + '@types/react-test-renderer': + specifier: 19.1.0 + version: 19.1.0 '@babel/core': specifier: ^7.25.0 version: 7.29.0 @@ -649,6 +664,15 @@ importers: packages: + '@types/react-test-renderer@19.1.0': + resolution: {integrity: sha512-XD0WZrHqjNrxA/MaR9O22w/RNidWR9YZmBdRGI7wcnWGrv/3dA8wKCJ8m63Sn+tLJhcjmuhOi629N66W6kgWzQ==} + + react-test-renderer@19.1.0: + resolution: {integrity: sha512-jXkSl3CpvPYEF+p/eGDLB4sPoDX8pKkYvRl9+rR8HxLY0X04vW7hCm1/0zHoUSjPZ3bDa+wXWNTDVIw/R8aDVw==} + peerDependencies: + react: 19.1.0 + + '@0no-co/graphql.web@1.2.0': resolution: {integrity: sha512-/1iHy9TTr63gE1YcR5idjx8UREz1s0kFhydf3bBLCXyqjhkIc6igAzTOx3zPifCwFR87tsh/4Pa9cNts6d2otw==} peerDependencies: @@ -9498,6 +9522,17 @@ packages: snapshots: + '@types/react-test-renderer@19.1.0': + dependencies: + '@types/react': 19.1.17 + + react-test-renderer@19.1.0(react@19.1.0): + dependencies: + react: 19.1.0 + react-is: 19.2.4 + scheduler: 0.26.0 + + '@0no-co/graphql.web@1.2.0(graphql@16.8.1)': optionalDependencies: graphql: 16.8.1