From 4d1860213fb3ab7053f970991aacf9d07cb95177 Mon Sep 17 00:00:00 2001 From: krkarma777 Date: Mon, 31 Aug 2026 11:22:55 +0900 Subject: [PATCH] feat: Add Intl.Segmenter modes (intl-word, grapheme) --- CHANGELOG.md | 8 ++++ README.md | 13 +++++- src/index.ts | 37 ++++++++++++++--- src/tokenize.ts | Bin 644 -> 1993 bytes test/segmenter.test.ts | 90 +++++++++++++++++++++++++++++++++++++++++ tsconfig.json | 2 +- 6 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 test/segmenter.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 238ce41..aa82966 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to this project are documented in this file. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project adheres to [Semantic Versioning](https://semver.org/). +## [Unreleased] + +### Added +- `intl-word` and `grapheme` modes via `Intl.Segmenter` (with a `locale` + option): locale-aware word diffs for unspaced scripts (Japanese, Chinese, + Thai) and cluster-safe character diffs (ZWJ emoji, combining sequences). + `refine` drops `intl-word` pairs to grapheme granularity. (#15) + ## [1.1.0] - 2026-08-31 ### Added diff --git a/README.md b/README.md index 1416d9e..f7f4ddc 100644 --- a/README.md +++ b/README.md @@ -70,13 +70,24 @@ Returns `DiffEntry[]` — the shortest edit script between `a` and `b`. | option | type | default | description | |---|---|---|---| -| `mode` | `'word' \| 'char' \| 'line'` | `'word'` | tokenization granularity | +| `mode` | `'word' \| 'char' \| 'line' \| 'intl-word' \| 'grapheme'` | `'word'` | tokenization granularity | +| `locale` | `string \| string[]` | runtime locale | BCP 47 locale(s) for the `Intl.Segmenter` modes | | `refine` | `boolean` | `false` | re-diff each delete/insert pair one level finer (`line`→word, `word`→char), e.g. `quick`→`quicker` reports just `+er` | | `heuristic` | `boolean` | `false` | cap the search cost like git does, keeping pathological inputs fast (the 227 ms worst case below drops to ~8 ms, +8% edit-script size); output stays identical to exact mode while the edit distance is small | - `word` — runs of Unicode letters/digits/underscore, whitespace runs, symbol runs - `char` — individual code points (surrogate-pair safe) - `line` — lines with their terminators attached +- `intl-word` — locale-aware words via `Intl.Segmenter`: splits unspaced scripts (Japanese, Chinese, Thai) that `word` mode sees as one token + + ```ts + diff('私は猫が好きです', '私は犬が好きです', { mode: 'intl-word', locale: 'ja' }); + // equal '私は' · delete '猫' · insert '犬' · equal 'が好きです' + ``` + +- `grapheme` — grapheme clusters via `Intl.Segmenter`: ZWJ emoji (👨‍👩‍👧) and combining sequences stay whole where `char` mode would split code points + +The `Intl.Segmenter` modes are opt-in because they're slower than the scanner modes; they throw a clear `TypeError` on runtimes without `Intl.Segmenter` (Node < 16, older browsers). ### `diffRanges(a, b, options?)` diff --git a/src/index.ts b/src/index.ts index cd92ef4..0b8e272 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,8 +7,16 @@ import { pushEntry, type DiffEntry, type DiffOperation } from './entries.ts'; export { tokenize, type DiffMode, type DiffEntry, type DiffOperation }; export interface DiffOptions { - /** Tokenization granularity. Defaults to 'word'. */ + /** + * Tokenization granularity. Defaults to 'word'. The scanner modes + * ('word' | 'char' | 'line') are the fastest; 'intl-word' and 'grapheme' + * use Intl.Segmenter for locale-aware word boundaries (unspaced scripts + * like Japanese/Chinese/Thai) and cluster-safe characters (ZWJ emoji, + * combining sequences). + */ mode?: DiffMode; + /** BCP 47 locale(s) for the Intl.Segmenter modes. Defaults to the runtime locale. */ + locale?: string | string[]; /** * Re-diffs each delete/insert pair one granularity finer ('line' pairs by * word, 'word' pairs by char), so replacing "quick" with "quicker" reports @@ -45,13 +53,28 @@ export function diff(a: string, b: string, options: DiffOptions = {}): DiffEntry } const mode = options.mode ?? 'word'; const heuristic = options.heuristic === true; - const entries = mode === 'char' ? diffChars(a, b, heuristic) : diffScanned(a, b, mode, heuristic); - if (options.refine === true && mode !== 'char') { - return refineEntries(entries, mode === 'line' ? 'word' : 'char', heuristic); + let entries: DiffEntry[]; + if (mode === 'char') { + entries = diffChars(a, b, heuristic); + } else if (mode === 'intl-word' || mode === 'grapheme') { + entries = diffTokens(tokenize(a, mode, options.locale), tokenize(b, mode, options.locale), { heuristic }); + } else { + entries = diffScanned(a, b, mode, heuristic); + } + const finer = REFINE_TARGET[mode]; + if (options.refine === true && finer !== undefined) { + return refineEntries(entries, finer, heuristic, options.locale); } return entries; } +/** Which granularity a refine pass drops to; char/grapheme are already finest. */ +const REFINE_TARGET: Partial> = { + line: 'word', + word: 'char', + 'intl-word': 'grapheme', +}; + /** * A changed region as code-unit offsets into the inputs: * a[aStart, aEnd) was replaced by b[bStart, bEnd). Either side (but never @@ -95,13 +118,15 @@ export function diffRanges(a: string, b: string, options: DiffOptions = {}): Dif } /** Re-diffs adjacent delete/insert pairs at a finer granularity. */ -function refineEntries(entries: DiffEntry[], finerMode: DiffMode, heuristic: boolean): DiffEntry[] { +function refineEntries( + entries: DiffEntry[], finerMode: DiffMode, heuristic: boolean, locale?: string | string[], +): DiffEntry[] { const out: DiffEntry[] = []; for (let i = 0; i < entries.length; i++) { const entry = entries[i]; const next = entries[i + 1]; if (entry.operation === 'delete' && next !== undefined && next.operation === 'insert') { - for (const sub of diff(entry.text, next.text, { mode: finerMode, heuristic })) { + for (const sub of diff(entry.text, next.text, { mode: finerMode, heuristic, locale })) { pushEntry(out, sub.operation, sub.text); } i++; diff --git a/src/tokenize.ts b/src/tokenize.ts index c564937116aa351b445028e884c2c7feacb1ac56..715f79f71c10d68553490f5eb66f06a2051071cb 100644 GIT binary patch literal 1993 zcma)7O>f&c5Z$wX#S{o2DRHFjWe-lhMv(T9Zj)U!E&|j}8&(=ytYuOqsdc@M{`(F| zNtV(cnu9|~^YJ+I-V=T=r7~dlCBkdITHVMDVG6-csw{vf2-2J>cPO|(ci_Sl=iU3Y zVr7n73@%2aE|NH&x z&;MM`=WpKM(zuevvOK)2>F37&+}58T&YqU~>}m07segQxtSg!p`q8f{ks2;IghK3JlVc&qTSggSe-shSW=*{|+wtdF_JS5> zE4apOpySuGK-b1CHcZ0VtLaiiNo3KsNk=X|%hb5uqA+vELU;LbbCY&(gE}3Hf2B+p zJSbI<>P`Xg+-lC*&eFeu!rYSoyL=cYF&U~iO?Dtru14z_9?IGT~9zg1k(riVj?@Yc3|`Nx^}AaaT?R7{ggM~_ZC zHI3HW;Rg4F@$ux)gIWJ}a=28A?PIRp|IqK5Jfg9#_^ae1490ZhN%P41M-bGH7Cn6? zTI~>TrpL1*MZ886QeTI}8eGR2H=l0o?KFF@-8_buZLp2uFqm}Rm_(QCTWjB%K2)uU zpUuN7r6mx#@U(Hol~fS=nU##byTpaAM% { + assert.deepEqual( + tokenize('私は猫が好きです', 'intl-word', 'ja'), + ['私', 'は', '猫', 'が', '好き', 'です'], + ); +}); + +test('tokenize: grapheme keeps ZWJ emoji and combining sequences whole', () => { + assert.deepEqual(tokenize('a👨‍👩‍👧b', 'grapheme'), ['a', '👨‍👩‍👧', 'b']); + assert.deepEqual(tokenize('', 'grapheme'), []); + assert.deepEqual(tokenize('', 'intl-word'), []); +}); + +test('intl-word: Japanese diff isolates the changed word (regular word mode cannot)', () => { + // Regular word mode sees one opaque letter-run, so everything changes. + const coarse = diff('私は猫が好きです', '私は犬が好きです'); + assert.deepEqual(coarse, [ + { operation: 'delete', text: '私は猫が好きです' }, + { operation: 'insert', text: '私は犬が好きです' }, + ]); + // intl-word isolates 猫 -> 犬. + assert.deepEqual(diff('私は猫が好きです', '私は犬が好きです', { mode: 'intl-word', locale: 'ja' }), [ + { operation: 'equal', text: '私は' }, + { operation: 'delete', text: '猫' }, + { operation: 'insert', text: '犬' }, + { operation: 'equal', text: 'が好きです' }, + ]); +}); + +test('grapheme: ZWJ emoji replaced as one cluster (char mode splits code points)', () => { + const a = 'x👨‍👩‍👧y'; + const b = 'x👨‍👩‍👦y'; + assert.deepEqual(diff(a, b, { mode: 'grapheme' }), [ + { operation: 'equal', text: 'x' }, + { operation: 'delete', text: '👨‍👩‍👧' }, + { operation: 'insert', text: '👨‍👩‍👦' }, + { operation: 'equal', text: 'y' }, + ]); +}); + +test('intl-word: refine drops to grapheme granularity', () => { + const entries = diff('color', 'colour', { mode: 'intl-word', refine: true }); + assert.deepEqual(entries, [ + { operation: 'equal', text: 'colo' }, + { operation: 'insert', text: 'u' }, + { operation: 'equal', text: 'r' }, + ]); +}); + +test('segmenter modes: round-trip fuzz with mixed scripts', () => { + let state = 20270101 >>> 0; + const rng = () => { + state = (state * 1664525 + 1013904223) >>> 0; + return state / 0x100000000; + }; + const pieces = ['猫', '好き', 'a', ' ', '👨‍👩‍👧', '한글', '\n', '.', 'देवनागरी']; + const make = () => { + const len = Math.floor(rng() * 25); + let s = ''; + for (let i = 0; i < len; i++) s += pieces[Math.floor(rng() * pieces.length)]; + return s; + }; + for (let iter = 0; iter < 100; iter++) { + const a = make(); + const b = make(); + for (const mode of ['intl-word', 'grapheme'] as const) { + const entries = diff(a, b, { mode }); + assert.equal(joinSide(entries, 'insert'), a, `a mismatch mode=${mode}`); + assert.equal(joinSide(entries, 'delete'), b, `b mismatch mode=${mode}`); + } + } +}); + +test('segmenter modes: heuristic flag composes', () => { + const a = '猫'.repeat(200) + '好き'.repeat(200); + const b = '犬'.repeat(180) + '嫌い'.repeat(180); + const entries = diff(a, b, { mode: 'grapheme', heuristic: true }); + assert.equal(joinSide(entries, 'insert'), a); + assert.equal(joinSide(entries, 'delete'), b); +}); diff --git a/tsconfig.json b/tsconfig.json index bcffce8..9549c93 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "target": "ES2020", "module": "ESNext", "moduleResolution": "bundler", - "lib": ["ES2020"], + "lib": ["ES2022"], "strict": true, "noEmit": true, "allowImportingTsExtensions": true,