From 7ef15c042a594ecf920f172bae4c7401badf9b4b Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Fri, 3 Jul 2026 18:47:51 -0700 Subject: [PATCH 1/4] fix(fts): use complete English stop-word list for ICU tokenizer The ICU stop-word path (StopWordFilter::all()) sourced its English list from the local Tantivy-style stopwords::ENGLISH constant (~33 words), which omits extremely common pronouns and function words: you, my, your, we, she, what, can, about, us, them, him, our. Because these are the highest-frequency tokens in English text, they survived stop-word removal and built pathologically large single-term posting lists. On large corpora (100M+ rows) with positions enabled, one such term's in-memory posting list exceeded u32::MAX bytes and panicked the whole FTS index build with 'posting list memory size overflowed u32'. Switch the English source to stop_words::get("en") (~198 words), matching how the other CJK/misc languages in all_stop_words() are already sourced from the stop-words crate. The leak is independent of stemming, so the new tests cover both stem=false and stem=true for icu and icu/split. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/scalar/inverted/tokenizer.rs | 31 +++++++++++++++++++ rust/lance-tokenizer/src/stop_word_filter.rs | 7 ++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs index 2c7a17465f8..248b3ea6ce9 100644 --- a/rust/lance-index/src/scalar/inverted/tokenizer.rs +++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs @@ -668,4 +668,35 @@ mod tests { } assert_eq!(tokens, vec!["lance".to_string(), "data".to_string()]); } + + // Common English pronouns/function words such as `you`/`my`/`your`/`we` + // must be removed by the ICU `all()` stop-word path. These are among the + // highest-frequency tokens, so leaking them builds pathologically large + // single-term posting lists (and previously overflowed the u32 posting-list + // size counter, panicking the whole index build). The leak is independent + // of stemming, so we assert it for both stem=false and stem=true. + #[rstest] + #[case::icu_no_stem("icu", false)] + #[case::icu_stem("icu", true)] + #[case::icu_split_no_stem("icu/split", false)] + #[case::icu_split_stem("icu/split", true)] + fn test_icu_common_english_stop_words_do_not_leak( + #[case] base_tokenizer: &str, + #[case] stem: bool, + ) { + let mut tokenizer = InvertedIndexParams::default() + .base_tokenizer(base_tokenizer.to_string()) + .stem(stem) + .remove_stop_words(true) + .build() + .unwrap(); + let mut stream = tokenizer.token_stream_for_search("you my your we lance data"); + let tokens: Vec = std::iter::from_fn(|| stream.next().map(|t| t.text.clone())) + .filter(|t| matches!(t.as_str(), "you" | "my" | "your" | "we")) + .collect(); + assert!( + tokens.is_empty(), + "common English stop words leaked through the icu pipeline (stem={stem}): {tokens:?}" + ); + } } diff --git a/rust/lance-tokenizer/src/stop_word_filter.rs b/rust/lance-tokenizer/src/stop_word_filter.rs index 2acf0b3dbd5..269e72a20e7 100644 --- a/rust/lance-tokenizer/src/stop_word_filter.rs +++ b/rust/lance-tokenizer/src/stop_word_filter.rs @@ -17,7 +17,12 @@ fn all_stop_words() -> impl Iterator { stop_words::get("ar"), stopwords::DANISH, stopwords::DUTCH, - stopwords::ENGLISH, + // Use the fuller `stop-words` crate English list (~198 words) rather + // than the local Tantivy-style list (~33 words), which omits extremely + // common pronouns/function words (you, my, your, we, she, what, ...). + // Those omissions let the highest-frequency English tokens through the + // ICU stop-word path and build pathologically large posting lists. + stop_words::get("en"), stopwords::FINNISH, stopwords::FRENCH, stopwords::GERMAN, From 2c9ec079167988157e6598d8a2e7a3f7eb5701bb Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Fri, 3 Jul 2026 18:51:41 -0700 Subject: [PATCH 2/4] test(fts): assert Chinese stop words are removed by ICU pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a regression test alongside the English one confirming common Chinese function words/particles (我 在 有 了 是 的 和) are removed by the ICU all() stop-word path while real content words (英语) survive. The Chinese list is sourced from stop_words::get("zh") (complete), so this already passed; the test locks it in. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/scalar/inverted/tokenizer.rs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs index 248b3ea6ce9..daea945c451 100644 --- a/rust/lance-index/src/scalar/inverted/tokenizer.rs +++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs @@ -699,4 +699,36 @@ mod tests { "common English stop words leaked through the icu pipeline (stem={stem}): {tokens:?}" ); } + + // Common Chinese function words/particles (了 是 在 的 和 有 我) are the + // highest-frequency Chinese tokens; like the English pronouns they must be + // removed by the ICU `all()` stop-word path so they don't build huge + // posting lists. Real content words (英语 = "English", 数据 = "data") must + // survive. ICU dictionary segmentation splits the input into words, so this + // exercises the CJK stop-word path end to end. + #[rstest] + #[case::icu("icu")] + #[case::icu_split("icu/split")] + fn test_icu_common_chinese_stop_words_do_not_leak(#[case] base_tokenizer: &str) { + let mut tokenizer = InvertedIndexParams::default() + .base_tokenizer(base_tokenizer.to_string()) + .stem(true) + .remove_stop_words(true) + .build() + .unwrap(); + let mut stream = tokenizer.token_stream_for_search("我 在 有 了 是 的 和 英语 数据"); + let tokens: Vec = + std::iter::from_fn(|| stream.next().map(|t| t.text.clone())).collect(); + let stop = ["我", "在", "有", "了", "是", "的", "和"]; + let leaked: Vec<&String> = tokens.iter().filter(|t| stop.contains(&t.as_str())).collect(); + assert!( + leaked.is_empty(), + "common Chinese stop words leaked through the icu pipeline: {leaked:?} (all tokens: {tokens:?})" + ); + // The real content words must still be indexed. + assert!( + tokens.iter().any(|t| t == "英语"), + "content word 英语 was dropped: {tokens:?}" + ); + } } From e1475ae5846372144a8ad76922c5512c4e1217df Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Fri, 3 Jul 2026 18:54:20 -0700 Subject: [PATCH 3/4] style: cargo fmt --- rust/lance-index/src/scalar/inverted/tokenizer.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs index daea945c451..4310d887787 100644 --- a/rust/lance-index/src/scalar/inverted/tokenizer.rs +++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs @@ -720,7 +720,10 @@ mod tests { let tokens: Vec = std::iter::from_fn(|| stream.next().map(|t| t.text.clone())).collect(); let stop = ["我", "在", "有", "了", "是", "的", "和"]; - let leaked: Vec<&String> = tokens.iter().filter(|t| stop.contains(&t.as_str())).collect(); + let leaked: Vec<&String> = tokens + .iter() + .filter(|t| stop.contains(&t.as_str())) + .collect(); assert!( leaked.is_empty(), "common Chinese stop words leaked through the icu pipeline: {leaked:?} (all tokens: {tokens:?})" From a9fe4758cdf461c0ce22041205202cfdf757be57 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Fri, 3 Jul 2026 19:34:50 -0700 Subject: [PATCH 4/4] fix(fts): complete English stop-word list for the simple tokenizer path too The per-language StopWordFilter::new(English) path (used by non-ICU tokenizers such as `simple`, the recommended tokenizer for monolingual English) also sourced the truncated local stopwords::ENGLISH (~33 words), so common pronouns (you/my/your/we/...) leaked there as well. Switch it to stop_words::get("en") to match the ICU all() path fix, and extend the regression test to cover base_tokenizer=simple. Co-Authored-By: Claude Opus 4.8 (1M context) --- rust/lance-index/src/scalar/inverted/tokenizer.rs | 5 +++++ rust/lance-tokenizer/src/stop_word_filter.rs | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs index 4310d887787..5080bfe624f 100644 --- a/rust/lance-index/src/scalar/inverted/tokenizer.rs +++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs @@ -680,6 +680,11 @@ mod tests { #[case::icu_stem("icu", true)] #[case::icu_split_no_stem("icu/split", false)] #[case::icu_split_stem("icu/split", true)] + // `simple` is the recommended tokenizer for monolingual English corpora and + // uses StopWordFilter::new(English) rather than the ICU all() path, so it + // must be covered too. + #[case::simple_no_stem("simple", false)] + #[case::simple_stem("simple", true)] fn test_icu_common_english_stop_words_do_not_leak( #[case] base_tokenizer: &str, #[case] stem: bool, diff --git a/rust/lance-tokenizer/src/stop_word_filter.rs b/rust/lance-tokenizer/src/stop_word_filter.rs index 269e72a20e7..9a690b0ec06 100644 --- a/rust/lance-tokenizer/src/stop_word_filter.rs +++ b/rust/lance-tokenizer/src/stop_word_filter.rs @@ -56,7 +56,11 @@ impl StopWordFilter { Language::Arabic => stop_words::get("ar"), Language::Danish => stopwords::DANISH, Language::Dutch => stopwords::DUTCH, - Language::English => stopwords::ENGLISH, + // Use the fuller `stop-words` crate English list (~198 words); the + // local Tantivy-style list (~33 words) omits common pronouns/function + // words (you, my, your, we, ...) that would otherwise leak through + // stop-word removal and build pathologically large posting lists. + Language::English => stop_words::get("en"), Language::Finnish => stopwords::FINNISH, Language::French => stopwords::FRENCH, Language::German => stopwords::GERMAN,