diff --git a/rust/lance-index/src/scalar/inverted/tokenizer.rs b/rust/lance-index/src/scalar/inverted/tokenizer.rs index 2c7a17465f8..5080bfe624f 100644 --- a/rust/lance-index/src/scalar/inverted/tokenizer.rs +++ b/rust/lance-index/src/scalar/inverted/tokenizer.rs @@ -668,4 +668,75 @@ mod tests { } assert_eq!(tokens, vec!["lance".to_string(), "data".to_string()]); } + + // Common English pronouns/function words such as `you`/`my`/`your`/`we` + // must be removed by the ICU `all()` stop-word path. These are among the + // highest-frequency tokens, so leaking them builds pathologically large + // single-term posting lists (and previously overflowed the u32 posting-list + // size counter, panicking the whole index build). The leak is independent + // of stemming, so we assert it for both stem=false and stem=true. + #[rstest] + #[case::icu_no_stem("icu", false)] + #[case::icu_stem("icu", true)] + #[case::icu_split_no_stem("icu/split", false)] + #[case::icu_split_stem("icu/split", true)] + // `simple` is the recommended tokenizer for monolingual English corpora and + // uses StopWordFilter::new(English) rather than the ICU all() path, so it + // must be covered too. + #[case::simple_no_stem("simple", false)] + #[case::simple_stem("simple", true)] + fn test_icu_common_english_stop_words_do_not_leak( + #[case] base_tokenizer: &str, + #[case] stem: bool, + ) { + let mut tokenizer = InvertedIndexParams::default() + .base_tokenizer(base_tokenizer.to_string()) + .stem(stem) + .remove_stop_words(true) + .build() + .unwrap(); + let mut stream = tokenizer.token_stream_for_search("you my your we lance data"); + let tokens: Vec = std::iter::from_fn(|| stream.next().map(|t| t.text.clone())) + .filter(|t| matches!(t.as_str(), "you" | "my" | "your" | "we")) + .collect(); + assert!( + tokens.is_empty(), + "common English stop words leaked through the icu pipeline (stem={stem}): {tokens:?}" + ); + } + + // Common Chinese function words/particles (了 是 在 的 和 有 我) are the + // highest-frequency Chinese tokens; like the English pronouns they must be + // removed by the ICU `all()` stop-word path so they don't build huge + // posting lists. Real content words (英语 = "English", 数据 = "data") must + // survive. ICU dictionary segmentation splits the input into words, so this + // exercises the CJK stop-word path end to end. + #[rstest] + #[case::icu("icu")] + #[case::icu_split("icu/split")] + fn test_icu_common_chinese_stop_words_do_not_leak(#[case] base_tokenizer: &str) { + let mut tokenizer = InvertedIndexParams::default() + .base_tokenizer(base_tokenizer.to_string()) + .stem(true) + .remove_stop_words(true) + .build() + .unwrap(); + let mut stream = tokenizer.token_stream_for_search("我 在 有 了 是 的 和 英语 数据"); + let tokens: Vec = + std::iter::from_fn(|| stream.next().map(|t| t.text.clone())).collect(); + let stop = ["我", "在", "有", "了", "是", "的", "和"]; + let leaked: Vec<&String> = tokens + .iter() + .filter(|t| stop.contains(&t.as_str())) + .collect(); + assert!( + leaked.is_empty(), + "common Chinese stop words leaked through the icu pipeline: {leaked:?} (all tokens: {tokens:?})" + ); + // The real content words must still be indexed. + assert!( + tokens.iter().any(|t| t == "英语"), + "content word 英语 was dropped: {tokens:?}" + ); + } } diff --git a/rust/lance-tokenizer/src/stop_word_filter.rs b/rust/lance-tokenizer/src/stop_word_filter.rs index 2acf0b3dbd5..9a690b0ec06 100644 --- a/rust/lance-tokenizer/src/stop_word_filter.rs +++ b/rust/lance-tokenizer/src/stop_word_filter.rs @@ -17,7 +17,12 @@ fn all_stop_words() -> impl Iterator { stop_words::get("ar"), stopwords::DANISH, stopwords::DUTCH, - stopwords::ENGLISH, + // Use the fuller `stop-words` crate English list (~198 words) rather + // than the local Tantivy-style list (~33 words), which omits extremely + // common pronouns/function words (you, my, your, we, she, what, ...). + // Those omissions let the highest-frequency English tokens through the + // ICU stop-word path and build pathologically large posting lists. + stop_words::get("en"), stopwords::FINNISH, stopwords::FRENCH, stopwords::GERMAN, @@ -51,7 +56,11 @@ impl StopWordFilter { Language::Arabic => stop_words::get("ar"), Language::Danish => stopwords::DANISH, Language::Dutch => stopwords::DUTCH, - Language::English => stopwords::ENGLISH, + // Use the fuller `stop-words` crate English list (~198 words); the + // local Tantivy-style list (~33 words) omits common pronouns/function + // words (you, my, your, we, ...) that would otherwise leak through + // stop-word removal and build pathologically large posting lists. + Language::English => stop_words::get("en"), Language::Finnish => stopwords::FINNISH, Language::French => stopwords::FRENCH, Language::German => stopwords::GERMAN,