diff --git a/diffr/crates/diffr-core/Cargo.toml b/diffr/crates/diffr-core/Cargo.toml index d353d2ea6..c8f940769 100644 --- a/diffr/crates/diffr-core/Cargo.toml +++ b/diffr/crates/diffr-core/Cargo.toml @@ -137,3 +137,7 @@ toml = "0.8" [features] # Constructors and helpers the diffr package's tests build on. test-support = [] + +[[example]] +name = "initialization" +path = "src/initialization_example.rs" diff --git a/diffr/crates/diffr-core/src/config.rs b/diffr/crates/diffr-core/src/config.rs index 680df48b6..fe3754194 100644 --- a/diffr/crates/diffr-core/src/config.rs +++ b/diffr/crates/diffr-core/src/config.rs @@ -26,7 +26,7 @@ use crate::options::DiffOptions; use crate::parse::{guess_language::Language, tree_sitter_parser}; use crate::plugin::config::PluginsConfig; use crate::plugin::queries::{self, Queries}; -use query::AnnotationQuery; +use query::{AnnotationQuery, QuerySource}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; @@ -128,7 +128,7 @@ impl Default for ThemeConfig { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ConfigError(pub(crate) String); impl std::fmt::Display for ConfigError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -138,7 +138,9 @@ impl std::fmt::Display for ConfigError { impl std::error::Error for ConfigError {} pub struct Params { - languages: DftHashMap>>, + languages: DftHashMap, ConfigError>>>, + sources: DftHashMap>, + plugin_order: Vec, pub diff: DiffConfig, } @@ -147,10 +149,13 @@ pub struct LanguageParams { /// The fold and context queries of every enabled plugin, concatenated. pub(crate) query: AnnotationQuery, sub_languages: OnceLock< - Vec<( - &'static tree_sitter_parser::TreeSitterSubLanguage, - Arc, - )>, + Result< + Vec<( + &'static tree_sitter_parser::TreeSitterSubLanguage, + Arc, + )>, + ConfigError, + >, >, } @@ -161,7 +166,11 @@ impl LanguageParams { &'static tree_sitter_parser::TreeSitterSubLanguage, Arc, )] { - self.sub_languages.get().expect("resolved sub-languages") + self.sub_languages + .get() + .expect("resolved sub-languages") + .as_ref() + .expect("valid sub-languages") } } @@ -266,27 +275,34 @@ impl Config { /// Compile with `queries`, the enabled plugins' query files in /// `plugins.shape.order`. pub fn compile_queries(&self, queries: Vec<(String, Queries)>) -> Result { - let mut languages: DftHashMap<_, _> = Language::iter() + let params = self.prepare_queries(queries)?; + for &language in params.sources.keys() { + params.compiled_language(language)?; + } + Ok(params) + } + + /// Assemble query sources now, compiling each language on first use. Hosts + /// using this path report query errors with the file that needs them. + pub fn prepare(&self) -> Result { + self.prepare_queries(self.plugins.shape.queries()?) + } + + fn prepare_queries(&self, queries: Vec<(String, Queries)>) -> Result { + let languages: DftHashMap<_, _> = Language::iter() .map(|language| (language, OnceLock::new())) .collect(); + let mut by_language = DftHashMap::default(); for (name, sources) in queries::assemble(&queries)? { let language = Language::iter() .find(|language| format!("{language:?}").to_lowercase() == name) .ok_or_else(|| ConfigError(format!("unknown language: {name}")))?; - let parser = tree_sitter_parser::from_language(language); - let query = AnnotationQuery::compile(&parser.language, &sources)?; - check_tags(&query, &self.plugins.shape.order)?; - languages.insert( - language, - OnceLock::from(Arc::new(LanguageParams { - parser, - query, - sub_languages: OnceLock::new(), - })), - ); + by_language.insert(language, sources); } Ok(Params { languages, + sources: by_language, + plugin_order: self.plugins.shape.order.clone(), diff: self.diff, }) } @@ -314,26 +330,49 @@ fn check_tags(query: &AnnotationQuery, order: &[String]) -> Result<(), ConfigErr } impl Params { - pub fn language(&self, language: Language) -> &Arc { - let config = self.languages[&language].get_or_init(|| { - // Languages without annotation rules still support structural diffing. - // Keep their grammars lazy, as in the existing parser registry. - let parser = tree_sitter_parser::from_language(language); - Arc::new(LanguageParams { - parser, - query: AnnotationQuery::compile(&parser.language, &[]).expect("an empty query"), - sub_languages: OnceLock::new(), + fn compiled_language(&self, language: Language) -> Result<&Arc, ConfigError> { + self.languages[&language] + .get_or_init(|| { + // Languages without annotation rules still support structural diffing. + // Keep their grammars lazy, as in the existing parser registry. + let parser = tree_sitter_parser::from_language(language); + let sources = self + .sources + .get(&language) + .map(Vec::as_slice) + .unwrap_or(&[]); + let query = AnnotationQuery::compile(&parser.language, sources)?; + check_tags(&query, &self.plugin_order)?; + Ok(Arc::new(LanguageParams { + parser, + query, + sub_languages: OnceLock::new(), + })) }) - }); - config.sub_languages.get_or_init(|| { - config - .parser - .sub_languages - .iter() - .map(|sub| (sub, Arc::clone(self.language(sub.parse_as)))) - .collect() - }); + .as_ref() + .map_err(Clone::clone) + } + + /// Resolve this language and its embedded languages once. Prepared configs + /// can report query errors here; both successes and failures are cached. + pub fn language(&self, language: Language) -> Result<&Arc, ConfigError> { + let config = self.compiled_language(language)?; config + .sub_languages + .get_or_init(|| { + config + .parser + .sub_languages + .iter() + .map(|sub| { + self.language(sub.parse_as) + .map(|params| (sub, Arc::clone(params))) + }) + .collect() + }) + .as_ref() + .map_err(Clone::clone)?; + Ok(config) } } @@ -351,6 +390,30 @@ mod tests { use super::*; use crate::summary::DiffResult; + #[test] + fn prepared_queries_fail_only_when_their_language_is_used() { + let queries = vec![( + "removed-runs".to_owned(), + vec![crate::plugin::queries::PluginQuery { + language: "rust".into(), + name: "broken-rust.scm".into(), + text: "(not_a_rust_node) @fold".into(), + }], + )]; + let config = Config::default(); + assert!(config.compile_queries(queries.clone()).is_err()); + let params = config.prepare_queries(queries).unwrap(); + assert!( + DiffResult::try_from_sources_with_params("a.py", "x = 1", "x = 2", ¶ms).is_ok() + ); + for _ in 0..2 { + let error = + DiffResult::try_from_sources_with_params("a.rs", "fn a() {}", "fn b() {}", ¶ms) + .unwrap_err(); + assert!(error.to_string().contains("broken-rust.scm")); + } + } + #[test] fn diff_limits_default_and_layer_from_the_file() { let defaults = Config::default().diff; diff --git a/diffr/crates/diffr-core/src/diff/sliders.rs b/diffr/crates/diffr-core/src/diff/sliders.rs index 281542c49..3943e5c1e 100644 --- a/diffr/crates/diffr-core/src/diff/sliders.rs +++ b/diffr/crates/diffr-core/src/diff/sliders.rs @@ -800,7 +800,9 @@ mod tests { fn test_slider_two_steps() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs = parse(&arena, "A B", config, false).unwrap(); let rhs = parse(&arena, "A B X\n A B", config, false).unwrap(); @@ -826,7 +828,9 @@ mod tests { fn test_slider_partially_unchanged() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs = parse(&arena, "(A B) X \n (A B)", config, false).unwrap(); let rhs = parse(&arena, "((novel) A B)", config, false).unwrap(); diff --git a/diffr/crates/diffr-core/src/diff/unchanged.rs b/diffr/crates/diffr-core/src/diff/unchanged.rs index ec1d63879..bddf12066 100644 --- a/diffr/crates/diffr-core/src/diff/unchanged.rs +++ b/diffr/crates/diffr-core/src/diff/unchanged.rs @@ -497,7 +497,9 @@ mod tests { fn test_shrink_unchanged_at_start() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "unchanged A B", config, false).unwrap(); let rhs_nodes = parse(&arena, "unchanged X", config, false).unwrap(); @@ -524,7 +526,9 @@ mod tests { fn test_shrink_unchanged_at_end() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "A B unchanged", config, false).unwrap(); let rhs_nodes = parse(&arena, "X unchanged", config, false).unwrap(); @@ -551,7 +555,9 @@ mod tests { fn test_shrink_unchanged_nested() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -589,7 +595,9 @@ mod tests { fn test_split_unchanged_toplevel_at_start() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); // Make sure that the initial unchanged node exceeds TINY_TREE_THRESHOLD. let lhs_nodes = parse( @@ -631,7 +639,9 @@ mod tests { fn test_split_unchanged_toplevel_at_end() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -672,7 +682,9 @@ mod tests { fn test_split_preserves_outer_delimiters() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "(A)", config, false).unwrap(); let rhs_nodes = parse(&arena, "(B)", config, false).unwrap(); @@ -700,7 +712,9 @@ mod tests { fn test_split_unchanged_middle() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -742,7 +756,9 @@ mod tests { fn test_split_unchanged_multiple() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -771,7 +787,9 @@ mod tests { fn test_split_unchanged_outer_delimiter() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -803,7 +821,9 @@ mod tests { fn test_split_mostly_unchanged_toplevel() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse( &arena, @@ -829,7 +849,9 @@ mod tests { fn test_count_common_unique() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); // There are two subtrees that are unique on both sides and // shared between the two sides here: @@ -859,7 +881,9 @@ mod tests { fn test_similar_with_common_grandchildren() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "((novel-lhs 1 2 3 4 5)) x", config, false).unwrap(); let rhs_nodes = parse(&arena, "((novel-rhs 1 2 3 4 5)) y", config, false).unwrap(); @@ -874,7 +898,9 @@ mod tests { fn test_similar_ignore_delimiter() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess_language::Language::EmacsLisp); + let config = params + .language(guess_language::Language::EmacsLisp) + .unwrap(); let lhs_nodes = parse(&arena, "(novel-lhs 1 2 3 4 5) x", config, false).unwrap(); let rhs_nodes = parse(&arena, "[novel-rhs 1 2 3 4 5] y", config, false).unwrap(); diff --git a/diffr/crates/diffr-core/src/engine.rs b/diffr/crates/diffr-core/src/engine.rs index 5041e385e..92f5795b5 100644 --- a/diffr/crates/diffr-core/src/engine.rs +++ b/diffr/crates/diffr-core/src/engine.rs @@ -84,7 +84,7 @@ impl DiffResult { lhs: &str, rhs: &str, params: &Params, - ) -> Result { + ) -> anyhow::Result { Self::from_sources_with_options(path, lhs, rhs, params, &DiffOptions::default()) } @@ -94,7 +94,7 @@ impl DiffResult { rhs: &str, params: &Params, options: &DiffOptions, - ) -> Result { + ) -> anyhow::Result { diff_file_content(params, path, lhs, rhs, options, &[]) } } @@ -105,11 +105,13 @@ pub fn diff_file_content( rhs_src: &str, diff_options: &DiffOptions, overrides: &[(LanguageOverride, Vec)], -) -> Result { +) -> anyhow::Result { // A deleted file's language comes from what it was. let guess_src = if rhs_src.is_empty() { lhs_src } else { rhs_src }; let language = guess(Path::new(display_path), guess_src, overrides); - let lang_config = language.map(|lang| (lang, params.language(lang))); + let lang_config = language + .map(|lang| params.language(lang).map(|params| (lang, params))) + .transpose()?; // Highlights come from the same parse as the folds, so a side that // parsed has them whether or not the match runs. @@ -264,7 +266,8 @@ pub fn diff_file_content( path: display_path.to_owned(), side, conflict, - }); + } + .into()); } Err(tsp::ToSyntaxError::ExceededParseErrorLimit( tsp::ExceededParseErrorLimit { diff --git a/diffr/crates/diffr-core/src/initialization_example.rs b/diffr/crates/diffr-core/src/initialization_example.rs new file mode 100644 index 000000000..cc71580a4 --- /dev/null +++ b/diffr/crates/diffr-core/src/initialization_example.rs @@ -0,0 +1,45 @@ +//! Compare cold eager and lazy initialization in separate processes: +//! cargo run --release -p diffr-core --example initialization -- eager +//! cargo run --release -p diffr-core --example initialization -- lazy +use diffr_core::config::Config; +use diffr_core::options::DiffOptions; +use diffr_core::summary::DiffResult; +use std::time::Instant; + +fn main() -> anyhow::Result<()> { + let mode = std::env::args().nth(1).unwrap_or_else(|| "lazy".into()); + anyhow::ensure!( + matches!(mode.as_str(), "eager" | "lazy"), + "use eager or lazy" + ); + let start = Instant::now(); + let config = Config::default(); + let params = match mode.as_str() { + "eager" => config.compile()?, + _ => config.prepare()?, + }; + let initialization = start.elapsed(); + let options = DiffOptions { + syntax: true, + ..Default::default() + }; + let diff = || { + DiffResult::from_sources_with_options( + "example.tsx", + "export function Counter() { return ; }\n", + "export function Counter() { return ; }\n", + ¶ms, + &options, + ) + }; + let first = Instant::now(); + std::hint::black_box(diff()?); + let first_diff = first.elapsed(); + let warm = Instant::now(); + for _ in 0..10 { + std::hint::black_box(diff()?); + } + println!("{mode}: init={initialization:?}, first_diff={first_diff:?}, init_and_first={:?}, warm_diff={:?}", + initialization + first_diff, warm.elapsed() / 10); + Ok(()) +} diff --git a/diffr/crates/diffr-core/src/parse/tree_sitter_parser.rs b/diffr/crates/diffr-core/src/parse/tree_sitter_parser.rs index 32b8dd82d..7f50a81d3 100644 --- a/diffr/crates/diffr-core/src/parse/tree_sitter_parser.rs +++ b/diffr/crates/diffr-core/src/parse/tree_sitter_parser.rs @@ -2147,7 +2147,7 @@ mod tests { #[test] fn tsx_import_type_arrays_parse_without_errors() { let params = Params::default(); - let config = params.language(guess::Language::TypeScriptTsx); + let config = params.language(guess::Language::TypeScriptTsx).unwrap(); for source in [ "interface Props { entries?: import('./types').Entry[]; }", "type Entries = import('./types').Entry[][];", @@ -2177,7 +2177,7 @@ mod tests { fn test_parse() { let arena = Arena::new(); let params = Params::default(); - let css_config = params.language(guess::Language::Css); + let css_config = params.language(guess::Language::Css).unwrap(); parse(&arena, ".foo {}", css_config, false).unwrap(); } @@ -2185,7 +2185,7 @@ mod tests { fn test_parse_empty_file() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess::Language::EmacsLisp); + let config = params.language(guess::Language::EmacsLisp).unwrap(); let res = parse(&arena, "", config, false).unwrap(); let expected: Vec<&Syntax> = vec![]; @@ -2198,7 +2198,7 @@ mod tests { fn test_subtrees() { let arena = Arena::new(); let params = Params::default(); - let config = params.language(guess::Language::Html); + let config = params.language(guess::Language::Html).unwrap(); let res = parse(&arena, "", config, false).unwrap(); match res[0] { diff --git a/diffr/src/main.rs b/diffr/src/main.rs index 3a1e3819d..abdaf09c2 100644 --- a/diffr/src/main.rs +++ b/diffr/src/main.rs @@ -165,7 +165,9 @@ fn run_debug(mode: Mode, params: &Params) { let language = guess(path, &src, &language_overrides); match language { Some(lang) => { - let ts_lang = params.language(lang); + let ts_lang = params + .language(lang) + .expect("debug configuration was compiled"); let arena = Arena::new(); let ast = conflict_or_die(tsp::parse(&arena, &src, ts_lang, ignore_comments)); init_all_info(&ast, &[]); @@ -188,7 +190,9 @@ fn run_debug(mode: Mode, params: &Params) { let language = guess(path, &src, &language_overrides); match language { Some(lang) => { - let ts_lang = params.language(lang); + let ts_lang = params + .language(lang) + .expect("debug configuration was compiled"); let arena = Arena::new(); let ast = conflict_or_die(tsp::parse(&arena, &src, ts_lang, ignore_comments)); init_all_info(&ast, &[]);