Skip to content

Attach a span to most common errors describing the problematic token #2097

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+694 −328
Open

Attach a span to most common errors describing the problematic token #2097

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/ast/spans.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2511,7 +2511,7 @@ pub mod tests {
span,
})) => {
assert_eq!(":fooBar", s);
assert_eq!(&Span::new((3, 3).into(), (3, 10).into()), span);
assert_eq!(&Span::new((3, 3), (3, 10)), span);
}
_ => panic!("expected unnamed expression; got {col:?}"),
}
Expand Down
193 changes: 102 additions & 91 deletions src/parser/mod.rs
View file
Open in desktop

Large diffs are not rendered by default.

7 changes: 5 additions & 2 deletions src/tokenizer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -574,8 +574,11 @@ impl Span {
const EMPTY: Span = Self::empty();

/// Create a new span from a start and end [`Location`]
pub fn new(start: Location, end: Location) -> Span {
Span { start, end }
pub fn new(start: impl Into<Location>, end: impl Into<Location>) -> Span {
Span {
start: start.into(),
end: end.into(),
}
}

/// Returns an empty span `(0, 0) -> (0, 0)`
Expand Down
41 changes: 25 additions & 16 deletions tests/sqlparser_bigquery.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,13 +295,13 @@ fn parse_begin() {
bigquery()
.parse_sql_statements("BEGIN SELECT 1; SELECT 2 END")
.unwrap_err(),
ParserError::ParserError("Expected: ;, found: END".to_string())
ParserError::SpannedParserError("Expected: ;, found: END".to_string(), Span::empty())
);
assert_eq!(
bigquery()
.parse_sql_statements("BEGIN SELECT 1; EXCEPTION WHEN ERROR THEN SELECT 2 END")
.unwrap_err(),
ParserError::ParserError("Expected: ;, found: END".to_string())
ParserError::SpannedParserError("Expected: ;, found: END".to_string(), Span::empty())
);
}

Expand Down Expand Up @@ -2008,7 +2008,7 @@ fn parse_merge_invalid_statements() {
] {
let res = dialects.parse_sql_statements(sql);
assert_eq!(
ParserError::ParserError(err_msg.to_string()),
ParserError::SpannedParserError(err_msg.to_string(), Span::empty()),
res.unwrap_err()
);
}
Expand Down Expand Up @@ -2132,13 +2132,19 @@ fn parse_big_query_declare() {

let error_sql = "DECLARE x";
assert_eq!(
ParserError::ParserError("Expected: a data type name, found: EOF".to_owned()),
ParserError::SpannedParserError(
"Expected: a data type name, found: EOF".to_owned(),
Span::empty()
),
bigquery().parse_sql_statements(error_sql).unwrap_err()
);

let error_sql = "DECLARE x 42";
assert_eq!(
ParserError::ParserError("Expected: a data type name, found: 42".to_owned()),
ParserError::SpannedParserError(
"Expected: a data type name, found: 42".to_owned(),
Span::empty()
),
bigquery().parse_sql_statements(error_sql).unwrap_err()
);
}
Expand Down Expand Up @@ -2342,7 +2348,7 @@ fn test_bigquery_create_function() {
];
for (sql, error) in error_sqls {
assert_eq!(
ParserError::ParserError(error.to_owned()),
ParserError::SpannedParserError(error.to_owned(), Span::empty()),
bigquery().parse_sql_statements(sql).unwrap_err()
);
}
Expand Down Expand Up @@ -2372,7 +2378,7 @@ fn test_bigquery_trim() {
// missing comma separation
let error_sql = "SELECT TRIM('xyz' 'a')";
assert_eq!(
ParserError::ParserError("Expected: ), found: 'a'".to_owned()),
ParserError::SpannedParserError("Expected: ), found: 'a'".to_owned(), Span::empty()),
bigquery().parse_sql_statements(error_sql).unwrap_err()
);
}
Expand Down Expand Up @@ -2535,16 +2541,17 @@ fn test_struct_trailing_and_nested_bracket() {

// Bad case with missing closing bracket
assert_eq!(
ParserError::ParserError("Expected: >, found: )".to_owned()),
ParserError::SpannedParserError("Expected: >, found: )".to_owned(), Span::empty()),
bigquery()
.parse_sql_statements("CREATE TABLE my_table(f1 STRUCT<a STRING, b INT64)")
.unwrap_err()
);

// Bad case with redundant closing bracket
assert_eq!(
ParserError::ParserError(
"unmatched > after parsing data type STRUCT<a STRING, b INT64>)".to_owned()
ParserError::SpannedParserError(
"unmatched > after parsing data type STRUCT<a STRING, b INT64>".to_owned(),
Span::empty()
),
bigquery()
.parse_sql_statements("CREATE TABLE my_table(f1 STRUCT<a STRING, b INT64>>)")
Expand All @@ -2553,8 +2560,9 @@ fn test_struct_trailing_and_nested_bracket() {

// Base case with redundant closing bracket in nested struct
assert_eq!(
ParserError::ParserError(
"Expected: ',' or ')' after column definition, found: >".to_owned()
ParserError::SpannedParserError(
"Expected: ',' or ')' after column definition, found: >".to_owned(),
Span::empty()
),
bigquery()
.parse_sql_statements("CREATE TABLE my_table(f1 STRUCT<a STRUCT<b INT>>>, c INT64)")
Expand All @@ -2566,24 +2574,25 @@ fn test_struct_trailing_and_nested_bracket() {
bigquery_and_generic()
.parse_sql_statements(sql)
.unwrap_err(),
ParserError::ParserError("unmatched > in STRUCT literal".to_string())
ParserError::SpannedParserError("unmatched > in STRUCT literal".to_string(), Span::empty())
);

let sql = "SELECT STRUCT<STRUCT<INT64>>>(NULL)";
assert_eq!(
bigquery_and_generic()
.parse_sql_statements(sql)
.unwrap_err(),
ParserError::ParserError("Expected: (, found: >".to_string())
ParserError::SpannedParserError("Expected: (, found: >".to_string(), Span::empty())
);

let sql = "CREATE TABLE table (x STRUCT<STRUCT<INT64>>>)";
assert_eq!(
bigquery_and_generic()
.parse_sql_statements(sql)
.unwrap_err(),
ParserError::ParserError(
"Expected: ',' or ')' after column definition, found: >".to_string()
ParserError::SpannedParserError(
"Expected: ',' or ')' after column definition, found: >".to_string(),
Span::empty()
)
);
}
Expand Down
73 changes: 56 additions & 17 deletions tests/sqlparser_clickhouse.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ use sqlparser::ast::Value::Boolean;
use sqlparser::ast::*;
use sqlparser::dialect::ClickHouseDialect;
use sqlparser::dialect::GenericDialect;
use sqlparser::parser::ParserError::ParserError;
use sqlparser::parser::ParserError;

#[test]
fn parse_map_access_expr() {
Expand Down Expand Up @@ -289,13 +289,19 @@ fn parse_alter_table_attach_and_detach_partition() {
clickhouse_and_generic()
.parse_sql_statements(format!("ALTER TABLE t0 {operation} PARTITION").as_str())
.unwrap_err(),
ParserError("Expected: an expression, found: EOF".to_string())
ParserError::SpannedParserError(
"Expected: an expression, found: EOF".to_string(),
Span::empty()
)
);
assert_eq!(
clickhouse_and_generic()
.parse_sql_statements(format!("ALTER TABLE t0 {operation} PART").as_str())
.unwrap_err(),
ParserError("Expected: an expression, found: EOF".to_string())
ParserError::SpannedParserError(
"Expected: an expression, found: EOF".to_string(),
Span::empty()
)
);
}
}
Expand Down Expand Up @@ -358,19 +364,22 @@ fn parse_alter_table_add_projection() {
clickhouse_and_generic()
.parse_sql_statements("ALTER TABLE t0 ADD PROJECTION my_name")
.unwrap_err(),
ParserError("Expected: (, found: EOF".to_string())
ParserError::SpannedParserError("Expected: (, found: EOF".to_string(), Span::empty())
);
assert_eq!(
clickhouse_and_generic()
.parse_sql_statements("ALTER TABLE t0 ADD PROJECTION my_name ()")
.unwrap_err(),
ParserError("Expected: SELECT, found: )".to_string())
ParserError::SpannedParserError("Expected: SELECT, found: )".to_string(), Span::empty())
);
assert_eq!(
clickhouse_and_generic()
.parse_sql_statements("ALTER TABLE t0 ADD PROJECTION my_name (SELECT)")
.unwrap_err(),
ParserError("Expected: an expression, found: )".to_string())
ParserError::SpannedParserError(
"Expected: an expression, found: )".to_string(),
Span::empty()
)
);
}

Expand Down Expand Up @@ -400,7 +409,10 @@ fn parse_alter_table_drop_projection() {
clickhouse_and_generic()
.parse_sql_statements("ALTER TABLE t0 DROP PROJECTION")
.unwrap_err(),
ParserError("Expected: identifier, found: EOF".to_string())
ParserError::SpannedParserError(
"Expected: identifier, found: EOF".to_string(),
Span::empty()
)
);
}

Expand Down Expand Up @@ -447,7 +459,10 @@ fn parse_alter_table_clear_and_materialize_projection() {
clickhouse_and_generic()
.parse_sql_statements(format!("ALTER TABLE t0 {keyword} PROJECTION",).as_str())
.unwrap_err(),
ParserError("Expected: identifier, found: EOF".to_string())
ParserError::SpannedParserError(
"Expected: identifier, found: EOF".to_string(),
Span::empty()
)
);

assert_eq!(
Expand All @@ -456,7 +471,10 @@ fn parse_alter_table_clear_and_materialize_projection() {
format!("ALTER TABLE t0 {keyword} PROJECTION my_name IN PARTITION",).as_str()
)
.unwrap_err(),
ParserError("Expected: identifier, found: EOF".to_string())
ParserError::SpannedParserError(
"Expected: identifier, found: EOF".to_string(),
Span::empty()
)
);

assert_eq!(
Expand All @@ -465,7 +483,10 @@ fn parse_alter_table_clear_and_materialize_projection() {
format!("ALTER TABLE t0 {keyword} PROJECTION my_name IN",).as_str()
)
.unwrap_err(),
ParserError("Expected: end of statement, found: IN".to_string())
ParserError::SpannedParserError(
"Expected: end of statement, found: IN".to_string(),
Span::empty()
)
);
}
}
Expand Down Expand Up @@ -513,19 +534,28 @@ fn parse_optimize_table() {
clickhouse_and_generic()
.parse_sql_statements("OPTIMIZE TABLE t0 DEDUPLICATE BY")
.unwrap_err(),
ParserError("Expected: an expression, found: EOF".to_string())
ParserError::SpannedParserError(
"Expected: an expression, found: EOF".to_string(),
Span::empty()
)
);
assert_eq!(
clickhouse_and_generic()
.parse_sql_statements("OPTIMIZE TABLE t0 PARTITION")
.unwrap_err(),
ParserError("Expected: an expression, found: EOF".to_string())
ParserError::SpannedParserError(
"Expected: an expression, found: EOF".to_string(),
Span::empty()
)
);
assert_eq!(
clickhouse_and_generic()
.parse_sql_statements("OPTIMIZE TABLE t0 PARTITION ID")
.unwrap_err(),
ParserError("Expected: identifier, found: EOF".to_string())
ParserError::SpannedParserError(
"Expected: identifier, found: EOF".to_string(),
Span::empty()
)
);
}

Expand Down Expand Up @@ -1063,7 +1093,7 @@ fn parse_settings_in_query() {
clickhouse_and_generic()
.parse_sql_statements(sql)
.unwrap_err(),
ParserError(error_msg.to_string())
ParserError::SpannedParserError(error_msg.to_string(), Span::empty())
);
}
}
Expand Down Expand Up @@ -1568,23 +1598,32 @@ fn parse_freeze_and_unfreeze_partition() {
clickhouse_and_generic()
.parse_sql_statements(format!("ALTER TABLE t0 {operation_name} PARTITION").as_str())
.unwrap_err(),
ParserError("Expected: an expression, found: EOF".to_string())
ParserError::SpannedParserError(
"Expected: an expression, found: EOF".to_string(),
Span::empty()
)
);
assert_eq!(
clickhouse_and_generic()
.parse_sql_statements(
format!("ALTER TABLE t0 {operation_name} PARTITION p0 WITH").as_str()
)
.unwrap_err(),
ParserError("Expected: NAME, found: EOF".to_string())
ParserError::SpannedParserError(
"Expected: NAME, found: EOF".to_string(),
Span::empty()
)
);
assert_eq!(
clickhouse_and_generic()
.parse_sql_statements(
format!("ALTER TABLE t0 {operation_name} PARTITION p0 WITH NAME").as_str()
)
.unwrap_err(),
ParserError("Expected: identifier, found: EOF".to_string())
ParserError::SpannedParserError(
"Expected: identifier, found: EOF".to_string(),
Span::empty()
)
);
}
}
Expand Down
Loading