refactor: Add dedicated whitespace module

rusty_parser/src/built_ins/line_input.rs

@@ -3,17 +3,15 @@ use rusty_pc::*;
 use crate::built_ins::common::{encode_opt_file_handle_arg, opt_file_handle_comma_p};
 use crate::input::StringView;
 use crate::pc_specific::*;
-use crate::tokens::whitespace_ignoring;
 use crate::{BuiltInSub, ParserError, *};
 // LINE INPUT variable$
 // LINE INPUT #file-number%, variable$
 pub fn parse() -> impl Parser<StringView, Output = Statement, Error = ParserError> {
-    seq4(
+    seq3(
         keyword_pair(Keyword::Line, Keyword::Input),
-        whitespace_ignoring(),
-        opt_file_handle_comma_p(),
+        demand_lead_ws(opt_file_handle_comma_p()),
         expression_pos_p().or_expected("#file-number or variable"),
-        |_, _, opt_file_number_pos, variable| {
+        |_, opt_file_number_pos, variable| {
             let mut args: Expressions = encode_opt_file_handle_arg(opt_file_number_pos);
             // add the LINE INPUT variable
             args.push(variable);

rusty_parser/src/built_ins/open.rs

@@ -1,10 +1,9 @@
 use rusty_common::*;
-use rusty_pc::and::IgnoringBothCombiner;
 use rusty_pc::*;
 
 use crate::input::StringView;
 use crate::pc_specific::*;
-use crate::tokens::{equal_sign_ws, whitespace_ignoring};
+use crate::tokens::equal_sign_ws;
 use crate::{BuiltInSub, ParserError, *};
 pub fn parse() -> impl Parser<StringView, Output = Statement, Error = ParserError> {
     seq6(
@@ -65,7 +64,7 @@ fn parse_file_number_p() -> impl Parser<StringView, Output = ExpressionPos, Erro
 
 fn parse_len_p() -> impl Parser<StringView, Output = ExpressionPos, Error = ParserError> {
     seq3(
-        whitespace_ignoring().and(keyword_ignoring(Keyword::Len), IgnoringBothCombiner),
+        lead_ws(keyword_ignoring(Keyword::Len)),
         equal_sign_ws(),
         expression_pos_p().or_expected("expression after LEN ="),
         |_, _, e| e,

rusty_parser/src/core/declaration.rs

@@ -1,11 +1,9 @@
-use rusty_pc::and::opt_and_keep_right;
 use rusty_pc::*;
 
 use crate::core::name::{bare_name_p, name_p};
 use crate::core::param_name::parameter_pos_p;
 use crate::input::StringView;
 use crate::pc_specific::*;
-use crate::tokens::whitespace_ignoring;
 use crate::{ParserError, *};
 
 // Declaration           ::= DECLARE<ws+>(FunctionDeclaration|SubDeclaration)
@@ -53,12 +51,7 @@ pub fn sub_declaration_p()
 
 // result ::= "" | "(" ")" | "(" parameter (,parameter)* ")"
 fn declaration_parameters_p() -> impl Parser<StringView, Output = Parameters, Error = ParserError> {
-    // TODO remove the need for the double .or_default()
-    opt_and_keep_right(
-        whitespace_ignoring(),
-        in_parenthesis(csv(parameter_pos_p()).or_default()),
-    )
-    .or_default()
+    lead_opt_ws(in_parenthesis(csv(parameter_pos_p()).or_default())).or_default()
 }
 
 #[cfg(test)]

rusty_parser/src/core/def_type.rs

@@ -3,7 +3,7 @@ use rusty_pc::*;
 use crate::error::ParserError;
 use crate::input::StringView;
 use crate::pc_specific::*;
-use crate::tokens::{TokenType, any_token_of, minus_sign, whitespace_ignoring};
+use crate::tokens::{TokenType, any_token_of, minus_sign};
 use crate::{Keyword, LetterRange, TypeQualifier};
 
 /// Represents a definition of default type, such as DEFINT A-Z.
@@ -34,11 +34,10 @@ impl DefType {
 // Letter       ::= [a-zA-Z]
 
 pub fn def_type_p() -> impl Parser<StringView, Output = DefType, Error = ParserError> {
-    seq3(
+    seq2(
         def_keyword_p(),
-        whitespace_ignoring(),
-        letter_ranges(),
-        |l, _, r| DefType::new(l, r),
+        demand_lead_ws(letter_ranges()),
+        DefType::new,
     )
 }
 

rusty_parser/src/core/do_loop.rs

@@ -4,7 +4,6 @@ use crate::core::expression::ws_expr_pos_p;
 use crate::core::statements::zero_or_more_statements;
 use crate::input::StringView;
 use crate::pc_specific::*;
-use crate::tokens::whitespace_ignoring;
 use crate::{ParserError, *};
 
 pub fn do_loop_p() -> impl Parser<StringView, Output = Statement, Error = ParserError> {
@@ -19,7 +18,7 @@ pub fn do_loop_p() -> impl Parser<StringView, Output = Statement, Error = Parser
 
 fn do_condition_top() -> impl Parser<StringView, Output = DoLoop, Error = ParserError> {
     seq4(
-        whitespace_ignoring().and_keep_right(keyword_of!(Keyword::Until, Keyword::While)),
+        lead_ws(keyword_of!(Keyword::Until, Keyword::While)),
         ws_expr_pos_p().or_expected("expression"),
         zero_or_more_statements!(Keyword::Loop),
         keyword(Keyword::Loop),

rusty_parser/src/core/expression.rs

@@ -554,7 +554,6 @@ fn eager_expression_pos_p() -> impl Parser<StringView, Output = ExpressionPos, E
 }
 
 mod single_or_double_literal {
-    use rusty_pc::and::opt_and_tuple;
     use rusty_pc::*;
 
     use crate::input::StringView;
@@ -570,36 +569,36 @@ mod single_or_double_literal {
     // TODO support more qualifiers besides '#'
 
     pub fn parser() -> impl Parser<StringView, Output = ExpressionPos, Error = ParserError> {
-        // TODO this is difficult to understand
-        opt_and_tuple(
-            // read integer digits optionally (might start with . e.g. `.123`)
-            digits(),
+        // read integer digits optionally (might start with . e.g. `.123`)
+        digits()
+            .to_option()
             // read dot and demand digits after decimal point
             // if dot is missing, the parser returns an empty result
             // the "deal breaker" is therefore the dot
-            dot().and_keep_right(digits().to_fatal()),
-        )
-        // and parse optionally a type qualifier such as `#`
-        .and_tuple(pound().to_option())
-        // done parsing, flat map everything
-        .and_then(|((opt_integer_digits, frac_digits), opt_pound)| {
-            let left = opt_integer_digits
-                .map(|token| token.to_string())
-                .unwrap_or_else(|| "0".to_owned());
-            let s = format!("{}.{}", left, frac_digits.as_str());
-            if opt_pound.is_some() {
-                match s.parse::<f64>() {
-                    Ok(f) => Ok(Expression::DoubleLiteral(f)),
-                    Err(err) => Err(err.into()),
-                }
-            } else {
-                match s.parse::<f32>() {
-                    Ok(f) => Ok(Expression::SingleLiteral(f)),
-                    Err(err) => Err(err.into()),
+            .and_keep_left(dot())
+            // demand digits after decimal point
+            .and_tuple(digits().to_fatal())
+            // and parse optionally a type qualifier such as `#`
+            .and_tuple(pound().to_option())
+            // done parsing, flat map everything
+            .and_then(|((opt_integer_digits, frac_digits), opt_pound)| {
+                let left = opt_integer_digits
+                    .map(|token| token.to_string())
+                    .unwrap_or_else(|| "0".to_owned());
+                let s = format!("{}.{}", left, frac_digits.as_str());
+                if opt_pound.is_some() {
+                    match s.parse::<f64>() {
+                        Ok(f) => Ok(Expression::DoubleLiteral(f)),
+                        Err(err) => Err(err.into()),
+                    }
+                } else {
+                    match s.parse::<f32>() {
+                        Ok(f) => Ok(Expression::SingleLiteral(f)),
+                        Err(err) => Err(err.into()),
+                    }
                 }
-            }
-        })
-        .with_pos()
+            })
+            .with_pos()
     }
 }
 
@@ -998,16 +997,16 @@ mod built_in_function_call {
 
 mod binary_expression {
     use rusty_common::Positioned;
-    use rusty_pc::and::{TupleCombiner, opt_and_keep_right};
+    use rusty_pc::and::TupleCombiner;
     use rusty_pc::*;
 
     use super::{
         built_in_function_call, expression_pos_p, guard, integer_or_long_literal, parenthesis, property, single_or_double_literal, string_literal, unary_expression
     };
     use crate::error::ParserError;
     use crate::input::StringView;
-    use crate::pc_specific::{OrExpected, WithPos};
-    use crate::tokens::{TokenType, any_token, whitespace_ignoring};
+    use crate::pc_specific::{OrExpected, WithPos, lead_opt_ws, lead_ws};
+    use crate::tokens::{TokenType, any_token};
     use crate::*;
 
     // result ::= <non-bin-expr> <operator> <expr>
@@ -1078,14 +1077,9 @@ mod binary_expression {
     -> impl Parser<StringView, bool, Output = Positioned<Operator>, Error = ParserError> {
         IifParser::new(
             // no whitespace needed
-            opt_and_keep_right(whitespace_ignoring(), operator_p()),
+            lead_opt_ws(operator_p()),
             // whitespace needed
-            whitespace_ignoring()
-                .and_keep_right(operator_p())
-                .or(opt_and_keep_right(
-                    whitespace_ignoring(),
-                    symbol_operator_p(),
-                )),
+            lead_ws(operator_p()).or(lead_opt_ws(symbol_operator_p())),
         )
     }
 
@@ -1201,7 +1195,7 @@ pub mod file_handle {
     use crate::error::ParserError;
     use crate::input::StringView;
     use crate::pc_specific::*;
-    use crate::tokens::{TokenType, any_token_of, pound, whitespace_ignoring};
+    use crate::tokens::{TokenType, any_token_of, pound};
     use crate::*;
 
     pub fn file_handle_p()
@@ -1230,7 +1224,7 @@ pub mod file_handle {
     }
 
     fn ws_file_handle() -> impl Parser<StringView, Output = ExpressionPos, Error = ParserError> {
-        whitespace_ignoring().and_keep_right(file_handle_as_expression_pos_p())
+        lead_ws(file_handle_as_expression_pos_p())
     }
 }
 
@@ -1239,8 +1233,8 @@ pub mod guard {
 
     use crate::ParserError;
     use crate::input::StringView;
-    use crate::pc_specific::WithExpected;
-    use crate::tokens::{any_symbol_of, any_token_of, whitespace_ignoring};
+    use crate::pc_specific::{WithExpected, whitespace_ignoring};
+    use crate::tokens::{any_symbol_of, any_token_of};
 
     /// `result ::= " " | "("`
     ///

rusty_parser/src/core/for_loop.rs

@@ -6,7 +6,7 @@ use crate::core::statements::zero_or_more_statements;
 use crate::error::ParserError;
 use crate::input::StringView;
 use crate::pc_specific::*;
-use crate::tokens::{equal_sign_ws, whitespace_ignoring};
+use crate::tokens::equal_sign_ws;
 use crate::*;
 
 // FOR I = 0 TO 5 STEP 1
@@ -65,7 +65,7 @@ fn parse_for_p()
 }
 
 fn next_counter_p() -> impl Parser<StringView, Output = ExpressionPos, Error = ParserError> {
-    whitespace_ignoring().and_keep_right(property::parser())
+    lead_ws(property::parser())
 }
 
 #[cfg(test)]

rusty_parser/src/core/global_statement.rs

@@ -159,7 +159,7 @@ fn main_program() -> impl Parser<StringView, Output = Program, Error = ParserErr
 }
 
 fn next_statements() -> impl Parser<StringView, Output = Program, Error = ParserError> {
-    next_statement().padded_by_ws().zero_or_more()
+    padded_by_ws(next_statement()).zero_or_more()
 }
 
 fn next_statement() -> impl Parser<StringView, Output = GlobalStatementPos, Error = ParserError> {

rusty_parser/src/core/go_sub.rs

@@ -3,7 +3,6 @@ use rusty_pc::*;
 use crate::core::name::bare_name_p;
 use crate::input::StringView;
 use crate::pc_specific::*;
-use crate::tokens::whitespace_ignoring;
 use crate::{Keyword, ParserError, Statement};
 
 pub fn statement_go_sub_p() -> impl Parser<StringView, Output = Statement, Error = ParserError> {
@@ -13,13 +12,9 @@ pub fn statement_go_sub_p() -> impl Parser<StringView, Output = Statement, Error
 }
 
 pub fn statement_return_p() -> impl Parser<StringView, Output = Statement, Error = ParserError> {
-    seq2(
-        keyword(Keyword::Return),
-        whitespace_ignoring()
-            .and_keep_right(bare_name_p())
-            .to_option(),
-        |_, name| Statement::Return(name),
-    )
+    keyword(Keyword::Return)
+        .and_keep_right(lead_ws(bare_name_p()).to_option())
+        .map(Statement::Return)
 }
 
 #[cfg(test)]

rusty_parser/src/core/if_block.rs

@@ -1,4 +1,3 @@
-use rusty_pc::and::IgnoringBothCombiner;
 use rusty_pc::*;
 
 use crate::core::comment::comment_p;
@@ -9,7 +8,6 @@ use crate::core::single_line_statements::{
 use crate::core::statements::zero_or_more_statements;
 use crate::input::StringView;
 use crate::pc_specific::*;
-use crate::tokens::whitespace_ignoring;
 use crate::{ParserError, *};
 
 pub fn if_block_p() -> impl Parser<StringView, Output = Statement, Error = ParserError> {
@@ -58,14 +56,11 @@ fn single_line_if_else_p() -> impl Parser<
 }
 
 fn single_line_comment_p() -> impl Parser<StringView, Output = Statements, Error = ParserError> {
-    whitespace_ignoring()
-        .to_option()
-        .and(comment_p().with_pos(), |_, s| vec![s])
+    lead_opt_ws(comment_p().with_pos()).map(|s| vec![s])
 }
 
 fn single_line_else_p() -> impl Parser<StringView, Output = Statements, Error = ParserError> {
-    whitespace_ignoring()
-        .and(keyword(Keyword::Else), IgnoringBothCombiner)
+    lead_ws(keyword(Keyword::Else))
         .and_keep_right(single_line_statements_p().or_expected("Statements for single line ELSE"))
 }
 

rusty_parser/src/core/implementation.rs

@@ -1,12 +1,10 @@
-use rusty_pc::and::IgnoringBothCombiner;
 use rusty_pc::*;
 
 use crate::core::declaration::{function_declaration_p, sub_declaration_p};
 use crate::core::statements::zero_or_more_statements;
 use crate::input::StringView;
 use crate::pc_specific::*;
-use crate::tokens::whitespace_ignoring;
-use crate::{ParserError, *};
+use crate::{FunctionImplementation, GlobalStatement, Keyword, ParserError, SubImplementation};
 
 // FunctionImplementation ::= <FunctionDeclaration> eol <Statements> eol END<ws+>FUNCTION
 // SubImplementation      ::= <SubDeclaration> eol <Statements> eol END<ws+>SUB
@@ -60,18 +58,18 @@ where
 }
 
 fn ws_static() -> impl Parser<StringView, Output = (), Error = ParserError> {
-    whitespace_ignoring()
-        .to_option()
-        .and(keyword(Keyword::Static), IgnoringBothCombiner)
+    lead_opt_ws(keyword_ignoring(Keyword::Static))
 }
 
 #[cfg(test)]
 mod tests {
     use rusty_common::*;
 
     use super::*;
-    use crate::assert_parser_err;
     use crate::test_utils::*;
+    use crate::{
+        Expression, ExpressionType, Operator, ParamType, Parameter, Statement, assert_parser_err, parse
+    };
 
     #[test]
     fn test_function_implementation() {

rusty_parser/src/core/on_error.rs

@@ -6,18 +6,15 @@ use crate::core::name::bare_name_p;
 use crate::error::ParserError;
 use crate::input::StringView;
 use crate::pc_specific::*;
-use crate::tokens::whitespace_ignoring;
 use crate::{Expression, Keyword, OnErrorOption, Statement};
 
 pub fn statement_on_error_go_to_p()
 -> impl Parser<StringView, Output = Statement, Error = ParserError> {
-    seq2(
-        keyword_pair(Keyword::On, Keyword::Error),
-        whitespace_ignoring(),
-        |_, _| (),
-    )
-    .and_keep_right(next().or(goto()).or_expected("GOTO or RESUME"))
-    .map(Statement::OnError)
+    keyword_pair(Keyword::On, Keyword::Error)
+        .and_keep_right(demand_lead_ws(
+            next().or(goto()).or_expected("GOTO or RESUME"),
+        ))
+        .map(Statement::OnError)
 }
 
 fn next() -> impl Parser<StringView, Output = OnErrorOption, Error = ParserError> {

rusty_parser/src/core/opt_second_expression.rs

@@ -4,8 +4,7 @@ use rusty_pc::{IifParser, Parser, ParserErrorTrait};
 use crate::core::expression::ws_expr_pos_p;
 use crate::error::ParserError;
 use crate::input::StringView;
-use crate::pc_specific::keyword;
-use crate::tokens::whitespace_ignoring;
+use crate::pc_specific::{keyword, whitespace_ignoring};
 use crate::{ExpressionPos, Keyword};
 
 /// Parses an optional second expression that follows the first expression