From a521a8606415d190c84e2c1b1a3b556fec97d0c4 Mon Sep 17 00:00:00 2001 From: rameel Date: Sat, 7 Nov 2026 22:29:49 +0500 Subject: [PATCH 01/10] docs: add a brief parsers guide --- docs/README.md | 211 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 docs/README.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..c9a6de2 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,211 @@ +# Parser guide + +A `Parser` reads text and produces a value of type `T`. Small parsers match +individual tokens; combinators join them into lists, expressions, and grammars. +The examples use these imports: + +```csharp +using Ramstack.Parsing; +using static Ramstack.Parsing.Parser; +``` + +## Basic parsers + +| Parser | Purpose | +| --- | --- | +| `L('a')` | Match one specific character and return it. | +| `L("hello")` | Match a literal string. Comparison is ordinal by default; a `StringComparison` argument can change it. | +| `OneOf("abc")` | Match one character from the supplied string. | +| `OneOf("if", "else")` | Match one of several literal strings. For ordinal comparisons, prefer the longest matching literal. | +| `Set('a', 'z')` | Match a character in an inclusive range. | +| `Set("a-zA-Z0-9_")` | Match a character from a character class. | +| `L(UnicodeCategory...)`, `L(GeneralUnicodeCategory...)` | Match a character by its Unicode category. | +| `Any` | Match any single character. Fail at the end of input. | +| `S` | Consume zero or more whitespace characters, including line breaks. | +| `Eol` | Consume a line break, or succeed at the end of input. | +| `Eof` | Require the end of input without consuming anything. | +| `Regex(regex)` | Match a regular expression at the current position; its pattern must start with `^`. Available on .NET 7+. | + +`Character` provides ready-made character parsers: `Letter`, `Uppercase`, +`Lowercase`, `Digit`, `LetterOrDigit`, `HexDigit`, `WhiteSpace`, and other Unicode +groups. These return `char`; `S`, `Eol`, and `Eof` return `Unit`, representing +success without a meaningful value. + +```csharp +var identifier = Seq( + Set("a-zA-Z_"), + Set("a-zA-Z0-9_").ZeroOrMore() +).Text(); +// "item_42" => "item_42" +``` + +## Literals + +`L` matches fixed text. The `Literal` class parses a value written in text form. + +| Parser | Purpose | +| --- | --- | +| `Literal.Number()` | Parse a numeric value, such as `int`, `double`, or `decimal`. | +| `Literal.Number(kind)` | Select a numeric format with `NumberKind`: `Auto`, `Integer`, `Float`, `HexNumber`, or `BinaryNumber` (.NET 8+). Supported formats depend on `T`. | +| `Literal.DoubleQuotedString` | Parse a double-quoted string, remove its quotes, and decode escapes. | +| `Literal.SingleQuotedString` | Parse a single-quoted string with the same value handling. | +| `Literal.QuotedCharacter` | Parse one character enclosed in single quotes. | +| `Literal.EscapeSequence` | Decode an escape sequence such as `\n` into a character. | +| `Literal.UnicodeEscapeSequence` | Decode a Unicode escape such as `\u0041`. | + +```csharp +var integer = Literal.Number(); // "42" => 42 +var text = Literal.DoubleQuotedString; // "\"hello\"" => "hello" +``` + +## Combining parsers + +| Combinator | Purpose | +| --- | --- | +| `Seq(a, b, ...)` | Run parsers in order and return their values as a tuple. Sequences of `Unit` parsers return `Unit`. | +| `a.Then(b)` | Run both parsers; keep the value from `b`. | +| `a.ThenIgnore(b)` | Run both parsers; keep the value from `a`. | +| `p.Between(open, close)` | Parse an opening delimiter, `p`, and a closing delimiter; keep the value from `p`. | +| `p.Between(around)` | Use the same parser before and after `p`. | +| `Choice(a, b, ...)`, `a.Or(b)` | Try alternatives in order and return the first successful result. Alternatives must have the same result type. | + +```csharp +var boolean = Choice( + L("true").Do(_ => true), + L("false").Do(_ => false)); + +var parenthesized = Literal.Number().Between(L('('), L(')')); +// "(42)" => 42 +``` + +Whitespace is handled explicitly: `p.ThenIgnore(S)` accepts whitespace after +`p`, while `p.Between(S, S)` accepts it on both sides. Alternatives are ordered: +`L("a").Or(L("ab"))` accepts the prefix `"a"` of `"ab"`. + +## Repetition and lists + +| Combinator | Purpose | +| --- | --- | +| `p.ZeroOrMore()`, `p.Many()` | Read zero or more items. An empty result is allowed. | +| `p.OneOrMore()` | Read at least one item. | +| `p.AtLeast(n)` | Read at least `n` items. | +| `p.Repeat(n)` | Read exactly `n` items. | +| `p.Repeat(min, max)` | Read between `min` and `max` items, inclusive. | +| `p.Separated(separator)` | Read items separated by another parser; discard separator values. | +| `p.Until(terminator)` | Read items until the terminator matches. Require the terminator, but leave it unconsumed. | +| `p.Optional()` | Accept an absent item; return `OptionalValue` with `HasValue` and `Value`. | +| `p.DefaultOnFail(value)` | On ordinary failure, succeed with a fallback value. With no argument, use `default(T)`. | + +Repetition normally returns `List`. Repeating a `Parser` with methods +such as `ZeroOrMore` or `Repeat` returns `Unit` instead. + +`Separated` allows an empty list by default. Use `min` and `max` to constrain its +size and `allowTrailing: true` to accept a final separator. + +```csharp +var item = Literal.Number().ThenIgnore(S); +var comma = L(',').ThenIgnore(S); +var list = item.Separated(comma) + .Between(L('[').ThenIgnore(S), L(']')) + .ThenIgnore(Eof); +// "[1, 2, 3]" => List { 1, 2, 3 }; "[]" => an empty list + +var comment = Any.Until(L("*/")).Text() + .Between(L("/*"), L("*/")); +// "/*note*/" => "note"; Between consumes the terminator left by Until +``` + +## Working with results + +| Combinator | Purpose | +| --- | --- | +| `p.Do(value => ...)` | Transform the parsed value. Overloads unpack tuples produced by `Seq`. | +| `p.Map(match => ...)` | Transform the matched source segment using its text, index, and length. | +| `p.Map((match, value) => ...)` | Transform both the matched segment and the parsed value. | +| `p.Text()` | Return the matched source text as a string. | +| `p.Void()` | Discard the value and return `Unit`. | +| `Return(value)` | Succeed with a constant value without consuming input. | +| `p.Where(predicate, expected)` | Accept the parsed value only when a condition holds; otherwise report the expected description. | + +```csharp +var pair = Seq(Literal.Number(), L(':'), Literal.Number()) + .Do((left, _, right) => (Left: left, Right: right)); +// "12:34" => (Left: 12, Right: 34) +``` + +## Recursive parsers + +Use recursion for nested structures such as parentheses, arrays, or syntax trees. +`Deferred()` creates a reference whose `.Parser` is assigned once the rule is +built. `Recursive(self => ...)` defines a rule using a self-reference in a +factory function. + +```csharp +var nested = Recursive(self => + Literal.Number().Or(self.Between(L('('), L(')')))); +// "42", "(42)", and "((42))" => 42 + +var deferred = Deferred(); +deferred.Parser = Literal.Number() + .Or(deferred.Between(L('('), L(')'))); +``` + +Initialize a deferred rule before parsing or using it in operations that discard +its value, such as `Void`, `And`, or `Not`. Recursive calls must follow input +consumption: a rule that immediately calls itself is left-recursive and is not +supported. + +## Operators and expressions + +| Combinator | Purpose | +| --- | --- | +| `p.FoldL(op, reduce)` | Parse an operand followed by operator/operand pairs; combine from the left. `Fold` is an alias. | +| `p.FoldR(op, reduce)` | Combine operator/operand pairs from the right. | +| `p.Prefixed(op, reduce)` | Parse zero or more prefix operators before an operand; apply the closest operator first. | + +```csharp +var number = Literal.Number().ThenIgnore(S); +var subtraction = number.FoldL(L('-').ThenIgnore(S), (l, r, _) => l - r); +// "10 - 3 - 2" => (10 - 3) - 2 => 5 +``` + +Build precedence in layers: for example, use a product parser as the operand +parser for a sum. See the [calculator](../samples/CalcExpr/README.md). + +## Lookahead and errors + +| Operation | Purpose | +| --- | --- | +| `And(p)` | Require `p` to match without consuming input. Return `Unit`. | +| `Not(p)` | Require `p` to fail without consuming input. Return `Unit`. | +| `p.As("name")` | Give a parser a readable name for diagnostics. | +| `Error("expected")` | Fail normally and report an expectation; alternatives may still be tried. | +| `Fail("message")` | Force parsing to terminate with an error message. | +| `FatalError("message")` | Force termination from a callback. | + +`Optional` and `DefaultOnFail` handle ordinary failures; they do not recover from +forced termination. + +## Running a parser + +`p.Parse(source)` returns `ParseResult`: check `Success` before reading `Value`. +`Length` reports the consumed length on success; `ErrorMessage` describes a +failure. Forced termination produces an error message; other callback exceptions +are also captured in `Exception`. + +`p.TryParse(source, out value)` returns a boolean and suppresses diagnostics. +Forced termination returns `false`, while other callback exceptions propagate. +Both entry points accept `ReadOnlySpan`, including strings via conversion. + +```csharp +var parser = Literal.Number().Between(S, S).ThenIgnore(Eof); +var result = parser.Parse(" 42 "); // Success: true, Value: 42 +var invalid = parser.Parse("42!"); // Success: false +``` + +A successful parse can match only a prefix. Add `ThenIgnore(Eof)` when the entire +input must match. Build parsers once and reuse them. + +For complete grammars, see [JSON](../samples/Json/README.md) and +[TinyC](../samples/TinyC/README.md). Installation is covered in the +[project README](../README.md). From 00310183f8c91a81eb6fca590afa09e73a278fcf Mon Sep 17 00:00:00 2001 From: rameel Date: Sat, 7 Nov 2026 23:39:21 +0500 Subject: [PATCH 02/10] docs: split parser guide --- docs/README.md | 207 ++--------------------------------- docs/basic-parsers.md | 54 +++++++++ docs/combinators.md | 52 +++++++++ docs/parsing-and-errors.md | 39 +++++++ docs/recursive-parsers.md | 44 ++++++++ docs/repetition-and-lists.md | 36 ++++++ 6 files changed, 236 insertions(+), 196 deletions(-) create mode 100644 docs/basic-parsers.md create mode 100644 docs/combinators.md create mode 100644 docs/parsing-and-errors.md create mode 100644 docs/recursive-parsers.md create mode 100644 docs/repetition-and-lists.md diff --git a/docs/README.md b/docs/README.md index c9a6de2..7a1794c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,203 +9,18 @@ using Ramstack.Parsing; using static Ramstack.Parsing.Parser; ``` -## Basic parsers +## Contents -| Parser | Purpose | -| --- | --- | -| `L('a')` | Match one specific character and return it. | -| `L("hello")` | Match a literal string. Comparison is ordinal by default; a `StringComparison` argument can change it. | -| `OneOf("abc")` | Match one character from the supplied string. | -| `OneOf("if", "else")` | Match one of several literal strings. For ordinal comparisons, prefer the longest matching literal. | -| `Set('a', 'z')` | Match a character in an inclusive range. | -| `Set("a-zA-Z0-9_")` | Match a character from a character class. | -| `L(UnicodeCategory...)`, `L(GeneralUnicodeCategory...)` | Match a character by its Unicode category. | -| `Any` | Match any single character. Fail at the end of input. | -| `S` | Consume zero or more whitespace characters, including line breaks. | -| `Eol` | Consume a line break, or succeed at the end of input. | -| `Eof` | Require the end of input without consuming anything. | -| `Regex(regex)` | Match a regular expression at the current position; its pattern must start with `^`. Available on .NET 7+. | +- [Basic parsers and literals](basic-parsers.md) — Match characters, fixed text, and values such as numbers and quoted strings. +- [Combining parsers and transforming values](combinators.md) — Build sequences and alternatives, then transform or discard their results. +- [Repetition and lists](repetition-and-lists.md) — Read repeated items, separated lists, optional values, and text up to a terminator. +- [Recursive parsers and expressions](recursive-parsers.md) — Describe nested structures and expressions with associativity and precedence. +- [Running parsers, lookahead, and errors](parsing-and-errors.md) — Run a parser, inspect its result, check upcoming input, and report failures. -`Character` provides ready-made character parsers: `Letter`, `Uppercase`, -`Lowercase`, `Digit`, `LetterOrDigit`, `HexDigit`, `WhiteSpace`, and other Unicode -groups. These return `char`; `S`, `Eol`, and `Eof` return `Unit`, representing -success without a meaningful value. +## Complete examples -```csharp -var identifier = Seq( - Set("a-zA-Z_"), - Set("a-zA-Z0-9_").ZeroOrMore() -).Text(); -// "item_42" => "item_42" -``` - -## Literals - -`L` matches fixed text. The `Literal` class parses a value written in text form. - -| Parser | Purpose | -| --- | --- | -| `Literal.Number()` | Parse a numeric value, such as `int`, `double`, or `decimal`. | -| `Literal.Number(kind)` | Select a numeric format with `NumberKind`: `Auto`, `Integer`, `Float`, `HexNumber`, or `BinaryNumber` (.NET 8+). Supported formats depend on `T`. | -| `Literal.DoubleQuotedString` | Parse a double-quoted string, remove its quotes, and decode escapes. | -| `Literal.SingleQuotedString` | Parse a single-quoted string with the same value handling. | -| `Literal.QuotedCharacter` | Parse one character enclosed in single quotes. | -| `Literal.EscapeSequence` | Decode an escape sequence such as `\n` into a character. | -| `Literal.UnicodeEscapeSequence` | Decode a Unicode escape such as `\u0041`. | - -```csharp -var integer = Literal.Number(); // "42" => 42 -var text = Literal.DoubleQuotedString; // "\"hello\"" => "hello" -``` - -## Combining parsers - -| Combinator | Purpose | -| --- | --- | -| `Seq(a, b, ...)` | Run parsers in order and return their values as a tuple. Sequences of `Unit` parsers return `Unit`. | -| `a.Then(b)` | Run both parsers; keep the value from `b`. | -| `a.ThenIgnore(b)` | Run both parsers; keep the value from `a`. | -| `p.Between(open, close)` | Parse an opening delimiter, `p`, and a closing delimiter; keep the value from `p`. | -| `p.Between(around)` | Use the same parser before and after `p`. | -| `Choice(a, b, ...)`, `a.Or(b)` | Try alternatives in order and return the first successful result. Alternatives must have the same result type. | - -```csharp -var boolean = Choice( - L("true").Do(_ => true), - L("false").Do(_ => false)); - -var parenthesized = Literal.Number().Between(L('('), L(')')); -// "(42)" => 42 -``` - -Whitespace is handled explicitly: `p.ThenIgnore(S)` accepts whitespace after -`p`, while `p.Between(S, S)` accepts it on both sides. Alternatives are ordered: -`L("a").Or(L("ab"))` accepts the prefix `"a"` of `"ab"`. - -## Repetition and lists - -| Combinator | Purpose | -| --- | --- | -| `p.ZeroOrMore()`, `p.Many()` | Read zero or more items. An empty result is allowed. | -| `p.OneOrMore()` | Read at least one item. | -| `p.AtLeast(n)` | Read at least `n` items. | -| `p.Repeat(n)` | Read exactly `n` items. | -| `p.Repeat(min, max)` | Read between `min` and `max` items, inclusive. | -| `p.Separated(separator)` | Read items separated by another parser; discard separator values. | -| `p.Until(terminator)` | Read items until the terminator matches. Require the terminator, but leave it unconsumed. | -| `p.Optional()` | Accept an absent item; return `OptionalValue` with `HasValue` and `Value`. | -| `p.DefaultOnFail(value)` | On ordinary failure, succeed with a fallback value. With no argument, use `default(T)`. | - -Repetition normally returns `List`. Repeating a `Parser` with methods -such as `ZeroOrMore` or `Repeat` returns `Unit` instead. - -`Separated` allows an empty list by default. Use `min` and `max` to constrain its -size and `allowTrailing: true` to accept a final separator. - -```csharp -var item = Literal.Number().ThenIgnore(S); -var comma = L(',').ThenIgnore(S); -var list = item.Separated(comma) - .Between(L('[').ThenIgnore(S), L(']')) - .ThenIgnore(Eof); -// "[1, 2, 3]" => List { 1, 2, 3 }; "[]" => an empty list - -var comment = Any.Until(L("*/")).Text() - .Between(L("/*"), L("*/")); -// "/*note*/" => "note"; Between consumes the terminator left by Until -``` - -## Working with results - -| Combinator | Purpose | -| --- | --- | -| `p.Do(value => ...)` | Transform the parsed value. Overloads unpack tuples produced by `Seq`. | -| `p.Map(match => ...)` | Transform the matched source segment using its text, index, and length. | -| `p.Map((match, value) => ...)` | Transform both the matched segment and the parsed value. | -| `p.Text()` | Return the matched source text as a string. | -| `p.Void()` | Discard the value and return `Unit`. | -| `Return(value)` | Succeed with a constant value without consuming input. | -| `p.Where(predicate, expected)` | Accept the parsed value only when a condition holds; otherwise report the expected description. | - -```csharp -var pair = Seq(Literal.Number(), L(':'), Literal.Number()) - .Do((left, _, right) => (Left: left, Right: right)); -// "12:34" => (Left: 12, Right: 34) -``` - -## Recursive parsers - -Use recursion for nested structures such as parentheses, arrays, or syntax trees. -`Deferred()` creates a reference whose `.Parser` is assigned once the rule is -built. `Recursive(self => ...)` defines a rule using a self-reference in a -factory function. - -```csharp -var nested = Recursive(self => - Literal.Number().Or(self.Between(L('('), L(')')))); -// "42", "(42)", and "((42))" => 42 - -var deferred = Deferred(); -deferred.Parser = Literal.Number() - .Or(deferred.Between(L('('), L(')'))); -``` - -Initialize a deferred rule before parsing or using it in operations that discard -its value, such as `Void`, `And`, or `Not`. Recursive calls must follow input -consumption: a rule that immediately calls itself is left-recursive and is not -supported. - -## Operators and expressions - -| Combinator | Purpose | -| --- | --- | -| `p.FoldL(op, reduce)` | Parse an operand followed by operator/operand pairs; combine from the left. `Fold` is an alias. | -| `p.FoldR(op, reduce)` | Combine operator/operand pairs from the right. | -| `p.Prefixed(op, reduce)` | Parse zero or more prefix operators before an operand; apply the closest operator first. | - -```csharp -var number = Literal.Number().ThenIgnore(S); -var subtraction = number.FoldL(L('-').ThenIgnore(S), (l, r, _) => l - r); -// "10 - 3 - 2" => (10 - 3) - 2 => 5 -``` - -Build precedence in layers: for example, use a product parser as the operand -parser for a sum. See the [calculator](../samples/CalcExpr/README.md). - -## Lookahead and errors - -| Operation | Purpose | -| --- | --- | -| `And(p)` | Require `p` to match without consuming input. Return `Unit`. | -| `Not(p)` | Require `p` to fail without consuming input. Return `Unit`. | -| `p.As("name")` | Give a parser a readable name for diagnostics. | -| `Error("expected")` | Fail normally and report an expectation; alternatives may still be tried. | -| `Fail("message")` | Force parsing to terminate with an error message. | -| `FatalError("message")` | Force termination from a callback. | - -`Optional` and `DefaultOnFail` handle ordinary failures; they do not recover from -forced termination. - -## Running a parser - -`p.Parse(source)` returns `ParseResult`: check `Success` before reading `Value`. -`Length` reports the consumed length on success; `ErrorMessage` describes a -failure. Forced termination produces an error message; other callback exceptions -are also captured in `Exception`. - -`p.TryParse(source, out value)` returns a boolean and suppresses diagnostics. -Forced termination returns `false`, while other callback exceptions propagate. -Both entry points accept `ReadOnlySpan`, including strings via conversion. - -```csharp -var parser = Literal.Number().Between(S, S).ThenIgnore(Eof); -var result = parser.Parse(" 42 "); // Success: true, Value: 42 -var invalid = parser.Parse("42!"); // Success: false -``` - -A successful parse can match only a prefix. Add `ThenIgnore(Eof)` when the entire -input must match. Build parsers once and reuse them. +- [Calculator](../samples/CalcExpr/README.md): arithmetic expressions and precedence. +- [JSON](../samples/Json/README.md): nested objects and arrays. +- [TinyC](../samples/TinyC/README.md): a small language grammar. -For complete grammars, see [JSON](../samples/Json/README.md) and -[TinyC](../samples/TinyC/README.md). Installation is covered in the -[project README](../README.md). +Installation is covered in the [project README](../README.md). diff --git a/docs/basic-parsers.md b/docs/basic-parsers.md new file mode 100644 index 0000000..a755f1f --- /dev/null +++ b/docs/basic-parsers.md @@ -0,0 +1,54 @@ +# Basic parsers and literals + +[Parser guide](README.md) + +Match characters, fixed text, and values such as numbers and quoted strings. + +## Basic parsers + +| Parser | Purpose | +| ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `L('a')` | Match one specific character and return it. | +| `L("hello")` | Match a literal string. Comparison is ordinal by default; a `StringComparison` argument can change it. | +| `OneOf("abc")` | Match one character from the supplied string. | +| `OneOf("if", "else")` | Match one of several literal strings. For ordinal comparisons, prefer the longest matching literal. | +| `Set('a', 'z')` | Match a character in an inclusive range. | +| `Set("a-zA-Z0-9_")` | Match a character from a character class. | +| `L(UnicodeCategory...)`, `L(GeneralUnicodeCategory...)` | Match a character by its Unicode category. | +| `Any` | Match any single character. Fail at the end of input. | +| `S` | Consume zero or more whitespace characters, including line breaks. | +| `Eol` | Consume a line break, or succeed at the end of input. | +| `Eof` | Require the end of input without consuming anything. | +| `Regex(regex)` | Match a regular expression at the current position; its pattern must start with `^`. Available on .NET 7+. | + +`Character` provides ready-made character parsers: `Letter`, `Uppercase`, +`Lowercase`, `Digit`, `LetterOrDigit`, `HexDigit`, `WhiteSpace`, and other Unicode +groups. These return `char`; `S`, `Eol`, and `Eof` return `Unit`, representing +success without a meaningful value. + +```csharp +var identifier = Seq( + Set("a-zA-Z_"), + Set("a-zA-Z0-9_").ZeroOrMore() +).Text(); +// "item_42" => "item_42" +``` + +## Literals + +`L` matches fixed text. The `Literal` class parses a value written in text form. + +| Parser | Purpose | +| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Literal.Number()` | Parse a numeric value, such as `int`, `double`, or `decimal`. | +| `Literal.Number(kind)` | Select a numeric format with `NumberKind`: `Auto`, `Integer`, `Float`, `HexNumber`, or `BinaryNumber` (.NET 8+). Supported formats depend on `T`. | +| `Literal.DoubleQuotedString` | Parse a double-quoted string, remove its quotes, and decode escapes. | +| `Literal.SingleQuotedString` | Parse a single-quoted string with the same value handling. | +| `Literal.QuotedCharacter` | Parse one character enclosed in single quotes. | +| `Literal.EscapeSequence` | Decode an escape sequence such as `\n` into a character. | +| `Literal.UnicodeEscapeSequence` | Decode a Unicode escape such as `\u0041`. | + +```csharp +var integer = Literal.Number(); // "42" => 42 +var text = Literal.DoubleQuotedString; // "\"hello\"" => "hello" +``` diff --git a/docs/combinators.md b/docs/combinators.md new file mode 100644 index 0000000..b77b5ab --- /dev/null +++ b/docs/combinators.md @@ -0,0 +1,52 @@ +# Combining parsers and transforming values + +[Parser guide](README.md) + +Build sequences and alternatives, then transform or discard their results. + +## Combining parsers + +| Combinator | Purpose | +| ------------------------------ | -------------------------------------------------------------------------------------------------------------- | +| `Seq(a, b, ...)` | Run parsers in order and return their values as a tuple. Sequences of `Unit` parsers return `Unit`. | +| `a.Then(b)` | Run both parsers; keep the value from `b`. | +| `a.ThenIgnore(b)` | Run both parsers; keep the value from `a`. | +| `p.Between(open, close)` | Parse an opening delimiter, `p`, and a closing delimiter; keep the value from `p`. | +| `p.Between(around)` | Use the same parser before and after `p`. | +| `Choice(a, b, ...)`, `a.Or(b)` | Try alternatives in order and return the first successful result. Alternatives must have the same result type. | + +```csharp +var boolean = Choice( + L("true").Do(_ => true), + L("false").Do(_ => false)); + +var parenthesized = Literal + .Number() + .Between(L('('), L(')')); +// "(42)" => 42 +``` + +Whitespace is handled explicitly: `p.ThenIgnore(S)` accepts whitespace after +`p`, while `p.Between(S, S)` accepts it on both sides. Alternatives are ordered: +`L("a").Or(L("ab"))` accepts the prefix `"a"` of `"ab"`. + +## Working with results + +| Combinator | Purpose | +| ------------------------------ | ----------------------------------------------------------------------------------------------- | +| `p.Do(value => ...)` | Transform the parsed value. Overloads unpack tuples produced by `Seq`. | +| `p.Map(match => ...)` | Transform the matched source segment using its text, index, and length. | +| `p.Map((match, value) => ...)` | Transform both the matched segment and the parsed value. | +| `p.Text()` | Return the matched source text as a string. | +| `p.Void()` | Discard the value and return `Unit`. | +| `Return(value)` | Succeed with a constant value without consuming input. | +| `p.Where(predicate, expected)` | Accept the parsed value only when a condition holds; otherwise report the expected description. | + +```csharp +var pair = Seq( + Literal.Number(), + L(':'), + Literal.Number() + ).Do((left, _, right) => (Left: left, Right: right)); +// "12:34" => (Left: 12, Right: 34) +``` diff --git a/docs/parsing-and-errors.md b/docs/parsing-and-errors.md new file mode 100644 index 0000000..fdfb43e --- /dev/null +++ b/docs/parsing-and-errors.md @@ -0,0 +1,39 @@ +# Running parsers, lookahead, and errors + +[Parser guide](README.md) + +Run a parser, inspect its result, check upcoming input, and report failures. + +## Running a parser + +`p.Parse(source)` returns `ParseResult`: check `Success` before reading `Value`. +`Length` reports the consumed length on success; `ErrorMessage` describes a +failure. Forced termination produces an error message; other callback exceptions +are also captured in `Exception`. + +`p.TryParse(source, out value)` returns a boolean and suppresses diagnostics. +Forced termination returns `false`, while other callback exceptions propagate. +Both entry points accept `ReadOnlySpan`, including strings via conversion. + +```csharp +var parser = Literal.Number().Between(S, S).ThenIgnore(Eof); +var result = parser.Parse(" 42 "); // Success: true, Value: 42 +var invalid = parser.Parse("42!"); // Success: false +``` + +A successful parse can match only a prefix. Add `ThenIgnore(Eof)` when the entire +input must match. Build parsers once and reuse them. + +## Lookahead and errors + +| Operation | Purpose | +| ----------------------- | ------------------------------------------------------------------------- | +| `And(p)` | Require `p` to match without consuming input. Return `Unit`. | +| `Not(p)` | Require `p` to fail without consuming input. Return `Unit`. | +| `p.As("name")` | Give a parser a readable name for diagnostics. | +| `Error("expected")` | Fail normally and report an expectation; alternatives may still be tried. | +| `Fail("message")` | Force parsing to terminate with an error message. | +| `FatalError("message")` | Force termination from a callback. | + +`Optional` and `DefaultOnFail` handle ordinary failures; they do not recover from +forced termination. diff --git a/docs/recursive-parsers.md b/docs/recursive-parsers.md new file mode 100644 index 0000000..f2bd5cf --- /dev/null +++ b/docs/recursive-parsers.md @@ -0,0 +1,44 @@ +# Recursive parsers and expressions + +[Parser guide](README.md) + +Describe nested structures and expressions with associativity and precedence. + +## Recursive parsers + +Use recursion for nested structures such as parentheses, arrays, or syntax trees. +`Deferred()` creates a reference whose `.Parser` is assigned once the rule is +built. `Recursive(self => ...)` defines a rule using a self-reference in a +factory function. + +```csharp +var nested = Recursive(self => + Literal.Number().Or(self.Between(L('('), L(')')))); +// "42", "(42)", and "((42))" => 42 + +var deferred = Deferred(); +deferred.Parser = Literal.Number() + .Or(deferred.Between(L('('), L(')'))); +``` + +Initialize a deferred rule before parsing or using it in operations that discard +its value, such as `Void`, `And`, or `Not`. Recursive calls must follow input +consumption: a rule that immediately calls itself is left-recursive and is not +supported. + +## Operators and expressions + +| Combinator | Purpose | +| ------------------------ | ----------------------------------------------------------------------------------------------- | +| `p.FoldL(op, reduce)` | Parse an operand followed by operator/operand pairs; combine from the left. `Fold` is an alias. | +| `p.FoldR(op, reduce)` | Combine operator/operand pairs from the right. | +| `p.Prefixed(op, reduce)` | Parse zero or more prefix operators before an operand; apply the closest operator first. | + +```csharp +var number = Literal.Number().ThenIgnore(S); +var subtraction = number.FoldL(L('-').ThenIgnore(S), (l, r, _) => l - r); +// "10 - 3 - 2" => (10 - 3) - 2 => 5 +``` + +Build precedence in layers: for example, use a product parser as the operand +parser for a sum. See the [calculator](../samples/CalcExpr/README.md). diff --git a/docs/repetition-and-lists.md b/docs/repetition-and-lists.md new file mode 100644 index 0000000..a98a2ea --- /dev/null +++ b/docs/repetition-and-lists.md @@ -0,0 +1,36 @@ +# Repetition and lists + +[Parser guide](README.md) + +Read repeated items, separated lists, optional values, and text up to a terminator. + +| Combinator | Purpose | +| ---------------------------- | ----------------------------------------------------------------------------------------- | +| `p.ZeroOrMore()`, `p.Many()` | Read zero or more items. An empty result is allowed. | +| `p.OneOrMore()` | Read at least one item. | +| `p.AtLeast(n)` | Read at least `n` items. | +| `p.Repeat(n)` | Read exactly `n` items. | +| `p.Repeat(min, max)` | Read between `min` and `max` items, inclusive. | +| `p.Separated(separator)` | Read items separated by another parser; discard separator values. | +| `p.Until(terminator)` | Read items until the terminator matches. Require the terminator, but leave it unconsumed. | +| `p.Optional()` | Accept an absent item; return `OptionalValue` with `HasValue` and `Value`. | +| `p.DefaultOnFail(value)` | On ordinary failure, succeed with a fallback value. With no argument, use `default(T)`. | + +Repetition normally returns `List`. Repeating a `Parser` with methods +such as `ZeroOrMore` or `Repeat` returns `Unit` instead. + +`Separated` allows an empty list by default. Use `min` and `max` to constrain its +size and `allowTrailing: true` to accept a final separator. + +```csharp +var item = Literal.Number().ThenIgnore(S); +var comma = L(',').ThenIgnore(S); +var list = item.Separated(comma) + .Between(L('[').ThenIgnore(S), L(']')) + .ThenIgnore(Eof); +// "[1, 2, 3]" => List { 1, 2, 3 }; "[]" => an empty list + +var comment = Any.Until(L("*/")).Text() + .Between(L("/*"), L("*/")); +// "/*note*/" => "note"; Between consumes the terminator left by Until +``` From 1e1bf75e7c6599be305b0ee72b30f172c0f5d3eb Mon Sep 17 00:00:00 2001 From: rameel Date: Sun, 8 Nov 2026 00:13:44 +0500 Subject: [PATCH 03/10] docs: add examples --- docs/README.md | 3 +++ docs/basic-parsers.md | 26 ++++++++++++++++++++- docs/combinators.md | 41 +++++++++++++++++++++++++++++---- docs/parsing-and-errors.md | 35 ++++++++++++++++++++++++++++ docs/recursive-parsers.md | 44 +++++++++++++++++++++++++----------- docs/repetition-and-lists.md | 33 ++++++++++++++++++++++++--- 6 files changed, 160 insertions(+), 22 deletions(-) diff --git a/docs/README.md b/docs/README.md index 7a1794c..93f3f68 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,10 +1,13 @@ # Parser guide +> Draft: a brief overview of the available parsers and combinators. + A `Parser` reads text and produces a value of type `T`. Small parsers match individual tokens; combinators join them into lists, expressions, and grammars. The examples use these imports: ```csharp +using System; using Ramstack.Parsing; using static Ramstack.Parsing.Parser; ``` diff --git a/docs/basic-parsers.md b/docs/basic-parsers.md index a755f1f..3dfdab0 100644 --- a/docs/basic-parsers.md +++ b/docs/basic-parsers.md @@ -26,12 +26,26 @@ Match characters, fixed text, and values such as numbers and quoted strings. groups. These return `char`; `S`, `Eol`, and `Eof` return `Unit`, representing success without a meaningful value. +`OneOf` distinguishes a set of characters from a set of strings: + +```csharp +var sign = OneOf("+-"); +var comparison = OneOf("=", "=="); + +Console.WriteLine(sign.Parse("+").Value); // + +Console.WriteLine(comparison.Parse("==").Value); // == +``` + +Combine character classes to describe an identifier: + ```csharp var identifier = Seq( Set("a-zA-Z_"), Set("a-zA-Z0-9_").ZeroOrMore() ).Text(); -// "item_42" => "item_42" + +Console.WriteLine(identifier.Parse("item_42").Value); // item_42 +Console.WriteLine(identifier.Parse("42item").Success); // False ``` ## Literals @@ -52,3 +66,13 @@ var identifier = Seq( var integer = Literal.Number(); // "42" => 42 var text = Literal.DoubleQuotedString; // "\"hello\"" => "hello" ``` + +Numeric formats and escape sequences produce values directly: + +```csharp +var hex = Literal.Number(NumberKind.HexNumber); +Console.WriteLine(hex.Parse("2A").Value); // 42 + +var quoted = Literal.DoubleQuotedString.Parse("\"A\\u0042\""); +Console.WriteLine(quoted.Value); // AB +``` diff --git a/docs/combinators.md b/docs/combinators.md index b77b5ab..420fd91 100644 --- a/docs/combinators.md +++ b/docs/combinators.md @@ -20,15 +20,30 @@ var boolean = Choice( L("true").Do(_ => true), L("false").Do(_ => false)); +// "(42)" => 42 var parenthesized = Literal .Number() .Between(L('('), L(')')); -// "(42)" => 42 ``` -Whitespace is handled explicitly: `p.ThenIgnore(S)` accepts whitespace after -`p`, while `p.Between(S, S)` accepts it on both sides. Alternatives are ordered: -`L("a").Or(L("ab"))` accepts the prefix `"a"` of `"ab"`. +Whitespace is handled explicitly: `p.ThenIgnore(S)` accepts whitespace after `p`, +while `p.Between(S, S)` accepts it on both sides. + +Alternatives are ordered: `L("a").Or(L("ab"))` accepts the prefix `"a"` of `"ab"`. + +The choice of sequence combinator determines which values survive: + +```csharp +var number = Literal.Number(); +var labeled = L("count=").Then(number); +var percent = number.ThenIgnore(L('%')); + +Console.WriteLine(labeled.Parse("count=12").Value); // 12 +Console.WriteLine(percent.Parse("12%").Value); // 12 + +Console.WriteLine( + Seq(L('a'), L('b')).Parse("ab").Value); // (a, b) +``` ## Working with results @@ -43,10 +58,26 @@ Whitespace is handled explicitly: `p.ThenIgnore(S)` accepts whitespace after | `p.Where(predicate, expected)` | Accept the parsed value only when a condition holds; otherwise report the expected description. | ```csharp +// "12:34" => (Left: 12, Right: 34) var pair = Seq( Literal.Number(), L(':'), Literal.Number() ).Do((left, _, right) => (Left: left, Right: right)); -// "12:34" => (Left: 12, Right: 34) +``` + +Use `Map` when the matched text matters, and `Where` to validate a value: + +```csharp +var digits = Set("0-9").OneOrMore().Map(match => match.Length); +Console.WriteLine(digits.Parse("1234").Value); // 4 + +var port = Literal.Number() + .Where( + value => value >= 1 && value <= 65535, + "port from 1 to 65535") + .ThenIgnore(Eof); + +Console.WriteLine(port.Parse("8080").Value); // 8080 +Console.WriteLine(port.Parse("0").Success); // False ``` diff --git a/docs/parsing-and-errors.md b/docs/parsing-and-errors.md index fdfb43e..4e2af97 100644 --- a/docs/parsing-and-errors.md +++ b/docs/parsing-and-errors.md @@ -24,6 +24,16 @@ var invalid = parser.Parse("42!"); // Success: false A successful parse can match only a prefix. Add `ThenIgnore(Eof)` when the entire input must match. Build parsers once and reuse them. +Use the boolean API when diagnostics are unnecessary: + +```csharp +var integer = Literal.Number().ThenIgnore(Eof); +if (integer.TryParse("42", out var value)) +{ + Console.WriteLine(value); // 42 +} +``` + ## Lookahead and errors | Operation | Purpose | @@ -37,3 +47,28 @@ input must match. Build parsers once and reuse them. `Optional` and `DefaultOnFail` handle ordinary failures; they do not recover from forced termination. + +Lookahead can prevent a keyword from matching the beginning of an identifier: + +```csharp +var identifierPart = Character.LetterOrDigit.Or(L('_')); +var keyword = L("if").ThenIgnore(Not(identifierPart)); + +Console.WriteLine(keyword.Parse("if(").Value); // if +Console.WriteLine(keyword.Parse("iffy").Success); // False +Console.WriteLine(keyword.Parse("if_else").Success); // False +``` + +Provide a custom error when a required rule has no match: + +```csharp +var required = Literal + .Number() + .Or(Fail("An integer is required.")); + +var result = required.Parse("oops"); + +Console.WriteLine(result.Success); // False +Console.WriteLine(result.ErrorMessage); // (1:1) An integer is required. +Console.WriteLine(required.TryParse("oops", out _)); // False +``` diff --git a/docs/recursive-parsers.md b/docs/recursive-parsers.md index f2bd5cf..cbc26b7 100644 --- a/docs/recursive-parsers.md +++ b/docs/recursive-parsers.md @@ -7,24 +7,28 @@ Describe nested structures and expressions with associativity and precedence. ## Recursive parsers Use recursion for nested structures such as parentheses, arrays, or syntax trees. -`Deferred()` creates a reference whose `.Parser` is assigned once the rule is -built. `Recursive(self => ...)` defines a rule using a self-reference in a -factory function. +`Deferred()` creates a reference whose `.Parser` is assigned once the rule is built. +`Recursive(self => ...)` defines a rule using a self-reference in a factory function. ```csharp -var nested = Recursive(self => - Literal.Number().Or(self.Between(L('('), L(')')))); // "42", "(42)", and "((42))" => 42 +var nested = Recursive(self => + Literal + .Number() + .Or(self.Between(L('('), L(')')))); + +Console.WriteLine(nested.Parse("((42))").Value); // 42 +Console.WriteLine(nested.Parse("((42)").Success); // False var deferred = Deferred(); -deferred.Parser = Literal.Number() +deferred.Parser = Literal + .Number() .Or(deferred.Between(L('('), L(')'))); ``` -Initialize a deferred rule before parsing or using it in operations that discard -its value, such as `Void`, `And`, or `Not`. Recursive calls must follow input -consumption: a rule that immediately calls itself is left-recursive and is not -supported. +Initialize a deferred rule before parsing or using it in operations that discard its value, +such as `Void`, `And`, or `Not`. Recursive calls must follow input consumption: +a rule that immediately calls itself is left-recursive and is not supported. ## Operators and expressions @@ -35,10 +39,24 @@ supported. | `p.Prefixed(op, reduce)` | Parse zero or more prefix operators before an operand; apply the closest operator first. | ```csharp +// "10 - 3 - 2" => (10 - 3) - 2 => 5 var number = Literal.Number().ThenIgnore(S); var subtraction = number.FoldL(L('-').ThenIgnore(S), (l, r, _) => l - r); -// "10 - 3 - 2" => (10 - 3) - 2 => 5 ``` -Build precedence in layers: for example, use a product parser as the operand -parser for a sum. See the [calculator](../samples/CalcExpr/README.md). +Build precedence in layers: for example, use a product parser as the operand parser for a sum. +See the [calculator](../samples/CalcExpr/README.md). + +Right-associative folding groups exponentiation from the right. +Prefix operators are applied before the resulting operand is returned: + +```csharp +var power = Literal + .Number() + .FoldR(L('^'), (left, right, _) => Math.Pow(left, right)); + +Console.WriteLine(power.Parse("2^3^2").Value); // 512: 2^(3^2) + +var unary = Literal.Number().Prefixed(L('-'), (value, _) => -value); +Console.WriteLine(unary.Parse("--5").Value); // 5 +``` diff --git a/docs/repetition-and-lists.md b/docs/repetition-and-lists.md index a98a2ea..1fb5879 100644 --- a/docs/repetition-and-lists.md +++ b/docs/repetition-and-lists.md @@ -22,15 +22,42 @@ such as `ZeroOrMore` or `Repeat` returns `Unit` instead. `Separated` allows an empty list by default. Use `min` and `max` to constrain its size and `allowTrailing: true` to accept a final separator. +## Repeated items + +Use `Text` to turn repeated character matches into a single string: + +```csharp +var code = Set("0-9").Repeat(4).Text().ThenIgnore(Eof); + +Console.WriteLine(code.Parse("1234").Value); // 1234 +Console.WriteLine(code.Parse("123").Success); // False +Console.WriteLine(code.Parse("12345").Success); // False +``` + +## Lists and terminators + ```csharp +// "[1, 2, 3]" => List { 1, 2, 3 }; "[]" => an empty list var item = Literal.Number().ThenIgnore(S); var comma = L(',').ThenIgnore(S); var list = item.Separated(comma) .Between(L('[').ThenIgnore(S), L(']')) .ThenIgnore(Eof); -// "[1, 2, 3]" => List { 1, 2, 3 }; "[]" => an empty list -var comment = Any.Until(L("*/")).Text() - .Between(L("/*"), L("*/")); // "/*note*/" => "note"; Between consumes the terminator left by Until +var comment = Any.Until(L("*/")).Text().Between(L("/*"), L("*/")); +``` + +## Optional values + +`Optional` preserves whether a value was present. `DefaultOnFail` supplies a replacement instead: + +```csharp +var marker = L('!').Optional().ThenIgnore(Eof); +Console.WriteLine(marker.Parse("!").Value.HasValue); // True +Console.WriteLine(marker.Parse("").Value.HasValue); // False + +var count = Literal.Number().DefaultOnFail(10).ThenIgnore(Eof); +Console.WriteLine(count.Parse("").Value); // 10 +Console.WriteLine(count.Parse("12").Value); // 12 ``` From 56b249d62bd62c7ad88643ca856cd86441e0a1c0 Mon Sep 17 00:00:00 2001 From: rameel Date: Wed, 11 Nov 2026 01:28:29 +0500 Subject: [PATCH 04/10] docs: add usage details --- docs/basic-parsers.md | 63 +++++++++++++++++++++++++++++++++ docs/combinators.md | 55 +++++++++++++++++++++++++++++ docs/parsing-and-errors.md | 67 +++++++++++++++++++++++++++++++++--- docs/recursive-parsers.md | 40 ++++++++++++++++++++- docs/repetition-and-lists.md | 56 ++++++++++++++++++++++++++++++ 5 files changed, 275 insertions(+), 6 deletions(-) diff --git a/docs/basic-parsers.md b/docs/basic-parsers.md index 3dfdab0..3ef836f 100644 --- a/docs/basic-parsers.md +++ b/docs/basic-parsers.md @@ -36,6 +36,16 @@ Console.WriteLine(sign.Parse("+").Value); // + Console.WriteLine(comparison.Parse("==").Value); // == ``` +`L` and the string overloads of `OneOf` return the supplied literal, +even if a case-insensitive match has different spelling in the source. +Use `Text` for the original spelling: + +```csharp +var word = L("hello", StringComparison.OrdinalIgnoreCase); +Console.WriteLine(word.Parse("HELLO").Value); // hello +Console.WriteLine(word.Text().Parse("HELLO").Value); // HELLO +``` + Combine character classes to describe an identifier: ```csharp @@ -48,6 +58,34 @@ Console.WriteLine(identifier.Parse("item_42").Value); // item_42 Console.WriteLine(identifier.Parse("42item").Success); // False ``` +### Character classes and Unicode + +Write `Set` patterns without surrounding square brackets. Use verbatim C# strings +for patterns with backslashes: + +| Pattern | Meaning | +| -------------------------- | -------------------------------------------------------------------------------- | +| `a-z`, `0-9` | An inclusive character range. | +| `abc_` | Any of the listed characters. | +| `\p{L}`, `\p{Nd}`, `\P{L}` | A Unicode letter, a decimal digit, or a character outside the letter categories. | +| `\d`, `\w`, `\s` | A Unicode decimal digit, word character, or whitespace character. | +| `\n`, `\r`, `\t`, `\u0041` | A character escape or a four-digit Unicode escape. | +| `\-`, `\\` | A literal hyphen or backslash. | + +`OneOf("a-z")` means `a`, `-`, or `z`; `Set("a-z")` means a range. +`Set` is a character-class parser: `^`, `[`, `]`, and `.` match themselves. +For example, `Set("^a")` accepts `^` or `a`. It has no regex anchors, class subtraction, +or named Unicode blocks such as `\p{IsGreek}`. + +```csharp +Console.WriteLine(Set(@"\p{L}").Parse("Ж").Value); // Ж +Console.WriteLine(Character.Digit.Parse("٣").Success); // True +Console.WriteLine(Set("0-9").Parse("٣").Success); // False +``` + +Character parsers read one UTF-16 code unit (`char`), so an emoji can require two matches. +`S` includes line breaks; use `OneOf(" \t").ZeroOrMore().Void()` when only horizontal spaces are allowed. + ## Literals `L` matches fixed text. The `Literal` class parses a value written in text form. @@ -76,3 +114,28 @@ Console.WriteLine(hex.Parse("2A").Value); // 42 var quoted = Literal.DoubleQuotedString.Parse("\"A\\u0042\""); Console.WriteLine(quoted.Value); // AB ``` + +Numbers use invariant culture: a decimal point is `.`. +Integer input permits an optional sign and ASCII digits; +floating-point input also permits a fraction and exponent, such as `1.5e2`. +fraction requires digits on both sides of the point. An incomplete suffix +such as `1e+` stays unread, so use `Eof` or a delimiter to reject it. +An integer outside the target type's range fails normally. + +`HexNumber` and `BinaryNumber` read digits without a sign or a base prefix. +Parse a prefix separately, for example `L("0x").Then(hex)`. +Binary `BigInteger` requires a target of .NET 9 or later. + +Quoted strings return decoded contents; `Text()` returns the original spelling, +including quotes and escapes: + +```csharp +var text = Literal.DoubleQuotedString; +Console.WriteLine(text.Parse("\"A\\u0042\"").Value); // AB +Console.WriteLine(text.Text().Parse("\"A\\u0042\"").Value); // "A\u0042" +Console.WriteLine(Literal.QuotedCharacter.Parse("'ab'").Success); // False +``` + +Quoted strings require a closing quote and reject unescaped CR/LF and unknown escapes. +Unicode escapes use `\u` plus exactly four hexadecimal digits; `\x` and `\U` forms are unsupported. +These are general-purpose literals: their number and string syntax does not enforce a strict JSON grammar. diff --git a/docs/combinators.md b/docs/combinators.md index 420fd91..f625bb2 100644 --- a/docs/combinators.md +++ b/docs/combinators.md @@ -30,6 +30,13 @@ Whitespace is handled explicitly: `p.ThenIgnore(S)` accepts whitespace after `p` while `p.Between(S, S)` accepts it on both sides. Alternatives are ordered: `L("a").Or(L("ab"))` accepts the prefix `"a"` of `"ab"`. +Once an alternative succeeds, a later failure outside the choice does not make +it try another alternative. Put longer literals first, or use the string +overloads of `OneOf` for the longest ordinal match. + +`Seq`, `Then`, `ThenIgnore`, and `Between` restore their starting position on +ordinary failure, so another branch can try the same input. Restoring the +position does not undo changes made by callbacks. The choice of sequence combinator determines which values survive: @@ -81,3 +88,51 @@ var port = Literal.Number() Console.WriteLine(port.Parse("8080").Value); // 8080 Console.WriteLine(port.Parse("0").Success); // False ``` + +`Do(n => n < 10)` produces a successful boolean result, even when the value is `false`. +Use `Where` to reject a value. Its predicate also runs when the parsed value is discarded. + +### Matched text and source locations + +The matched segment used by `Text` and `Map` can be smaller than the consumed input +reported by `ParseResult.Length`: + +| Parser | Matched segment | +| ------------------------------- | ----------------------------------------- | +| `Seq`, repetition, lists, folds | The whole successful sequence. | +| `a.Then(b)` | The segment from `b`. | +| `a.ThenIgnore(b)` | The segment from `a`. | +| `p.Between(open, close)` | The segment from `p`, without delimiters. | + +```csharp +var token = L("hello").ThenIgnore(S); +Console.WriteLine(token.Text().Parse("hello ").Value); // hello +Console.WriteLine(token.Parse("hello ").Length); // 8 +Console.WriteLine(Seq(L("hello"), S).Text().Parse("hello ").Value!.Length); // 8 +``` + +`Map` receives a `Match` with `Index`, `Length`, and a `ReadOnlySpan` in `Value`. +The two-argument form can attach a source location to a parsed value: + +```csharp +var located = L("id=") + .Then(Literal.Number() + .Map((match, value) => + (Value: value, Start: match.Index, Width: match.Length))); + +Console.WriteLine(located.Parse("id=042").Value); // (42, 3, 3) +``` + +`Match` is a ref struct and cannot be stored in an ordinary heap object. +Keep its offsets or copy the text with `ToString()` if the result needs to outlive the parse. + +### Callbacks and discarded values + +`Void`, `Then`, `ThenIgnore`, `Between`, lookahead, and separators discard values +that are not needed. `Text` and the one-argument `Map` also discard the inner +value. This can avoid creating intermediate lists or other result objects. + +The library may skip `Do`, `Map`, or fold callbacks when their result is discarded. +A callback that runs can belong to an attempt that later fails. Keep callbacks +free of external side effects; put validation in `Where` and perform actions +after the top-level parse succeeds. diff --git a/docs/parsing-and-errors.md b/docs/parsing-and-errors.md index 4e2af97..6073f38 100644 --- a/docs/parsing-and-errors.md +++ b/docs/parsing-and-errors.md @@ -11,6 +11,14 @@ Run a parser, inspect its result, check upcoming input, and report failures. failure. Forced termination produces an error message; other callback exceptions are also captured in `Exception`. +| Property | Meaning | +| -------------- | ---------------------------------------------------------------------------------------------------------- | +| `Success` | Whether parsing succeeded. | +| `Value` | The parsed value on success; `default(T)` on failure. | +| `Length` | Consumed UTF-16 code units, including discarded spaces and delimiters; zero on failure. | +| `ErrorMessage` | An error with a line and column, or `Parsing failed.` when no expectation was recorded; `null` on success. | +| `Exception` | The caught exception; `null` for ordinary failures and forced parse errors. | + `p.TryParse(source, out value)` returns a boolean and suppresses diagnostics. Forced termination returns `false`, while other callback exceptions propagate. Both entry points accept `ReadOnlySpan`, including strings via conversion. @@ -21,8 +29,16 @@ var result = parser.Parse(" 42 "); // Success: true, Value: 42 var invalid = parser.Parse("42!"); // Success: false ``` -A successful parse can match only a prefix. Add `ThenIgnore(Eof)` when the entire -input must match. Build parsers once and reuse them. +A successful parse can match only a prefix. Add `ThenIgnore(Eof)` when the entire input must match. +Build parsers once and reuse them. + +```csharp +var number = Literal.Number(); +var prefix = number.Parse("123abc"); +Console.WriteLine(prefix.Value); // 123 +Console.WriteLine(prefix.Length); // 3 +Console.WriteLine(number.ThenIgnore(Eof).Parse("123abc").Success); // False +``` Use the boolean API when diagnostics are unnecessary: @@ -45,8 +61,7 @@ if (integer.TryParse("42", out var value)) | `Fail("message")` | Force parsing to terminate with an error message. | | `FatalError("message")` | Force termination from a callback. | -`Optional` and `DefaultOnFail` handle ordinary failures; they do not recover from -forced termination. +`Optional` and `DefaultOnFail` handle ordinary failures; they do not recover from forced termination. Lookahead can prevent a keyword from matching the beginning of an identifier: @@ -65,10 +80,52 @@ Provide a custom error when a required rule has no match: var required = Literal .Number() .Or(Fail("An integer is required.")); - + var result = required.Parse("oops"); Console.WriteLine(result.Success); // False Console.WriteLine(result.ErrorMessage); // (1:1) An integer is required. Console.WriteLine(required.TryParse("oops", out _)); // False ``` + +### Names and error positions + +Ordinary failures report expectations at the furthest recorded position. +Line and column numbers start at 1; columns count UTF-16 code units, with a tab counting +as one column. `As` replaces inner expectations with a name for the whole rule: + +```csharp +var name = Seq( + Set("A-Za-z_"), + Set("A-Za-z0-9_").ZeroOrMore() + ).Text().As("identifier"); + +Console.WriteLine(name.Parse("123").ErrorMessage); // (1:1) Expected identifier + +var pair = Seq(L('a'), L('b')); +Console.WriteLine(pair.Parse("ax").ErrorMessage); // (1:2) Expected 'b' +``` + +Name useful grammar units; naming an entire document can hide which inner token was missing. +Lookahead records no expectations of its own; use `As` to give it a useful failure message. +A fatal error reports the current position instead. + +### Ordinary failures and exceptions + +| Situation | `Parse(source)` | `TryParse(source, out value)` | +| ------------------------------ | -------------------------------------------------- | --------------------------------------------------- | +| Ordinary mismatch | Failed result with diagnostics. | Returns `false` and sets the value to `default(T)`. | +| `Fail` or `FatalError` | Failed result with the supplied message. | Returns `false` and sets the value to `default(T)`. | +| Other exception during parsing | Failed result with `ErrorMessage` and `Exception`. | The exception propagates to the caller. | + +`Error` is an ordinary failure, so a choice can continue after it: + +```csharp +var fallback = Error("letter").Or(L('1')); +Console.WriteLine(fallback.Parse("1").Value); // 1 +Console.WriteLine(fallback.Parse("?").ErrorMessage); // (1:1) Expected letter or '1' +``` + +`ErrorMessage` includes the exception text and stack trace when `Parse` catches a callback exception. +Parser construction errors, such as invalid repetition bounds or assigning a deferred rule twice, +occur before parsing and propagate directly to the caller. diff --git a/docs/recursive-parsers.md b/docs/recursive-parsers.md index cbc26b7..8b95088 100644 --- a/docs/recursive-parsers.md +++ b/docs/recursive-parsers.md @@ -30,6 +30,18 @@ Initialize a deferred rule before parsing or using it in operations that discard such as `Void`, `And`, or `Not`. Recursive calls must follow input consumption: a rule that immediately calls itself is left-recursive and is not supported. +The same initialization rule applies when a combinator discards the placeholder +internally: `deferred.Then(other)` needs the definition first. Keeping the value, +as in `deferred.Between(open, close)`, is allowed while building its definition. +Assign `.Parser` exactly once; a second assignment throws `InvalidOperationException`. +For mutually recursive rules, create all placeholders first, assign their definitions, +then build combinations that discard their values. + +`Recursive` follows the same rules inside its factory. Use it for a single self-reference +and `Deferred` for several rules that refer to each other. For operators, use folds +to avoid left recursion. Deeply nested valid input still uses the call stack; +the library does not impose a nesting limit. + ## Operators and expressions | Combinator | Purpose | @@ -54,9 +66,35 @@ Prefix operators are applied before the resulting operand is returned: var power = Literal .Number() .FoldR(L('^'), (left, right, _) => Math.Pow(left, right)); - + Console.WriteLine(power.Parse("2^3^2").Value); // 512: 2^(3^2) var unary = Literal.Number().Prefixed(L('-'), (value, _) => -value); Console.WriteLine(unary.Parse("--5").Value); // 5 ``` + +Both folds require an initial operand. If an operator has no valid operand after it, +the fold restores the position to before that operator and returns the value built so far. +An outer `Eof` rejects the unfinished suffix: + +```csharp +var sum = Literal.Number().FoldL(L('+'), (left, right, _) => left + right); +Console.WriteLine(sum.Parse("1+").Value); // 1 +Console.WriteLine(sum.Parse("1+").Length); // 1 +Console.WriteLine(sum.ThenIgnore(Eof).Parse("1+").Success); // False +``` + +Build precedence with one fold per level: + +```csharp +var number = Literal.Number().ThenIgnore(S); +var product = number.FoldL(L('*').ThenIgnore(S), (left, right, _) => left * right); +var sum = product.FoldL(L('+').ThenIgnore(S), (left, right, _) => left + right); +var expression = sum.Between(S, Eof); + +Console.WriteLine(expression.Parse("2 + 3 * 4").Value); // 14 +``` + +Folds discard an operator/operand pair that consumes no input, without calling the reduction function. +`Prefixed` similarly discards an empty operator match. If its operand is missing, +`Prefixed` fails and restores the starting position, including any prefix operators it read. diff --git a/docs/repetition-and-lists.md b/docs/repetition-and-lists.md index 1fb5879..f293c7e 100644 --- a/docs/repetition-and-lists.md +++ b/docs/repetition-and-lists.md @@ -34,6 +34,22 @@ Console.WriteLine(code.Parse("123").Success); // False Console.WriteLine(code.Parse("12345").Success); // False ``` +Repetition is greedy: it reads as much as possible and does not give matches +back to a later parser. Use `Until` when a following marker must remain unread: + +```csharp +var greedy = Any.OneOrMore().ThenIgnore(L(".txt")); +var filename = Any.Until(L(".txt")).Text().ThenIgnore(L(".txt")); + +Console.WriteLine(greedy.Parse("notes.txt").Success); // False +Console.WriteLine(filename.Parse("notes.txt").Value); // notes +``` + +Repetition bounds must satisfy `min >= 0`, `max > 0`, and `min <= max`. +`Repeat(0)` is invalid; `AtLeast(0)` works like `ZeroOrMore`. Invalid bounds throw +when the parser is built. If the minimum is not met, repetition fails and restores +its starting position. + ## Lists and terminators ```csharp @@ -48,6 +64,25 @@ var list = item.Separated(comma) var comment = Any.Until(L("*/")).Text().Between(L("/*"), L("*/")); ``` +`Separated` counts items, not separators. With `allowTrailing: false`, a separator +without a following item is left unread. A closing delimiter or `Eof` then rejects +it. Without either check, the list can still succeed with a shorter prefix. + +```csharp +var list = Literal.Number() + .Separated(L(','), allowTrailing: true, min: 1, max: 3) + .ThenIgnore(Eof); + +Console.WriteLine(list.Parse("1,2,").Success); // True +Console.WriteLine(list.Parse("").Success); // False +Console.WriteLine(list.Parse("1,2,3,4").Success); // False +``` + +`Until` checks the terminator before each item. Missing the terminator is a failure, +even at the end of input, unless the terminator accepts that position. +For a line comment, `Any.Until(Eol).Text().ThenIgnore(Eol)` accepts both a line +break and the end of the input. The block-comment example above does not handle nested comments. + ## Optional values `Optional` preserves whether a value was present. `DefaultOnFail` supplies a replacement instead: @@ -61,3 +96,24 @@ var count = Literal.Number().DefaultOnFail(10).ThenIgnore(Eof); Console.WriteLine(count.Parse("").Value); // 10 Console.WriteLine(count.Parse("12").Value); // 12 ``` + +Check `HasValue` before reading an optional value. `DefaultOnFail` leaves invalid +text unread: the `count` parser above still rejects `"missing"` because of `Eof`. +Neither method catches forced termination or callback exceptions. + +## Empty matches in repetition + +Prefer an item parser that consumes text whenever it succeeds. +Making the whole item optional can introduce an extra empty item: + +```csharp +var items = L('x').Optional().ZeroOrMore(); +Console.WriteLine(items.Parse("xxb").Value!.Count); // 3 +Console.WriteLine(L('x').ZeroOrMore().Parse("xxb").Value!.Count); // 2 +``` + +`Repeat` stops after an empty match once its minimum is met; that match can count +as an item. `Separated` stops when an item and its following separator together +consume nothing, once the minimum is met. `Until` fails if its item consumes +nothing before the terminator matches. These checks prevent endless loops, +but an optional part usually belongs inside an item that otherwise consumes text. From dabe92131dd7e507a9a0708243fe6504a2cd434a Mon Sep 17 00:00:00 2001 From: rameel Date: Wed, 11 Nov 2026 01:57:12 +0500 Subject: [PATCH 05/10] docs: briefly document custom parser --- docs/README.md | 24 +++++++++++++ docs/custom-parsers.md | 81 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 docs/custom-parsers.md diff --git a/docs/README.md b/docs/README.md index 93f3f68..2cf2491 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,6 +12,12 @@ using Ramstack.Parsing; using static Ramstack.Parsing.Parser; ``` +Each snippet is independent unless stated otherwise. +Comments after `Console.WriteLine` show the expected output. +Small examples may parse only a prefix; complete grammars use `Eof`. +Check `Success` before using `Value` in application code; +examples read known successful results directly. + ## Contents - [Basic parsers and literals](basic-parsers.md) — Match characters, fixed text, and values such as numbers and quoted strings. @@ -19,6 +25,24 @@ using static Ramstack.Parsing.Parser; - [Repetition and lists](repetition-and-lists.md) — Read repeated items, separated lists, optional values, and text up to a terminator. - [Recursive parsers and expressions](recursive-parsers.md) — Describe nested structures and expressions with associativity and precedence. - [Running parsers, lookahead, and errors](parsing-and-errors.md) — Run a parser, inspect its result, check upcoming input, and report failures. +- [Writing your own parser](custom-parsers.md) — Implement matching logic with `Parser` and `ParseContext`. + +## Find a method by task + +| Task | Use | +| -------------------------------------------------------------- | ----------------------------------------------------- | +| Match fixed text or a character class | `L`, `OneOf`, `Set`, `Character` | +| Read a number or quoted text | `Literal` | +| Keep every value, the next value, or the first value | `Seq`, `Then`, `ThenIgnore` | +| Try alternatives or accept an absent value | `Choice`, `Or`, `Optional`, `DefaultOnFail` | +| Read repeated items, a separated list, or a terminated section | `Repeat`, `OneOrMore`, `Separated`, `Until` | +| Transform a value, capture source text, or validate | `Do`, `Map`, `Text`, `Where` | +| Describe nesting or operators | `Deferred`, `Recursive`, `FoldL`, `FoldR`, `Prefixed` | +| Check upcoming input or improve an error message | `And`, `Not`, `As` | + +Build parsers once and reuse them. Each top-level call creates its own parse context. +Keep callbacks focused on building values; do external work after a successful parse. +See [callbacks and discarded values](combinators.md#callbacks-and-discarded-values). ## Complete examples diff --git a/docs/custom-parsers.md b/docs/custom-parsers.md new file mode 100644 index 0000000..00966bc --- /dev/null +++ b/docs/custom-parsers.md @@ -0,0 +1,81 @@ +# Writing your own parser + +[Parser guide](README.md) + +Use the built-in combinators for ordinary grammars. When matching needs custom +logic, derive from `Parser` and implement `TryParse(ref ParseContext, out T)`. + +## ParseContext + +`ParseContext` holds the input and current position shared by the parsers in one parse. +The low-level `TryParse(ref context, out value)` overload does not catch exceptions; +use its boolean return value to decide whether the output is valid. + +| Member | Purpose | +| -------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| `Source`, `Remaining` | Read the whole input span or the unread suffix. | +| `Position` | Read the current zero-based offset. | +| `MatchedSegment` | Read the most recently marked `Match`. | +| `Advance(count)` | Consume characters and mark them as the matched segment. Use a count within `Remaining.Length`. | +| `SetMatched(index, length)` | Mark a source segment without changing the position. | +| `BookmarkPosition()`, `RestorePosition(bookmark)` | Save and restore the position. Restoration also clears the matched segment. | +| `ReportExpected(text)` | Record an expectation at the current position. | +| `SuppressDiagnostics()`, `RestoreDiagnosticState(state)` | Temporarily suppress diagnostics and restore their previous state. | + +Position restoration does not undo diagnostics or callback side effects. +Use the boolean return value to check success: the context can retain expectations +from an earlier branch that failed. `ParseContext` and `Match` are ref structs +and cannot be stored in ordinary heap objects. + +## A parser for two ASCII digits + +Add this class to your application: + +```csharp +public sealed class TwoDigitParser : Parser +{ + public TwoDigitParser() : base("two ASCII digits") + { + } + + public override bool TryParse(ref ParseContext context, out int value) + { + var source = context.Remaining; + if (source.Length >= 2 + && source[0] >= '0' && source[0] <= '9' + && source[1] >= '0' && source[1] <= '9') + { + value = (source[0] - '0') * 10 + source[1] - '0'; + context.Advance(2); + return true; + } + + value = default; + context.ReportExpected(Name); + + return false; + } +} +``` + +Then use it with the same combinators as a built-in parser: + +```csharp +var number = new TwoDigitParser().ThenIgnore(Eof); +Console.WriteLine(number.Parse("42").Value); // 42 +Console.WriteLine(number.Parse("4x").ErrorMessage); // (1:1) Expected two ASCII digits +Console.WriteLine(number.Parse("123").Success); // False +``` + +On success, set the value, advance the position, and mark the matched segment. +For a single segment, `Advance` does both position and match updates. + +On failure, set the value to `default`, leave the position at the starting point, +and clear the matched segment. If nothing was consumed, `Advance(0)` clears it. +If input was consumed, restore a bookmark. Record an expectation before restoring +when it belongs at the position where matching failed. + +The base class supplies wrappers for `As` and `Void`, so custom implementations +of naming and void conversion are usually unnecessary. Any optimized void +conversion must preserve the same accepted input and validation as the value +parser. From 8a04be8424dea55e3acc243c3c9b555731a53460 Mon Sep 17 00:00:00 2001 From: rameel Date: Fri, 13 Nov 2026 15:53:57 +0500 Subject: [PATCH 06/10] docs: rewrite the parser guide --- docs/README.md | 127 ++++++++++----- docs/basic-parsers.md | 291 +++++++++++++++++++++++------------ docs/combinators.md | 138 ----------------- docs/combining-parsers.md | 235 ++++++++++++++++++++++++++++ docs/common-mistakes.md | 177 +++++++++++++++++++++ docs/custom-parsers.md | 119 +++++++++----- docs/errors.md | 204 ++++++++++++++++++++++++ docs/examples.md | 144 +++++++++++++++++ docs/getting-started.md | 147 ++++++++++++++++++ docs/literals.md | 209 +++++++++++++++++++++++++ docs/parsing-and-errors.md | 131 ---------------- docs/performance-tips.md | 54 +++++++ docs/recursive-parsers.md | 238 ++++++++++++++++++++-------- docs/repetition-and-lists.md | 194 +++++++++++++---------- docs/working-with-values.md | 163 ++++++++++++++++++++ 15 files changed, 1988 insertions(+), 583 deletions(-) delete mode 100644 docs/combinators.md create mode 100644 docs/combining-parsers.md create mode 100644 docs/common-mistakes.md create mode 100644 docs/errors.md create mode 100644 docs/examples.md create mode 100644 docs/getting-started.md create mode 100644 docs/literals.md delete mode 100644 docs/parsing-and-errors.md create mode 100644 docs/performance-tips.md create mode 100644 docs/working-with-values.md diff --git a/docs/README.md b/docs/README.md index 2cf2491..89c9e87 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,53 +1,102 @@ -# Parser guide +# Ramstack.Parsing documentation -> Draft: a brief overview of the available parsers and combinators. +Ramstack.Parsing is a parser combinator library for .NET. A parser reads text and +returns a value. You build larger parsers by joining small ones: a character, +a word, a number, a list, or a whole expression. -A `Parser` reads text and produces a value of type `T`. Small parsers match -individual tokens; combinators join them into lists, expressions, and grammars. -The examples use these imports: +Start with [Getting started](getting-started.md). Then use the other pages as a +guide to the methods you need. + +| Page | What it covers | +| --- | --- | +| [Getting started](getting-started.md) | Install the library, run a parser, read results, handle spaces, and require the whole input. | +| [Basic parsers](basic-parsers.md) | `L`, `OneOf`, `Set`, Unicode categories, `Character`, `Any`, `S`, `Eol`, `Eof`, and `Regex`. | +| [Combining parsers](combining-parsers.md) | `Seq`, `Then`, `ThenIgnore`, `Between`, `Choice`, `Or`, `Optional`, `DefaultOnFail`, `Return`, `And`, `Not`, and `Where`. | +| [Repetition and lists](repetition-and-lists.md) | `ZeroOrMore`, `Many`, `OneOrMore`, `AtLeast`, `Repeat`, `Separated`, and `Until`. | +| [Working with values](working-with-values.md) | `Do`, `Map`, `Text`, `Void`, `Match`, and `Unit`. | +| [Numbers and quoted text](literals.md) | `Literal.Number`, `NumberKind`, quoted strings, quoted characters, and escape sequences. | +| [Recursive parsers and expressions](recursive-parsers.md) | `Deferred`, `Recursive`, `FoldL`, `FoldR`, `Prefixed`, and operator precedence. | +| [Errors](errors.md) | Error messages, `As`, `Error`, `Fail`, `FatalError`, and exceptions. | +| [Writing your own parser](custom-parsers.md) | `ParseContext` and a custom `Parser` class. | +| [Complete examples](examples.md) | A list of integers, a setting, a calculator, and links to the project samples. | +| [Common mistakes](common-mistakes.md) | Problems that new users often meet, and how to fix them. | +| [Performance tips](performance-tips.md) | How to keep parsers fast. | + +## Find a method by task + +| I want to... | Use | +| --- | --- | +| Read an exact character or word | `L` | +| Read one of several words | `OneOf` | +| Read one character from a range or a set | `Set`, `Character` | +| Read a number | `Literal.Number` | +| Read a quoted string | `Literal.DoubleQuotedString`, `Literal.SingleQuotedString` | +| Skip spaces | `S` | +| Require the end of the input | `Eof` | +| Run parsers one after another | `Seq`, `Then`, `ThenIgnore`, `Between` | +| Try alternatives | `Choice`, `Or` | +| Make a part optional | `Optional`, `DefaultOnFail` | +| Repeat a parser | `ZeroOrMore`, `OneOrMore`, `Repeat`, `AtLeast` | +| Read a list with separators | `Separated` | +| Read until an end marker | `Until` | +| Change the value | `Do` | +| Get the source text | `Text` | +| Check a value | `Where` | +| Look ahead without reading | `And`, `Not` | +| Read operators such as `+` or `^` | `FoldL`, `FoldR` | +| Read prefix operators such as `-` or `!` | `Prefixed` | +| Let a parser use itself | `Deferred`, `Recursive` | +| Give a parser a better error name | `As` | + +## Words used in these docs + +- **Failure**: a parser does not match the text. This is normal. A choice tries + the next alternative, and a repetition stops. +- **Fatal error**: `Fail` or `FatalError` stops the whole parse. Nothing recovers from it. +- **Callback**: a function that you pass to a method, such as the one in `Do`. +- **Matched segment**: the part of the input that a parser matched. + `Text` and `Map` read it. +- **Void parser**: a parser that throws its value away. `Unit` is the empty value + that it returns. +- **Prefix**: the text at the start of the input. + +## Reading the examples + +The examples use these imports. Add them to a console app before you copy a +snippet. Each snippet is separate unless the page says otherwise. ```csharp using System; +using System.Collections.Generic; +using System.Globalization; using Ramstack.Parsing; using static Ramstack.Parsing.Parser; ``` -Each snippet is independent unless stated otherwise. -Comments after `Console.WriteLine` show the expected output. -Small examples may parse only a prefix; complete grammars use `Eof`. -Check `Success` before using `Value` in application code; -examples read known successful results directly. - -## Contents - -- [Basic parsers and literals](basic-parsers.md) — Match characters, fixed text, and values such as numbers and quoted strings. -- [Combining parsers and transforming values](combinators.md) — Build sequences and alternatives, then transform or discard their results. -- [Repetition and lists](repetition-and-lists.md) — Read repeated items, separated lists, optional values, and text up to a terminator. -- [Recursive parsers and expressions](recursive-parsers.md) — Describe nested structures and expressions with associativity and precedence. -- [Running parsers, lookahead, and errors](parsing-and-errors.md) — Run a parser, inspect its result, check upcoming input, and report failures. -- [Writing your own parser](custom-parsers.md) — Implement matching logic with `Parser` and `ParseContext`. - -## Find a method by task +Comments after `Console.WriteLine` show the expected output. Small examples +often read only the start of the input, to show how a method works. The +complete examples use `Eof` to check the whole input. -| Task | Use | -| -------------------------------------------------------------- | ----------------------------------------------------- | -| Match fixed text or a character class | `L`, `OneOf`, `Set`, `Character` | -| Read a number or quoted text | `Literal` | -| Keep every value, the next value, or the first value | `Seq`, `Then`, `ThenIgnore` | -| Try alternatives or accept an absent value | `Choice`, `Or`, `Optional`, `DefaultOnFail` | -| Read repeated items, a separated list, or a terminated section | `Repeat`, `OneOrMore`, `Separated`, `Until` | -| Transform a value, capture source text, or validate | `Do`, `Map`, `Text`, `Where` | -| Describe nesting or operators | `Deferred`, `Recursive`, `FoldL`, `FoldR`, `Prefixed` | -| Check upcoming input or improve an error message | `And`, `Not`, `As` | +`ParseResult.Value` has a nullable type. When an example knows that the parse +succeeded, it writes `result.Value!`. -Build parsers once and reuse them. Each top-level call creates its own parse context. -Keep callbacks focused on building values; do external work after a successful parse. -See [callbacks and discarded values](combinators.md#callbacks-and-discarded-values). +The library targets .NET 6, 7, 8, and 9. It also works in later .NET versions. +`Parser.Regex` needs .NET 7 or later. `NumberKind.BinaryNumber` needs .NET 8 or +later (.NET 9 for `BigInteger`). All other examples compile with C# 10, the +default for .NET 6. -## Complete examples +## Common rules -- [Calculator](../samples/CalcExpr/README.md): arithmetic expressions and precedence. -- [JSON](../samples/Json/README.md): nested objects and arrays. -- [TinyC](../samples/TinyC/README.md): a small language grammar. +- Parsing starts at the current position. A parser does not search ahead for a match. +- Spaces are part of the input. Add `S` or your own whitespace parser where needed. +- A successful parse can leave unread text. Add `ThenIgnore(Eof)` to reject it. +- A choice uses the first alternative that succeeds. Put longer and more specific + alternatives first. +- Repetition is greedy. It reads as much as it can and never gives text back. +- Keep the functions that build values free of side effects. They may run during + an attempt that later fails, or be skipped when the value is thrown away. +- Build a parser once and reuse it. Finish setting up recursive parsers before use. -Installation is covered in the [project README](../README.md). +See [Common mistakes](common-mistakes.md) for examples of what goes wrong when you +forget these rules. See the [project README](../README.md) for a short overview +and benchmark links. diff --git a/docs/basic-parsers.md b/docs/basic-parsers.md index 3ef836f..06a22ac 100644 --- a/docs/basic-parsers.md +++ b/docs/basic-parsers.md @@ -1,141 +1,240 @@ -# Basic parsers and literals +# Basic parsers -[Parser guide](README.md) +[Documentation index](README.md) -Match characters, fixed text, and values such as numbers and quoted strings. +These parsers match text at the current position. They do not skip spaces or +require the end of the input unless you add those parsers. See the index for the +[imports used in examples](README.md#reading-the-examples). -## Basic parsers +## L: one character or one literal string -| Parser | Purpose | -| ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | -| `L('a')` | Match one specific character and return it. | -| `L("hello")` | Match a literal string. Comparison is ordinal by default; a `StringComparison` argument can change it. | -| `OneOf("abc")` | Match one character from the supplied string. | -| `OneOf("if", "else")` | Match one of several literal strings. For ordinal comparisons, prefer the longest matching literal. | -| `Set('a', 'z')` | Match a character in an inclusive range. | -| `Set("a-zA-Z0-9_")` | Match a character from a character class. | -| `L(UnicodeCategory...)`, `L(GeneralUnicodeCategory...)` | Match a character by its Unicode category. | -| `Any` | Match any single character. Fail at the end of input. | -| `S` | Consume zero or more whitespace characters, including line breaks. | -| `Eol` | Consume a line break, or succeed at the end of input. | -| `Eof` | Require the end of input without consuming anything. | -| `Regex(regex)` | Match a regular expression at the current position; its pattern must start with `^`. Available on .NET 7+. | - -`Character` provides ready-made character parsers: `Letter`, `Uppercase`, -`Lowercase`, `Digit`, `LetterOrDigit`, `HexDigit`, `WhiteSpace`, and other Unicode -groups. These return `char`; `S`, `Eol`, and `Eof` return `Unit`, representing -success without a meaningful value. - -`OneOf` distinguishes a set of characters from a set of strings: +`L(char)` reads one exact character. `L(string, comparison)` reads an exact +string. Both return the literal they matched. ```csharp -var sign = OneOf("+-"); -var comparison = OneOf("=", "=="); - -Console.WriteLine(sign.Parse("+").Value); // + -Console.WriteLine(comparison.Parse("==").Value); // == +Console.WriteLine(L(':').Parse(":rest").Value); // : +Console.WriteLine(L("hello").Parse("hello!").Value); // hello +Console.WriteLine(L("hello").Parse("Hello").Success); // False ``` -`L` and the string overloads of `OneOf` return the supplied literal, -even if a case-insensitive match has different spelling in the source. -Use `Text` for the original spelling: +String matching uses `StringComparison.Ordinal` by default. Pass a comparison +to change it: ```csharp var word = L("hello", StringComparison.OrdinalIgnoreCase); + Console.WriteLine(word.Parse("HELLO").Value); // hello Console.WriteLine(word.Text().Parse("HELLO").Value); // HELLO ``` -Combine character classes to describe an identifier: +The returned value is the literal you gave to `L`, even when the source has +different letter case. Use `Text()` to get the source spelling. + +Use ordinal comparisons (the default) for programming languages and file formats. +A culture-based comparison reads the current culture when you build the parser. +Changing the culture later does not change that parser. It can also match text +that is shorter or longer than the literal, and in rare cases it matches without +reading any text. Avoid it inside repeated parsers. + +`L("")` is not allowed. Use `Return(value)` for a parser that reads nothing. + +## OneOf: characters or words + +`OneOf("+-")` reads one character from the string. It returns a `char`. +The string is a list of characters, so `OneOf("a-z")` means `a`, `-`, or `z`. +Use `Set("a-z")` for a range. + +`OneOf("yes", "no")` or `OneOf(new[] { "yes", "no" })` reads one whole +string and returns a `string`. There are also overloads for three or four +strings, and the string overloads accept a `StringComparison`. + +```csharp +Console.WriteLine(OneOf("+-").Parse("-1").Value); // - +Console.WriteLine(OneOf("yes", "no").Parse("yes!").Value); // yes + +var comparison = OneOf("<", "<=", ">", ">="); +Console.WriteLine(comparison.Parse("<=").Value); // <= +``` + +With ordinal comparisons, `OneOf` tries longer words first. So `OneOf("<", "<=")` +reads `<=` when the input has it. The order in which you wrote the words does not +matter. `Choice` is different: it tries alternatives in the order you gave them. + +With culture-based comparisons, `OneOf` tries the words in descending sort order. +Words that compare equal are merged, and only the first one is kept. This also +applies to ignore-case comparisons: ```csharp -var identifier = Seq( - Set("a-zA-Z_"), - Set("a-zA-Z0-9_").ZeroOrMore() -).Text(); +var ab = OneOf("Ab", "ab", StringComparison.OrdinalIgnoreCase); -Console.WriteLine(identifier.Parse("item_42").Value); // item_42 -Console.WriteLine(identifier.Parse("42item").Success); // False +Console.WriteLine(ab.Parse("ab").Value); // Ab +Console.WriteLine(ab.Text().Parse("ab").Value); // ab ``` -### Character classes and Unicode +As with `L`, the result is the word you supplied. Use `Text()` to get the source +text. -Write `Set` patterns without surrounding square brackets. Use verbatim C# strings -for patterns with backslashes: +The character string, the array of words, and each word must be non-empty. -| Pattern | Meaning | -| -------------------------- | -------------------------------------------------------------------------------- | -| `a-z`, `0-9` | An inclusive character range. | -| `abc_` | Any of the listed characters. | -| `\p{L}`, `\p{Nd}`, `\P{L}` | A Unicode letter, a decimal digit, or a character outside the letter categories. | -| `\d`, `\w`, `\s` | A Unicode decimal digit, word character, or whitespace character. | -| `\n`, `\r`, `\t`, `\u0041` | A character escape or a four-digit Unicode escape. | -| `\-`, `\\` | A literal hyphen or backslash. | +## Set: a character range or class -`OneOf("a-z")` means `a`, `-`, or `z`; `Set("a-z")` means a range. -`Set` is a character-class parser: `^`, `[`, `]`, and `.` match themselves. -For example, `Set("^a")` accepts `^` or `a`. It has no regex anchors, class subtraction, -or named Unicode blocks such as `\p{IsGreek}`. +`Set(lo, hi)` reads one character in an inclusive range. `Set(pattern)` reads +one character from a class written without square brackets. ```csharp -Console.WriteLine(Set(@"\p{L}").Parse("Ж").Value); // Ж -Console.WriteLine(Character.Digit.Parse("٣").Success); // True -Console.WriteLine(Set("0-9").Parse("٣").Success); // False +Console.WriteLine(Set('a', 'z').Parse("cat").Value); // c +Console.WriteLine(Set("A-Za-z0-9_").Parse("_id").Value); // _ +Console.WriteLine(Set(@"\p{L}").Parse("é").Value); // é +Console.WriteLine(Set(@"\d").Parse("5").Value); // 5 ``` -Character parsers read one UTF-16 code unit (`char`), so an emoji can require two matches. -`S` includes line breaks; use `OneOf(" \t").ZeroOrMore().Void()` when only horizontal spaces are allowed. +| Pattern | Matches | +| --- | --- | +| `a-z`, `0-9` | An inclusive range. | +| `abc_` | Any of these literal characters. | +| `\p{L}`, `\p{Nd}` | A Unicode general category. | +| `\P{L}` | A character outside that Unicode general category. | +| `\d`, `\D` | A Unicode decimal digit, or a character outside that class. | +| `\w`, `\W` | A word character, or a character outside that class. | +| `\s`, `\S` | Whitespace, or a character outside that class. | +| `\n`, `\r`, `\t`, `\a`, `\b`, `\v`, `\f`, `\e`, `\0` | A character escape. Here `\b` means backspace and `\0` is the NUL character. | +| `\\`, `\-` | A literal backslash or hyphen. | +| `\u0041` | A character given by exactly four hexadecimal digits. | + +Use a verbatim C# string, such as `@"\d"`, or double the backslash in a normal +string, such as `"\\d"`. + +A hyphen makes a range only when it sits between two characters. At the start or +at the end of the pattern it is an ordinary hyphen, so `Set("+-")` reads `+` or +`-`. In the middle, write `\-`: + +```csharp +Console.WriteLine(Set("+-").Parse("-").Success); // True +Console.WriteLine(Set(@"a\-z").Parse("-").Success); // True +Console.WriteLine(Set(@"a\-z").Parse("m").Success); // False +``` + +`Set` is a character-class parser, not a full regular expression. These +characters have no special meaning in it: `^`, `[`, `]`, and `.`. They match +themselves. + +```csharp +Console.WriteLine(Set("^a").Parse("^").Success); // True +Console.WriteLine(Set("^a").Parse("b").Success); // False +Console.WriteLine(Set("[a-z]").Parse("[").Success); // True +``` -## Literals +So `Set("^a")` does not mean "any character except `a`". `Set` also has no class +subtraction, no anchors, and no named Unicode blocks such as `\p{IsGreek}`. A bad +pattern, such as `\q`, `\.`, or a backward range like `z-a`, throws +`ArgumentException` when you build the parser. To read a character other than a +comma, use `Not(L(',')).Then(Any)`. -`L` matches fixed text. The `Literal` class parses a value written in text form. +## Unicode categories and Character -| Parser | Purpose | -| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Literal.Number()` | Parse a numeric value, such as `int`, `double`, or `decimal`. | -| `Literal.Number(kind)` | Select a numeric format with `NumberKind`: `Auto`, `Integer`, `Float`, `HexNumber`, or `BinaryNumber` (.NET 8+). Supported formats depend on `T`. | -| `Literal.DoubleQuotedString` | Parse a double-quoted string, remove its quotes, and decode escapes. | -| `Literal.SingleQuotedString` | Parse a single-quoted string with the same value handling. | -| `Literal.QuotedCharacter` | Parse one character enclosed in single quotes. | -| `Literal.EscapeSequence` | Decode an escape sequence such as `\n` into a character. | -| `Literal.UnicodeEscapeSequence` | Decode a Unicode escape such as `\u0041`. | +`L(UnicodeCategory)` reads a character from one .NET Unicode category. +`L(GeneralUnicodeCategory)` accepts flags, including groups such as `Letter`, +`Number`, and `Punctuation`. ```csharp -var integer = Literal.Number(); // "42" => 42 -var text = Literal.DoubleQuotedString; // "\"hello\"" => "hello" +var letter = L(UnicodeCategory.LowercaseLetter); +var letterOrDigit = L( + GeneralUnicodeCategory.Letter | GeneralUnicodeCategory.DecimalDigitNumber); + +Console.WriteLine(letter.Parse("é").Value); // é +Console.WriteLine(letterOrDigit.Parse("7").Value); // 7 ``` -Numeric formats and escape sequences produce values directly: +`GeneralUnicodeCategory.None` is not a valid category for `L`. It throws +`ArgumentException`. + +The `Character` class provides ready-made `Parser` properties: + +| Property | What it reads | +| --- | --- | +| `Letter` | Any Unicode letter. | +| `Uppercase`, `Lowercase` | An uppercase letter, or a lowercase letter. | +| `Digit` | A decimal digit from any script. | +| `Number` | A character in any Unicode number category. | +| `LetterOrDigit` | A letter or a decimal digit. | +| `HexDigit` | An ASCII hexadecimal digit: `0-9`, `A-F`, or `a-f`. | +| `Symbol`, `Separator`, `Punctuation`, `Control` | A character in the matching Unicode category group. | +| `WhiteSpace` | One whitespace character. | +| `Surrogate`, `HighSurrogate`, `LowSurrogate` | One UTF-16 surrogate code unit. | ```csharp -var hex = Literal.Number(NumberKind.HexNumber); -Console.WriteLine(hex.Parse("2A").Value); // 42 +Console.WriteLine(Character.Letter.Parse("Ж").Value); // Ж +Console.WriteLine(Character.HexDigit.Parse("F").Value); // F +Console.WriteLine(Character.Digit.Parse("٣").Success); // True +Console.WriteLine(Set("0-9").Parse("٣").Success); // False +``` + +Each time you read a `Character` property, it creates a new parser. Keep it in +a variable or a field if you use it in several places. + +These parsers read one C# `char` at a time. Some characters, such as emoji, use +two `char` values. `Any` and the `Character` parsers do not join the pair. -var quoted = Literal.DoubleQuotedString.Parse("\"A\\u0042\""); -Console.WriteLine(quoted.Value); // AB +## Any, S, Eol, and Eof + +| Parser | Result | What it does | +| --- | --- | --- | +| `Any` | `char` | Reads any one UTF-16 code unit, including a line break. Fails at the end of the input. | +| `S` | `Unit` | Reads zero or more whitespace characters, using `char.IsWhiteSpace`. Always succeeds. | +| `Eol` | `Unit` | Reads a line break. At the end of the input it succeeds without reading anything. | +| `Eof` | `Unit` | Succeeds only at the end of the input, without reading anything. | + +`Unit` means that the parser has no useful value to return. + +```csharp +Console.WriteLine(Any.Parse("abc").Value); // a +Console.WriteLine(S.Parse(" \tvalue").Length); // 2 +Console.WriteLine(Eol.Parse("\r\nnext").Length); // 2 +Console.WriteLine(Eol.Parse("").Success); // True +Console.WriteLine(Eof.Parse("x").Success); // False ``` -Numbers use invariant culture: a decimal point is `.`. -Integer input permits an optional sign and ASCII digits; -floating-point input also permits a fraction and exponent, such as `1.5e2`. -fraction requires digits on both sides of the point. An incomplete suffix -such as `1e+` stays unread, so use `Eof` or a delimiter to reject it. -An integer outside the target type's range fails normally. +`Eol` reads CRLF as one line break. It also reads LF, CR, U+0085, U+2028, +and U+2029. Because it can succeed without reading anything at the end of the +input, do not use it alone in a loop that must move forward. -`HexNumber` and `BinaryNumber` read digits without a sign or a base prefix. -Parse a prefix separately, for example `L("0x").Then(hex)`. -Binary `BigInteger` requires a target of .NET 9 or later. +## Regex (.NET 7 or later) -Quoted strings return decoded contents; `Text()` returns the original spelling, -including quotes and escapes: +`Regex(System.Text.RegularExpressions.Regex)` turns a .NET regular expression +into a `Parser`. The pattern must start with the character `^`. Without it, +`Regex` throws `ArgumentException`. The match must start at the current position. ```csharp -var text = Literal.DoubleQuotedString; -Console.WriteLine(text.Parse("\"A\\u0042\"").Value); // AB -Console.WriteLine(text.Text().Parse("\"A\\u0042\"").Value); // "A\u0042" -Console.WriteLine(Literal.QuotedCharacter.Parse("'ab'").Success); // False +var code = Regex(new System.Text.RegularExpressions.Regex("^[A-Z]{2}[0-9]{3}")) + .ThenIgnore(Eof); + +Console.WriteLine(code.Parse("AB123").Value); // AB123 +Console.WriteLine(code.Parse("xAB123").Success); // False +``` + +The parser returns the matched text. It fails when no text is left, even if the +pattern could match an empty string. When some text is left, a pattern such as +`^a*` can match zero characters. Then the parser succeeds and reads nothing. Use +`Eof` when the whole input must match. + +The regular expression sees only the text that is left, not the text that was +already read. A lookbehind such as `(?<=a)b` cannot see earlier characters, and +`\b` at the start always sees a word boundary: + +```csharp +var lookbehind = L('a').Then(Regex(new System.Text.RegularExpressions.Regex("^(?<=a)b"))); +Console.WriteLine(lookbehind.Parse("ab").Success); // False +``` + +The error message uses the pattern text. Give the parser a name with `As` to +get a message that is easier to read: + +```csharp +var code = Regex(new System.Text.RegularExpressions.Regex("^[A-Z]{2}[0-9]{3}")); +Console.WriteLine(code.Parse("1").ErrorMessage); // (1:1) Expected ^[A-Z]{2}[0-9]{3} + +var named = code.As("product code"); +Console.WriteLine(named.Parse("1").ErrorMessage); // (1:1) Expected product code ``` -Quoted strings require a closing quote and reject unescaped CR/LF and unknown escapes. -Unicode escapes use `\u` plus exactly four hexadecimal digits; `\x` and `\U` forms are unsupported. -These are general-purpose literals: their number and string syntax does not enforce a strict JSON grammar. +Next: [Combining parsers](combining-parsers.md). diff --git a/docs/combinators.md b/docs/combinators.md deleted file mode 100644 index f625bb2..0000000 --- a/docs/combinators.md +++ /dev/null @@ -1,138 +0,0 @@ -# Combining parsers and transforming values - -[Parser guide](README.md) - -Build sequences and alternatives, then transform or discard their results. - -## Combining parsers - -| Combinator | Purpose | -| ------------------------------ | -------------------------------------------------------------------------------------------------------------- | -| `Seq(a, b, ...)` | Run parsers in order and return their values as a tuple. Sequences of `Unit` parsers return `Unit`. | -| `a.Then(b)` | Run both parsers; keep the value from `b`. | -| `a.ThenIgnore(b)` | Run both parsers; keep the value from `a`. | -| `p.Between(open, close)` | Parse an opening delimiter, `p`, and a closing delimiter; keep the value from `p`. | -| `p.Between(around)` | Use the same parser before and after `p`. | -| `Choice(a, b, ...)`, `a.Or(b)` | Try alternatives in order and return the first successful result. Alternatives must have the same result type. | - -```csharp -var boolean = Choice( - L("true").Do(_ => true), - L("false").Do(_ => false)); - -// "(42)" => 42 -var parenthesized = Literal - .Number() - .Between(L('('), L(')')); -``` - -Whitespace is handled explicitly: `p.ThenIgnore(S)` accepts whitespace after `p`, -while `p.Between(S, S)` accepts it on both sides. - -Alternatives are ordered: `L("a").Or(L("ab"))` accepts the prefix `"a"` of `"ab"`. -Once an alternative succeeds, a later failure outside the choice does not make -it try another alternative. Put longer literals first, or use the string -overloads of `OneOf` for the longest ordinal match. - -`Seq`, `Then`, `ThenIgnore`, and `Between` restore their starting position on -ordinary failure, so another branch can try the same input. Restoring the -position does not undo changes made by callbacks. - -The choice of sequence combinator determines which values survive: - -```csharp -var number = Literal.Number(); -var labeled = L("count=").Then(number); -var percent = number.ThenIgnore(L('%')); - -Console.WriteLine(labeled.Parse("count=12").Value); // 12 -Console.WriteLine(percent.Parse("12%").Value); // 12 - -Console.WriteLine( - Seq(L('a'), L('b')).Parse("ab").Value); // (a, b) -``` - -## Working with results - -| Combinator | Purpose | -| ------------------------------ | ----------------------------------------------------------------------------------------------- | -| `p.Do(value => ...)` | Transform the parsed value. Overloads unpack tuples produced by `Seq`. | -| `p.Map(match => ...)` | Transform the matched source segment using its text, index, and length. | -| `p.Map((match, value) => ...)` | Transform both the matched segment and the parsed value. | -| `p.Text()` | Return the matched source text as a string. | -| `p.Void()` | Discard the value and return `Unit`. | -| `Return(value)` | Succeed with a constant value without consuming input. | -| `p.Where(predicate, expected)` | Accept the parsed value only when a condition holds; otherwise report the expected description. | - -```csharp -// "12:34" => (Left: 12, Right: 34) -var pair = Seq( - Literal.Number(), - L(':'), - Literal.Number() - ).Do((left, _, right) => (Left: left, Right: right)); -``` - -Use `Map` when the matched text matters, and `Where` to validate a value: - -```csharp -var digits = Set("0-9").OneOrMore().Map(match => match.Length); -Console.WriteLine(digits.Parse("1234").Value); // 4 - -var port = Literal.Number() - .Where( - value => value >= 1 && value <= 65535, - "port from 1 to 65535") - .ThenIgnore(Eof); - -Console.WriteLine(port.Parse("8080").Value); // 8080 -Console.WriteLine(port.Parse("0").Success); // False -``` - -`Do(n => n < 10)` produces a successful boolean result, even when the value is `false`. -Use `Where` to reject a value. Its predicate also runs when the parsed value is discarded. - -### Matched text and source locations - -The matched segment used by `Text` and `Map` can be smaller than the consumed input -reported by `ParseResult.Length`: - -| Parser | Matched segment | -| ------------------------------- | ----------------------------------------- | -| `Seq`, repetition, lists, folds | The whole successful sequence. | -| `a.Then(b)` | The segment from `b`. | -| `a.ThenIgnore(b)` | The segment from `a`. | -| `p.Between(open, close)` | The segment from `p`, without delimiters. | - -```csharp -var token = L("hello").ThenIgnore(S); -Console.WriteLine(token.Text().Parse("hello ").Value); // hello -Console.WriteLine(token.Parse("hello ").Length); // 8 -Console.WriteLine(Seq(L("hello"), S).Text().Parse("hello ").Value!.Length); // 8 -``` - -`Map` receives a `Match` with `Index`, `Length`, and a `ReadOnlySpan` in `Value`. -The two-argument form can attach a source location to a parsed value: - -```csharp -var located = L("id=") - .Then(Literal.Number() - .Map((match, value) => - (Value: value, Start: match.Index, Width: match.Length))); - -Console.WriteLine(located.Parse("id=042").Value); // (42, 3, 3) -``` - -`Match` is a ref struct and cannot be stored in an ordinary heap object. -Keep its offsets or copy the text with `ToString()` if the result needs to outlive the parse. - -### Callbacks and discarded values - -`Void`, `Then`, `ThenIgnore`, `Between`, lookahead, and separators discard values -that are not needed. `Text` and the one-argument `Map` also discard the inner -value. This can avoid creating intermediate lists or other result objects. - -The library may skip `Do`, `Map`, or fold callbacks when their result is discarded. -A callback that runs can belong to an attempt that later fails. Keep callbacks -free of external side effects; put validation in `Where` and perform actions -after the top-level parse succeeds. diff --git a/docs/combining-parsers.md b/docs/combining-parsers.md new file mode 100644 index 0000000..358d3a3 --- /dev/null +++ b/docs/combining-parsers.md @@ -0,0 +1,235 @@ +# Combining parsers + +[Documentation index](README.md) + +A combinator takes existing parsers and returns a new parser. The examples +below use the [standard imports](README.md#reading-the-examples). + +## Seq: keep every result + +`Seq(p1, p2, ...)` runs parsers in order. All must succeed. It returns a tuple +with their values, in the same order. There are overloads for 2 through 10 +parsers. If all the parsers return `Unit`, the result is `Unit` too. + +```csharp +var pair = Seq( + Literal.Number(), + L(':'), + Literal.Number()); + +var result = pair.Parse("12:34"); + +if (result.Success) +{ + var (left, separator, right) = result.Value; + Console.WriteLine($"{left} {separator} {right}"); // 12 : 34 +} +``` + +Use `.Do((a, b, ...) => ...)` to turn a tuple into the value you need: + +```csharp +var pair = Seq(Literal.Number(), L(':'), Literal.Number()) + .Do((left, _, right) => left + right) + .ThenIgnore(Eof); + +Console.WriteLine(pair.Parse("12:34").Value); // 46 +``` + +If one part fails, `Seq` moves the input position back to where the sequence +began. An alternative can then try the same text. This does not undo side effects +in your callbacks. + +## Then and ThenIgnore: keep one result + +Both methods require two parsers to succeed in order: + +| Method | Value kept | +| --- | --- | +| `first.Then(second)` | The second value. | +| `first.ThenIgnore(second)` | The first value. | + +```csharp +var number = L("id=").Then(Literal.Number()).ThenIgnore(L(';')); + +Console.WriteLine(number.Parse("id=42;").Value); // 42 + +var word = L("hello").ThenIgnore(S); +Console.WriteLine(word.Parse("hello ").Value); // hello +Console.WriteLine(word.Parse("hello ").Length); // 8 +``` + +Use these methods for punctuation, prefixes, whitespace, and `Eof`. If either +part fails, they move the input position back to the start. The ignored parser +becomes a void parser, so its callbacks can be skipped. + +## Between: surround a value + +`parser.Between(before, after)` reads an opening parser, the value, and a closing +parser. Only the middle value is returned. `parser.Between(around)` uses the same +parser on both sides. + +```csharp +var bracketed = Literal.Number().Between(L('['), L(']')); +var padded = Literal.Number().Between(S); + +Console.WriteLine(bracketed.Parse("[42]").Value); // 42 +Console.WriteLine(padded.Parse(" 42 ").Value); // 42 +Console.WriteLine(bracketed.Parse("[42").Success); // False +``` + +`Between` requires both surrounding parsers. If any part fails, it moves the +input position back to the start. The surrounding values are thrown away, and +their callbacks can be skipped. + +## Choice and Or: try alternatives + +`Choice(p1, p2, ...)` tries alternatives in the given order and keeps the first +one that succeeds. `p1.Or(p2)` is the two-parser form. All alternatives must +produce the same type. A choice needs at least one alternative. + +```csharp +var color = Choice(L("red"), L("green"), L("blue")); +Console.WriteLine(color.Parse("green").Value); // green + +var answer = L("yes").Or(L("no")); +Console.WriteLine(answer.Parse("no").Value); // no + +var shortFirst = Choice(L("<"), L("<=")); +Console.WriteLine(shortFirst.Parse("<=").Value); // < +Console.WriteLine(shortFirst.ThenIgnore(Eof).Parse("<=").Success); // False +``` + +Once an alternative succeeds, a later failure outside the choice does not make +it try the other alternatives. Put `L("<=")` before `L("<")`, or use +`OneOf("<", "<=")`, which tries longer words first. + +A normal failure lets the choice try the next alternative. A fatal error (`Fail` +or `FatalError`) stops the whole parse. A choice does not recover from it. + +## Optional: keep track of whether a value was present + +`parser.Optional()` returns `Parser>`. When the parser fails +normally, `Optional` succeeds without reading anything and returns +`HasValue == false`. When the parser matches, `HasValue` is `true` and `Value` +holds the result. + +```csharp +var sign = L('-').Optional(); + +var present = sign.Parse("-42").Value; +var absent = sign.Parse("42").Value; + +Console.WriteLine(present.HasValue); // True +Console.WriteLine(present.Value); // - +Console.WriteLine(absent.HasValue); // False +``` + +Check `HasValue` before you use `Value`. Absence is different from successfully +parsing `default(T)` or `null`. If you call `Optional()` on a parser that already +returns `OptionalValue`, you get the same parser back. It does not add a second +wrapper. + +## DefaultOnFail: supply a fallback value + +`parser.DefaultOnFail(value)` returns the parsed value on success and the +fallback when the parser fails normally. The overload without an argument uses +`default(T)`. + +```csharp +var count = Literal.Number().DefaultOnFail(10); + +Console.WriteLine(count.Parse("3").Value); // 3 +Console.WriteLine(count.Parse("").Value); // 10 +Console.WriteLine(count.Parse("missing").Length); // 0 + +var zero = Literal.Number().DefaultOnFail(); +Console.WriteLine(zero.Parse("").Value); // 0 +``` + +This method does not skip invalid text. When the parser fails, the text stays +unread. For example, if you add `ThenIgnore(Eof)`, then `"missing"` fails. Use +`Optional` when you need to tell absence from an actual fallback value. + +Neither `Optional` nor `DefaultOnFail` catches fatal errors or exceptions from +your functions. + +## Return: succeed without reading text + +`Return(value)` always succeeds and returns that value without moving the +input position. It is useful as an explicit fallback in a choice. + +```csharp +var enabled = L("on").Do(_ => true).Or(Return(false)); + +Console.WriteLine(enabled.Parse("on").Value); // True +Console.WriteLine(enabled.Parse("").Value); // False +Console.WriteLine(Return(42).Parse("").Length); // 0 +``` + +A `Return` alternative hides all later alternatives, because it always succeeds. +Usually it belongs last. It also reads no text, which matters in repeated and +recursive parsers. + +## And and Not: check what comes next + +These are lookahead parsers: they test a parser and leave the input position +where it was. Both return `Unit`. They do not record error messages for the +parser that they check. + +| Method | Succeeds when | +| --- | --- | +| `And(parser)` | The supplied parser succeeds. | +| `Not(parser)` | The supplied parser fails normally. | + +```csharp +var beforeColon = Set("a-z").OneOrMore().Text().ThenIgnore(And(L(':'))); +var result = beforeColon.Parse("name:value"); + +Console.WriteLine(result.Value); // name +Console.WriteLine(result.Length); // 4 + +var keyword = L("if").ThenIgnore(Not(Set("A-Za-z0-9_"))).As("'if'"); +Console.WriteLine(keyword.Parse("if(").Success); // True +Console.WriteLine(keyword.Parse("iffy").Success); // False +``` + +`L("if")` alone also matches the start of `"iffy"`. The negative check adds a +keyword boundary. A failed lookahead records no error message of its own. Without +a name, the result is only `Parsing failed.`. Use `As` to give the parser a useful +error name: + +```csharp +var unnamed = L("if").ThenIgnore(Not(Set("A-Za-z0-9_"))); +var named = unnamed.As("'if'"); + +Console.WriteLine(unnamed.Parse("iffy").ErrorMessage); // Parsing failed. +Console.WriteLine(named.Parse("iffy").ErrorMessage); // (1:1) Expected 'if' +``` + +Lookahead turns its inner parser into a void parser, so callbacks inside it can +be skipped. A fatal error or an exception is not turned into a match or a +failure. It still stops the parse. + +## Where: check a parsed value + +`parser.Where(predicate, expected)` accepts the result only if the predicate +returns `true`. If it returns `false`, the parser moves the input position back +to the start and reports `expected` there. The description must be non-empty. + +```csharp +var port = Literal.Number() + .Where(n => n >= 1 && n <= 65535, "valid port") + .ThenIgnore(Eof); + +Console.WriteLine(port.Parse("8080").Success); // True +Console.WriteLine(port.Parse("0").Success); // False +Console.WriteLine(port.Parse("0").ErrorMessage); // (1:1) Expected valid port +``` + +Use `Where` for ranges, allowed values, and other checks on a parsed value. A +rejection is a normal failure, so `Or` can try another parser. `Where` still +runs its predicate when its value is thrown away. An exception from the +predicate is an exception, not a rejection. + +Next: [Repetition and lists](repetition-and-lists.md). diff --git a/docs/common-mistakes.md b/docs/common-mistakes.md new file mode 100644 index 0000000..8763afc --- /dev/null +++ b/docs/common-mistakes.md @@ -0,0 +1,177 @@ +# Common mistakes + +[Documentation index](README.md) + +Each section shows a mistake, what happens, and how to fix it. The examples use +the [standard imports](README.md#reading-the-examples). + +## Forgetting Eof + +A parser needs to match only the start of the input. It succeeds and ignores the +rest. + +```csharp +var number = Literal.Number(); +var wrong = number.Parse("12abc"); + +Console.WriteLine(wrong.Success); // True +Console.WriteLine(wrong.Length); // 2 + +var right = number.ThenIgnore(Eof); +Console.WriteLine(right.Parse("12abc").Success); // False +``` + +Add `ThenIgnore(Eof)` to the outermost parser. + +## Forgetting spaces + +The library never skips spaces by itself. + +```csharp +var wrong = Seq( + Literal.Number(), + L(','), + Literal.Number() + ).ThenIgnore(Eof); + +Console.WriteLine(wrong.Parse("1, 2").Success); // False + +var right = Seq( + Literal.Number().ThenIgnore(S), + L(',').ThenIgnore(S), + Literal.Number() + ).ThenIgnore(Eof); + +Console.WriteLine(right.Parse("1, 2").Success); // True +``` + +Read the spaces after each token with `ThenIgnore(S)`. See +[Handle whitespace](getting-started.md#handle-whitespace). + +## Putting a short alternative first + +A choice uses the first alternative that succeeds. It does not look for the +longest one. + +```csharp +var wrong = Choice(L("<"), L("<=")).ThenIgnore(Eof); +Console.WriteLine(wrong.Parse("<=").Success); // False + +var right = OneOf("<", "<=").ThenIgnore(Eof); +Console.WriteLine(right.Parse("<=").Success); // True +``` + +`OneOf` tries longer words first. With `Choice`, put `L("<=")` before `L("<")`. + +## Matching a keyword that is only the start of a word + +```csharp +var wrong = L("if"); +Console.WriteLine(wrong.Parse("iffy").Success); // True + +var right = L("if").ThenIgnore(Not(Set("A-Za-z0-9_"))); +Console.WriteLine(right.Parse("iffy").Success); // False +``` + +Add a check that no identifier character follows the keyword. + +## Expecting a repetition to give text back + +Repetition is greedy. It reads as much as it can, and it never gives text back. + +```csharp +var wrong = Any.OneOrMore().Text().ThenIgnore(L(".txt")); +Console.WriteLine(wrong.Parse("notes.txt").Success); // False + +var right = Any.Until(L(".txt")).Text().ThenIgnore(L(".txt")); +Console.WriteLine(right.Parse("notes.txt").Value); // notes +``` + +`Any.OneOrMore()` reads the whole input, so nothing is left for `.txt`. `Until` +stops before the terminator. + +## Putting Text in the wrong place + +`Text()` returns the matched segment of the parser it is attached to. Some +methods keep the segment of only one of their parts. + +```csharp +var wrong = Set("A-Za-z_") + .ThenIgnore(Set("A-Za-z0-9_").ZeroOrMore()); + +Console.WriteLine(wrong.Parse("user42").Value); // u + +var right = Seq( + Set("A-Za-z_"), + Set("A-Za-z0-9_").ZeroOrMore()) + .Text(); + +Console.WriteLine(right.Parse("user42").Value); // user42 +``` + +`ThenIgnore` keeps the value and the segment of its first parser. `Seq` marks the +whole sequence. See [Text: return source text](working-with-values.md#text-return-source-text). + +## Making the whole item optional + +A repetition stops when its item reads no text. The empty match still counts as +an item. + +```csharp +var wrong = L('x').Optional().ZeroOrMore(); +Console.WriteLine(wrong.Parse("xxb").Value!.Count); // 3 + +var right = L('x').ZeroOrMore(); +Console.WriteLine(right.Parse("xxb").Value!.Count); // 2 +``` + +The `wrong` parser returns two items for the two `x` characters and a third, empty +item at `b`. Use an item that reads text every time it succeeds. If only a part of +the item is optional, put `Optional` inside the item. + +## Checking a value with Do + +`Do` changes a value. It does not reject one. + +```csharp +var wrong = Literal.Number().Do(n => n < 10); + +Console.WriteLine(wrong.Parse("50").Success); // True +Console.WriteLine(wrong.Parse("50").Value); // False + +var right = Literal.Number().Where(n => n < 10, "number below 10"); +Console.WriteLine(right.Parse("50").Success); // False +``` + +Use `Where` to check a value. + +## Using Value without checking Success + +On failure, `Value` is `default(T)`. For `int` that is `0`, which looks like a +real result. + +```csharp +var result = Literal.Number().Parse("abc"); + +Console.WriteLine(result.Value); // 0 +Console.WriteLine(result.Success); // False +``` + +Always check `Success` first, or use `TryParse`. + +## Counting or logging inside a callback + +The functions that you pass to `Do`, `Map`, and folds can run during an attempt +that later fails. They can also be skipped when the value is thrown away. So a +counter or a log line inside a callback can be wrong. Build values in callbacks, +and do external work after a successful parse. See +[Void and Unit](working-with-values.md#void-and-unit-throw-a-value-away). + +## Left recursion + +A parser that calls itself before it reads any text never stops. The app crashes +with a stack overflow, and `try/catch` cannot catch it. Use `FoldL` or `FoldR` +for operators, or rewrite the grammar. See +[Avoid left recursion](recursive-parsers.md#avoid-left-recursion). + +Next: [Performance tips](performance-tips.md). diff --git a/docs/custom-parsers.md b/docs/custom-parsers.md index 00966bc..cc3599d 100644 --- a/docs/custom-parsers.md +++ b/docs/custom-parsers.md @@ -1,35 +1,62 @@ # Writing your own parser -[Parser guide](README.md) +[Documentation index](README.md) -Use the built-in combinators for ordinary grammars. When matching needs custom -logic, derive from `Parser` and implement `TryParse(ref ParseContext, out T)`. +Most grammars need only the built-in methods. This page is for the rare case when +you need your own matching logic. It also shows `ParseContext`, the object that +holds the input and the current position. The examples use the +[standard imports](README.md#reading-the-examples). -## ParseContext +## ParseContext: share a position between parsers -`ParseContext` holds the input and current position shared by the parsers in one parse. -The low-level `TryParse(ref context, out value)` overload does not catch exceptions; -use its boolean return value to decide whether the output is valid. +The low-level `TryParse(ref ParseContext, out value)` overload uses a context that +you supply. Unlike the top-level methods, it does not catch exceptions, and it +does not reset the output on every kind of failure. Use its Boolean result to +decide whether the output is valid. -| Member | Purpose | -| -------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| `Source`, `Remaining` | Read the whole input span or the unread suffix. | -| `Position` | Read the current zero-based offset. | -| `MatchedSegment` | Read the most recently marked `Match`. | -| `Advance(count)` | Consume characters and mark them as the matched segment. Use a count within `Remaining.Length`. | -| `SetMatched(index, length)` | Mark a source segment without changing the position. | -| `BookmarkPosition()`, `RestorePosition(bookmark)` | Save and restore the position. Restoration also clears the matched segment. | -| `ReportExpected(text)` | Record an expectation at the current position. | -| `SuppressDiagnostics()`, `RestoreDiagnosticState(state)` | Temporarily suppress diagnostics and restore their previous state. | - -Position restoration does not undo diagnostics or callback side effects. -Use the boolean return value to check success: the context can retain expectations -from an earlier branch that failed. `ParseContext` and `Match` are ref structs -and cannot be stored in ordinary heap objects. +```csharp +var context = new ParseContext("12:34"); +var number = Literal.Number(); -## A parser for two ASCII digits +if (number.TryParse(ref context, out var first) + && L(':').TryParse(ref context, out _) + && number.TryParse(ref context, out var second)) +{ + Console.WriteLine($"{first}, {second}"); // 12, 34 + Console.WriteLine(context.Position); // 5 +} +``` -Add this class to your application: +| Member | Use | +| --- | --- | +| `Source` | Read the whole input span. | +| `Remaining` | Read the span from the current position to the end. | +| `Position` | Read the current zero-based offset. | +| `MatchedSegment` | Read the segment that a parser marked most recently. | +| `Advance(count)` | Read `count` characters and mark them as the matched segment. Pass a count from 0 to `Remaining.Length`. Other values are not checked: the context moves to the end of the input instead of throwing. | +| `SetMatched(index, length)` | Mark the segment that `Map` and `Text` receive, without changing the position. Values outside the source are cut to fit. | +| `BookmarkPosition()` | Save a `PositionBookmark` so that you can go back. | +| `RestorePosition(bookmark)` | Go back to the saved position and make the matched segment empty. It does not undo error messages or the work of callbacks. | +| `ReportExpected(text)` / `ReportExpected(string[])` | Record what was expected at the current position, when error messages are on. | +| `DiagnosticState` | Get or set `Normal` or `Suppressed`. Error messages are recorded only in the `Normal` state. | +| `SuppressDiagnostics()` | Turn error messages off and return the previous state. | +| `SuppressDiagnosticsIfNamed(name)` | Turn error messages off when the name is not `null`, and return the previous state. | +| `RestoreDiagnosticState(state)` | Restore a saved state. | +| `ToString()` | Format the recorded expectations with a line and column. An empty string means that nothing was recorded. | + +Error messages may remain from an attempt that failed, even when a later parser +succeeds. Do not use `context.ToString()` to decide whether parsing succeeded. Use +the Boolean result from `TryParse`. + +`ParseContext` and `Match` are ref structs that hold spans. Use them only inside +the current synchronous parse. You cannot store them in normal heap objects. + +## Write a custom parser + +To write your own matching logic, derive from `Parser` and implement +`TryParse`. + +This parser reads exactly two ASCII digits and returns their integer value: ```csharp public sealed class TwoDigitParser : Parser @@ -41,6 +68,7 @@ public sealed class TwoDigitParser : Parser public override bool TryParse(ref ParseContext context, out int value) { var source = context.Remaining; + if (source.Length >= 2 && source[0] >= '0' && source[0] <= '9' && source[1] >= '0' && source[1] <= '9') @@ -52,30 +80,45 @@ public sealed class TwoDigitParser : Parser value = default; context.ReportExpected(Name); - + context.Advance(0); return false; } } ``` -Then use it with the same combinators as a built-in parser: +After you add that class to your app, use it like any other parser: ```csharp var number = new TwoDigitParser().ThenIgnore(Eof); -Console.WriteLine(number.Parse("42").Value); // 42 + +Console.WriteLine(number.Parse("42").Value); // 42 Console.WriteLine(number.Parse("4x").ErrorMessage); // (1:1) Expected two ASCII digits -Console.WriteLine(number.Parse("123").Success); // False +Console.WriteLine(number.Parse("123").Success); // False ``` -On success, set the value, advance the position, and mark the matched segment. -For a single segment, `Advance` does both position and match updates. +The rules for `TryParse`: + +- **On success**, move the context forward by the length you read, and set the + value. Mark the segment that `Map` and `Text` should see. `Advance` does both + for a single segment. +- **On failure**, leave the position where the parser started, set the output to + `default`, and return `false`. If you did not move, call `context.Advance(0)`. + This makes the matched segment empty, so a failed attempt does not leave an older + match behind. If you already moved, go back with a bookmark: `RestorePosition` + also makes the segment empty. If the expectation belongs at the failed position, + record it with `ReportExpected` before you go back. +- **Fatal error**: call `FatalError(message)` only when the parse must stop + completely. A mismatch is a failure, not a fatal error. + +## Naming and void conversion + +The base class has two methods that other methods call: `ToNamedParser(string? name)` +is used by `As`, and `ToVoidParser()` is used by `Void` and by many combinators. +The default versions wrap your parser, which is enough for most custom parsers. -On failure, set the value to `default`, leave the position at the starting point, -and clear the matched segment. If nothing was consumed, `Advance(0)` clears it. -If input was consumed, restore a bookmark. Record an expectation before restoring -when it belongs at the position where matching failed. +You can override them (as `protected override`) to make your parser faster, for +example to skip building a value that is not needed. If you do, keep the same +matching behavior and the same checks as the normal version. A void parser must +accept and reject exactly the same input. -The base class supplies wrappers for `As` and `Void`, so custom implementations -of naming and void conversion are usually unnecessary. Any optimized void -conversion must preserve the same accepted input and validation as the value -parser. +Next: [Complete examples](examples.md). diff --git a/docs/errors.md b/docs/errors.md new file mode 100644 index 0000000..42f1650 --- /dev/null +++ b/docs/errors.md @@ -0,0 +1,204 @@ +# Errors + +[Documentation index](README.md) + +There are three kinds of problems when you parse: + +- A **failure** is a normal part of parsing. A choice may try another parser, or a + repetition may stop. +- A **fatal error** stops the whole parse. You create it with `Fail` or + `FatalError`. Nothing recovers from it. +- An **exception** from your own function is a third case. + +The examples use the [standard imports](README.md#reading-the-examples). + +## How error messages look + +When `Parse` fails, it builds an error message from what the parsers expected at +the furthest position they reached. The message has the form +`(line:column) Expected ...`. Line and column numbers start at 1. A column counts +`char` values, and a tab counts as one column. If several things are expected, +the message joins them: + +```csharp +var abc = Choice(L("a"), L("b"), L("c")); +Console.WriteLine(abc.Parse("x").ErrorMessage); // (1:1) Expected 'a', 'b', or 'c' + +var ab = Seq(L("a"), L("b")); +Console.WriteLine(ab.Parse("ax").ErrorMessage); // (1:2) Expected 'b' +``` + +The message points to the furthest position that any parser reached. It is not +always the place where the whole attempt started: + +```csharp +var sum = Literal.Number().FoldL(L('+'), (a, b, _) => a + b).ThenIgnore(Eof); + +Console.WriteLine(sum.Parse("1+").ErrorMessage); // (1:3) Expected number +``` + +The fold read `1+` and then looked for a number at column 3. That message is more +useful than "expected end of input" at column 2. + +Every built-in parser has a default name for its messages: + +| Parser | Name in the message | +| --- | --- | +| `L('a')`, `L("hello")` | `'a'`, `'hello'` | +| `OneOf("yes", "no")` | `'yes'` and `'no'`, as two separate items. | +| `OneOf("+-")`, `Set("a-z")` | The sorted set: `[+-]`, `[a-z]`. A pattern such as `A-Za-z0-9_` shows as `[0-9A-Z_a-z]`. | +| `Literal.Number()` | `number` | +| `Literal.Number()` | `floating-point number` | +| `NumberKind.HexNumber`, `NumberKind.BinaryNumber` | `hexadecimal number`, `binary number` | +| `Literal.DoubleQuotedString`, `Literal.SingleQuotedString` | `double-quoted string`, `single-quoted string` | +| `Literal.QuotedCharacter` | `quoted-character` | +| `Literal.EscapeSequence`, `Literal.UnicodeEscapeSequence` | `escape sequence`, `unicode escape` | +| `Eof`, `Eol`, `Any` | `end of input`, `end of line`, `any character` | +| `Character.Letter`, `Character.Digit`, `Character.WhiteSpace` | `letter`, `digit`, `whitespace` | +| `Regex(...)` | The text of the pattern. | + +```csharp +Console.WriteLine(L("hello").Parse("x").ErrorMessage); // (1:1) Expected 'hello' +Console.WriteLine(Set("A-Za-z0-9_").Parse("!").ErrorMessage); // (1:1) Expected [0-9A-Z_a-z] +Console.WriteLine(Literal.Number().Parse("x").ErrorMessage); // (1:1) Expected floating-point number +Console.WriteLine(L("hello").ThenIgnore(Eof).Parse("hello world").ErrorMessage); // (1:6) Expected end of input +``` + +Use `As` (below) to replace a default name with a better one. + +If a parser fails and no parser recorded what it expected, `Parse` returns +`Parsing failed.`. This happens, for example, when `Not` fails, because `Not` and +`And` do not record error messages: + +```csharp +Console.WriteLine(Not(L("a")).Parse("a").ErrorMessage); // Parsing failed. +``` + +## As: give a parser a useful name + +`parser.As(name)` returns a parser with a name for error messages. The name must +be non-empty. `As` leaves the original parser unchanged, and the result type +stays the same. + +```csharp +var identifier = Seq( + Set("A-Za-z_"), + Set("A-Za-z0-9_").ZeroOrMore()) + .Text() + .As("identifier"); + +Console.WriteLine(identifier.Parse("123").ErrorMessage); // (1:1) Expected identifier +``` + +A named parser hides the error messages of the parsers inside it. When it fails, +it reports its own name. Name the useful units of your grammar, such as +`identifier` or `number`, to make errors easier to read. Naming a very large +parser can hide helpful details about the exact token that was missing. + +## Error: report a failure + +`Error(expected)` always fails normally and records what was expected. It reads +no text, and an alternative can still succeed after it. + +```csharp +var letter = Error("letter"); +Console.WriteLine(letter.Parse("1").ErrorMessage); // (1:1) Expected letter + +var fallback = Error("letter").Or(L('1')); +Console.WriteLine(fallback.Parse("1").Value); // 1 +``` + +It is useful for adding an explicit "missing" message to one branch of a +grammar. `Error` does not make a whole repetition or optional parser fail. + +## Fail and FatalError: stop parsing + +`Fail(message)` creates a parser that stops the parse when it is reached. +`FatalError(message)` does the same from inside your own function. The message must +be non-empty. A choice, an optional parser, or a repetition does not recover from a +fatal error. + +```csharp +var stopped = Fail("Stopped here.").Or(L('x')); + +Console.WriteLine(stopped.Parse("x").ErrorMessage); // (1:1) Stopped here. +Console.WriteLine(stopped.TryParse("x", out _)); // False +``` + +```csharp +var number = Literal.Number().Do(n => +{ + if (n == 0) + FatalError("Zero is not allowed."); + + return n; +}); + +Console.WriteLine(number.Parse("0").ErrorMessage); // (1:2) Zero is not allowed. +``` + +The position in a fatal error is the current position. In the second example, +the number has already been read, so the column is 2. A callback can be skipped +when its value is thrown away, so use `Where` for checks that must always run. + +## When your own code throws + +`Parse` and `TryParse` treat exceptions differently: + +| Situation | `Parse(source)` | `TryParse(source, out value)` | +| --- | --- | --- | +| Failure | Failed result with `ErrorMessage`. `Exception` is `null`. | Returns `false`. | +| `Fail` or `FatalError` | Failed result with your message and the current location. `Exception` is `null`. | Returns `false`. | +| Any other exception thrown while parsing | Failed result with `ErrorMessage` and the caught `Exception`. | The exception reaches the caller. | + +A failed top-level result has `Length == 0` and a default value. A failed +top-level `TryParse` call also sets the output to `default(T)`. + +```csharp +var parser = L('x').Do( + _ => throw new InvalidOperationException("Cannot build the value.")); + +var result = parser.Parse("x"); +Console.WriteLine(result.Success); // False +Console.WriteLine(result.Exception is InvalidOperationException); // True + +var startsLikeThis = result.ErrorMessage!.StartsWith( + "(1:2) System.InvalidOperationException: Cannot build the value."); +Console.WriteLine(startsLikeThis); // True + +try +{ + parser.TryParse("x", out _); +} +catch (InvalidOperationException) +{ + Console.WriteLine("Caught by the caller."); // Caught by the caller. +} +``` + +When `Parse` catches an exception, `ErrorMessage` contains the location, the +exception type, its message, and the stack trace (see `startsLikeThis` above). Do +not show this text to end users. Show your own message, and log `result.Exception` +instead. + +## Mistakes while building a parser + +These rules apply while a parser runs. Mistakes in how you build a parser throw an +exception right away, and the exception reaches the caller: + +| Call | Exception | +| --- | --- | +| `Repeat(0)`, `Repeat(0, 0)`, `Separated(..., max: 0)` | `ArgumentOutOfRangeException` | +| `Repeat(3, 2)` or `Separated(..., min: 3, max: 2)` | `ArgumentOutOfRangeException` | +| `Set('z', 'a')` | `ArgumentOutOfRangeException` | +| `Set("")`, `Set("z-a")`, `Set(@"\q")`, `Set(@"\p{IsGreek}")` | `ArgumentException` | +| `L("")`, `OneOf("")`, `OneOf("yes", "")`, `Choice()` with no parsers | `ArgumentException` | +| `As("")`, `Where(..., "")`, `Fail("")` | `ArgumentException` | +| `Regex(...)` with a pattern that does not start with `^` | `ArgumentException` | +| `Literal.Number` with a kind that does not fit `T`, or `BinaryNumber` for `BigInteger` in an app that targets .NET 8 | `ArgumentException` | +| `Literal.Number()`, or another unsupported type | `InvalidOperationException` | +| Assigning `Deferred.Parser` twice | `InvalidOperationException` | +| Assigning `null` to `Deferred.Parser` | `ArgumentNullException` | +| Using a `Deferred` without a parser where its value is thrown away | `InvalidOperationException` | + +Next: [Writing your own parser](custom-parsers.md). diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..10dd04d --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,144 @@ +# Complete examples + +[Documentation index](README.md) + +Each example below can be pasted into a console app with the +[standard imports](README.md#reading-the-examples). Each one checks the whole +input. Build the parser once, then reuse it for multiple inputs. + +## List of integers + +Read a bracketed list with optional spaces. Empty lists are allowed; trailing +commas are rejected. + +```csharp +var item = Literal.Number().ThenIgnore(S); +var comma = L(',').ThenIgnore(S); + +var list = item + .Separated(comma) + .Between( + L('[').ThenIgnore(S), + L(']').ThenIgnore(S)) + .Between(S, Eof); + +var result = list.Parse(" [1, -2, 3] "); +if (result.Success) +{ + Console.WriteLine(string.Join(", ", result.Value!)); // 1, -2, 3 +} + +Console.WriteLine(list.Parse("[]").Value!.Count); // 0 +Console.WriteLine(list.Parse("[1,]").Success); // False +Console.WriteLine(list.Parse("[1] extra").Success); // False +``` + +`Separated` handles items and commas. `Between` requires the brackets. +Whitespace is read after tokens and at the start. To allow a trailing comma, +set `allowTrailing: true` on `Separated`. + +## A setting with a quoted value + +Read one setting such as `name = "Ada"`. The key is an ASCII identifier and the +value uses the library's double-quoted string syntax. + +```csharp +var identifier = Seq( + Set("A-Za-z_"), + Set("A-Za-z0-9_").ZeroOrMore()) + .Text() + .As("setting name"); + +var setting = Seq( + identifier.ThenIgnore(S), + L('=').ThenIgnore(S), + Literal.DoubleQuotedString.ThenIgnore(S)) + .Do((key, _, value) => KeyValuePair.Create(key, value)) + .Between(S, Eof); + +var result = setting.Parse(" name = \"Ada\" "); +if (result.Success) +{ + Console.WriteLine(result.Value.Key); // name + Console.WriteLine(result.Value.Value); // Ada +} + +Console.WriteLine(setting.Parse("name = Ada").Success); // False +Console.WriteLine(setting.Parse("9name = \"Ada\"").Success); // False +``` + +`Seq(...).Text()` keeps the whole identifier. `Do` turns the sequence into a +key/value pair. This parser reads one setting. It does not define a whole +configuration-file format. + +## Calculator + +This example reads numbers, parentheses, unary minus, and the four basic +arithmetic operators. Multiplication and division have higher precedence than +addition and subtraction. Operators at the same level group from the left. + +```csharp +var calculator = CreateCalculator(); + +Console.WriteLine(calculator.Parse("2 + 3 * (4 - 1)").Value); // 11 +Console.WriteLine(calculator.Parse("-(2 + 3)").Value); // -5 +Console.WriteLine(calculator.Parse("- 5").Value); // -5 +Console.WriteLine(calculator.Parse("10 - 3 - 2").Value); // 5 +Console.WriteLine(calculator.Parse("1 + (").Success); // False +Console.WriteLine(calculator.Parse("2 + 3 extra").Success); // False + +static Parser CreateCalculator() +{ + var expression = Deferred(); + + var number = Literal.Number("number").ThenIgnore(S); + + var parenthesized = expression.Between( + L('(').ThenIgnore(S), + L(')').ThenIgnore(S)); + + var primary = parenthesized.Or(number); + + var unary = Seq( + L('-').Optional(), + S, + primary) + .Do((sign, _, value) => sign.HasValue ? -value : value); + + var product = unary.FoldL( + OneOf("*/").ThenIgnore(S), + (left, right, op) => op == '*' ? left * right : left / right); + + expression.Parser = product.FoldL( + OneOf("+-").ThenIgnore(S), + (left, right, op) => op == '+' ? left + right : left - right); + + return expression.Between(S, Eof); +} +``` + +The deferred parser lets parentheses refer back to the full expression. `FoldL` +builds each precedence level. `Number` reads its own sign, so `-5` is +already a number. The unary parser is still needed for `-(2 + 3)` and for `- 5` +with a space. + +This example uses `double` arithmetic. Dividing by zero gives infinity or `NaN`; +it does not throw. The example has no variables, functions, or power operator. +See [Recursive parsers](recursive-parsers.md) for `FoldR` and a power example. + +## Read the project samples + +The repository has larger working grammars: + +| Sample | Useful for | +| --- | --- | +| [Calculator source](../samples/CalcExpr/ExpressionParser.cs) and [grammar](../samples/CalcExpr/README.md) | Recursive expressions and operator precedence. | +| [JSON source](../samples/Json/JsonParser.cs) and [grammar](../samples/Json/README.md) | Recursive arrays and objects, lists, and values with different .NET types. | +| [TinyC source](../samples/TinyC/TinyCParser.cs) and [grammar](../samples/TinyC/README.md) | Statements, comments, keyword boundaries, and syntax trees. | + +The JSON sample uses the library's general number and string parsers. Treat it +as a grammar example, not as a strict JSON validator. Those parsers accept forms +that JSON does not allow, such as a leading `+` on a number and extra string +escapes. + +Next: [Common mistakes](common-mistakes.md). diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..0e06f21 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,147 @@ +# Getting started + +[Documentation index](README.md) + +## Install + +Add the package to a .NET 6 or later app: + +```shell +dotnet add package Ramstack.Parsing +``` + +## Build and run a parser + +`Parser` reads text and produces a value of type `T`. For example, +`Parser` produces an integer, and `Parser` produces a string. +A method that combines parsers returns another parser, so you can build the +grammar one step at a time. + +This complete console example reads a port number: + +```csharp +using System; +using Ramstack.Parsing; +using static Ramstack.Parsing.Parser; + +var port = Literal.Number() + .Where(n => n >= 1 && n <= 65535, "port from 1 to 65535") + .Between(S, S) + .ThenIgnore(Eof); + +var result = port.Parse(" 8080 "); + +if (result.Success) +{ + Console.WriteLine(result.Value); // 8080 +} +else +{ + Console.WriteLine(result.ErrorMessage); +} + +Console.WriteLine(port.Parse("70000").Success); // False +Console.WriteLine(port.Parse("8080abc").Success); // False +``` + +`Number` reads the number. `Where` checks its range. `Between(S, S)` +allows spaces on either side. `ThenIgnore(Eof)` checks that nothing is left. + +## Read a result + +`Parse` returns `ParseResult`: + +| Property | Meaning | +| --- | --- | +| `Success` | Whether the parser succeeded. Check this before you use `Value`. | +| `Value` | The result on success; otherwise `default(T)`. Its type is nullable, so the examples write `result.Value!` when they know the parse succeeded. A successful result can also be `null`, if your grammar returns `null`. | +| `Length` | The number of UTF-16 characters read on success, including spaces and delimiters that the parser threw away. It is zero on failure. | +| `ErrorMessage` | The error text with a line and column, or `Parsing failed.` when nothing more is known. It is `null` on success. | +| `Exception` | The exception caught during parsing, such as one thrown by a function you passed to `Do`. It is `null` for ordinary failures. | + +`result.ToString()` returns the value's text on success and the error message on +failure. Use the individual properties when your code needs to tell them apart. + +If a function that you pass throws an exception, `ErrorMessage` contains the full +exception text, including the stack trace. Do not show it to end users. See +[Errors](errors.md#when-your-own-code-throws). + +## Parse a prefix or the whole input + +By default, a parser only needs to match a prefix: the text at the start of the +input. It can succeed and leave unread text. + +```csharp +var number = Literal.Number(); +var prefix = number.Parse("123abc"); + +Console.WriteLine(prefix.Success); // True +Console.WriteLine(prefix.Value); // 123 +Console.WriteLine(prefix.Length); // 3 + +var wholeNumber = number.ThenIgnore(Eof); +Console.WriteLine(wholeNumber.Parse("123abc").Success); // False +``` + +Use prefix parsing when you read one part of a larger grammar. Use `Eof` in the +outermost parser when you read a whole value, expression, or document. + +## Handle whitespace + +The library does not skip spaces for you. `S` reads zero or more whitespace +characters, including tabs and line breaks, and returns `Unit`. + +```csharp +var strict = L("hello").ThenIgnore(Eof); +var withSpaces = L("hello").Between(S, S).ThenIgnore(Eof); + +Console.WriteLine(strict.Parse(" hello ").Success); // False +Console.WriteLine(withSpaces.Parse(" hello ").Success); // True +``` + +A common pattern is `token.ThenIgnore(S)`. It reads a token and any spaces after +it, and keeps the token's value. Use `Between(S, Eof)` for a complete grammar +whose tokens already read their trailing spaces. + +If line breaks matter, use a narrower parser such as `OneOf(" \t").ZeroOrMore()` +instead of `S`. + +## Use TryParse when you do not need an error message + +`TryParse` returns a Boolean and writes the value through an `out` parameter. +It does not build error messages, so it does less work. + +```csharp +var number = Literal.Number().ThenIgnore(Eof); + +if (number.TryParse("42", out var value)) +{ + Console.WriteLine(value); // 42 +} + +Console.WriteLine(number.TryParse("bad", out _)); // False +``` + +Both `Parse` and `TryParse` take `ReadOnlySpan`. You can pass a string +directly, or pass a slice without first making a new string: + +```csharp +var number = Literal.Number().ThenIgnore(Eof); +ReadOnlySpan source = "id=42;".AsSpan(); + +Console.WriteLine(number.Parse(source.Slice(3, 2)).Value); // 42 +``` + +`TryParse` returns `false` for ordinary failures and for fatal errors (`Fail` and +`FatalError`). Exceptions thrown by your own functions reach the caller. `Parse` +catches those exceptions and stores them in the result. See +[Errors](errors.md#when-your-own-code-throws). + +## Reuse the parser + +Build parsers outside the code that handles each input. A finished parser can +be kept in a `static readonly` field and used many times. Each call to `Parse` +creates its own position and error messages. If your functions use shared data, +make that data safe for several threads. + +Next: [Basic parsers](basic-parsers.md). diff --git a/docs/literals.md b/docs/literals.md new file mode 100644 index 0000000..57a149b --- /dev/null +++ b/docs/literals.md @@ -0,0 +1,209 @@ +# Numbers and quoted text + +[Documentation index](README.md) + +The `Literal` class reads common values and converts their text into .NET +values. It is separate from `Parser.L`, which matches a fixed character or +string. The examples use the [standard imports](README.md#reading-the-examples). + +## `Literal.Number` + +`Literal.Number(kind)` reads a number and converts it to `T`. +`Literal.Number(name, kind)` also sets the name used in error messages. +The default kind is `NumberKind.Auto`. + +```csharp +Console.WriteLine(Literal.Number().Parse("-42").Value); // -42 +Console.WriteLine(Literal.Number().Parse("1.5e2").Value); // 150 + +var quantity = Literal.Number("quantity").ThenIgnore(Eof); +Console.WriteLine(quantity.Parse("oops").ErrorMessage); // (1:1) Expected quantity +``` + +Without a name, the error message says `number` for integers, `floating-point number` +for floating-point types, `hexadecimal number` for `HexNumber`, and `binary number` +for `BinaryNumber`. + +The parser uses `CultureInfo.InvariantCulture`. Decimal input uses `.` even if +the current culture uses a comma. Whitespace is not read as part of the number. +Add `S` where the grammar allows it. + +### Supported types and kinds + +| Target type | Supported kinds | +| --- | --- | +| `sbyte`, `byte`, `short`, `ushort`, `int`, `uint`, `long`, `ulong`, `nint`, `nuint` | `Auto`, `Integer`, `HexNumber`; also `BinaryNumber` on .NET 8 or later. | +| `Int128`, `UInt128` (.NET 7 or later) | The same integer kinds. | +| `BigInteger` | `Auto`, `Integer`, `HexNumber`; also `BinaryNumber` in apps that target .NET 9 or later (see below). | +| `float`, `double`, `decimal`, `Half` | `Auto`, `Float`. | + +`BigInteger` is in `System.Numerics`. On .NET 7 or later, `T` also has an +`INumber` constraint, but the method still accepts only the types in the table. +An unsupported type, such as `char`, throws `InvalidOperationException` when you +build the parser. A kind that does not fit the type, such as `Float` for `int`, +throws `ArgumentException`. + +### Decimal integers and floating-point values + +`Integer` reads an optional `+` or `-`, followed by one or more ASCII digits. +`Float` reads the same part, followed by an optional fraction and exponent: + +```text +[+-]?[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)? +``` + +Leading zeros are allowed. A fraction needs digits on both sides of the dot. +An exponent needs at least one digit. This grammar does not accept `.5`, `NaN`, +or `Infinity`. + +Like other parsers, a number can match a prefix. An incomplete fraction or +exponent is left unread: + +```csharp +var number = Literal.Number(); + +Console.WriteLine(number.Parse("1.").Value); // 1 +Console.WriteLine(number.Parse("1e+").Length); // 1 +Console.WriteLine(number.Parse(".5").Success); // False +Console.WriteLine(number.ThenIgnore(Eof).Parse("1e+").Success); // False + +Console.WriteLine(Literal.Number().Parse("256").Success); // False +``` + +Use `Eof`, a delimiter, or a boundary check to reject an invalid suffix. +An integer outside the range of the target type fails normally. A floating-point +value follows the parsing rules of the .NET type. For example, `double` turns +`1e999` into infinity, and `decimal` fails. Use `Where` for any extra range or +finite-value checks that your format needs. + +### Hexadecimal and binary integers + +`HexNumber` reads one or more hexadecimal digits. `BinaryNumber`, available on +.NET 8 or later, reads one or more `0` or `1` digits. Neither accepts a sign or +a base prefix. Add the prefix as a separate parser: + +```csharp +var hex = Literal.Number(NumberKind.HexNumber).ThenIgnore(Eof); +var prefixedHex = L("0x").Then(hex); + +Console.WriteLine(hex.Parse("2A").Value); // 42 +Console.WriteLine(hex.Parse("0x2A").Success); // False +Console.WriteLine(prefixedHex.Parse("0x2A").Value); // 42 +``` + +```csharp +// .NET 8 or later +var binary = Literal.Number(NumberKind.BinaryNumber).ThenIgnore(Eof); +Console.WriteLine(binary.Parse("101010").Value); // 42 +Console.WriteLine(binary.Parse("0b1010").Success); // False +``` + +The conversion follows the .NET rules for hexadecimal and binary numbers. For a +signed type of fixed size, the top bit of the full width is the sign bit. For +example, `Number(HexNumber)` reads `FFFFFFFF` as -1. + +`BigInteger` has no fixed size, so the first digit decides the sign. A hexadecimal +digit from `8` to `F`, or a binary `1`, at the start makes the number negative. +Add a leading `0` to get a positive number: + +```csharp +using System.Numerics; + +var big = Literal.Number(NumberKind.HexNumber); +Console.WriteLine(big.Parse("FF").Value); // -1 +Console.WriteLine(big.Parse("0FF").Value); // 255 +``` + +```csharp +// .NET 9 or later +using System.Numerics; + +var bigBinary = Literal.Number(NumberKind.BinaryNumber); +Console.WriteLine(bigBinary.Parse("101010").Value); // -22 +Console.WriteLine(bigBinary.Parse("0101010").Value); // 42 +``` + +`BigInteger` can read binary numbers only in apps that target .NET 9 or later. +.NET 8 cannot parse binary `BigInteger` values. In an app that targets .NET 8, +`Literal.Number(NumberKind.BinaryNumber)` throws `ArgumentException` +when you build the parser. This also happens when the app runs on a newer +runtime. Use another type, or read the digits yourself, if you must support +.NET 8. + +## Quoted strings + +`Literal.DoubleQuotedString` and `Literal.SingleQuotedString` read a quoted +string and return its decoded contents. The quotes are required, but they are not +part of the returned value. Empty strings are allowed. + +```csharp +Console.WriteLine(Literal.DoubleQuotedString.Parse("\"hello\"").Value); // hello +Console.WriteLine(Literal.SingleQuotedString.Parse("'hello'").Value); // hello +Console.WriteLine(Literal.DoubleQuotedString.Parse("\"\"").Value!.Length); // 0 + +var escaped = Literal.DoubleQuotedString.Parse("\"A\\tB\""); +Console.WriteLine(escaped.Value == "A\tB"); // True + +var raw = Literal.DoubleQuotedString.Text(); +Console.WriteLine(raw.Parse("\"A\\tB\"").Value); // "A\tB" +``` + +The decoded value contains a tab. `Text()` returns the original spelling, +including the quotes and the backslash escape. The length read includes the +quotes and all source characters used for escape sequences. + +A quoted string fails when it has an unescaped CR or LF, a missing closing quote, +or an unknown escape. These are general-purpose string parsers. Their escapes and +control characters do not follow a strict JSON string grammar. + +## QuotedCharacter + +`Literal.QuotedCharacter` reads exactly one character inside single quotes, +including the supported escape forms, and returns a `char`. + +```csharp +Console.WriteLine(Literal.QuotedCharacter.Parse("'A'").Value); // A +Console.WriteLine(Literal.QuotedCharacter.Parse("'\\u0041'").Value); // A +Console.WriteLine(Literal.QuotedCharacter.Parse("'ab'").Success); // False +Console.WriteLine(Literal.QuotedCharacter.Parse("''").Success); // False +``` + +## EscapeSequence and UnicodeEscapeSequence + +These parsers read an escape without surrounding quotes: + +```csharp +Console.WriteLine(Literal.EscapeSequence.Parse(@"\n").Value == '\n'); // True +Console.WriteLine(Literal.UnicodeEscapeSequence.Parse(@"\u0041").Value); // A +Console.WriteLine(Literal.EscapeSequence.Parse(@"\q").Success); // False +``` + +`EscapeSequence` accepts these two-character forms. Quoted strings and quoted +characters use the same table: + +| Source escape | Decoded character | +| --- | --- | +| `\0` | NUL, U+0000. | +| `\a` | Bell, U+0007. | +| `\b` | Backspace, U+0008. | +| `\t` | Tab, U+0009. | +| `\n` | Line feed, U+000A. | +| `\v` | Vertical tab, U+000B. | +| `\f` | Form feed, U+000C. | +| `\r` | Carriage return, U+000D. | +| `\e` | Escape, U+001B. | +| `\"`, `\'`, `\/`, `\\` | A literal double quote, single quote, slash, or backslash. | + +`UnicodeEscapeSequence` accepts `\u` followed by exactly four hexadecimal +digits. It reads six source characters and returns one UTF-16 code unit. +It does not support `\xNN` or `\UNNNNNNNN`. It does not check surrogate pairs, +so it can return a lone surrogate as a `char`. + +To accept either escape form without quotes, use a choice: + +```csharp +var escape = Literal.EscapeSequence.Or(Literal.UnicodeEscapeSequence); +Console.WriteLine(escape.Parse(@"\u0042").Value); // B +``` + +Next: [Recursive parsers and expressions](recursive-parsers.md). diff --git a/docs/parsing-and-errors.md b/docs/parsing-and-errors.md deleted file mode 100644 index 6073f38..0000000 --- a/docs/parsing-and-errors.md +++ /dev/null @@ -1,131 +0,0 @@ -# Running parsers, lookahead, and errors - -[Parser guide](README.md) - -Run a parser, inspect its result, check upcoming input, and report failures. - -## Running a parser - -`p.Parse(source)` returns `ParseResult`: check `Success` before reading `Value`. -`Length` reports the consumed length on success; `ErrorMessage` describes a -failure. Forced termination produces an error message; other callback exceptions -are also captured in `Exception`. - -| Property | Meaning | -| -------------- | ---------------------------------------------------------------------------------------------------------- | -| `Success` | Whether parsing succeeded. | -| `Value` | The parsed value on success; `default(T)` on failure. | -| `Length` | Consumed UTF-16 code units, including discarded spaces and delimiters; zero on failure. | -| `ErrorMessage` | An error with a line and column, or `Parsing failed.` when no expectation was recorded; `null` on success. | -| `Exception` | The caught exception; `null` for ordinary failures and forced parse errors. | - -`p.TryParse(source, out value)` returns a boolean and suppresses diagnostics. -Forced termination returns `false`, while other callback exceptions propagate. -Both entry points accept `ReadOnlySpan`, including strings via conversion. - -```csharp -var parser = Literal.Number().Between(S, S).ThenIgnore(Eof); -var result = parser.Parse(" 42 "); // Success: true, Value: 42 -var invalid = parser.Parse("42!"); // Success: false -``` - -A successful parse can match only a prefix. Add `ThenIgnore(Eof)` when the entire input must match. -Build parsers once and reuse them. - -```csharp -var number = Literal.Number(); -var prefix = number.Parse("123abc"); -Console.WriteLine(prefix.Value); // 123 -Console.WriteLine(prefix.Length); // 3 -Console.WriteLine(number.ThenIgnore(Eof).Parse("123abc").Success); // False -``` - -Use the boolean API when diagnostics are unnecessary: - -```csharp -var integer = Literal.Number().ThenIgnore(Eof); -if (integer.TryParse("42", out var value)) -{ - Console.WriteLine(value); // 42 -} -``` - -## Lookahead and errors - -| Operation | Purpose | -| ----------------------- | ------------------------------------------------------------------------- | -| `And(p)` | Require `p` to match without consuming input. Return `Unit`. | -| `Not(p)` | Require `p` to fail without consuming input. Return `Unit`. | -| `p.As("name")` | Give a parser a readable name for diagnostics. | -| `Error("expected")` | Fail normally and report an expectation; alternatives may still be tried. | -| `Fail("message")` | Force parsing to terminate with an error message. | -| `FatalError("message")` | Force termination from a callback. | - -`Optional` and `DefaultOnFail` handle ordinary failures; they do not recover from forced termination. - -Lookahead can prevent a keyword from matching the beginning of an identifier: - -```csharp -var identifierPart = Character.LetterOrDigit.Or(L('_')); -var keyword = L("if").ThenIgnore(Not(identifierPart)); - -Console.WriteLine(keyword.Parse("if(").Value); // if -Console.WriteLine(keyword.Parse("iffy").Success); // False -Console.WriteLine(keyword.Parse("if_else").Success); // False -``` - -Provide a custom error when a required rule has no match: - -```csharp -var required = Literal - .Number() - .Or(Fail("An integer is required.")); - -var result = required.Parse("oops"); - -Console.WriteLine(result.Success); // False -Console.WriteLine(result.ErrorMessage); // (1:1) An integer is required. -Console.WriteLine(required.TryParse("oops", out _)); // False -``` - -### Names and error positions - -Ordinary failures report expectations at the furthest recorded position. -Line and column numbers start at 1; columns count UTF-16 code units, with a tab counting -as one column. `As` replaces inner expectations with a name for the whole rule: - -```csharp -var name = Seq( - Set("A-Za-z_"), - Set("A-Za-z0-9_").ZeroOrMore() - ).Text().As("identifier"); - -Console.WriteLine(name.Parse("123").ErrorMessage); // (1:1) Expected identifier - -var pair = Seq(L('a'), L('b')); -Console.WriteLine(pair.Parse("ax").ErrorMessage); // (1:2) Expected 'b' -``` - -Name useful grammar units; naming an entire document can hide which inner token was missing. -Lookahead records no expectations of its own; use `As` to give it a useful failure message. -A fatal error reports the current position instead. - -### Ordinary failures and exceptions - -| Situation | `Parse(source)` | `TryParse(source, out value)` | -| ------------------------------ | -------------------------------------------------- | --------------------------------------------------- | -| Ordinary mismatch | Failed result with diagnostics. | Returns `false` and sets the value to `default(T)`. | -| `Fail` or `FatalError` | Failed result with the supplied message. | Returns `false` and sets the value to `default(T)`. | -| Other exception during parsing | Failed result with `ErrorMessage` and `Exception`. | The exception propagates to the caller. | - -`Error` is an ordinary failure, so a choice can continue after it: - -```csharp -var fallback = Error("letter").Or(L('1')); -Console.WriteLine(fallback.Parse("1").Value); // 1 -Console.WriteLine(fallback.Parse("?").ErrorMessage); // (1:1) Expected letter or '1' -``` - -`ErrorMessage` includes the exception text and stack trace when `Parse` catches a callback exception. -Parser construction errors, such as invalid repetition bounds or assigning a deferred rule twice, -occur before parsing and propagate directly to the caller. diff --git a/docs/performance-tips.md b/docs/performance-tips.md new file mode 100644 index 0000000..1676f59 --- /dev/null +++ b/docs/performance-tips.md @@ -0,0 +1,54 @@ +# Performance tips + +[Documentation index](README.md) + +These tips follow from how the library works. Results depend on your grammar, +your input, and your hardware, so measure before you tune. The project +[benchmarks](https://github.com/rameel/ramstack.parsing/blob/main/benchmarks/README.md) +show how to do that. + +## Build parsers once + +Building a parser does work, for example it prepares tables for character sets. +Build it once, keep it in a `static readonly` field, and reuse it. Each call to +`Parse` or `TryParse` creates its own position, so one parser can serve many +inputs. + +The `Character` properties create a new parser each time you read them. Keep them +in a field too. + +## Use TryParse when you do not need an error message + +`TryParse` does not build error messages, so it does less work than `Parse`. +Use `Parse` when you need the message, or use `TryParse` first and run `Parse` +again only when it fails. + +## Throw values away + +A parser that only reads or checks input does not need to build a value. +`Then`, `ThenIgnore`, `Between`, `And`, `Not`, separators, and `Void()` throw +values away, and the library can skip the work of building them. Write the +punctuation of your grammar with these methods. + +## Use Text instead of a list of characters + +`Set("a-z").OneOrMore()` returns a `List`. If you want the text, add +`.Text()`. It gives you a string and does not build the list: + +```csharp +var word = Set("a-z").OneOrMore().Text(); +Console.WriteLine(word.Parse("hello world").Value); // hello +``` + +Repeating a single character set (`Set`, or `OneOf` with a string of characters) +uses a special scan that is made for this case. Prefer it to repeating a more +complex parser when you only need a run of characters. + +## Prefer ordinal comparisons + +`L` and `OneOf` compare strings with `StringComparison.Ordinal` by default. This +is the simplest and the most predictable choice. A culture-based comparison can +match text of a different length than the literal, and it depends on the culture +that was current when you built the parser. + +Next: [Documentation index](README.md). diff --git a/docs/recursive-parsers.md b/docs/recursive-parsers.md index 8b95088..3a3d243 100644 --- a/docs/recursive-parsers.md +++ b/docs/recursive-parsers.md @@ -1,100 +1,214 @@ # Recursive parsers and expressions -[Parser guide](README.md) +[Documentation index](README.md) -Describe nested structures and expressions with associativity and precedence. +A recursive parser refers to itself, directly or through other parsers. This is +useful for nested parentheses, arrays, objects, and statements. Folds handle a +common case: a series of values joined by operators. The examples use the +[standard imports](README.md#reading-the-examples). -## Recursive parsers +## Deferred: define a parser later -Use recursion for nested structures such as parentheses, arrays, or syntax trees. -`Deferred()` creates a reference whose `.Parser` is assigned once the rule is built. -`Recursive(self => ...)` defines a rule using a self-reference in a factory function. +`Deferred()` creates a placeholder. You can use it while you build other +parsers, and then assign its `Parser` property once. ```csharp -// "42", "(42)", and "((42))" => 42 -var nested = Recursive(self => - Literal - .Number() - .Or(self.Between(L('('), L(')')))); - -Console.WriteLine(nested.Parse("((42))").Value); // 42 -Console.WriteLine(nested.Parse("((42)").Success); // False - -var deferred = Deferred(); -deferred.Parser = Literal - .Number() - .Or(deferred.Between(L('('), L(')'))); +var value = Deferred(); + +value.Parser = Choice( + Literal.Number(), + value.Between(L('('), L(')'))); + +var complete = value.ThenIgnore(Eof); + +Console.WriteLine(complete.Parse("42").Value); // 42 +Console.WriteLine(complete.Parse("((42))").Value); // 42 +Console.WriteLine(complete.Parse("(42").Success); // False ``` -Initialize a deferred rule before parsing or using it in operations that discard its value, -such as `Void`, `And`, or `Not`. Recursive calls must follow input consumption: -a rule that immediately calls itself is left-recursive and is not supported. +The placeholder holds a reference to the real parser. In the parenthesized +branch, the opening parenthesis is read before the recursive parser is called +again. -The same initialization rule applies when a combinator discards the placeholder -internally: `deferred.Then(other)` needs the definition first. Keeping the value, -as in `deferred.Between(open, close)`, is allowed while building its definition. -Assign `.Parser` exactly once; a second assignment throws `InvalidOperationException`. -For mutually recursive rules, create all placeholders first, assign their definitions, -then build combinations that discard their values. +Finish the setup before you parse: + +- Assign a non-null parser exactly once. A second assignment throws + `InvalidOperationException`. +- If you parse with a placeholder that has no parser yet, the parse stops with a + fatal error. +- A placeholder with no parser yet cannot be used where its value is thrown away. + This includes `Void()`, the first parser in `Then`, and the parsers inside `And` + or `Not`. Building such a parser throws `InvalidOperationException` at once. +- A placeholder with no parser yet can be used where its value is kept. This + includes the middle parser of `Between` and a part of `Seq`. The example above + is built this way. + +```csharp +var item = Deferred(); + +var kept = item.Between(L('('), L(')')); // Works: the value is kept. + +try +{ + item.Then(L(';')); // Fails: the value would be thrown away. +} +catch (InvalidOperationException) +{ + Console.WriteLine("Assign item.Parser first."); // Assign item.Parser first. +} +``` -`Recursive` follows the same rules inside its factory. Use it for a single self-reference -and `Deferred` for several rules that refer to each other. For operators, use folds -to avoid left recursion. Deeply nested valid input still uses the call stack; -the library does not impose a nesting limit. +For parsers that call each other, create all the placeholders first. Then assign +all their definitions. Build any parser that throws a value away after that. -## Operators and expressions +## Recursive: define a self-reference with a function -| Combinator | Purpose | -| ------------------------ | ----------------------------------------------------------------------------------------------- | -| `p.FoldL(op, reduce)` | Parse an operand followed by operator/operand pairs; combine from the left. `Fold` is an alias. | -| `p.FoldR(op, reduce)` | Combine operator/operand pairs from the right. | -| `p.Prefixed(op, reduce)` | Parse zero or more prefix operators before an operand; apply the closest operator first. | +`Recursive(factory)` passes a reference to the unfinished parser into the +factory, and uses the parser that the factory returns as its definition. ```csharp -// "10 - 3 - 2" => (10 - 3) - 2 => 5 -var number = Literal.Number().ThenIgnore(S); -var subtraction = number.FoldL(L('-').ThenIgnore(S), (l, r, _) => l - r); +var value = Recursive(self => Choice( + Literal.Number(), + self.Between(L('('), L(')')))) + .ThenIgnore(Eof); + +Console.WriteLine(value.Parse("((7))").Value); // 7 ``` -Build precedence in layers: for example, use a product parser as the operand parser for a sum. -See the [calculator](../samples/CalcExpr/README.md). +Use `Recursive` for a single parser, and `Deferred` when several parsers need to +refer to each other. The factory must return a non-null parser. The rules above +for a placeholder with no parser yet also apply inside the factory. + +## Avoid left recursion + +A parser must read some text before it calls itself on the same path. A grammar +such as `expression = expression '+' number | number` calls itself before it +reads anything. It never stops: the app crashes with a stack overflow, and +`try/catch` cannot catch it. `Deferred` and `Recursive` do not prevent this. + +For binary operators, use `FoldL` or `FoldR`. For prefix operators such as unary +minus, use `Prefixed`. For other recursive parsers, rewrite +the grammar so that the recursion comes after some input is read. Very deeply +nested valid input also uses the call stack. The library sets no limit on the +depth. + +## FoldL: combine operators from the left -Right-associative folding groups exponentiation from the right. -Prefix operators are applied before the resulting operand is returned: +`operand.FoldL(operator, reduce)` reads one operand, then zero or more +operator/operand pairs. The `reduce` function receives `(left, right, operatorValue)`. ```csharp -var power = Literal - .Number() - .FoldR(L('^'), (left, right, _) => Math.Pow(left, right)); +var subtraction = Literal.Number() + .FoldL(L('-'), (left, right, _) => left - right) + .ThenIgnore(Eof); -Console.WriteLine(power.Parse("2^3^2").Value); // 512: 2^(3^2) +Console.WriteLine(subtraction.Parse("10-3-2").Value); // 5 +``` + +This groups the input as `(10 - 3) - 2`. Use it for left-associative operators +such as ordinary subtraction and division. `Fold` is an alias for `FoldL`. Prefer +`FoldL`, so that the grouping is clear. + +## FoldR: combine operators from the right + +`FoldR` reads the same kind of chain, but combines it from the right: + +```csharp +var power = Literal.Number() + .FoldR(L('^'), (left, right, _) => Math.Pow(left, right)) + .ThenIgnore(Eof); -var unary = Literal.Number().Prefixed(L('-'), (value, _) => -value); -Console.WriteLine(unary.Parse("--5").Value); // 5 +Console.WriteLine(power.Parse("2^3^2").Value); // 512 ``` -Both folds require an initial operand. If an operator has no valid operand after it, -the fold restores the position to before that operator and returns the value built so far. -An outer `Eof` rejects the unfinished suffix: +This means `2 ^ (3 ^ 2)`. Use right grouping for operators such as powers or +assignment, when your language defines them that way. + +Both folds require the first operand. If an operator is not followed by a valid +operand, the position goes back to before that operator, and the fold returns the +value built so far. Add `Eof` or another outer parser to reject the unfinished +end: ```csharp var sum = Literal.Number().FoldL(L('+'), (left, right, _) => left + right); + Console.WriteLine(sum.Parse("1+").Value); // 1 Console.WriteLine(sum.Parse("1+").Length); // 1 Console.WriteLine(sum.ThenIgnore(Eof).Parse("1+").Success); // False ``` -Build precedence with one fold per level: +If a whole operator/operand pair reads no text, the fold throws that pair away +and stops. It does not call the `reduce` function for that pair. The `reduce` +function can also be skipped when the folded value is thrown away, so keep +validation in the operand parsers. + +## Prefixed: read operators before an operand + +`operand.Prefixed(operator, apply)` reads zero or more operators, then one operand. +The `apply` function receives `(operand, operatorValue)`. It runs once for each +operator, starting with the operator closest to the operand. ```csharp -var number = Literal.Number().ThenIgnore(S); -var product = number.FoldL(L('*').ThenIgnore(S), (left, right, _) => left * right); -var sum = product.FoldL(L('+').ThenIgnore(S), (left, right, _) => left + right); -var expression = sum.Between(S, Eof); +var negation = Set('0', '9').Do(c => c - '0') + .Prefixed(L('-'), (value, _) => -value) + .ThenIgnore(Eof); +Console.WriteLine(negation.Parse("5").Value); // 5 +Console.WriteLine(negation.Parse("---5").Value); // -5 +``` + +With several kinds of operators, the closest one is applied first. In `!-x`, the +`-` is applied to `x`, and then the `!` is applied to that result: + +```csharp +var tree = Set('a', 'z').Map(m => m.ToString()) + .Prefixed(OneOf("-!"), (value, op) => $"({op}{value})"); + +Console.WriteLine(tree.Parse("!-x").Value); // (!(-x)) +``` + +`Prefixed` reads the operators in a loop, so a very long chain of operators does +not use the call stack. If the operand is missing, `Prefixed` fails and the position +goes back to where it started, including the operators that it read. If an +operator reads no text, `Prefixed` stops looking for operators and does not call +`apply` for it. Like the folds, it skips `apply` when the value is thrown away. + +Use the parser as the operand of a binary level. The binary `-` and the prefix `-` +do not clash, because each one is tried at a different place: + +```csharp +var digit = Set('0', '9').Do(c => c - '0').ThenIgnore(S); +var unary = digit.Prefixed(L('-').ThenIgnore(S), (value, _) => -value); + +var sum = unary.FoldL( + OneOf("+-").ThenIgnore(S), + (left, right, op) => op == '+' ? left + right : left - right); + +Console.WriteLine(sum.Parse("5 - - 3").Value); // 8 +``` + +## Operator precedence + +Build one parser for each precedence level. Use the tighter level as the operand +parser of the looser level: + +```csharp +var number = Literal.Number().ThenIgnore(S); + +var product = number.FoldL( + OneOf("*/").ThenIgnore(S), + (left, right, op) => op == '*' ? left * right : left / right); + +var sum = product.FoldL( + OneOf("+-").ThenIgnore(S), + (left, right, op) => op == '+' ? left + right : left - right); + +var expression = sum.Between(S, Eof); Console.WriteLine(expression.Parse("2 + 3 * 4").Value); // 14 ``` -Folds discard an operator/operand pair that consumes no input, without calling the reduction function. -`Prefixed` similarly discards an empty operator match. If its operand is missing, -`Prefixed` fails and restores the starting position, including any prefix operators it read. +Here multiplication and division are handled before addition and subtraction. +For parentheses, the primary parser needs a reference back to the whole +expression. See the [complete calculator](examples.md#calculator). + +Next: [Errors](errors.md). diff --git a/docs/repetition-and-lists.md b/docs/repetition-and-lists.md index f293c7e..27af452 100644 --- a/docs/repetition-and-lists.md +++ b/docs/repetition-and-lists.md @@ -1,119 +1,155 @@ # Repetition and lists -[Parser guide](README.md) +[Documentation index](README.md) -Read repeated items, separated lists, optional values, and text up to a terminator. +These methods run a parser more than once. They normally return `List`, with +the values in source order. See the [standard imports](README.md#reading-the-examples). -| Combinator | Purpose | -| ---------------------------- | ----------------------------------------------------------------------------------------- | -| `p.ZeroOrMore()`, `p.Many()` | Read zero or more items. An empty result is allowed. | -| `p.OneOrMore()` | Read at least one item. | -| `p.AtLeast(n)` | Read at least `n` items. | -| `p.Repeat(n)` | Read exactly `n` items. | -| `p.Repeat(min, max)` | Read between `min` and `max` items, inclusive. | -| `p.Separated(separator)` | Read items separated by another parser; discard separator values. | -| `p.Until(terminator)` | Read items until the terminator matches. Require the terminator, but leave it unconsumed. | -| `p.Optional()` | Accept an absent item; return `OptionalValue` with `HasValue` and `Value`. | -| `p.DefaultOnFail(value)` | On ordinary failure, succeed with a fallback value. With no argument, use `default(T)`. | +## ZeroOrMore, Many, and OneOrMore -Repetition normally returns `List`. Repeating a `Parser` with methods -such as `ZeroOrMore` or `Repeat` returns `Unit` instead. +`ZeroOrMore()` reads as many matches as it can, including none. `Many()` is an +alias for the same method. `OneOrMore()` requires at least one match. -`Separated` allows an empty list by default. Use `min` and `max` to constrain its -size and `allowTrailing: true` to accept a final separator. +```csharp +var digits = Set("0-9"); -## Repeated items +Console.WriteLine(digits.ZeroOrMore().Parse("").Value!.Count); // 0 +Console.WriteLine(digits.Many().Text().Parse("123x").Value); // 123 +Console.WriteLine(digits.OneOrMore().Parse("").Success); // False +Console.WriteLine(digits.OneOrMore().Text().Parse("123x").Value); // 123 +``` -Use `Text` to turn repeated character matches into a single string: +Use `.Text()` when you want the matched text instead of a list of characters. +Use `.Void()` when you only need to check or skip the matches. -```csharp -var code = Set("0-9").Repeat(4).Text().ThenIgnore(Eof); +For a `Parser`, these repetition methods have overloads that return +`Parser` directly. This avoids a list of empty values: -Console.WriteLine(code.Parse("1234").Value); // 1234 -Console.WriteLine(code.Parse("123").Success); // False -Console.WriteLine(code.Parse("12345").Success); // False +```csharp +Parser spaces = L(' ').Void().ZeroOrMore(); +Console.WriteLine(spaces.Parse(" x").Length); // 3 ``` -Repetition is greedy: it reads as much as possible and does not give matches -back to a later parser. Use `Until` when a following marker must remain unread: +## Repeat and AtLeast + +| Method | Accepted number of matches | +| --- | --- | +| `Repeat(count)` | Exactly `count`. | +| `Repeat(min, max)` | From `min` to `max`, inclusive. | +| `AtLeast(count)` | At least `count`, with no practical upper limit. | ```csharp -var greedy = Any.OneOrMore().ThenIgnore(L(".txt")); -var filename = Any.Until(L(".txt")).Text().ThenIgnore(L(".txt")); +var threeDigits = Set("0-9").Repeat(3).Text().ThenIgnore(Eof); +Console.WriteLine(threeDigits.Parse("123").Value); // 123 +Console.WriteLine(threeDigits.Parse("12").Success); // False +Console.WriteLine(threeDigits.Parse("1234").Success); // False -Console.WriteLine(greedy.Parse("notes.txt").Success); // False -Console.WriteLine(filename.Parse("notes.txt").Value); // notes +var shortWord = Set("a-z").Repeat(2, 4).Text(); +Console.WriteLine(shortWord.Parse("abcdef").Value); // abcd + +var atLeastTwo = L('x').AtLeast(2).Text(); +Console.WriteLine(atLeastTwo.Parse("xxxx!").Value); // xxxx ``` -Repetition bounds must satisfy `min >= 0`, `max > 0`, and `min <= max`. -`Repeat(0)` is invalid; `AtLeast(0)` works like `ZeroOrMore`. Invalid bounds throw -when the parser is built. If the minimum is not met, repetition fails and restores -its starting position. +The same methods have `Parser` overloads. The counts must obey `min >= 0`, +`max > 0`, and `min <= max`. `Repeat(0)` and `Repeat(0, 0)` are not allowed. +`AtLeast(0)` is allowed and works like `ZeroOrMore()`. A wrong count throws +`ArgumentOutOfRangeException` when you build the parser. + +Repetition is greedy: it reads until the parser fails or the maximum is reached. +It never gives matches back to help a later parser succeed. For example, +`Set("a-z").OneOrMore().Then(L('z'))` cannot match `"abz"`: the repeated parser +has already read the `z`. + +If fewer than the minimum number of items match, the repetition fails and moves +the input position back to the start. -## Lists and terminators +## Separated: lists with delimiters + +`item.Separated(separator, allowTrailing: false, min: 0, max: int.MaxValue)` +reads items with a separator between them. It returns the items and throws the +separators away. By default, an empty list is allowed and a trailing separator is +left unread. ```csharp -// "[1, 2, 3]" => List { 1, 2, 3 }; "[]" => an empty list var item = Literal.Number().ThenIgnore(S); var comma = L(',').ThenIgnore(S); -var list = item.Separated(comma) - .Between(L('[').ThenIgnore(S), L(']')) - .ThenIgnore(Eof); -// "/*note*/" => "note"; Between consumes the terminator left by Until -var comment = Any.Until(L("*/")).Text().Between(L("/*"), L("*/")); -``` +var list = item.Separated(comma).Between(S, Eof); +var result = list.Parse("1, 2, 3"); -`Separated` counts items, not separators. With `allowTrailing: false`, a separator -without a following item is left unread. A closing delimiter or `Eof` then rejects -it. Without either check, the list can still succeed with a shorter prefix. +Console.WriteLine(string.Join(", ", result.Value!)); // 1, 2, 3 +Console.WriteLine(list.Parse("").Value!.Count); // 0 +Console.WriteLine(list.Parse("1,").Success); // False -```csharp -var list = Literal.Number() - .Separated(L(','), allowTrailing: true, min: 1, max: 3) - .ThenIgnore(Eof); +var withTrailingComma = item + .Separated(comma, allowTrailing: true, min: 1, max: 3) + .Between(S, Eof); -Console.WriteLine(list.Parse("1,2,").Success); // True -Console.WriteLine(list.Parse("").Success); // False -Console.WriteLine(list.Parse("1,2,3,4").Success); // False +Console.WriteLine(withTrailingComma.Parse("1, 2,").Success); // True +Console.WriteLine(withTrailingComma.Parse("").Success); // False +Console.WriteLine(withTrailingComma.Parse("1,2,3,4").Success); // False ``` -`Until` checks the terminator before each item. Missing the terminator is a failure, -even at the end of input, unless the terminator accepts that position. -For a line comment, `Any.Until(Eol).Text().ThenIgnore(Eol)` accepts both a line -break and the end of the input. The block-comment example above does not handle nested comments. +`min` and `max` count items, not separators. The limits follow the same rules +as `Repeat`. Suppose a separator was read but no next item matches. With +`allowTrailing: false`, the position goes back to before that separator. With +`allowTrailing: true`, the separator stays read, even after the last allowed item. +Without `Eof` or a closing delimiter, a list can succeed and leave more input. + +To read a bracketed list, add `Between` with opening and closing parsers. See the +[complete integer list example](examples.md#list-of-integers). -## Optional values +## Until: stop before a terminator -`Optional` preserves whether a value was present. `DefaultOnFail` supplies a replacement instead: +`item.Until(terminator)` checks the terminator before each item. It succeeds as +soon as the terminator matches, and returns a list of items. The terminator is +required, but it is not read. Its value is thrown away. ```csharp -var marker = L('!').Optional().ThenIgnore(Eof); -Console.WriteLine(marker.Parse("!").Value.HasValue); // True -Console.WriteLine(marker.Parse("").Value.HasValue); // False +var body = Any.Until(L("*/")).Text(); +var prefix = body.Parse("hello*/tail"); -var count = Literal.Number().DefaultOnFail(10).ThenIgnore(Eof); -Console.WriteLine(count.Parse("").Value); // 10 -Console.WriteLine(count.Parse("12").Value); // 12 -``` +Console.WriteLine(prefix.Value); // hello +Console.WriteLine(prefix.Length); // 5 -Check `HasValue` before reading an optional value. `DefaultOnFail` leaves invalid -text unread: the `count` parser above still rejects `"missing"` because of `Eof`. -Neither method catches forced termination or callback exceptions. +var comment = body.Between(L("/*"), L("*/")).ThenIgnore(Eof); +Console.WriteLine(comment.Parse("/*hello*/").Value); // hello +Console.WriteLine(comment.Parse("/**/").Value!.Length); // 0 +Console.WriteLine(comment.Parse("/*hello").Success); // False +``` -## Empty matches in repetition +Read the terminator separately with `ThenIgnore`, `Seq`, or `Between`. Reaching +the end of the input is a failure, unless the terminator itself accepts it. When +`Until` fails, it moves the input position back to the start. This simple +comment parser does not support nested comments. -Prefer an item parser that consumes text whenever it succeeds. -Making the whole item optional can introduce an extra empty item: +For a line comment, `Eol` also accepts the end of the input: ```csharp -var items = L('x').Optional().ZeroOrMore(); -Console.WriteLine(items.Parse("xxb").Value!.Count); // 3 -Console.WriteLine(L('x').ZeroOrMore().Parse("xxb").Value!.Count); // 2 +var comment = L("//") + .Then(Any.Until(Eol).Text()) + .ThenIgnore(Eol); + +Console.WriteLine(comment.Parse("// hello\nnext").Value); // hello +Console.WriteLine(comment.Parse("// last line").Value); // last line ``` -`Repeat` stops after an empty match once its minimum is met; that match can count -as an item. `Separated` stops when an item and its following separator together -consume nothing, once the minimum is met. `Until` fails if its item consumes -nothing before the terminator matches. These checks prevent endless loops, -but an optional part usually belongs inside an item that otherwise consumes text. +## Parsers that can match no text + +`S`, `Eof`, `Return`, lookahead, and optional parsers can succeed without moving +the position. The repetition methods protect against endless loops, but the +results can be surprising: + +- `Repeat` stops after a match that reads no text, once the minimum is met. That + empty match can count as an item. +- `Separated` stops once its minimum is met, if an item and the separator after it + together read no text. That item is included in the result. +- `Until` fails if its item reads no text before the terminator has matched. +- A fold throws away an operator/operand pair that reads no text, and stops. + +Prefer an item parser that reads some text every time it succeeds. If part of the +item is optional, put `Optional` inside the item. Do not make the whole item +optional. See [Common mistakes](common-mistakes.md#making-the-whole-item-optional). + +Next: [Working with values](working-with-values.md). diff --git a/docs/working-with-values.md b/docs/working-with-values.md new file mode 100644 index 0000000..41e9506 --- /dev/null +++ b/docs/working-with-values.md @@ -0,0 +1,163 @@ +# Working with values + +[Documentation index](README.md) + +A parser can return a value, keep the source text, or only check that the input +matches. These methods let you choose. The examples use the +[standard imports](README.md#reading-the-examples). + +| Method | Callback receives | Result | +| --- | --- | --- | +| `Do(value => ...)` | The parsed value. | The callback's return value. | +| `Map(match => ...)` | The matched segment. | The callback's return value. The original parsed value is thrown away. | +| `Map((match, value) => ...)` | Both the matched segment and the parsed value. | The callback's return value. | +| `Text()` | No callback needed. | The matched segment as a string. | +| `Void()` | No callback needed. | `Unit`. The value is thrown away. | + +The *matched segment* is the part of the input that a parser matched. A callback +is a function that you pass to a method. + +## Do: change the parsed value + +`Do` changes a value after the parser succeeds. Despite its name, it takes a +function that returns a value, not an `Action`. + +```csharp +var doubled = Literal.Number().Do(n => n * 2); +Console.WriteLine(doubled.Parse("21").Value); // 42 + +var pair = Seq(Literal.Number(), L(':'), Literal.Number()) + .Do((hours, _, minutes) => new TimeSpan(hours, minutes, 0)); + +Console.WriteLine(pair.Parse("2:30").Value); // 02:30:00 +``` + +The tuple overloads let a callback take 2 through 10 values as separate +parameters. They are useful after `Seq`. Use `Where` if you want to reject a +parsed value. If you return `false` from `Do`, `false` just becomes the +successful result. See [Common mistakes](common-mistakes.md#checking-a-value-with-do). + +## Map: use the matched segment + +The one-argument form maps the matched text and throws away the old value. For +example, you can convert digits directly from their span: + +```csharp +var number = Set("0-9").OneOrMore() + .Map(m => int.Parse(m.Value, NumberStyles.None, CultureInfo.InvariantCulture)); + +Console.WriteLine(number.Parse("123").Value); // 123 +``` + +For ordinary numbers, prefer `Literal.Number()`. It treats a conversion +problem as a normal failure. The `int.Parse` call above throws an exception when +the number is too large. + +The two-argument form keeps the old value too. It is useful for adding source +locations or keeping the original spelling: + +```csharp +var number = Literal.Number() + .Map((m, value) => (Value: value, Start: m.Index, Width: m.Length)); + +var located = L("id=").Then(number); +var result = located.Parse("id=042"); + +Console.WriteLine(result.Value.Value); // 42 +Console.WriteLine(result.Value.Start); // 3 +Console.WriteLine(result.Value.Width); // 3 +``` + +## Match: a view of the source + +`Match` is a `readonly ref struct`. It gives a view of a part of the source +without first making a string: + +| Member | Meaning | +| --- | --- | +| `Index` | Zero-based start offset in the source passed to this parse. | +| `Length` | Number of UTF-16 code units in the segment. | +| `Value` | A `ReadOnlySpan` over the segment. | +| `ToString()` | Copies the segment into a string. | + +`Match` can also be converted to `ReadOnlySpan` or deconstructed into +`(index, length)`. You cannot store it in a normal heap object. If you need the +text after the callback ends, copy the text, or store the index and length. + +## Text: return source text + +`Text()` is a shortcut for mapping the matched segment to a string. It keeps +the exact source spelling and does not decode escape sequences. + +```csharp +var identifier = Set("A-Za-z_") + .ThenIgnore(Set("A-Za-z0-9_").ZeroOrMore()); + +Console.WriteLine(identifier.Parse("user42").Value); // u + +// Seq marks the whole sequence as its matched segment. +var fullIdentifier = Seq( + Set("A-Za-z_"), + Set("A-Za-z0-9_").ZeroOrMore()).Text(); + +Console.WriteLine(fullIdentifier.Parse("user42").Value); // user42 +``` + +The matched segment and the amount of input read are different things: + +- `Seq`, repetition, lists, and folds mark their whole successful span. +- `Then` keeps the segment from its second parser. +- `ThenIgnore` keeps the segment from its first parser. +- `Between` keeps the segment from the middle parser. + +For example, a token can read spaces but leave them out of `Text()`: + +```csharp +var token = L("hello").ThenIgnore(S); +Console.WriteLine(token.Text().Parse("hello ").Value); // hello +Console.WriteLine(token.Parse("hello ").Length); // 8 + +var withSpaces = Seq(L("hello"), S).Text(); +Console.WriteLine(withSpaces.Parse("hello ").Value!.Length); // 8 +``` + +Put `Text` or `Map` at the point in the grammar whose segment you want. +`ParseResult.Length` always counts the input read by the whole parse. + +## Void and Unit: throw a value away + +`Void()` returns a `Parser`. `Unit.Value` is an empty value. It is used when +a parser only needs to read or check input. + +```csharp +var punctuation = Seq(L('('), L(')')).Void(); +Console.WriteLine(punctuation.Parse("()").Success); // True +Console.WriteLine(punctuation.Parse("()").Length); // 2 +``` + +Throwing values away can avoid creating strings, lists, tuples, and other result +objects. `Then`, `ThenIgnore`, `Between`, `And`, `Not`, and separators also +throw away the values that they do not need. + +The library may skip `Do`, `Map`, or fold callbacks when their result is +thrown away. The one-argument `Map` and `Text` also throw away the inner value. +For example, the callback below is skipped: + +```csharp +var calls = 0; +var token = L('x').Do(c => +{ + calls++; + return c; +}); + +Console.WriteLine(token.Void().Parse("x").Success); // True +Console.WriteLine(calls); // 0 +``` + +Keep callbacks focused on building values. Put checks in `Where`, and do +external actions after a successful top-level parse. A callback that does run may +run during an attempt that later fails. Moving the input position back does not +undo changes to external state. + +Next: [Numbers and quoted text](literals.md). From 13fc624c463954c5684d00d63ae13b3fa4753a32 Mon Sep 17 00:00:00 2001 From: rameel Date: Fri, 13 Nov 2026 16:07:17 +0500 Subject: [PATCH 07/10] docs: state ParseContext argument requirements instead of clamping --- docs/custom-parsers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/custom-parsers.md b/docs/custom-parsers.md index cc3599d..bea78e1 100644 --- a/docs/custom-parsers.md +++ b/docs/custom-parsers.md @@ -33,8 +33,8 @@ if (number.TryParse(ref context, out var first) | `Remaining` | Read the span from the current position to the end. | | `Position` | Read the current zero-based offset. | | `MatchedSegment` | Read the segment that a parser marked most recently. | -| `Advance(count)` | Read `count` characters and mark them as the matched segment. Pass a count from 0 to `Remaining.Length`. Other values are not checked: the context moves to the end of the input instead of throwing. | -| `SetMatched(index, length)` | Mark the segment that `Map` and `Text` receive, without changing the position. Values outside the source are cut to fit. | +| `Advance(count)` | Read `count` characters and mark them as the matched segment. `count` must be from 0 to `Remaining.Length`. | +| `SetMatched(index, length)` | Mark the segment that `Map` and `Text` receive, without changing the position. The segment must lie inside `Source`. | | `BookmarkPosition()` | Save a `PositionBookmark` so that you can go back. | | `RestorePosition(bookmark)` | Go back to the saved position and make the matched segment empty. It does not undo error messages or the work of callbacks. | | `ReportExpected(text)` / `ReportExpected(string[])` | Record what was expected at the current position, when error messages are on. | From 3ebdf595cb6f24fb882c26119a0304dfa7f07eff Mon Sep 17 00:00:00 2001 From: rameel Date: Fri, 13 Nov 2026 16:40:38 +0500 Subject: [PATCH 08/10] docs: tidy up --- docs/README.md | 9 ++--- docs/basic-parsers.md | 8 ++--- docs/combining-parsers.md | 39 ++++----------------- docs/common-mistakes.md | 21 ++++++++---- docs/errors.md | 1 + docs/literals.md | 7 ++-- docs/performance-tips.md | 17 +++++----- docs/recursive-parsers.md | 6 ++-- docs/working-with-values.md | 68 ++++++++++++++++++++++++++++--------- 9 files changed, 95 insertions(+), 81 deletions(-) diff --git a/docs/README.md b/docs/README.md index 89c9e87..19acedd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,10 +11,10 @@ guide to the methods you need. | --- | --- | | [Getting started](getting-started.md) | Install the library, run a parser, read results, handle spaces, and require the whole input. | | [Basic parsers](basic-parsers.md) | `L`, `OneOf`, `Set`, Unicode categories, `Character`, `Any`, `S`, `Eol`, `Eof`, and `Regex`. | -| [Combining parsers](combining-parsers.md) | `Seq`, `Then`, `ThenIgnore`, `Between`, `Choice`, `Or`, `Optional`, `DefaultOnFail`, `Return`, `And`, `Not`, and `Where`. | -| [Repetition and lists](repetition-and-lists.md) | `ZeroOrMore`, `Many`, `OneOrMore`, `AtLeast`, `Repeat`, `Separated`, and `Until`. | -| [Working with values](working-with-values.md) | `Do`, `Map`, `Text`, `Void`, `Match`, and `Unit`. | | [Numbers and quoted text](literals.md) | `Literal.Number`, `NumberKind`, quoted strings, quoted characters, and escape sequences. | +| [Combining parsers](combining-parsers.md) | `Seq`, `Then`, `ThenIgnore`, `Between`, `Choice`, `Or`, `Optional`, `DefaultOnFail`, `Return`, `And`, and `Not`. | +| [Repetition and lists](repetition-and-lists.md) | `ZeroOrMore`, `Many`, `OneOrMore`, `AtLeast`, `Repeat`, `Separated`, and `Until`. | +| [Working with values](working-with-values.md) | `Do`, `Where`, `Map`, `Text`, `Void`, `Match`, `Unit`, and when callbacks run. | | [Recursive parsers and expressions](recursive-parsers.md) | `Deferred`, `Recursive`, `FoldL`, `FoldR`, `Prefixed`, and operator precedence. | | [Errors](errors.md) | Error messages, `As`, `Error`, `Fail`, `FatalError`, and exceptions. | | [Writing your own parser](custom-parsers.md) | `ParseContext` and a custom `Parser` class. | @@ -94,7 +94,8 @@ default for .NET 6. alternatives first. - Repetition is greedy. It reads as much as it can and never gives text back. - Keep the functions that build values free of side effects. They may run during - an attempt that later fails, or be skipped when the value is thrown away. + an attempt that later fails, or be skipped when the value is thrown away. See + [Callbacks and thrown-away values](working-with-values.md#callbacks-and-thrown-away-values). - Build a parser once and reuse it. Finish setting up recursive parsers before use. See [Common mistakes](common-mistakes.md) for examples of what goes wrong when you diff --git a/docs/basic-parsers.md b/docs/basic-parsers.md index 06a22ac..51c4285 100644 --- a/docs/basic-parsers.md +++ b/docs/basic-parsers.md @@ -58,7 +58,8 @@ Console.WriteLine(comparison.Parse("<=").Value); // <= With ordinal comparisons, `OneOf` tries longer words first. So `OneOf("<", "<=")` reads `<=` when the input has it. The order in which you wrote the words does not -matter. `Choice` is different: it tries alternatives in the order you gave them. +matter. [`Choice`](combining-parsers.md#choice-and-or-try-alternatives) is +different: it tries alternatives in the order you gave them. With culture-based comparisons, `OneOf` tries the words in descending sort order. Words that compare equal are merged, and only the first one is kept. This also @@ -169,9 +170,6 @@ Console.WriteLine(Character.Digit.Parse("٣").Success); // True Console.WriteLine(Set("0-9").Parse("٣").Success); // False ``` -Each time you read a `Character` property, it creates a new parser. Keep it in -a variable or a field if you use it in several places. - These parsers read one C# `char` at a time. Some characters, such as emoji, use two `char` values. `Any` and the `Character` parsers do not join the pair. @@ -237,4 +235,4 @@ var named = code.As("product code"); Console.WriteLine(named.Parse("1").ErrorMessage); // (1:1) Expected product code ``` -Next: [Combining parsers](combining-parsers.md). +Next: [Numbers and quoted text](literals.md). diff --git a/docs/combining-parsers.md b/docs/combining-parsers.md index 358d3a3..9610023 100644 --- a/docs/combining-parsers.md +++ b/docs/combining-parsers.md @@ -37,8 +37,7 @@ Console.WriteLine(pair.Parse("12:34").Value); // 46 ``` If one part fails, `Seq` moves the input position back to where the sequence -began. An alternative can then try the same text. This does not undo side effects -in your callbacks. +began. An alternative can then try the same text. ## Then and ThenIgnore: keep one result @@ -60,8 +59,7 @@ Console.WriteLine(word.Parse("hello ").Length); // 8 ``` Use these methods for punctuation, prefixes, whitespace, and `Eof`. If either -part fails, they move the input position back to the start. The ignored parser -becomes a void parser, so its callbacks can be skipped. +part fails, they move the input position back to the start. ## Between: surround a value @@ -79,8 +77,7 @@ Console.WriteLine(bracketed.Parse("[42").Success); // False ``` `Between` requires both surrounding parsers. If any part fails, it moves the -input position back to the start. The surrounding values are thrown away, and -their callbacks can be skipped. +input position back to the start. The surrounding values are thrown away. ## Choice and Or: try alternatives @@ -104,8 +101,9 @@ Once an alternative succeeds, a later failure outside the choice does not make it try the other alternatives. Put `L("<=")` before `L("<")`, or use `OneOf("<", "<=")`, which tries longer words first. -A normal failure lets the choice try the next alternative. A fatal error (`Fail` -or `FatalError`) stops the whole parse. A choice does not recover from it. +A normal failure lets the choice try the next alternative. A +[fatal error](errors.md#fail-and-fatalerror-stop-parsing) stops the whole parse, +and the choice does not try the remaining alternatives. ## Optional: keep track of whether a value was present @@ -207,29 +205,4 @@ Console.WriteLine(unnamed.Parse("iffy").ErrorMessage); // Parsing failed. Console.WriteLine(named.Parse("iffy").ErrorMessage); // (1:1) Expected 'if' ``` -Lookahead turns its inner parser into a void parser, so callbacks inside it can -be skipped. A fatal error or an exception is not turned into a match or a -failure. It still stops the parse. - -## Where: check a parsed value - -`parser.Where(predicate, expected)` accepts the result only if the predicate -returns `true`. If it returns `false`, the parser moves the input position back -to the start and reports `expected` there. The description must be non-empty. - -```csharp -var port = Literal.Number() - .Where(n => n >= 1 && n <= 65535, "valid port") - .ThenIgnore(Eof); - -Console.WriteLine(port.Parse("8080").Success); // True -Console.WriteLine(port.Parse("0").Success); // False -Console.WriteLine(port.Parse("0").ErrorMessage); // (1:1) Expected valid port -``` - -Use `Where` for ranges, allowed values, and other checks on a parsed value. A -rejection is a normal failure, so `Or` can try another parser. `Where` still -runs its predicate when its value is thrown away. An exception from the -predicate is an exception, not a rejection. - Next: [Repetition and lists](repetition-and-lists.md). diff --git a/docs/common-mistakes.md b/docs/common-mistakes.md index 8763afc..ecc7834 100644 --- a/docs/common-mistakes.md +++ b/docs/common-mistakes.md @@ -21,7 +21,8 @@ var right = number.ThenIgnore(Eof); Console.WriteLine(right.Parse("12abc").Success); // False ``` -Add `ThenIgnore(Eof)` to the outermost parser. +Add `ThenIgnore(Eof)` to the outermost parser. See +[Parse a prefix or the whole input](getting-started.md#parse-a-prefix-or-the-whole-input). ## Forgetting spaces @@ -62,6 +63,7 @@ Console.WriteLine(right.Parse("<=").Success); // True ``` `OneOf` tries longer words first. With `Choice`, put `L("<=")` before `L("<")`. +See [Choice and Or](combining-parsers.md#choice-and-or-try-alternatives). ## Matching a keyword that is only the start of a word @@ -73,7 +75,8 @@ var right = L("if").ThenIgnore(Not(Set("A-Za-z0-9_"))); Console.WriteLine(right.Parse("iffy").Success); // False ``` -Add a check that no identifier character follows the keyword. +Add a check that no identifier character follows the keyword. See +[And and Not](combining-parsers.md#and-and-not-check-what-comes-next). ## Expecting a repetition to give text back @@ -88,7 +91,8 @@ Console.WriteLine(right.Parse("notes.txt").Value); // notes ``` `Any.OneOrMore()` reads the whole input, so nothing is left for `.txt`. `Until` -stops before the terminator. +stops before the terminator. See +[Until](repetition-and-lists.md#until-stop-before-a-terminator). ## Putting Text in the wrong place @@ -127,7 +131,8 @@ Console.WriteLine(right.Parse("xxb").Value!.Count); // 2 The `wrong` parser returns two items for the two `x` characters and a third, empty item at `b`. Use an item that reads text every time it succeeds. If only a part of -the item is optional, put `Optional` inside the item. +the item is optional, put `Optional` inside the item. See +[Parsers that can match no text](repetition-and-lists.md#parsers-that-can-match-no-text). ## Checking a value with Do @@ -143,7 +148,8 @@ var right = Literal.Number().Where(n => n < 10, "number below 10"); Console.WriteLine(right.Parse("50").Success); // False ``` -Use `Where` to check a value. +Use `Where` to check a value. See +[Where](working-with-values.md#where-check-a-parsed-value). ## Using Value without checking Success @@ -157,7 +163,8 @@ Console.WriteLine(result.Value); // 0 Console.WriteLine(result.Success); // False ``` -Always check `Success` first, or use `TryParse`. +Always check `Success` first, or use `TryParse`. See +[Read a result](getting-started.md#read-a-result). ## Counting or logging inside a callback @@ -165,7 +172,7 @@ The functions that you pass to `Do`, `Map`, and folds can run during an attempt that later fails. They can also be skipped when the value is thrown away. So a counter or a log line inside a callback can be wrong. Build values in callbacks, and do external work after a successful parse. See -[Void and Unit](working-with-values.md#void-and-unit-throw-a-value-away). +[Callbacks and thrown-away values](working-with-values.md#callbacks-and-thrown-away-values). ## Left recursion diff --git a/docs/errors.md b/docs/errors.md index 42f1650..d8fb0ca 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -140,6 +140,7 @@ Console.WriteLine(number.Parse("0").ErrorMessage); // (1:2) Zero is not allowed. The position in a fatal error is the current position. In the second example, the number has already been read, so the column is 2. A callback can be skipped when its value is thrown away, so use `Where` for checks that must always run. +See [Callbacks and thrown-away values](working-with-values.md#callbacks-and-thrown-away-values). ## When your own code throws diff --git a/docs/literals.md b/docs/literals.md index 57a149b..5a48b18 100644 --- a/docs/literals.md +++ b/docs/literals.md @@ -73,8 +73,9 @@ Console.WriteLine(Literal.Number().Parse("256").Success); // False Use `Eof`, a delimiter, or a boundary check to reject an invalid suffix. An integer outside the range of the target type fails normally. A floating-point value follows the parsing rules of the .NET type. For example, `double` turns -`1e999` into infinity, and `decimal` fails. Use `Where` for any extra range or -finite-value checks that your format needs. +`1e999` into infinity, and `decimal` fails. Use +[`Where`](working-with-values.md#where-check-a-parsed-value) for any extra range +or finite-value checks that your format needs. ### Hexadecimal and binary integers @@ -206,4 +207,4 @@ var escape = Literal.EscapeSequence.Or(Literal.UnicodeEscapeSequence); Console.WriteLine(escape.Parse(@"\u0042").Value); // B ``` -Next: [Recursive parsers and expressions](recursive-parsers.md). +Next: [Combining parsers](combining-parsers.md). diff --git a/docs/performance-tips.md b/docs/performance-tips.md index 1676f59..039d9d0 100644 --- a/docs/performance-tips.md +++ b/docs/performance-tips.md @@ -10,12 +10,10 @@ show how to do that. ## Build parsers once Building a parser does work, for example it prepares tables for character sets. -Build it once, keep it in a `static readonly` field, and reuse it. Each call to -`Parse` or `TryParse` creates its own position, so one parser can serve many -inputs. +Build it once and reuse it. See [Reuse the parser](getting-started.md#reuse-the-parser). -The `Character` properties create a new parser each time you read them. Keep them -in a field too. +A `Character` property can build a new parser each time you read it. If you use +one in several places, keep it in a field too. ## Use TryParse when you do not need an error message @@ -28,7 +26,8 @@ again only when it fails. A parser that only reads or checks input does not need to build a value. `Then`, `ThenIgnore`, `Between`, `And`, `Not`, separators, and `Void()` throw values away, and the library can skip the work of building them. Write the -punctuation of your grammar with these methods. +punctuation of your grammar with these methods. See +[Callbacks and thrown-away values](working-with-values.md#callbacks-and-thrown-away-values). ## Use Text instead of a list of characters @@ -40,9 +39,9 @@ var word = Set("a-z").OneOrMore().Text(); Console.WriteLine(word.Parse("hello world").Value); // hello ``` -Repeating a single character set (`Set`, or `OneOf` with a string of characters) -uses a special scan that is made for this case. Prefer it to repeating a more -complex parser when you only need a run of characters. +Repetition of a single character set (`Set`, or `OneOf` with a string of +characters) is optimized for this case. Prefer it to repeating a more complex +parser when you only need a run of characters. ## Prefer ordinal comparisons diff --git a/docs/recursive-parsers.md b/docs/recursive-parsers.md index 3a3d243..8a4f06e 100644 --- a/docs/recursive-parsers.md +++ b/docs/recursive-parsers.md @@ -138,9 +138,7 @@ Console.WriteLine(sum.ThenIgnore(Eof).Parse("1+").Success); // False ``` If a whole operator/operand pair reads no text, the fold throws that pair away -and stops. It does not call the `reduce` function for that pair. The `reduce` -function can also be skipped when the folded value is thrown away, so keep -validation in the operand parsers. +and stops. It does not call the `reduce` function for that pair. ## Prefixed: read operators before an operand @@ -171,7 +169,7 @@ Console.WriteLine(tree.Parse("!-x").Value); // (!(-x)) not use the call stack. If the operand is missing, `Prefixed` fails and the position goes back to where it started, including the operators that it read. If an operator reads no text, `Prefixed` stops looking for operators and does not call -`apply` for it. Like the folds, it skips `apply` when the value is thrown away. +`apply` for it. Use the parser as the operand of a binary level. The binary `-` and the prefix `-` do not clash, because each one is tried at a different place: diff --git a/docs/working-with-values.md b/docs/working-with-values.md index 41e9506..7d27e8d 100644 --- a/docs/working-with-values.md +++ b/docs/working-with-values.md @@ -9,13 +9,13 @@ matches. These methods let you choose. The examples use the | Method | Callback receives | Result | | --- | --- | --- | | `Do(value => ...)` | The parsed value. | The callback's return value. | +| `Where(value => ..., expected)` | The parsed value. | The same value, or a normal failure when the callback returns `false`. | | `Map(match => ...)` | The matched segment. | The callback's return value. The original parsed value is thrown away. | | `Map((match, value) => ...)` | Both the matched segment and the parsed value. | The callback's return value. | | `Text()` | No callback needed. | The matched segment as a string. | | `Void()` | No callback needed. | `Unit`. The value is thrown away. | -The *matched segment* is the part of the input that a parser matched. A callback -is a function that you pass to a method. +The *matched segment* is the part of the input that a parser matched. ## Do: change the parsed value @@ -33,9 +33,30 @@ Console.WriteLine(pair.Parse("2:30").Value); // 02:30:00 ``` The tuple overloads let a callback take 2 through 10 values as separate -parameters. They are useful after `Seq`. Use `Where` if you want to reject a -parsed value. If you return `false` from `Do`, `false` just becomes the -successful result. See [Common mistakes](common-mistakes.md#checking-a-value-with-do). +parameters. They are useful after `Seq`. If you return `false` from `Do`, +`false` just becomes the successful result. To reject a value, use `Where`. + +## Where: check a parsed value + +`parser.Where(predicate, expected)` accepts the result only if the predicate +returns `true`. If it returns `false`, the parser moves the input position back +to the start and reports `expected` there. The description must be non-empty. + +```csharp +var port = Literal.Number() + .Where(n => n >= 1 && n <= 65535, "valid port") + .ThenIgnore(Eof); + +Console.WriteLine(port.Parse("8080").Success); // True +Console.WriteLine(port.Parse("0").Success); // False +Console.WriteLine(port.Parse("0").ErrorMessage); // (1:1) Expected valid port +``` + +Use `Where` for ranges, allowed values, and other checks on a parsed value. A +rejection is a normal failure, so `Or` can try another parser. An exception from +the predicate is an exception, not a rejection. Unlike `Do`, the predicate runs +even when the value is thrown away; see +[below](#callbacks-and-thrown-away-values). ## Map: use the matched segment @@ -135,13 +156,20 @@ Console.WriteLine(punctuation.Parse("()").Success); // True Console.WriteLine(punctuation.Parse("()").Length); // 2 ``` -Throwing values away can avoid creating strings, lists, tuples, and other result -objects. `Then`, `ThenIgnore`, `Between`, `And`, `Not`, and separators also -throw away the values that they do not need. +## Callbacks and thrown-away values -The library may skip `Do`, `Map`, or fold callbacks when their result is -thrown away. The one-argument `Map` and `Text` also throw away the inner value. -For example, the callback below is skipped: +`Void` is not the only method that throws a value away. Others do it too: + +- `Then` throws away the first value, and `ThenIgnore` the second. +- `Between` throws away the surrounding values. +- `And` and `Not` throw away the value of the parser they check. +- `Separated` throws away the separators, and `Until` the terminator. +- `Text` and the one-argument `Map` throw away the inner value. + +This avoids creating strings, lists, tuples, and other result objects that +nobody reads. It also means that the library may skip a callback in `Do`, `Map`, +a fold, or `Prefixed` when its result is thrown away. For example, the callback +below is skipped: ```csharp var calls = 0; @@ -155,9 +183,17 @@ Console.WriteLine(token.Void().Parse("x").Success); // True Console.WriteLine(calls); // 0 ``` -Keep callbacks focused on building values. Put checks in `Where`, and do -external actions after a successful top-level parse. A callback that does run may -run during an attempt that later fails. Moving the input position back does not -undo changes to external state. +`Where` is different. Its predicate decides whether the parser matches, so it +runs even when the value is thrown away: + +```csharp +var nonZero = Set("0-9").Where(c => c != '0', "non-zero digit"); +Console.WriteLine(nonZero.Void().Parse("0").Success); // False +``` + +A callback that does run may run during an attempt that later fails. Moving the +input position back does not undo changes to external state. So keep callbacks +focused on building values, put checks in `Where`, and do external actions, such +as logging or counting, after a successful top-level parse. -Next: [Numbers and quoted text](literals.md). +Next: [Recursive parsers and expressions](recursive-parsers.md). From 9336d5cb32cd87973b50f0f3c047223463a2a189 Mon Sep 17 00:00:00 2001 From: rameel Date: Fri, 13 Nov 2026 17:05:45 +0500 Subject: [PATCH 09/10] docs: align markdown tables --- docs/README.md | 74 ++++++++++++++++++------------------ docs/basic-parsers.md | 54 +++++++++++++------------- docs/combining-parsers.md | 14 +++---- docs/custom-parsers.md | 32 ++++++++-------- docs/errors.md | 66 ++++++++++++++++---------------- docs/examples.md | 10 ++--- docs/getting-started.md | 14 +++---- docs/literals.md | 34 ++++++++--------- docs/repetition-and-lists.md | 10 ++--- docs/working-with-values.md | 28 +++++++------- 10 files changed, 168 insertions(+), 168 deletions(-) diff --git a/docs/README.md b/docs/README.md index 19acedd..d8ccb51 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,46 +7,46 @@ a word, a number, a list, or a whole expression. Start with [Getting started](getting-started.md). Then use the other pages as a guide to the methods you need. -| Page | What it covers | -| --- | --- | -| [Getting started](getting-started.md) | Install the library, run a parser, read results, handle spaces, and require the whole input. | -| [Basic parsers](basic-parsers.md) | `L`, `OneOf`, `Set`, Unicode categories, `Character`, `Any`, `S`, `Eol`, `Eof`, and `Regex`. | -| [Numbers and quoted text](literals.md) | `Literal.Number`, `NumberKind`, quoted strings, quoted characters, and escape sequences. | -| [Combining parsers](combining-parsers.md) | `Seq`, `Then`, `ThenIgnore`, `Between`, `Choice`, `Or`, `Optional`, `DefaultOnFail`, `Return`, `And`, and `Not`. | -| [Repetition and lists](repetition-and-lists.md) | `ZeroOrMore`, `Many`, `OneOrMore`, `AtLeast`, `Repeat`, `Separated`, and `Until`. | -| [Working with values](working-with-values.md) | `Do`, `Where`, `Map`, `Text`, `Void`, `Match`, `Unit`, and when callbacks run. | -| [Recursive parsers and expressions](recursive-parsers.md) | `Deferred`, `Recursive`, `FoldL`, `FoldR`, `Prefixed`, and operator precedence. | -| [Errors](errors.md) | Error messages, `As`, `Error`, `Fail`, `FatalError`, and exceptions. | -| [Writing your own parser](custom-parsers.md) | `ParseContext` and a custom `Parser` class. | -| [Complete examples](examples.md) | A list of integers, a setting, a calculator, and links to the project samples. | -| [Common mistakes](common-mistakes.md) | Problems that new users often meet, and how to fix them. | -| [Performance tips](performance-tips.md) | How to keep parsers fast. | +| Page | What it covers | +| --------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | +| [Getting started](getting-started.md) | Install the library, run a parser, read results, handle spaces, and require the whole input. | +| [Basic parsers](basic-parsers.md) | `L`, `OneOf`, `Set`, Unicode categories, `Character`, `Any`, `S`, `Eol`, `Eof`, and `Regex`. | +| [Numbers and quoted text](literals.md) | `Literal.Number`, `NumberKind`, quoted strings, quoted characters, and escape sequences. | +| [Combining parsers](combining-parsers.md) | `Seq`, `Then`, `ThenIgnore`, `Between`, `Choice`, `Or`, `Optional`, `DefaultOnFail`, `Return`, `And`, and `Not`. | +| [Repetition and lists](repetition-and-lists.md) | `ZeroOrMore`, `Many`, `OneOrMore`, `AtLeast`, `Repeat`, `Separated`, and `Until`. | +| [Working with values](working-with-values.md) | `Do`, `Where`, `Map`, `Text`, `Void`, `Match`, `Unit`, and when callbacks run. | +| [Recursive parsers and expressions](recursive-parsers.md) | `Deferred`, `Recursive`, `FoldL`, `FoldR`, `Prefixed`, and operator precedence. | +| [Errors](errors.md) | Error messages, `As`, `Error`, `Fail`, `FatalError`, and exceptions. | +| [Writing your own parser](custom-parsers.md) | `ParseContext` and a custom `Parser` class. | +| [Complete examples](examples.md) | A list of integers, a setting, a calculator, and links to the project samples. | +| [Common mistakes](common-mistakes.md) | Problems that new users often meet, and how to fix them. | +| [Performance tips](performance-tips.md) | How to keep parsers fast. | ## Find a method by task -| I want to... | Use | -| --- | --- | -| Read an exact character or word | `L` | -| Read one of several words | `OneOf` | -| Read one character from a range or a set | `Set`, `Character` | -| Read a number | `Literal.Number` | -| Read a quoted string | `Literal.DoubleQuotedString`, `Literal.SingleQuotedString` | -| Skip spaces | `S` | -| Require the end of the input | `Eof` | -| Run parsers one after another | `Seq`, `Then`, `ThenIgnore`, `Between` | -| Try alternatives | `Choice`, `Or` | -| Make a part optional | `Optional`, `DefaultOnFail` | -| Repeat a parser | `ZeroOrMore`, `OneOrMore`, `Repeat`, `AtLeast` | -| Read a list with separators | `Separated` | -| Read until an end marker | `Until` | -| Change the value | `Do` | -| Get the source text | `Text` | -| Check a value | `Where` | -| Look ahead without reading | `And`, `Not` | -| Read operators such as `+` or `^` | `FoldL`, `FoldR` | -| Read prefix operators such as `-` or `!` | `Prefixed` | -| Let a parser use itself | `Deferred`, `Recursive` | -| Give a parser a better error name | `As` | +| I want to... | Use | +| ---------------------------------------- | ---------------------------------------------------------- | +| Read an exact character or word | `L` | +| Read one of several words | `OneOf` | +| Read one character from a range or a set | `Set`, `Character` | +| Read a number | `Literal.Number` | +| Read a quoted string | `Literal.DoubleQuotedString`, `Literal.SingleQuotedString` | +| Skip spaces | `S` | +| Require the end of the input | `Eof` | +| Run parsers one after another | `Seq`, `Then`, `ThenIgnore`, `Between` | +| Try alternatives | `Choice`, `Or` | +| Make a part optional | `Optional`, `DefaultOnFail` | +| Repeat a parser | `ZeroOrMore`, `OneOrMore`, `Repeat`, `AtLeast` | +| Read a list with separators | `Separated` | +| Read until an end marker | `Until` | +| Change the value | `Do` | +| Get the source text | `Text` | +| Check a value | `Where` | +| Look ahead without reading | `And`, `Not` | +| Read operators such as `+` or `^` | `FoldL`, `FoldR` | +| Read prefix operators such as `-` or `!` | `Prefixed` | +| Let a parser use itself | `Deferred`, `Recursive` | +| Give a parser a better error name | `As` | ## Words used in these docs diff --git a/docs/basic-parsers.md b/docs/basic-parsers.md index 51c4285..4d07fa3 100644 --- a/docs/basic-parsers.md +++ b/docs/basic-parsers.md @@ -89,18 +89,18 @@ Console.WriteLine(Set(@"\p{L}").Parse("é").Value); // é Console.WriteLine(Set(@"\d").Parse("5").Value); // 5 ``` -| Pattern | Matches | -| --- | --- | -| `a-z`, `0-9` | An inclusive range. | -| `abc_` | Any of these literal characters. | -| `\p{L}`, `\p{Nd}` | A Unicode general category. | -| `\P{L}` | A character outside that Unicode general category. | -| `\d`, `\D` | A Unicode decimal digit, or a character outside that class. | -| `\w`, `\W` | A word character, or a character outside that class. | -| `\s`, `\S` | Whitespace, or a character outside that class. | +| Pattern | Matches | +| ---------------------------------------------------- | ---------------------------------------------------------------------------- | +| `a-z`, `0-9` | An inclusive range. | +| `abc_` | Any of these literal characters. | +| `\p{L}`, `\p{Nd}` | A Unicode general category. | +| `\P{L}` | A character outside that Unicode general category. | +| `\d`, `\D` | A Unicode decimal digit, or a character outside that class. | +| `\w`, `\W` | A word character, or a character outside that class. | +| `\s`, `\S` | Whitespace, or a character outside that class. | | `\n`, `\r`, `\t`, `\a`, `\b`, `\v`, `\f`, `\e`, `\0` | A character escape. Here `\b` means backspace and `\0` is the NUL character. | -| `\\`, `\-` | A literal backslash or hyphen. | -| `\u0041` | A character given by exactly four hexadecimal digits. | +| `\\`, `\-` | A literal backslash or hyphen. | +| `\u0041` | A character given by exactly four hexadecimal digits. | Use a verbatim C# string, such as `@"\d"`, or double the backslash in a normal string, such as `"\\d"`. @@ -151,17 +151,17 @@ Console.WriteLine(letterOrDigit.Parse("7").Value); // 7 The `Character` class provides ready-made `Parser` properties: -| Property | What it reads | -| --- | --- | -| `Letter` | Any Unicode letter. | -| `Uppercase`, `Lowercase` | An uppercase letter, or a lowercase letter. | -| `Digit` | A decimal digit from any script. | -| `Number` | A character in any Unicode number category. | -| `LetterOrDigit` | A letter or a decimal digit. | -| `HexDigit` | An ASCII hexadecimal digit: `0-9`, `A-F`, or `a-f`. | +| Property | What it reads | +| ----------------------------------------------- | --------------------------------------------------- | +| `Letter` | Any Unicode letter. | +| `Uppercase`, `Lowercase` | An uppercase letter, or a lowercase letter. | +| `Digit` | A decimal digit from any script. | +| `Number` | A character in any Unicode number category. | +| `LetterOrDigit` | A letter or a decimal digit. | +| `HexDigit` | An ASCII hexadecimal digit: `0-9`, `A-F`, or `a-f`. | | `Symbol`, `Separator`, `Punctuation`, `Control` | A character in the matching Unicode category group. | -| `WhiteSpace` | One whitespace character. | -| `Surrogate`, `HighSurrogate`, `LowSurrogate` | One UTF-16 surrogate code unit. | +| `WhiteSpace` | One whitespace character. | +| `Surrogate`, `HighSurrogate`, `LowSurrogate` | One UTF-16 surrogate code unit. | ```csharp Console.WriteLine(Character.Letter.Parse("Ж").Value); // Ж @@ -175,12 +175,12 @@ two `char` values. `Any` and the `Character` parsers do not join the pair. ## Any, S, Eol, and Eof -| Parser | Result | What it does | -| --- | --- | --- | -| `Any` | `char` | Reads any one UTF-16 code unit, including a line break. Fails at the end of the input. | -| `S` | `Unit` | Reads zero or more whitespace characters, using `char.IsWhiteSpace`. Always succeeds. | -| `Eol` | `Unit` | Reads a line break. At the end of the input it succeeds without reading anything. | -| `Eof` | `Unit` | Succeeds only at the end of the input, without reading anything. | +| Parser | Result | What it does | +| ------ | ------ | -------------------------------------------------------------------------------------- | +| `Any` | `char` | Reads any one UTF-16 code unit, including a line break. Fails at the end of the input. | +| `S` | `Unit` | Reads zero or more whitespace characters, using `char.IsWhiteSpace`. Always succeeds. | +| `Eol` | `Unit` | Reads a line break. At the end of the input it succeeds without reading anything. | +| `Eof` | `Unit` | Succeeds only at the end of the input, without reading anything. | `Unit` means that the parser has no useful value to return. diff --git a/docs/combining-parsers.md b/docs/combining-parsers.md index 9610023..bdce8c2 100644 --- a/docs/combining-parsers.md +++ b/docs/combining-parsers.md @@ -43,10 +43,10 @@ began. An alternative can then try the same text. Both methods require two parsers to succeed in order: -| Method | Value kept | -| --- | --- | -| `first.Then(second)` | The second value. | -| `first.ThenIgnore(second)` | The first value. | +| Method | Value kept | +| -------------------------- | ----------------- | +| `first.Then(second)` | The second value. | +| `first.ThenIgnore(second)` | The first value. | ```csharp var number = L("id=").Then(Literal.Number()).ThenIgnore(L(';')); @@ -175,9 +175,9 @@ These are lookahead parsers: they test a parser and leave the input position where it was. Both return `Unit`. They do not record error messages for the parser that they check. -| Method | Succeeds when | -| --- | --- | -| `And(parser)` | The supplied parser succeeds. | +| Method | Succeeds when | +| ------------- | ----------------------------------- | +| `And(parser)` | The supplied parser succeeds. | | `Not(parser)` | The supplied parser fails normally. | ```csharp diff --git a/docs/custom-parsers.md b/docs/custom-parsers.md index bea78e1..3bd879c 100644 --- a/docs/custom-parsers.md +++ b/docs/custom-parsers.md @@ -27,22 +27,22 @@ if (number.TryParse(ref context, out var first) } ``` -| Member | Use | -| --- | --- | -| `Source` | Read the whole input span. | -| `Remaining` | Read the span from the current position to the end. | -| `Position` | Read the current zero-based offset. | -| `MatchedSegment` | Read the segment that a parser marked most recently. | -| `Advance(count)` | Read `count` characters and mark them as the matched segment. `count` must be from 0 to `Remaining.Length`. | -| `SetMatched(index, length)` | Mark the segment that `Map` and `Text` receive, without changing the position. The segment must lie inside `Source`. | -| `BookmarkPosition()` | Save a `PositionBookmark` so that you can go back. | -| `RestorePosition(bookmark)` | Go back to the saved position and make the matched segment empty. It does not undo error messages or the work of callbacks. | -| `ReportExpected(text)` / `ReportExpected(string[])` | Record what was expected at the current position, when error messages are on. | -| `DiagnosticState` | Get or set `Normal` or `Suppressed`. Error messages are recorded only in the `Normal` state. | -| `SuppressDiagnostics()` | Turn error messages off and return the previous state. | -| `SuppressDiagnosticsIfNamed(name)` | Turn error messages off when the name is not `null`, and return the previous state. | -| `RestoreDiagnosticState(state)` | Restore a saved state. | -| `ToString()` | Format the recorded expectations with a line and column. An empty string means that nothing was recorded. | +| Member | Use | +| --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | +| `Source` | Read the whole input span. | +| `Remaining` | Read the span from the current position to the end. | +| `Position` | Read the current zero-based offset. | +| `MatchedSegment` | Read the segment that a parser marked most recently. | +| `Advance(count)` | Read `count` characters and mark them as the matched segment. `count` must be from 0 to `Remaining.Length`. | +| `SetMatched(index, length)` | Mark the segment that `Map` and `Text` receive, without changing the position. The segment must lie inside `Source`. | +| `BookmarkPosition()` | Save a `PositionBookmark` so that you can go back. | +| `RestorePosition(bookmark)` | Go back to the saved position and make the matched segment empty. It does not undo error messages or the work of callbacks. | +| `ReportExpected(text)` / `ReportExpected(string[])` | Record what was expected at the current position, when error messages are on. | +| `DiagnosticState` | Get or set `Normal` or `Suppressed`. Error messages are recorded only in the `Normal` state. | +| `SuppressDiagnostics()` | Turn error messages off and return the previous state. | +| `SuppressDiagnosticsIfNamed(name)` | Turn error messages off when the name is not `null`, and return the previous state. | +| `RestoreDiagnosticState(state)` | Restore a saved state. | +| `ToString()` | Format the recorded expectations with a line and column. An empty string means that nothing was recorded. | Error messages may remain from an attempt that failed, even when a later parser succeeds. Do not use `context.ToString()` to decide whether parsing succeeded. Use diff --git a/docs/errors.md b/docs/errors.md index d8fb0ca..b7f37f0 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -42,20 +42,20 @@ useful than "expected end of input" at column 2. Every built-in parser has a default name for its messages: -| Parser | Name in the message | -| --- | --- | -| `L('a')`, `L("hello")` | `'a'`, `'hello'` | -| `OneOf("yes", "no")` | `'yes'` and `'no'`, as two separate items. | -| `OneOf("+-")`, `Set("a-z")` | The sorted set: `[+-]`, `[a-z]`. A pattern such as `A-Za-z0-9_` shows as `[0-9A-Z_a-z]`. | -| `Literal.Number()` | `number` | -| `Literal.Number()` | `floating-point number` | -| `NumberKind.HexNumber`, `NumberKind.BinaryNumber` | `hexadecimal number`, `binary number` | -| `Literal.DoubleQuotedString`, `Literal.SingleQuotedString` | `double-quoted string`, `single-quoted string` | -| `Literal.QuotedCharacter` | `quoted-character` | -| `Literal.EscapeSequence`, `Literal.UnicodeEscapeSequence` | `escape sequence`, `unicode escape` | -| `Eof`, `Eol`, `Any` | `end of input`, `end of line`, `any character` | -| `Character.Letter`, `Character.Digit`, `Character.WhiteSpace` | `letter`, `digit`, `whitespace` | -| `Regex(...)` | The text of the pattern. | +| Parser | Name in the message | +| ------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | +| `L('a')`, `L("hello")` | `'a'`, `'hello'` | +| `OneOf("yes", "no")` | `'yes'` and `'no'`, as two separate items. | +| `OneOf("+-")`, `Set("a-z")` | The sorted set: `[+-]`, `[a-z]`. A pattern such as `A-Za-z0-9_` shows as `[0-9A-Z_a-z]`. | +| `Literal.Number()` | `number` | +| `Literal.Number()` | `floating-point number` | +| `NumberKind.HexNumber`, `NumberKind.BinaryNumber` | `hexadecimal number`, `binary number` | +| `Literal.DoubleQuotedString`, `Literal.SingleQuotedString` | `double-quoted string`, `single-quoted string` | +| `Literal.QuotedCharacter` | `quoted-character` | +| `Literal.EscapeSequence`, `Literal.UnicodeEscapeSequence` | `escape sequence`, `unicode escape` | +| `Eof`, `Eol`, `Any` | `end of input`, `end of line`, `any character` | +| `Character.Letter`, `Character.Digit`, `Character.WhiteSpace` | `letter`, `digit`, `whitespace` | +| `Regex(...)` | The text of the pattern. | ```csharp Console.WriteLine(L("hello").Parse("x").ErrorMessage); // (1:1) Expected 'hello' @@ -146,11 +146,11 @@ See [Callbacks and thrown-away values](working-with-values.md#callbacks-and-thro `Parse` and `TryParse` treat exceptions differently: -| Situation | `Parse(source)` | `TryParse(source, out value)` | -| --- | --- | --- | -| Failure | Failed result with `ErrorMessage`. `Exception` is `null`. | Returns `false`. | -| `Fail` or `FatalError` | Failed result with your message and the current location. `Exception` is `null`. | Returns `false`. | -| Any other exception thrown while parsing | Failed result with `ErrorMessage` and the caught `Exception`. | The exception reaches the caller. | +| Situation | `Parse(source)` | `TryParse(source, out value)` | +| ---------------------------------------- | -------------------------------------------------------------------------------- | --------------------------------- | +| Failure | Failed result with `ErrorMessage`. `Exception` is `null`. | Returns `false`. | +| `Fail` or `FatalError` | Failed result with your message and the current location. `Exception` is `null`. | Returns `false`. | +| Any other exception thrown while parsing | Failed result with `ErrorMessage` and the caught `Exception`. | The exception reaches the caller. | A failed top-level result has `Length == 0` and a default value. A failed top-level `TryParse` call also sets the output to `default(T)`. @@ -187,19 +187,19 @@ instead. These rules apply while a parser runs. Mistakes in how you build a parser throw an exception right away, and the exception reaches the caller: -| Call | Exception | -| --- | --- | -| `Repeat(0)`, `Repeat(0, 0)`, `Separated(..., max: 0)` | `ArgumentOutOfRangeException` | -| `Repeat(3, 2)` or `Separated(..., min: 3, max: 2)` | `ArgumentOutOfRangeException` | -| `Set('z', 'a')` | `ArgumentOutOfRangeException` | -| `Set("")`, `Set("z-a")`, `Set(@"\q")`, `Set(@"\p{IsGreek}")` | `ArgumentException` | -| `L("")`, `OneOf("")`, `OneOf("yes", "")`, `Choice()` with no parsers | `ArgumentException` | -| `As("")`, `Where(..., "")`, `Fail("")` | `ArgumentException` | -| `Regex(...)` with a pattern that does not start with `^` | `ArgumentException` | -| `Literal.Number` with a kind that does not fit `T`, or `BinaryNumber` for `BigInteger` in an app that targets .NET 8 | `ArgumentException` | -| `Literal.Number()`, or another unsupported type | `InvalidOperationException` | -| Assigning `Deferred.Parser` twice | `InvalidOperationException` | -| Assigning `null` to `Deferred.Parser` | `ArgumentNullException` | -| Using a `Deferred` without a parser where its value is thrown away | `InvalidOperationException` | +| Call | Exception | +| ----------------------------------------------------------------------------------------------------------------------- | ----------------------------- | +| `Repeat(0)`, `Repeat(0, 0)`, `Separated(..., max: 0)` | `ArgumentOutOfRangeException` | +| `Repeat(3, 2)` or `Separated(..., min: 3, max: 2)` | `ArgumentOutOfRangeException` | +| `Set('z', 'a')` | `ArgumentOutOfRangeException` | +| `Set("")`, `Set("z-a")`, `Set(@"\q")`, `Set(@"\p{IsGreek}")` | `ArgumentException` | +| `L("")`, `OneOf("")`, `OneOf("yes", "")`, `Choice()` with no parsers | `ArgumentException` | +| `As("")`, `Where(..., "")`, `Fail("")` | `ArgumentException` | +| `Regex(...)` with a pattern that does not start with `^` | `ArgumentException` | +| `Literal.Number` with a kind that does not fit `T`, or `BinaryNumber` for `BigInteger` in an app that targets .NET 8 | `ArgumentException` | +| `Literal.Number()`, or another unsupported type | `InvalidOperationException` | +| Assigning `Deferred.Parser` twice | `InvalidOperationException` | +| Assigning `null` to `Deferred.Parser` | `ArgumentNullException` | +| Using a `Deferred` without a parser where its value is thrown away | `InvalidOperationException` | Next: [Writing your own parser](custom-parsers.md). diff --git a/docs/examples.md b/docs/examples.md index 10dd04d..7c96ae1 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -130,11 +130,11 @@ See [Recursive parsers](recursive-parsers.md) for `FoldR` and a power example. The repository has larger working grammars: -| Sample | Useful for | -| --- | --- | -| [Calculator source](../samples/CalcExpr/ExpressionParser.cs) and [grammar](../samples/CalcExpr/README.md) | Recursive expressions and operator precedence. | -| [JSON source](../samples/Json/JsonParser.cs) and [grammar](../samples/Json/README.md) | Recursive arrays and objects, lists, and values with different .NET types. | -| [TinyC source](../samples/TinyC/TinyCParser.cs) and [grammar](../samples/TinyC/README.md) | Statements, comments, keyword boundaries, and syntax trees. | +| Sample | Useful for | +| --------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| [Calculator source](../samples/CalcExpr/ExpressionParser.cs) and [grammar](../samples/CalcExpr/README.md) | Recursive expressions and operator precedence. | +| [JSON source](../samples/Json/JsonParser.cs) and [grammar](../samples/Json/README.md) | Recursive arrays and objects, lists, and values with different .NET types. | +| [TinyC source](../samples/TinyC/TinyCParser.cs) and [grammar](../samples/TinyC/README.md) | Statements, comments, keyword boundaries, and syntax trees. | The JSON sample uses the library's general number and string parsers. Treat it as a grammar example, not as a strict JSON validator. Those parsers accept forms diff --git a/docs/getting-started.md b/docs/getting-started.md index 0e06f21..fa648cd 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -51,13 +51,13 @@ allows spaces on either side. `ThenIgnore(Eof)` checks that nothing is left. `Parse` returns `ParseResult`: -| Property | Meaning | -| --- | --- | -| `Success` | Whether the parser succeeded. Check this before you use `Value`. | -| `Value` | The result on success; otherwise `default(T)`. Its type is nullable, so the examples write `result.Value!` when they know the parse succeeded. A successful result can also be `null`, if your grammar returns `null`. | -| `Length` | The number of UTF-16 characters read on success, including spaces and delimiters that the parser threw away. It is zero on failure. | -| `ErrorMessage` | The error text with a line and column, or `Parsing failed.` when nothing more is known. It is `null` on success. | -| `Exception` | The exception caught during parsing, such as one thrown by a function you passed to `Do`. It is `null` for ordinary failures. | +| Property | Meaning | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Success` | Whether the parser succeeded. Check this before you use `Value`. | +| `Value` | The result on success; otherwise `default(T)`. Its type is nullable, so the examples write `result.Value!` when they know the parse succeeded. A successful result can also be `null`, if your grammar returns `null`. | +| `Length` | The number of UTF-16 characters read on success, including spaces and delimiters that the parser threw away. It is zero on failure. | +| `ErrorMessage` | The error text with a line and column, or `Parsing failed.` when nothing more is known. It is `null` on success. | +| `Exception` | The exception caught during parsing, such as one thrown by a function you passed to `Do`. It is `null` for ordinary failures. | `result.ToString()` returns the value's text on success and the error message on failure. Use the individual properties when your code needs to tell them apart. diff --git a/docs/literals.md b/docs/literals.md index 5a48b18..8791897 100644 --- a/docs/literals.md +++ b/docs/literals.md @@ -30,12 +30,12 @@ Add `S` where the grammar allows it. ### Supported types and kinds -| Target type | Supported kinds | -| --- | --- | -| `sbyte`, `byte`, `short`, `ushort`, `int`, `uint`, `long`, `ulong`, `nint`, `nuint` | `Auto`, `Integer`, `HexNumber`; also `BinaryNumber` on .NET 8 or later. | -| `Int128`, `UInt128` (.NET 7 or later) | The same integer kinds. | -| `BigInteger` | `Auto`, `Integer`, `HexNumber`; also `BinaryNumber` in apps that target .NET 9 or later (see below). | -| `float`, `double`, `decimal`, `Half` | `Auto`, `Float`. | +| Target type | Supported kinds | +| ----------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| `sbyte`, `byte`, `short`, `ushort`, `int`, `uint`, `long`, `ulong`, `nint`, `nuint` | `Auto`, `Integer`, `HexNumber`; also `BinaryNumber` on .NET 8 or later. | +| `Int128`, `UInt128` (.NET 7 or later) | The same integer kinds. | +| `BigInteger` | `Auto`, `Integer`, `HexNumber`; also `BinaryNumber` in apps that target .NET 9 or later (see below). | +| `float`, `double`, `decimal`, `Half` | `Auto`, `Float`. | `BigInteger` is in `System.Numerics`. On .NET 7 or later, `T` also has an `INumber` constraint, but the method still accepts only the types in the table. @@ -182,17 +182,17 @@ Console.WriteLine(Literal.EscapeSequence.Parse(@"\q").Success); // False `EscapeSequence` accepts these two-character forms. Quoted strings and quoted characters use the same table: -| Source escape | Decoded character | -| --- | --- | -| `\0` | NUL, U+0000. | -| `\a` | Bell, U+0007. | -| `\b` | Backspace, U+0008. | -| `\t` | Tab, U+0009. | -| `\n` | Line feed, U+000A. | -| `\v` | Vertical tab, U+000B. | -| `\f` | Form feed, U+000C. | -| `\r` | Carriage return, U+000D. | -| `\e` | Escape, U+001B. | +| Source escape | Decoded character | +| ---------------------- | ---------------------------------------------------------- | +| `\0` | NUL, U+0000. | +| `\a` | Bell, U+0007. | +| `\b` | Backspace, U+0008. | +| `\t` | Tab, U+0009. | +| `\n` | Line feed, U+000A. | +| `\v` | Vertical tab, U+000B. | +| `\f` | Form feed, U+000C. | +| `\r` | Carriage return, U+000D. | +| `\e` | Escape, U+001B. | | `\"`, `\'`, `\/`, `\\` | A literal double quote, single quote, slash, or backslash. | `UnicodeEscapeSequence` accepts `\u` followed by exactly four hexadecimal diff --git a/docs/repetition-and-lists.md b/docs/repetition-and-lists.md index 27af452..8231a6b 100644 --- a/docs/repetition-and-lists.md +++ b/docs/repetition-and-lists.md @@ -32,11 +32,11 @@ Console.WriteLine(spaces.Parse(" x").Length); // 3 ## Repeat and AtLeast -| Method | Accepted number of matches | -| --- | --- | -| `Repeat(count)` | Exactly `count`. | -| `Repeat(min, max)` | From `min` to `max`, inclusive. | -| `AtLeast(count)` | At least `count`, with no practical upper limit. | +| Method | Accepted number of matches | +| ------------------ | ------------------------------------------------ | +| `Repeat(count)` | Exactly `count`. | +| `Repeat(min, max)` | From `min` to `max`, inclusive. | +| `AtLeast(count)` | At least `count`, with no practical upper limit. | ```csharp var threeDigits = Set("0-9").Repeat(3).Text().ThenIgnore(Eof); diff --git a/docs/working-with-values.md b/docs/working-with-values.md index 7d27e8d..bc22272 100644 --- a/docs/working-with-values.md +++ b/docs/working-with-values.md @@ -6,14 +6,14 @@ A parser can return a value, keep the source text, or only check that the input matches. These methods let you choose. The examples use the [standard imports](README.md#reading-the-examples). -| Method | Callback receives | Result | -| --- | --- | --- | -| `Do(value => ...)` | The parsed value. | The callback's return value. | -| `Where(value => ..., expected)` | The parsed value. | The same value, or a normal failure when the callback returns `false`. | -| `Map(match => ...)` | The matched segment. | The callback's return value. The original parsed value is thrown away. | -| `Map((match, value) => ...)` | Both the matched segment and the parsed value. | The callback's return value. | -| `Text()` | No callback needed. | The matched segment as a string. | -| `Void()` | No callback needed. | `Unit`. The value is thrown away. | +| Method | Callback receives | Result | +| ------------------------------- | ---------------------------------------------- | ---------------------------------------------------------------------- | +| `Do(value => ...)` | The parsed value. | The callback's return value. | +| `Where(value => ..., expected)` | The parsed value. | The same value, or a normal failure when the callback returns `false`. | +| `Map(match => ...)` | The matched segment. | The callback's return value. The original parsed value is thrown away. | +| `Map((match, value) => ...)` | Both the matched segment and the parsed value. | The callback's return value. | +| `Text()` | No callback needed. | The matched segment as a string. | +| `Void()` | No callback needed. | `Unit`. The value is thrown away. | The *matched segment* is the part of the input that a parser matched. @@ -94,12 +94,12 @@ Console.WriteLine(result.Value.Width); // 3 `Match` is a `readonly ref struct`. It gives a view of a part of the source without first making a string: -| Member | Meaning | -| --- | --- | -| `Index` | Zero-based start offset in the source passed to this parse. | -| `Length` | Number of UTF-16 code units in the segment. | -| `Value` | A `ReadOnlySpan` over the segment. | -| `ToString()` | Copies the segment into a string. | +| Member | Meaning | +| ------------ | ----------------------------------------------------------- | +| `Index` | Zero-based start offset in the source passed to this parse. | +| `Length` | Number of UTF-16 code units in the segment. | +| `Value` | A `ReadOnlySpan` over the segment. | +| `ToString()` | Copies the segment into a string. | `Match` can also be converted to `ReadOnlySpan` or deconstructed into `(index, length)`. You cannot store it in a normal heap object. If you need the From 56aed7f695e414c307a2af1f24c3c781336a862c Mon Sep 17 00:00:00 2001 From: rameel Date: Fri, 13 Nov 2026 17:14:49 +0500 Subject: [PATCH 10/10] docs: add a custom parser example that restores its position --- docs/custom-parsers.md | 66 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/docs/custom-parsers.md b/docs/custom-parsers.md index 3bd879c..5bf8bab 100644 --- a/docs/custom-parsers.md +++ b/docs/custom-parsers.md @@ -110,6 +110,72 @@ The rules for `TryParse`: - **Fatal error**: call `FatalError(message)` only when the parse must stop completely. A mismatch is a failure, not a fatal error. +## Call other parsers and go back on failure + +A custom parser can call other parsers with the same context. Each successful +call moves the position. So when a later step fails, the parser must go back to +where it started. Save a bookmark before the first step. + +This parser reads a 24-hour time such as `09:30`. It reuses `TwoDigitParser` +from the previous section and checks the range of the hours and minutes: + +```csharp +public sealed class ClockTimeParser : Parser +{ + private static readonly Parser TwoDigits = new TwoDigitParser(); + private static readonly Parser Colon = L(':'); + + public ClockTimeParser() : base("time") + { + } + + public override bool TryParse(ref ParseContext context, out TimeSpan value) + { + var start = context.BookmarkPosition(); + + if (TwoDigits.TryParse(ref context, out var hours) + && Colon.TryParse(ref context, out _) + && TwoDigits.TryParse(ref context, out var minutes) + && hours < 24 + && minutes < 60) + { + value = new TimeSpan(hours, minutes, 0); + context.SetMatched(start.Position, context.Position - start.Position); + return true; + } + + // Go back first, so that "time" is reported at the start of the input. + context.RestorePosition(start); + context.ReportExpected(Name); + value = default; + return false; + } +} +``` + +```csharp +var time = new ClockTimeParser().ThenIgnore(Eof); + +Console.WriteLine(time.Parse("09:30").Value); // 09:30:00 +Console.WriteLine(time.Parse("25:00").ErrorMessage); // (1:1) Expected time +Console.WriteLine(time.Parse("12-30").ErrorMessage); // (1:3) Expected ':' + +var text = new ClockTimeParser().Text(); +Console.WriteLine(text.Parse("09:30").Value); // 09:30 +``` + +Three details matter here: + +- Each inner parser marks its own segment. After the last call, the matched + segment holds only the minutes. `SetMatched` marks the whole time, so `Text()` + returns `09:30`. +- For `25:00`, the parser has already read all five characters when the range + check fails. `RestorePosition` moves back to the start and makes the matched + segment empty. +- The inner parsers record their own expectations, such as `':'` for `12-30`. + The message uses the furthest position, so that precise expectation wins over + `time` at the start. + ## Naming and void conversion The base class has two methods that other methods call: `ToNamedParser(string? name)`