Skip to content

Commit 1ee8683

Browse files
aibaarsaibaars
authored
Merge pull request #207 from aibaars/parser-improvements
Parser improvements
2 parents 0b107de + 6039244 commit 1ee8683

File tree

13 files changed

+360404
-308410
lines changed

13 files changed

+360404
-308410
lines changed

‎grammar.js‎

Lines changed: 78 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ const ALPHA_CHAR = /[^\x00-\x1F\s0-9:;`"'@$#.,|^&<=>+\-*/\\%?!~()\[\]{}]/;
3333

3434
module.exports = grammar({
3535
name: 'ruby',
36-
36+
inline: $ => [$._arg_rhs],
3737
externals: $ => [
3838
$._line_break,
3939

@@ -57,6 +57,7 @@ module.exports = grammar({
5757
$._block_ampersand,
5858
$._splat_star,
5959
$._unary_minus,
60+
$._unary_minus_num,
6061
$._binary_minus,
6162
$._binary_star,
6263
$._singleton_class_left_angle_left_langle,
@@ -175,6 +176,14 @@ module.exports = grammar({
175176
)
176177
),
177178

179+
rescue_modifier_expression: $ => prec(PREC.RESCUE,
180+
seq(
181+
field('body', $._expression),
182+
'rescue',
183+
field('handler', $._arg)
184+
)
185+
),
186+
178187
_body_expr: $ =>
179188
seq(
180189
'=',
@@ -199,7 +208,7 @@ module.exports = grammar({
199208
block_parameters: $ => seq(
200209
'|',
201210
seq(commaSep($._formal_parameter), optional(',')),
202-
optional(seq(';', sep1($.identifier, ','))), // Block shadow args e.g. {|; a, b| ...}
211+
optional(seq(';', sep1(field('locals', $.identifier), ','))), // Block shadow args e.g. {|; a, b| ...}
203212
'|'
204213
),
205214

@@ -443,6 +452,8 @@ module.exports = grammar({
443452
choice(
444453
alias($.identifier, $.hash_key_symbol),
445454
alias($.constant, $.hash_key_symbol),
455+
alias($.identifier_suffix, $.hash_key_symbol),
456+
alias($.constant_suffix, $.hash_key_symbol),
446457
$.string
447458
)
448459
),
@@ -498,6 +509,8 @@ module.exports = grammar({
498509
_pattern_literal: $ => choice(
499510
$._literal,
500511
$.string,
512+
$.subshell,
513+
$.heredoc_beginning,
501514
$.regex,
502515
$.string_array,
503516
$.symbol_array,
@@ -629,18 +642,24 @@ module.exports = grammar({
629642
),
630643

631644
_arg: $ => choice(
645+
alias($._unary_minus_pow, $.unary),
632646
$._primary,
633647
$.assignment,
634648
$.operator_assignment,
635649
$.conditional,
636650
$.range,
637651
$.binary,
638-
$.unary
652+
$.unary,
639653
),
640654

655+
_unary_minus_pow: $ => seq(field('operator', alias($._unary_minus_num, '-')), field('operand', alias($._pow, $.binary))),
656+
_pow: $ => prec.right(PREC.EXPONENTIAL, seq(field('left', $._simple_numeric), field('operator', alias($._binary_star_star, '**')), field('right', $._arg, $.binary))),
657+
641658
_primary: $ => choice(
642659
$.parenthesized_statements,
643660
$._lhs,
661+
alias($._function_identifier_call, $.call),
662+
$.call,
644663
$.array,
645664
$.string_array,
646665
$.symbol_array,
@@ -684,18 +703,18 @@ module.exports = grammar({
684703
']'
685704
)),
686705

687-
scope_resolution: $ => prec.left(1, seq(
706+
scope_resolution: $ => prec.left(PREC.CALL + 1, seq(
688707
choice(
689708
'::',
690709
seq(field('scope', $._primary), token.immediate('::'))
691710
),
692-
field('name', choice($.identifier, $.constant))
711+
field('name', $.constant)
693712
)),
694713

695714
_call: $ => prec.left(PREC.CALL, seq(
696715
field('receiver', $._primary),
697-
choice('.', '&.'),
698-
field('method', choice($.identifier, $.operator, $.constant, $.argument_list))
716+
field('operator', choice('.', '&.', token.immediate('::'))),
717+
field('method', choice($.identifier, $.operator, $.constant, $._function_identifier)),
699718
)),
700719

701720
command_call: $ => seq(
@@ -704,7 +723,7 @@ module.exports = grammar({
704723
$._chained_command_call,
705724
field('method', choice(
706725
$._variable,
707-
$.scope_resolution
726+
$._function_identifier
708727
)),
709728
),
710729
field('arguments', alias($.command_argument_list, $.argument_list))
@@ -713,7 +732,7 @@ module.exports = grammar({
713732
command_call_with_block: $ => {
714733
const receiver = choice(
715734
$._call,
716-
field('method', choice($._variable, $.scope_resolution))
735+
field('method', choice($._variable, $._function_identifier))
717736
)
718737
const arguments = field('arguments', alias($.command_argument_list, $.argument_list))
719738
const block = field('block', $.block)
@@ -726,25 +745,37 @@ module.exports = grammar({
726745

727746
_chained_command_call: $ => seq(
728747
field('receiver', alias($.command_call_with_block, $.call)),
729-
choice('.', '&.'),
730-
field('method', choice($.identifier, $.operator, $.constant, $.argument_list))
748+
field('operator', choice('.', '&.', token.immediate('::'))),
749+
field('method', choice($.identifier, $._function_identifier, $.operator, $.constant)),
731750
),
732751

733752
call: $ => {
734753
const receiver = choice(
735754
$._call,
736755
field('method', choice(
737-
$._variable,
738-
$.scope_resolution
756+
$._variable, $._function_identifier
739757
))
740758
)
759+
741760
const arguments = field('arguments', $.argument_list)
761+
const receiver_arguments =
762+
seq(
763+
choice(
764+
receiver,
765+
prec.left(PREC.CALL, seq(
766+
field('receiver', $._primary),
767+
field('operator', choice('.', '&.', token.immediate('::')))
768+
))
769+
),
770+
arguments
771+
)
772+
742773
const block = field('block', $.block)
743774
const doBlock = field('block', $.do_block)
744775
return choice(
745-
seq(receiver, arguments),
746-
seq(receiver, prec(PREC.CURLY_BLOCK, seq(arguments, block))),
747-
seq(receiver, prec(PREC.DO_BLOCK, seq(arguments, doBlock))),
776+
receiver_arguments,
777+
prec(PREC.CURLY_BLOCK, seq(receiver_arguments, block)),
778+
prec(PREC.DO_BLOCK, seq(receiver_arguments, doBlock)),
748779
prec(PREC.CURLY_BLOCK, seq(receiver, block)),
749780
prec(PREC.DO_BLOCK, seq(receiver, doBlock))
750781
)
@@ -794,36 +825,37 @@ module.exports = grammar({
794825
'}'
795826
)),
796827

828+
_arg_rhs: $ => choice($._arg, alias($.rescue_modifier_arg, $.rescue_modifier)),
797829
assignment: $ => prec.right(PREC.ASSIGN, choice(
798830
seq(
799831
field('left', choice($._lhs, $.left_assignment_list)),
800832
'=',
801833
field('right', choice(
802-
$._arg,
834+
$._arg_rhs,
803835
$.splat_argument,
804836
$.right_assignment_list
805837
))
806838
)
807839
)),
808840

809-
command_assignment: $ => prec.right(PREC.ASSIGN, choice(
841+
command_assignment: $ => prec.right(PREC.ASSIGN,
810842
seq(
811843
field('left', choice($._lhs, $.left_assignment_list)),
812844
'=',
813-
field('right', $._expression)
845+
field('right', choice($._expression, alias($.rescue_modifier_expression, $.rescue_modifier)))
814846
)
815-
)),
847+
),
816848

817849
operator_assignment: $ => prec.right(PREC.ASSIGN, seq(
818850
field('left', $._lhs),
819851
field('operator', choice('+=', '-=', '*=', '**=', '/=', '||=', '|=', '&&=', '&=', '%=', '>>=', '<<=', '^=')),
820-
field('right', $._arg)
852+
field('right', $._arg_rhs)
821853
)),
822854

823855
command_operator_assignment: $ => prec.right(PREC.ASSIGN, seq(
824856
field('left', $._lhs),
825857
field('operator', choice('+=', '-=', '*=', '**=', '/=', '||=', '|=', '&&=', '&=', '%=', '>>=', '<<=', '^=')),
826-
field('right', $._expression)
858+
field('right', choice($._expression, alias($.rescue_modifier_expression, $.rescue_modifier)))
827859
)),
828860

829861
conditional: $ => prec.right(PREC.CONDITIONAL, seq(
@@ -850,7 +882,7 @@ module.exports = grammar({
850882
[prec.left, PREC.AND, 'and'],
851883
[prec.left, PREC.OR, 'or'],
852884
[prec.left, PREC.BOOLEAN_OR, '||'],
853-
[prec.left, PREC.BOOLEAN_OR, '&&'],
885+
[prec.left, PREC.BOOLEAN_AND, '&&'],
854886
[prec.left, PREC.SHIFT, choice('<<', '>>')],
855887
[prec.left, PREC.COMPARISON, choice('<', '<=', '>', '>=')],
856888
[prec.left, PREC.BITWISE_AND, '&'],
@@ -878,7 +910,7 @@ module.exports = grammar({
878910
const operators = [
879911
[prec, PREC.DEFINED, 'defined?'],
880912
[prec.right, PREC.NOT, 'not'],
881-
[prec.right, PREC.UNARY_MINUS, choice(alias($._unary_minus, '-'), '+')],
913+
[prec.right, PREC.UNARY_MINUS, choice(alias($._unary_minus, '-'), alias($._binary_minus, '-'), '+')],
882914
[prec.right, PREC.COMPLEMENT, choice('!', '~')]
883915
];
884916
return choice(...operators.map(([fn, precedence, operator]) => fn(precedence, seq(
@@ -906,7 +938,7 @@ module.exports = grammar({
906938
)),
907939

908940
unary_literal: $ => prec.right(PREC.UNARY_MINUS, seq(
909-
field('operator', choice(alias($._unary_minus, '-'), '+')),
941+
field('operator', choice(alias($._unary_minus_num, '-'), '+')),
910942
field('operand', $._simple_numeric)
911943
)),
912944

@@ -940,6 +972,8 @@ module.exports = grammar({
940972

941973
rest_assignment: $ => prec(-1, seq('*', optional($._lhs))),
942974

975+
_function_identifier: $ => choice(alias($.identifier_suffix, $.identifier), alias($.constant_suffix, $.constant)),
976+
_function_identifier_call: $ => prec.right(field('method', $._function_identifier)),
943977
_lhs: $ => prec.left(choice(
944978
$._variable,
945979
$.true,
@@ -948,7 +982,6 @@ module.exports = grammar({
948982
$.scope_resolution,
949983
$.element_reference,
950984
alias($._call, $.call),
951-
$.call
952985
)),
953986

954987
_variable: $ => prec.right(choice(
@@ -966,6 +999,7 @@ module.exports = grammar({
966999

9671000
_method_name: $ => choice(
9681001
$.identifier,
1002+
$._function_identifier,
9691003
$.constant,
9701004
$.setter,
9711005
$.simple_symbol,
@@ -999,23 +1033,28 @@ module.exports = grammar({
9991033
/=e[^n]/,
10001034
/=en[^d]/
10011035
)),
1002-
/=end/
1036+
/=end.*/
10031037
)
10041038
))),
10051039

10061040
integer: $ => /0[bB][01](_?[01])*|0[oO]?[0-7](_?[0-7])*|(0[dD])?\d(_?\d)*|0[xX][0-9a-fA-F](_?[0-9a-fA-F])*/,
1007-
1041+
_int_or_float: $ => choice($.integer, $.float),
10081042
float: $ => /\d(_?\d)*(\.\d)?(_?\d)*([eE][\+-]?\d(_?\d)*)?/,
1009-
complex: $ => /(\d+)?(\+|-)?(\d+)i/,
1010-
rational: $ => seq(choice($.integer, $.float), 'r'),
1043+
complex: $ => choice(
1044+
seq($._int_or_float, token.immediate('i')),
1045+
seq(alias($._int_or_float, $.rational), token.immediate('ri')),
1046+
),
1047+
rational: $ => seq($._int_or_float, token.immediate('r')),
10111048
super: $ => 'super',
10121049
self: $ => 'self',
1013-
true: $ => token(choice('true', 'TRUE')),
1014-
false: $ => token(choice('false', 'FALSE')),
1015-
nil: $ => token(choice('nil', 'NIL')),
1016-
1017-
constant: $ => token(seq(/[A-Z]/, IDENTIFIER_CHARS, /(\?|\!)?/)),
1018-
identifier: $ => token(seq(LOWER_ALPHA_CHAR, IDENTIFIER_CHARS, /(\?|\!)?/)),
1050+
true: $ => 'true',
1051+
false: $ => 'false',
1052+
nil: $ => 'nil',
1053+
1054+
constant: $ => token(seq(/[A-Z]/, IDENTIFIER_CHARS)),
1055+
constant_suffix: $ => token(seq(/[A-Z]/, IDENTIFIER_CHARS, /[?!]/)),
1056+
identifier: $ => token(seq(LOWER_ALPHA_CHAR, IDENTIFIER_CHARS)),
1057+
identifier_suffix: $ => token(seq(LOWER_ALPHA_CHAR, IDENTIFIER_CHARS, /[?!]/)),
10191058
instance_variable: $ => token(seq('@', ALPHA_CHAR, IDENTIFIER_CHARS)),
10201059
class_variable: $ => token(seq('@@', ALPHA_CHAR, IDENTIFIER_CHARS)),
10211060

@@ -1129,7 +1168,9 @@ module.exports = grammar({
11291168
field('key', choice(
11301169
$.hash_key_symbol,
11311170
alias($.identifier, $.hash_key_symbol),
1132-
alias($.constant, $.hash_key_symbol)
1171+
alias($.constant, $.hash_key_symbol),
1172+
alias($.identifier_suffix, $.hash_key_symbol),
1173+
alias($.constant_suffix, $.hash_key_symbol),
11331174
)),
11341175
token.immediate(':'),
11351176
field('value', optional($._arg))

‎script/known_failures.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,7 @@ examples/ruby_spec/command_line/fixtures/freeze_flag_required_diff_enc.rb
33
examples/ruby_spec/core/enumerable/shared/inject.rb
44
examples/ruby_spec/language/fixtures/freeze_magic_comment_required_diff_enc.rb
55
examples/ruby_spec/language/string_spec.rb
6+
examples/ruby_spec/core/proc/ruby2_keywords_spec.rb
7+
examples/ruby_spec/language/fixtures/utf16-be-nobom.rb
8+
examples/ruby_spec/language/fixtures/utf16-le-nobom.rb
9+
examples/ruby_spec/language/lambda_spec.rb

‎script/parse-examples‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,19 @@ function checkout_at() {
1515
popd
1616
}
1717

18-
checkout_at "examples/ruby_spec" "ruby/spec" "c3e6b9017926f44a76e2b966c4dd35fa84c4cd3b"
18+
checkout_at "examples/ruby_spec" "ruby/spec" "aaf998fb8c92c4e63ad423a2e7ca6e6921818c6e"
1919

2020
# TODO: Fix these known issues:
2121
# - [ ] String literals delimited with `=`, e.g. `%=hi=`
2222
# - [ ] Issue with << operator mistaken for heredocs, e.g. `send(@method){|r,i| r<<i}`
2323
# - [ ] defined as local var, e.g. `defn.send(@method, defined)`
2424
# - [ ] Unicode character in symbols, variables, etc, e.g. `:êad`
2525
# - [ ] Unicode characters in constants, e.g. `CS_CONSTλ = :const_unicode`
26+
# - [ ] Keyword arguments in lambda `-> a: { }`
27+
# - [ ] Lambda with call as default argument `-> a=a() { a }`
2628
known_failures="$(cat script/known_failures.txt)"
2729

28-
tree-sitter parse -q \
30+
node_modules/tree-sitter-cli/cli.js parse -q \
2931
'examples/**/*.rb' \
3032
$(for file in $known_failures; do echo "!${file}"; done)
3133

0 commit comments

Comments
 (0)