diff --git a/lib/spitfire.ex b/lib/spitfire.ex index e680f0b..fa27ac8 100644 --- a/lib/spitfire.ex +++ b/lib/spitfire.ex @@ -319,6 +319,8 @@ defmodule Spitfire do defp do_parse_expression(parser, {associativity, precedence}, is_list, is_map, is_top) do stop_before_stab_op? = Map.get(parser, :stop_before_stab_op?, false) stop_before_map_op? = Map.get(parser, :stop_before_map_op?, false) + stop_before_when_op? = Map.get(parser, :stop_before_when_op?, false) + prefix_start = current_meta(parser) prefix = case current_token_type(parser) do @@ -371,6 +373,8 @@ defmodule Spitfire do case prefix do {left, parser} -> + parser = set_expression_start(parser, prefix_start) + terminals = if is_top do @terminals @@ -382,6 +386,7 @@ defmodule Spitfire do if is_valid do while (not stab_state_set?(parser) and not MapSet.member?(terminals, peek_token(parser))) && + (not stop_before_when_op? or peek_token_type(parser) != :when_op) && (current_token(parser) != :do and peek_token(parser) != :eol) && (not stop_before_map_op? or (peek_token_type(parser) != :assoc_op and @@ -496,6 +501,8 @@ defmodule Spitfire do nil -> meta = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) ctype = current_token_type(parser) parser = put_error(parser, {meta, "unknown token: #{ctype}"}) @@ -509,7 +516,7 @@ defmodule Spitfire do _ -> next_token(parser) end - {{:__block__, [{:error, true} | meta], []}, parser} + {{:__block__, [{:error, true}, {:range, range} | meta], []}, parser} end end @@ -520,15 +527,19 @@ defmodule Spitfire do cond do peek_token(parser) == :eof -> meta = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) parser = put_error(parser, {meta, "missing closing parentheses"}) - {{:__block__, [{:error, true} | meta], []}, next_token(parser)} + {{:__block__, [{:error, true}, {:range, range} | meta], []}, next_token(parser)} peek_token(parser) == :")" -> parser = parser |> next_token() |> eat_eoe() closing_paren_meta = current_meta(parser) - {{:__block__, [parens: opening_paren_meta ++ [closing: closing_paren_meta]], []}, parser} + closing_paren_end_meta = current_end_meta(parser) + range = build_range(opening_paren_meta, closing_paren_end_meta) + {{:__block__, [parens: opening_paren_meta ++ [closing: closing_paren_meta], range: range], []}, parser} true -> orig_meta = current_meta(parser) @@ -546,12 +557,14 @@ defmodule Spitfire do if current_token(parser) == :")" do closing_paren_meta = current_meta(parser) + closing_paren_end_meta = current_end_meta(parser) parser = Map.put(parser, :nesting, old_nesting) + range = build_range(opening_paren_meta, closing_paren_end_meta) if has_semicolon do - {{:__block__, [{:closing, closing_paren_meta} | opening_paren_meta], []}, parser} + {{:__block__, [{:closing, closing_paren_meta}, {:range, range} | opening_paren_meta], []}, parser} else - {{:__block__, [parens: opening_paren_meta ++ [closing: closing_paren_meta]], []}, parser} + {{:__block__, [parens: opening_paren_meta ++ [closing: closing_paren_meta], range: range], []}, parser} end else # Stop at -> to allow stab expressions @@ -576,12 +589,14 @@ defmodule Spitfire do |> eat_eoe() closing_paren_meta = current_meta(parser) + closing_paren_end_meta = current_end_meta(parser) ast = case expression do # unquote_splicing with one arg is wrapped in a block {:unquote_splicing, _, [_]} -> - {:__block__, [{:closing, current_meta(parser)} | orig_meta], [expression]} + range = build_range(orig_meta, closing_paren_end_meta) + {:__block__, [{:closing, current_meta(parser)}, {:range, range} | orig_meta], [expression]} # not/! with one arg is wrapped in a block {op, _, [_]} when op in [:not, :!] -> @@ -591,18 +606,25 @@ defmodule Spitfire do [expression] {f, meta, a} -> - {f, [parens: opening_paren_meta ++ [closing: closing_paren_meta]] ++ meta, a} + range = build_range(orig_meta, closing_paren_end_meta) + meta_without_range = Keyword.delete(meta, :range) + + {f, + [parens: opening_paren_meta ++ [closing: closing_paren_meta], range: range] ++ meta_without_range, + a} [{key, _value} | _] = kw when is_atom(key) -> if Map.get(parser, :stop_before_stab_op?, false) do - {:__block__, [parens: opening_paren_meta ++ [closing: closing_paren_meta]], [kw]} + range = build_range(orig_meta, closing_paren_end_meta) + {:__block__, [parens: opening_paren_meta ++ [closing: closing_paren_meta], range: range], [kw]} else kw end [{{_, _, _}, _value} | _] = kw -> if Map.get(parser, :stop_before_stab_op?, false) do - {:__block__, [parens: opening_paren_meta ++ [closing: closing_paren_meta]], [kw]} + range = build_range(orig_meta, closing_paren_end_meta) + {:__block__, [parens: opening_paren_meta ++ [closing: closing_paren_meta], range: range], [kw]} else kw end @@ -701,41 +723,50 @@ defmodule Spitfire do exprs _ -> - {:__block__, [{:closing, current_meta(parser)} | orig_meta], exprs} + closing = current_meta(parser) + closing_end = current_end_meta(parser) + range = build_range(orig_meta, closing_end) + {:__block__, [{:closing, closing}, {:range, range} | orig_meta], exprs} end {ast, parser} else if invalid_grouped_stab_trailing_comma? and peek_token(parser) == :"," do meta = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) parser = parser |> put_error({meta, "syntax error"}) |> Map.put(:nesting, old_nesting) - {{:__block__, [{:error, true} | meta], []}, next_token(parser)} + {{:__block__, [{:error, true}, {:range, range} | meta], []}, next_token(parser)} else meta = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) parser = parser |> put_error({meta, "missing closing parentheses"}) |> Map.put(:nesting, old_nesting) - {{:__block__, [{:error, true} | meta], []}, next_token(parser)} + {{:__block__, [{:error, true}, {:range, range} | meta], []}, next_token(parser)} end end true -> meta = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) parser = parser |> put_error({meta, "missing closing parentheses"}) |> Map.put(:nesting, old_nesting) - {{:__block__, [{:error, true} | meta], []}, next_token(parser)} + {{:__block__, [{:error, true}, {:range, range} | meta], []}, next_token(parser)} end end end @@ -744,16 +775,18 @@ defmodule Spitfire do defp parse_unexpected_semicolon(parser) do meta = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) parser = put_error(parser, {meta, "unexpected token: ;"}) parser = parser |> next_token() |> eat_eol() case current_token_type(parser) do type when type in [:eof, :end, :block_identifier, :")", :"]", :"}", :">>"] -> - {{:__block__, [{:error, true} | meta], []}, parser} + {{:__block__, [{:error, true}, {:range, range} | meta], []}, parser} _ -> {expr, parser} = parse_expression(parser, @lowest, false, false, true) - {{:__block__, [{:error, true} | meta], [expr]}, parser} + {{:__block__, [{:error, true}, {:range, range} | meta], [expr]}, parser} end end @@ -809,7 +842,16 @@ defmodule Spitfire do token = encode_literal(parser, token, meta) parser = parser |> next_token() |> eat_eoe() - {value, parser} = parse_expression(parser, @list_comma, false, false, false) + {value, parser} = + if Map.get(parser, :parsing_map_key, false) do + with_context(parser, %{stop_before_map_op?: true, stop_before_when_op?: false}, fn parser -> + parse_expression(parser, @list_comma, false, false, false) + end) + else + with_context(parser, %{stop_before_when_op?: false}, fn parser -> + parse_expression(parser, @list_comma, false, false, false) + end) + end {kvs, parser} = if stab_state_set?(parser), do: {[], parser}, else: parse_kw_list_continuation(parser) @@ -1029,7 +1071,9 @@ defmodule Spitfire do {rhs, parser} end - ast = {token, meta, [rhs]} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + ast = {token, [{:range, range} | meta], [rhs]} {ast, parser} end @@ -1077,7 +1121,9 @@ defmodule Spitfire do parser = parser |> next_token() |> eat_eoe() parser = Map.delete(parser, :inside_map_update_pairs) {rhs, parser} = parse_struct_type(parser) - {{token, meta, [rhs]}, parser} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + {{token, [{:range, range} | meta], [rhs]}, parser} end end @@ -1092,8 +1138,10 @@ defmodule Spitfire do second_meta = [line: first_meta[:line], column: first_meta[:column] + 1] {rhs, parser} = parse_lone_identifier(parser) + end_meta = current_end_meta(parser) + range = build_range(second_meta, end_meta) - ast = {:/, second_meta, [first_slash, rhs]} + ast = {:/, [{:range, range} | second_meta], [first_slash, rhs]} {ast, parser} end end @@ -1105,8 +1153,10 @@ defmodule Spitfire do parser = parser |> next_token() |> eat_eoe() {rhs, parser} = parse_lone_identifier(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) - ast = {:..., meta, [rhs]} + ast = {:..., [{:range, range} | meta], [rhs]} {ast, parser} end end @@ -1119,8 +1169,10 @@ defmodule Spitfire do {encoder, parser} = Map.pop(parser, :literal_encoder) {rhs, parser} = parse_int(parser) parser = Map.put(parser, :literal_encoder, encoder) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) - ast = {token, meta, [rhs]} + ast = {token, [{:range, range} | meta], [rhs]} {ast, parser} end @@ -1139,6 +1191,7 @@ defmodule Spitfire do trace "parse_stab_expression", trace_meta(parser) do token = current_token(parser) meta = current_meta(parser) + arrow_meta = meta newlines = case current_newlines(parser) do @@ -1166,8 +1219,17 @@ defmodule Spitfire do rhs = build_stab_rhs(exprs) + end_meta = + if exprs == [] do + [end_line: arrow_meta[:line], end_column: arrow_meta[:column] + 2] + else + current_end_meta(parser) + end + + range = build_range(meta, end_meta) + ast = - {token, newlines ++ meta, [[], rhs]} + {token, newlines ++ [{:range, range} | meta], [[], rhs]} parser = Map.put(parser, :nesting, old_nesting) @@ -1206,6 +1268,7 @@ defmodule Spitfire do :-> -> token = current_token(parser) meta = current_meta(parser) + arrow_meta = meta newlines = case current_newlines(parser) do @@ -1318,8 +1381,23 @@ defmodule Spitfire do [lhs] end + range_start = + case ast_start_meta(lhs) do + [] -> arrow_meta + start_meta -> start_meta + end + + end_meta = + if exprs == [] and not has_leading_semicolon do + [end_line: arrow_meta[:line], end_column: arrow_meta[:column] + 2] + else + current_end_meta(parser) + end + + range = build_range(range_start, end_meta) + ast = - {token, meta, [lhs, rhs]} + {token, [{:range, range} | meta], [lhs, rhs]} parser = Map.put(parser, :nesting, old_nesting) @@ -1358,10 +1436,21 @@ defmodule Spitfire do defp parse_comma(parser, lhs) do trace "parse_comma", trace_meta(parser) do + lhs_start = expression_start(parser) + precedence = if Map.get(parser, :stop_before_stab_op?, false), do: @list_comma, else: @comma + parser = parser |> next_token() |> eat_eoe() - {exprs, parser} = parse_comma_list(parser, @comma) - {{:comma, [], [lhs | exprs]}, eat_eoe(parser)} + {exprs, parser} = + if Map.get(parser, :stop_before_stab_op?, false) do + with_context(parser, %{stop_before_when_op?: true}, fn parser -> + parse_comma_list(parser, precedence) + end) + else + parse_comma_list(parser, precedence) + end + + {{:comma, [], [lhs | exprs]}, set_expression_start(eat_eoe(parser), lhs_start)} end end @@ -1370,6 +1459,7 @@ defmodule Spitfire do token = current_token(parser) meta = current_meta(parser) precedence = current_precedence(parser) + lhs_start = expression_start(parser) effective_precedence = if Map.get(parser, :parsing_map_key, false) do @@ -1416,7 +1506,9 @@ defmodule Spitfire do case rhs do {:__block__, [{:error, true} | _], []} -> parser = put_error(pre_parser, {meta, "malformed right-hand side of #{token} operator"}) - {{:__block__, [{:error, true} | meta], []}, parser} + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + {{:__block__, [{:error, true}, {:range, range} | meta], []}, parser} _ -> {rhs, parser} @@ -1433,6 +1525,9 @@ defmodule Spitfire do parser end + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + ast = case token do :"//" -> @@ -1441,55 +1536,55 @@ defmodule Spitfire do {:..//, lhs_meta, [left, middle, rhs]} _ -> - {token, newlines ++ meta, [lhs, rhs]} + {token, newlines ++ [{:range, range} | meta], [lhs, rhs]} end :"not in" -> - {:not, meta, [{:in, meta, [lhs, rhs]}]} + {:not, [{:range, range} | meta], [{:in, [{:range, range} | meta], [lhs, rhs]}]} :in -> case lhs do {op, _meta, [inner]} when op in [:!, :not] -> - in_ast = {:in, meta, [inner, rhs]} - {op, meta, [in_ast]} + in_ast = {:in, [{:range, range} | meta], [inner, rhs]} + {op, [{:range, range} | meta], [in_ast]} _ -> - {token, newlines ++ meta, [lhs, rhs]} + {token, newlines ++ [{:range, range} | meta], [lhs, rhs]} end :when -> case lhs do {:__block__, [{:parens, _} = paren_meta | _], []} -> # () when ... - empty parens, keep parens meta - {token, [paren_meta | newlines ++ meta], [rhs]} + {token, [paren_meta | newlines ++ [{:range, range} | meta]], [rhs]} {:__block__, _, []} -> # Empty block without parens - {token, newlines ++ meta, [rhs]} + {token, newlines ++ [{:range, range} | meta], [rhs]} {:__block__, [{:parens, _} = paren_meta | _], [[{key, _} | _] = kw]} when is_atom(key) -> # (a: 1) when ... - preserve parens meta for stab - {token, [paren_meta | newlines ++ meta], [kw, rhs]} + {token, [paren_meta | newlines ++ [{:range, range} | meta]], [kw, rhs]} {:__block__, [{:parens, _} = paren_meta | _], [[{{_, _, _}, _} | _] = kw]} -> # Parenthesized kw list with interpolated key - {token, [paren_meta | newlines ++ meta], [kw, rhs]} + {token, [paren_meta | newlines ++ [{:range, range} | meta]], [kw, rhs]} {:comma, [{:parens, _} = paren_meta | _], args} -> - {token, [paren_meta | newlines ++ meta], args ++ [rhs]} + {token, [paren_meta | newlines ++ [{:range, range} | meta]], args ++ [rhs]} {:comma, _, args} -> - {token, newlines ++ meta, args ++ [rhs]} + {token, newlines ++ [{:range, range} | meta], args ++ [rhs]} _ -> - {token, newlines ++ meta, [lhs, rhs]} + {token, newlines ++ [{:range, range} | meta], [lhs, rhs]} end _ -> - {token, newlines ++ meta, [lhs, rhs]} + {token, newlines ++ [{:range, range} | meta], [lhs, rhs]} end - {ast, parser} + {ast, set_expression_start(parser, lhs_start)} end end @@ -1497,6 +1592,7 @@ defmodule Spitfire do trace "parse_pipe_op_in_map", trace_meta(parser) do token = current_token(parser) meta = current_meta(parser) + lhs_start = expression_start(parser) newlines = case current_newlines(parser) || peek_newlines(parser, :eol) do @@ -1506,7 +1602,7 @@ defmodule Spitfire do rhs_parser = parser |> next_token() |> eat_eoe() - if rhs_has_binding_op?(rhs_parser) or + if (rhs_has_binding_op?(rhs_parser) and not rhs_has_assoc_op?(rhs_parser)) or (unmatched_expr?(lhs) and rhs_has_bare_comma?(rhs_parser)) do # When the RHS of `|` has low-precedence operators (::, when, <-, \\) or # the LHS is an unmatched_expr (do-end) and the RHS has no-parens commas, @@ -1514,8 +1610,15 @@ defmodule Spitfire do parse_infix_expression(parser, lhs) else {pairs, pairs_parser} = parse_map_update_pairs(rhs_parser) - ast = {token, newlines ++ meta, [lhs, pairs]} - {ast, pairs_parser} + + if low_precedence_assoc_pair?(pairs) do + parse_infix_expression(parser, lhs) + else + end_meta = current_end_meta(pairs_parser) + range = build_range(lhs_start, end_meta) + ast = {token, newlines ++ [{:range, range} | meta], [lhs, pairs]} + {ast, set_expression_start(pairs_parser, lhs_start)} + end end end end @@ -1525,14 +1628,21 @@ defmodule Spitfire do @low_prec_map_op_types MapSet.new([:type_op, :when_op, :in_match_op]) defp rhs_has_binding_op?(parser) do - scan_binding_op(eat_eoe(parser), 0) + scan_binding_op(eat_eoe(parser), 0, false) end - defp scan_binding_op(parser, nesting) do + defp rhs_has_assoc_op?(parser) do + scan_assoc_op(eat_eoe(parser), 0) + end + + defp scan_binding_op(parser, nesting, saw_low_prec_op?) do token = peek_token(parser) token_type = peek_token_type(parser) cond do + token_type == :type_op and nesting == 0 -> + scan_binding_op(next_token(parser), nesting, saw_low_prec_op?) + MapSet.member?(@low_prec_map_op_types, token_type) and nesting == 0 -> true @@ -1542,6 +1652,53 @@ defmodule Spitfire do token_type in [:kw_identifier, :kw_identifier_safe, :kw_identifier_unsafe] and nesting == 0 -> false + token == :"}" and nesting == 0 -> + saw_low_prec_op? + + token == :"," and nesting == 0 -> + saw_low_prec_op? + + token == :eof -> + saw_low_prec_op? + + token in [:"(", :"[", :"{", :"<<"] -> + scan_binding_op(next_token(parser), nesting + 1, saw_low_prec_op?) + + token in [:")", :"]", :"}", :">>"] -> + scan_binding_op(next_token(parser), max(nesting - 1, 0), saw_low_prec_op?) + + token == :do -> + skip_do_end_for_binding_op(next_token(parser), 1, nesting, saw_low_prec_op?) + + true -> + scan_binding_op(next_token(parser), nesting, saw_low_prec_op?) + end + end + + defp skip_do_end_for_binding_op(parser, 0, nesting, saw_low_prec_op?) do + scan_binding_op(parser, nesting, saw_low_prec_op?) + end + + defp skip_do_end_for_binding_op(parser, depth, nesting, saw_low_prec_op?) do + case peek_token(parser) do + :end -> skip_do_end_for_binding_op(next_token(parser), depth - 1, nesting, saw_low_prec_op?) + :do -> skip_do_end_for_binding_op(next_token(parser), depth + 1, nesting, saw_low_prec_op?) + :eof -> saw_low_prec_op? + _ -> skip_do_end_for_binding_op(next_token(parser), depth, nesting, saw_low_prec_op?) + end + end + + defp scan_assoc_op(parser, nesting) do + token = peek_token(parser) + token_type = peek_token_type(parser) + + cond do + token_type == :assoc_op and nesting == 0 -> + true + + token_type in [:kw_identifier, :kw_identifier_safe, :kw_identifier_unsafe] and nesting == 0 -> + false + token == :"}" and nesting == 0 -> false @@ -1552,32 +1709,44 @@ defmodule Spitfire do false token in [:"(", :"[", :"{", :"<<"] -> - scan_binding_op(next_token(parser), nesting + 1) + scan_assoc_op(next_token(parser), nesting + 1) token in [:")", :"]", :"}", :">>"] -> - scan_binding_op(next_token(parser), max(nesting - 1, 0)) + scan_assoc_op(next_token(parser), max(nesting - 1, 0)) token == :do -> - skip_do_end_for_binding_op(next_token(parser), 1, nesting) + skip_do_end_for_assoc_op(next_token(parser), 1, nesting) true -> - scan_binding_op(next_token(parser), nesting) + scan_assoc_op(next_token(parser), nesting) end end - defp skip_do_end_for_binding_op(parser, 0, nesting) do - scan_binding_op(parser, nesting) + defp skip_do_end_for_assoc_op(parser, 0, nesting) do + scan_assoc_op(parser, nesting) end - defp skip_do_end_for_binding_op(parser, depth, nesting) do + defp skip_do_end_for_assoc_op(parser, depth, nesting) do case peek_token(parser) do - :end -> skip_do_end_for_binding_op(next_token(parser), depth - 1, nesting) - :do -> skip_do_end_for_binding_op(next_token(parser), depth + 1, nesting) + :end -> skip_do_end_for_assoc_op(next_token(parser), depth - 1, nesting) + :do -> skip_do_end_for_assoc_op(next_token(parser), depth + 1, nesting) :eof -> false - _ -> skip_do_end_for_binding_op(next_token(parser), depth, nesting) + _ -> skip_do_end_for_assoc_op(next_token(parser), depth, nesting) end end + @low_prec_assoc_ops MapSet.new([:"::", :when, :<-, :\\]) + + defp low_precedence_assoc_pair?(pairs) do + Enum.any?(pairs, fn + {{op, meta, _args}, _value} when is_list(meta) -> + Keyword.has_key?(meta, :assoc) and MapSet.member?(@low_prec_assoc_ops, op) + + _ -> + false + end) + end + defp rhs_has_bare_comma?(parser) do rhs_scan_comma_before_assoc(eat_eoe(parser), 0, false) end @@ -1692,6 +1861,8 @@ defmodule Spitfire do defp parse_access_expression(parser, lhs) do trace "parse_access_expression", trace_meta(parser) do meta = current_meta(parser) + lhs_start = ast_start_meta(lhs) + lhs_start = if lhs_start == [], do: expression_start(parser), else: lhs_start # Capture newlines before eating them newlines = @@ -1716,11 +1887,13 @@ defmodule Spitfire do end closing = current_meta(parser) - meta = extra_meta ++ newlines ++ [{:closing, closing} | meta] + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + meta = extra_meta ++ newlines ++ [{:closing, closing}, {:range, range} | meta] ast = {{:., meta, [Access, :get]}, meta, [lhs, rhs]} - {ast, parser} + {ast, set_expression_start(parser, lhs_start)} end end @@ -1728,7 +1901,9 @@ defmodule Spitfire do trace "parse_range_expression", trace_meta(parser) do token = current_token(parser) meta = current_meta(parser) - {{token, meta, []}, parser} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + {{token, [{:range, range} | meta], []}, parser} end end @@ -1736,6 +1911,7 @@ defmodule Spitfire do trace "parse_range_expression (with lhs)", trace_meta(parser) do token = current_token(parser) meta = current_meta(parser) + lhs_start = expression_start(parser) precedence = current_precedence(parser) newlines = @@ -1754,10 +1930,16 @@ defmodule Spitfire do if not inside_range and peek_token_skip_eol(parser) == :ternary_op do parser = parser |> eat_eoe_at(1) |> next_token() |> next_token() |> eat_eoe() {rrhs, parser} = parse_expression(parser, precedence, false, false, false) + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) - {{:..//, newlines ++ meta, [lhs, rhs, rrhs]}, eat_eoe(parser)} + {{:..//, newlines ++ [{:range, range} | meta], [lhs, rhs, rrhs]}, + set_expression_start(eat_eoe(parser), lhs_start)} else - {{token, newlines ++ meta, [lhs, rhs]}, eat_eoe(parser)} + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + + {{token, newlines ++ [{:range, range} | meta], [lhs, rhs]}, set_expression_start(eat_eoe(parser), lhs_start)} end end end @@ -1822,7 +2004,10 @@ defmodule Spitfire do {rhs, parser} end - ast = {:/, second_meta, [first_slash, rhs]} + end_meta = current_end_meta(parser) + range = build_range(second_meta, end_meta) + + ast = {:/, [{:range, range} | second_meta], [first_slash, rhs]} {ast, parser} end end @@ -1864,6 +2049,8 @@ defmodule Spitfire do defp parse_do_block(%{current_token: {:do, meta}} = parser, lhs) do trace "parse_do_block", trace_meta(parser) do do_meta = current_meta(parser) + lhs_start = ast_start_meta(lhs) + lhs_start = if lhs_start == [], do: expression_start(parser), else: lhs_start type = encode_literal(parser, :do, meta) old_nesting = parser.nesting @@ -1941,12 +2128,12 @@ defmodule Spitfire do [] end - {parser, end_meta} = + {parser, end_meta, end_position_meta} = if peek_token_eat_eoe(parser) == :end do parser = parser |> next_token() |> eat_eoe() - {parser, current_meta(parser)} + {parser, current_meta(parser), current_end_meta(parser)} else - {put_error(parser, {do_meta, "missing `end` for do block"}), do_meta} + {put_error(parser, {do_meta, "missing `end` for do block"}), do_meta, do_meta} end exprs = exprs ++ extra_exprs @@ -1965,16 +2152,19 @@ defmodule Spitfire do ast = case lhs do {token, meta, nil} -> - {token, [do: do_meta, end: end_meta] ++ meta, [exprs]} + range = build_range(lhs_start, end_position_meta) + meta = Keyword.delete(meta, :range) + {token, [do: do_meta, end: end_meta, range: range] ++ meta, [exprs]} {token, meta, args} when is_list(args) -> # Remove no_parens when attaching do-block - meta = Keyword.delete(meta, :no_parens) - {token, [do: do_meta, end: end_meta] ++ meta, args ++ [exprs]} + meta = Keyword.drop(meta, [:no_parens, :range]) + range = build_range(lhs_start, end_position_meta) + {token, [do: do_meta, end: end_meta, range: range] ++ meta, args ++ [exprs]} end parser = Map.put(parser, :nesting, old_nesting) - {ast, parser} + {ast, set_expression_start(parser, lhs_start)} end end @@ -1983,6 +2173,8 @@ defmodule Spitfire do token = current_token(parser) precedence = current_precedence(parser) meta = current_meta(parser) + lhs_start = ast_start_meta(lhs) + lhs_start = if lhs_start == [], do: expression_start(parser), else: lhs_start case peek_token_type(parser) do # if the next token is an open brace, we are in a multi alias situation `alias Foo.{Bar, Baz}` @@ -2017,10 +2209,14 @@ defmodule Spitfire do newlines ++ [{:closing, current_meta(parser)} | dot_meta] end + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + outer_meta_with_range = put_range(outer_meta, range) + multis = - {{:., dot_meta, [lhs, :{}]}, outer_meta, multis} + {{:., put_range(dot_meta, range), [lhs, :{}]}, outer_meta_with_range, multis} - {multis, parser} + {multis, set_expression_start(parser, lhs_start)} # if the next token is an alias, then we are in a dot chain of aliases, eg: __MODULE__.Foo :alias -> @@ -2029,7 +2225,11 @@ defmodule Spitfire do {{:__aliases__, ameta, aliases}, parser} = parse_alias(parser) last = ameta[:last] - {{:__aliases__, [{:last, last} | meta], [lhs | aliases]}, parser} + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + + {{:__aliases__, [{:last, last} | put_range(meta, range)], [lhs | aliases]}, + set_expression_start(parser, lhs_start)} # if the next token is a bracket_identifier, then we know that the whole dot expression needs to be used as an argument for the access expression. eg, foo.bar[:baz] :bracket_identifier -> @@ -2041,20 +2241,24 @@ defmodule Spitfire do |> current_meta() |> push_delimiter(token_meta) - rhs = {{token, meta, [lhs, rhs]}, [no_parens: true] ++ ident_meta, []} + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + rhs = {{token, put_range(meta, range), [lhs, rhs]}, [{:no_parens, true} | put_range(ident_meta, range)], []} parser = next_token(parser) {ast, parser} = parse_access_expression(parser, rhs) - {ast, eat_eoe(parser)} + {ast, set_expression_start(eat_eoe(parser), lhs_start)} :paren_identifier -> parser = next_token(parser) {{rhs, next_meta, args}, parser} = parse_expression(parser, precedence, false, false, false) args = if args == nil, do: [], else: args - ast = {{token, meta, [lhs, rhs]}, next_meta, args} - {ast, parser} + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + ast = {{token, put_range(meta, range), [lhs, rhs]}, put_range(next_meta, range), args} + {ast, set_expression_start(parser, lhs_start)} type when type in [:identifier, :do_identifier, :op_identifier] -> parser = next_token(parser) @@ -2069,13 +2273,17 @@ defmodule Spitfire do lone? = type != :op_identifier && MapSet.member?(@peeks, peek_token(parser)) if lone? do - ast = {{token, meta, [lhs, rhs_name]}, [no_parens: true] ++ ident_meta, []} + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + + ast = + {{token, put_range(meta, range), [lhs, rhs_name]}, [{:no_parens, true} | put_range(ident_meta, range)], []} # Handle trailing do-block if parser.nesting == 0 && peek_token(parser) == :do do parse_do_block(next_token(parser), ast) else - {ast, parser} + {ast, set_expression_start(parser, lhs_start)} end else # No-parens call with args @@ -2109,7 +2317,9 @@ defmodule Spitfire do parser = pop_nesting(parser) # Handle trailing do-block - dot_call = {{token, meta, [lhs, rhs_name]}, ident_meta, args} + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + dot_call = {{token, put_range(meta, range), [lhs, rhs_name]}, put_range(ident_meta, range), args} cond do parser.nesting == 0 && current_token(parser) == :do -> @@ -2120,7 +2330,7 @@ defmodule Spitfire do true -> # NOTE(doorgan): we don't add ambiguous_op for dot calls, only for regular identifier calls - {dot_call, parser} + {dot_call, set_expression_start(parser, lhs_start)} end end @@ -2128,9 +2338,11 @@ defmodule Spitfire do parser = next_token(parser) next_meta = current_meta(parser) {rhs, parser} = parse_expression(parser, @lowest, false, false, false) - ast = {{token, meta, [lhs, rhs]}, next_meta, []} + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + ast = {{token, put_range(meta, range), [lhs, rhs]}, put_range(next_meta, range), []} - {ast, parser} + {ast, set_expression_start(parser, lhs_start)} end end end @@ -2271,10 +2483,15 @@ defmodule Spitfire do {parser, meta} = case current_token(parser) do :end -> - {parser, [{:closing, current_meta(parser)} | meta]} + closing_meta = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + {parser, [{:closing, closing_meta}, {:range, range} | meta]} _ -> - {put_error(parser, {meta, "missing closing end for anonymous function"}), meta} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + {put_error(parser, {meta, "missing closing end for anonymous function"}), [{:range, range} | meta]} end {{:fn, newlines ++ meta, exprs}, parser} @@ -2284,23 +2501,34 @@ defmodule Spitfire do defp parse_dot_call_expression(parser, lhs) do trace "parse_dot_call_expression", trace_meta(parser) do meta = current_meta(parser) + lhs_start = ast_start_meta(lhs) + lhs_start = if lhs_start == [], do: expression_start(parser), else: lhs_start parser = next_token(parser) newlines = get_newlines(parser) parser = eat_eoe(parser) - if peek_token(parser) == :")" do + if peek_token_skip_eoe(parser) == :")" do + parser = + while peek_token(parser) in [:eol, :";"] <- parser do + next_token(parser) + end + parser = next_token(parser) - closing = [closing: current_meta(parser)] - ast = {{:., meta, [lhs]}, newlines ++ closing ++ meta, []} - {ast, parser} + closing = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + ast = {{:., put_range(meta, range), [lhs]}, newlines ++ [{:closing, closing} | put_range(meta, range)], []} + {ast, set_expression_start(parser, lhs_start)} else {pairs, parser} = parse_comma_list(parser |> next_token() |> eat_eoe()) parser = parser |> next_token() |> eat_eoe() - closing = [closing: current_meta(parser)] - ast = {{:., meta, [lhs]}, newlines ++ closing ++ meta, pairs} + closing = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + ast = {{:., put_range(meta, range), [lhs]}, newlines ++ [{:closing, closing} | put_range(meta, range)], pairs} - {ast, parser} + {ast, set_expression_start(parser, lhs_start)} end end end @@ -2323,6 +2551,7 @@ defmodule Spitfire do trace "parse_atom (#{type})", trace_meta(parser) do meta = current_meta(parser) {args, parser} = parse_interpolation(parser, tokens) + range = build_range(meta, current_end_meta(parser)) binary_to_atom_op = case type do @@ -2332,7 +2561,8 @@ defmodule Spitfire do delimiter_meta = push_delimiter(meta, token_meta) - {{{:., meta, [:erlang, binary_to_atom_op]}, delimiter_meta, [{:<<>>, meta, args}, :utf8]}, parser} + {{{:., put_range(meta, range), [:erlang, binary_to_atom_op]}, put_range(delimiter_meta, range), + [{:<<>>, put_range(meta, range), args}, :utf8]}, parser} end end @@ -2344,14 +2574,14 @@ defmodule Spitfire do end end - defp parse_int(%{current_token: {:int, {_, _, int} = meta, _}} = parser) do + defp parse_int(%{current_token: {:int, {_, _, _, _, int} = meta, _}} = parser) do trace "parse_int", trace_meta(parser) do int = encode_literal(parser, int, meta) {int, parser} end end - defp parse_float(%{current_token: {:flt, {_, _, float} = meta, _}} = parser) do + defp parse_float(%{current_token: {:flt, {_, _, _, _, float} = meta, _}} = parser) do trace "parse_float", trace_meta(parser) do float = encode_literal(parser, float, meta) {float, parser} @@ -2385,7 +2615,10 @@ defmodule Spitfire do meta end - {{:<<>>, [{:delimiter, ~s|"""|} | meta], args}, parser} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + + {{:<<>>, [{:delimiter, ~s|"""|}, {:range, range} | meta], args}, parser} end end @@ -2393,61 +2626,19 @@ defmodule Spitfire do trace "parse_string (list_heredoc w/interpolation)", trace_meta(parser) do meta = current_meta(parser) - args = - for token <- tokens do - case token do - token when is_binary(token) -> - token - - {{line, col, _}, {cline, ccol, _}, tokens} -> - meta = [line: line, column: col] - # construct a new parser - ast = - if tokens == [] do - {:__block__, [], []} - else - parser = - %{ - tokens: tokens ++ [:eof], - current_token: nil, - peek_token: nil, - nesting: 0, - fuel: 150, - errors: [], - literal_encoder: parser.literal_encoder - } - |> next_token() - |> next_token() - - initial_meta = current_meta(parser) - parser = eat_eoe(parser) + {args, parser} = + Enum.map_reduce(tokens, parser, fn + token, parser when is_binary(token) -> + {token, parser} - if current_token(parser) == :eof do - {:__block__, initial_meta, []} - else - {exprs, _parser} = - while2 current_token(parser) != :eof <- parser do - {ast, parser} = parse_expression(parser, @lowest, false, false, true) - - parser = - if peek_token(parser) in [:eol, :";", :eof] do - next_token(parser) - else - parser - end - - ast = push_eoe(ast, current_eoe(parser)) - {ast, eat_eoe(parser)} - end + {{line, col, _, _, _}, {cline, ccol, _, _, _}, tokens}, parser -> + meta = [line: line, column: col] + range = build_range(meta, end_line: cline, end_column: ccol + 1) + {ast, parser} = parse_interpolation_ast(parser, tokens) - build_block_nr(exprs) - end - end - - {{:., meta, [Kernel, :to_string]}, [from_interpolation: true, closing: [line: cline, column: ccol]] ++ meta, - [ast]} - end - end + {{{:., meta, [Kernel, :to_string]}, + [from_interpolation: true, closing: [line: cline, column: ccol], range: range] ++ meta, [ast]}, parser} + end) extra_meta = if indentation != nil do @@ -2456,7 +2647,10 @@ defmodule Spitfire do [] end - {{{:., meta, [List, :to_charlist]}, [{:delimiter, ~s|'''|} | extra_meta ++ meta], [args]}, parser} + range = build_range(meta, current_end_meta(parser)) + + {{{:., put_range(meta, range), [List, :to_charlist]}, + [{:delimiter, ~s|'''|} | extra_meta ++ put_range(meta, range)], [args]}, parser} end end @@ -2472,12 +2666,14 @@ defmodule Spitfire do meta = current_meta(parser) {args, parser} = parse_interpolation(parser, tokens) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) - {{:<<>>, [{:delimiter, "\""} | meta], args}, parser} + {{:<<>>, [{:delimiter, "\""}, {:range, range} | meta], args}, parser} end end - defp parse_string(%{current_token: {:list_string, meta, [string]}} = parser) do + defp parse_string(%{current_token: {:list_string, meta, [string]}} = parser) when is_binary(string) do trace "parse_string (list_string)", trace_meta(parser) do string = encode_literal(parser, String.to_charlist(string), meta) {string, parser} @@ -2488,67 +2684,27 @@ defmodule Spitfire do trace "parse_string (list_string w/interpolation)", trace_meta(parser) do meta = current_meta(parser) - args = - for token <- tokens do - case token do - token when is_binary(token) -> - token - - {{line, col, _}, {cline, ccol, _}, tokens} -> - meta = [line: line, column: col] - # construct a new parser - ast = - if tokens == [] do - {:__block__, [], []} - else - parser = - %{ - tokens: tokens ++ [:eof], - current_token: nil, - peek_token: nil, - nesting: 0, - fuel: 150, - errors: [], - literal_encoder: parser.literal_encoder - } - |> next_token() - |> next_token() - - initial_meta = current_meta(parser) - parser = eat_eoe(parser) + {args, parser} = + Enum.map_reduce(tokens, parser, fn + token, parser when is_binary(token) -> + {token, parser} - if current_token(parser) == :eof do - {:__block__, initial_meta, []} - else - {exprs, _parser} = - while2 current_token(parser) != :eof <- parser do - {ast, parser} = parse_expression(parser, @lowest, false, false, true) - - parser = - if peek_token(parser) in [:eol, :";", :eof] do - next_token(parser) - else - parser - end - - ast = push_eoe(ast, current_eoe(parser)) - {ast, eat_eoe(parser)} - end + {{line, col, _, _, _}, {cline, ccol, _, _, _}, tokens}, parser -> + meta = [line: line, column: col] + range = build_range(meta, end_line: cline, end_column: ccol + 1) + {ast, parser} = parse_interpolation_ast(parser, tokens) - build_block_nr(exprs) - end - end + {{{:., meta, [Kernel, :to_string]}, + [from_interpolation: true, closing: [line: cline, column: ccol], range: range] ++ meta, [ast]}, parser} + end) - {{:., meta, [Kernel, :to_string]}, [from_interpolation: true, closing: [line: cline, column: ccol]] ++ meta, - [ast]} - end - end + range = build_range(meta, current_end_meta(parser)) - {{{:., meta, [List, :to_charlist]}, [{:delimiter, "'"} | meta], [args]}, parser} + {{{:., put_range(meta, range), [List, :to_charlist]}, [{:delimiter, "'"} | put_range(meta, range)], [args]}, parser} end end - defp parse_char(%{current_token: {:char, {_, _, _token} = meta, num}} = parser) do + defp parse_char(%{current_token: {:char, {_, _, _, _, _token} = meta, num}} = parser) do trace "parse_char", trace_meta(parser) do char = encode_literal(parser, num, meta) {char, parser} @@ -2568,7 +2724,10 @@ defmodule Spitfire do meta end - ast = {token, Keyword.put(meta, :delimiter, delimiter), [{:<<>>, bs_meta, args}, mods]} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + + ast = {token, [{:range, range}, {:delimiter, delimiter} | meta], [{:<<>>, bs_meta, args}, mods]} {ast, parser} end end @@ -2592,8 +2751,10 @@ defmodule Spitfire do end aliases = [alias | aliases] + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) - {{:__aliases__, [{:last, Process.get(:alias_last_meta)} | meta], aliases}, parser} + {{:__aliases__, [{:last, Process.get(:alias_last_meta)}, {:range, range} | meta], aliases}, parser} end after Process.delete(:alias_last_meta) @@ -2608,7 +2769,10 @@ defmodule Spitfire do cond do current_token(parser) == :">>" -> - {{:<<>>, newlines ++ [{:closing, current_meta(parser)} | meta], []}, parser} + closing = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + {{:<<>>, newlines ++ [{:range, range}, {:closing, closing} | meta], []}, parser} current_token(parser) in [:end, :"}", :")", :"]"] -> # if the current token is the wrong kind of ending delimiter, we revert to the previous parser @@ -2623,7 +2787,11 @@ defmodule Spitfire do |> put_in([:peek_token], parser.current_token) |> update_in([:tokens], &[parser.peek_token | &1]) - {{:<<>>, [{:closing, current_meta(parser)} | meta], []}, parser} + closing = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + + {{:<<>>, [{:range, range}, {:closing, closing} | meta], []}, parser} true -> old_comma_list_parsers = Process.get(:comma_list_parsers) @@ -2633,9 +2801,11 @@ defmodule Spitfire do :">>" -> parser = eat_eol_at(parser, 1) parser = next_token(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) - {{:<<>>, newlines ++ [{:closing, current_meta(parser)} | meta], wrap_trailing_bitstring_keywords(pairs)}, - eat_eol(parser)} + {{:<<>>, newlines ++ [{:range, range}, {:closing, current_meta(parser)} | meta], + wrap_trailing_bitstring_keywords(pairs)}, eat_eol(parser)} _ -> all_pairs = pairs |> Enum.reverse() |> Enum.zip(Process.get(:comma_list_parsers)) @@ -2665,8 +2835,10 @@ defmodule Spitfire do parser = put_error(parser, {meta, "missing closing brackets for bitstring"}) {pairs, _} = pairs |> Enum.reverse() |> Enum.unzip() + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) - {{:<<>>, newlines ++ [{:closing, current_meta(parser)} | meta], + {{:<<>>, newlines ++ [{:range, range}, {:closing, current_meta(parser)} | meta], wrap_trailing_bitstring_keywords(List.wrap(pairs))}, parser} end end @@ -2705,19 +2877,23 @@ defmodule Spitfire do current_token(parser) == :"}" -> closing = current_meta(parser) parser = Map.put(parser, :nesting, old_nesting) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) extra = if newlines do - [{:newlines, newlines}, {:closing, closing}] + [{:newlines, newlines}, {:closing, closing}, {:range, range}] else - [{:closing, closing}] + [{:closing, closing}, {:range, range}] end {{:%{}, extra ++ meta, []}, parser} peek_token(parser) == :eof -> parser = put_error(parser, {meta, "missing closing brace for map"}) - {{:%{}, meta, []}, parser} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + {{:%{}, [{:range, range} | meta], []}, parser} true -> # Clear inside_map_update_pairs so nested maps (e.g., %{outer | key: %{inner | k => v}}) @@ -2740,12 +2916,14 @@ defmodule Spitfire do closing = current_meta(parser) parser = Map.put(parser, :nesting, old_nesting) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) extra = if newlines do - [{:newlines, newlines}, {:closing, closing}] + [{:newlines, newlines}, {:closing, closing}, {:range, range}] else - [{:closing, closing}] + [{:closing, closing}, {:range, range}] end {{:%{}, extra ++ meta, pairs}, parser} @@ -2972,7 +3150,9 @@ defmodule Spitfire do peek_token(parser) == :";" or peek in [:stab_op, :do, :end, :block_identifier] or (is_binary_op?(peek) and peek not in [:dual_op, :ternary_op]) do - {{:..., current_meta(parser), []}, parser} + end_meta = current_end_meta(parser) + range = build_range(current_meta(parser), end_meta) + {{:..., [{:range, range} | current_meta(parser)], []}, parser} else meta = current_meta(parser) parser = next_token(parser) @@ -2992,7 +3172,10 @@ defmodule Spitfire do {rhs, parser} end - {{:..., meta, [rhs]}, parser} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + + {{:..., [{:range, range} | meta], [rhs]}, parser} end end end @@ -3059,7 +3242,14 @@ defmodule Spitfire do if current_token(parser) == :"}" do closing = current_meta(parser) - ast = {:%, meta, [type, {:%{}, newlines ++ [{:closing, closing} | brace_meta], []}]} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + map_range = build_range(brace_meta, end_meta) + + ast = + {:%, [{:range, range} | meta], + [type, {:%{}, newlines ++ [{:range, map_range}, {:closing, closing} | brace_meta], []}]} + parser = Map.put(parser, :nesting, old_nesting) {ast, parser} else @@ -3077,7 +3267,14 @@ defmodule Spitfire do end closing = current_meta(parser) - ast = {:%, meta, [type, {:%{}, newlines ++ [{:closing, closing} | brace_meta], pairs}]} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + map_range = build_range(brace_meta, end_meta) + + ast = + {:%, [{:range, range} | meta], + [type, {:%{}, newlines ++ [{:range, map_range}, {:closing, closing} | brace_meta], pairs}]} + parser = Map.put(parser, :nesting, old_nesting) {ast, parser} end @@ -3104,7 +3301,9 @@ defmodule Spitfire do {parser, brace_meta} end - ast = {:%, meta, [type, {:%{}, closing_meta, pairs}]} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + ast = {:%, [{:range, range} | meta], [type, {:%{}, closing_meta, pairs}]} parser = Map.put(parser, :nesting, old_nesting) {ast, parser} @@ -3114,7 +3313,9 @@ defmodule Spitfire do do: put_error(parser, {current_meta(parser), "missing opening brace for struct %#{struct_name}"}), else: parser - {{:%, meta, [type, {:%{}, [], []}]}, parser} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + {{:%, [{:range, range} | meta], [type, {:%{}, [], []}]}, parser} end end end @@ -3133,12 +3334,14 @@ defmodule Spitfire do current_token(parser) == :"}" -> closing = current_meta(parser) parser = Map.put(parser, :nesting, old_nesting) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) extra = if newlines do - [{:newlines, newlines}, {:closing, closing}] + [{:newlines, newlines}, {:closing, closing}, {:range, range}] else - [{:closing, closing}] + [{:closing, closing}, {:range, range}] end {{:{}, extra ++ meta, []}, parser} @@ -3157,7 +3360,9 @@ defmodule Spitfire do |> update_in([:tokens], &[parser.peek_token | &1]) parser = put_in(parser.nesting, old_nesting) - {{:{}, meta, []}, parser} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + {{:{}, [{:range, range} | meta], []}, parser} true -> old_comma_list_parsers = Process.get(:comma_list_parsers) @@ -3208,12 +3413,14 @@ defmodule Spitfire do else closing = current_meta(parser) parser = Map.put(parser, :nesting, old_nesting) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) extra = if newlines do - [{:newlines, newlines}, {:closing, closing}] + [{:newlines, newlines}, {:closing, closing}, {:range, range}] else - [{:closing, closing}] + [{:closing, closing}, {:range, range}] end {{:{}, extra ++ meta, List.wrap(pairs)}, parser} @@ -3314,17 +3521,22 @@ defmodule Spitfire do cond do peek_token(parser) == :eof -> closing = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) parser = put_error(parser, {closing, "missing closing parentheses"}) - {{token, newlines ++ [{:closing, closing} | meta], []}, parser} + {{token, newlines ++ [{:range, range}, {:closing, closing} | meta], []}, parser} peek_token(parser) == :")" -> parser = next_token(parser) closing = current_meta(parser) - ast = {token, newlines ++ [{:closing, closing} | meta], []} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + ast = {token, newlines ++ [{:range, range}, {:closing, closing} | meta], []} if peek_token(parser) == :do and parser.nesting == 0 do parser = next_token(parser) - parse_do_block(parser, ast) + ast_no_range = {token, newlines ++ [{:closing, closing} | meta], []} + parse_do_block(parser, ast_no_range) else {ast, parser} end @@ -3341,11 +3553,14 @@ defmodule Spitfire do if current_token(parser) == :")" do parser = Map.put(parser, :nesting, old_nesting) closing = current_meta(parser) - ast = {token, newlines ++ [{:closing, closing} | meta], []} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + ast = {token, newlines ++ [{:range, range}, {:closing, closing} | meta], []} if peek_token(parser) == :do and parser.nesting == 0 do parser = next_token(parser) - parse_do_block(parser, ast) + ast_no_range = {token, newlines ++ [{:closing, closing} | meta], []} + parse_do_block(parser, ast_no_range) else {ast, parser} end @@ -3360,19 +3575,24 @@ defmodule Spitfire do :")" -> parser = next_token(parser) closing = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) - ast = {token, newlines ++ [{:closing, closing} | meta], List.wrap(pairs)} + ast = {token, newlines ++ [{:range, range}, {:closing, closing} | meta], List.wrap(pairs)} if peek_token(parser) == :do and parser.nesting == 0 do parser = next_token(parser) - parse_do_block(parser, ast) + ast_no_range = {token, newlines ++ [{:closing, closing} | meta], List.wrap(pairs)} + parse_do_block(parser, ast_no_range) else {ast, parser} end _ -> + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) parser = put_error(parser, {error_meta, "missing closing parentheses for function invocation"}) - {{token, newlines ++ meta, List.wrap(pairs)}, parser} + {{token, newlines ++ [{:range, range} | meta], List.wrap(pairs)}, parser} end end end @@ -3412,6 +3632,8 @@ defmodule Spitfire do args = [front | args] parser = pop_nesting(parser) + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) {ast, parser} = cond do @@ -3424,9 +3646,9 @@ defmodule Spitfire do true -> meta = if identifier == :op_identifier && match?([_], args) do - [{:ambiguous_op, nil} | meta] + [{:ambiguous_op, nil}, {:range, range} | meta] else - meta + [{:range, range} | meta] end {{token, meta, args}, parser} @@ -3441,7 +3663,9 @@ defmodule Spitfire do defp parse_do_identifier(%{current_token: {:do_identifier, _, token}} = parser) do trace "parse_do_identifier - nesting[#{parser.nesting}]", trace_meta(parser) do meta = current_meta(parser) + end_meta = current_end_meta(parser) parser = next_token(parser) + range = build_range(meta, end_meta) # if nesting is 0, that means we are not currently an argument for a function call # and can assume we are a "lone do_identifier" and parse the block @@ -3452,7 +3676,7 @@ defmodule Spitfire do if parser.nesting == 0 do parse_do_block(parser, {token, meta, []}) else - {{token, meta, nil}, parser} + {{token, [{:range, range} | meta], nil}, parser} end end end @@ -3461,6 +3685,8 @@ defmodule Spitfire do trace "parse_call_expression", trace_meta(parser) do # this might be wrong, but its how Code.string_to_quoted works {_, meta, _} = lhs + lhs_start = ast_start_meta(lhs) + lhs_start = if lhs_start == [], do: expression_start(parser), else: lhs_start newlines = get_newlines(parser) @@ -3468,12 +3694,16 @@ defmodule Spitfire do peek_token(parser) == :eof -> parser = put_error(parser, {meta, "missing closing parentheses for function invocation"}) closing = current_meta(parser) - {{lhs, newlines ++ [{:closing, closing} | meta], []}, parser} + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + {{lhs, newlines ++ [{:closing, closing} | put_range(meta, range)], []}, set_expression_start(parser, lhs_start)} peek_token(parser) == :")" -> parser = next_token(parser) closing = current_meta(parser) - {{lhs, newlines ++ [{:closing, closing} | meta], []}, parser} + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) + {{lhs, newlines ++ [{:closing, closing} | put_range(meta, range)], []}, set_expression_start(parser, lhs_start)} true -> {pairs, parser} = @@ -3494,8 +3724,11 @@ defmodule Spitfire do end closing = current_meta(parser) + end_meta = current_end_meta(parser) + range = build_range(lhs_start, end_meta) - {{lhs, newlines ++ [{:closing, closing} | meta], List.wrap(pairs)}, parser} + {{lhs, newlines ++ [{:closing, closing} | put_range(meta, range)], List.wrap(pairs)}, + set_expression_start(parser, lhs_start)} end end end @@ -3507,7 +3740,10 @@ defmodule Spitfire do |> current_meta() |> push_delimiter(token_meta) - {{token, meta, nil}, parser} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + + {{token, [{:range, range} | meta], nil}, parser} end end @@ -3561,7 +3797,10 @@ defmodule Spitfire do parse_lone_identifier(parser) end - {{token, meta, [rhs]}, parser} + end_meta = current_end_meta(parser) + range = build_range(meta, end_meta) + + {{token, [{:range, range} | meta], [rhs]}, parser} end end @@ -3601,69 +3840,70 @@ defmodule Spitfire do defp parse_interpolation(parser, tokens) do trace "parse_interpolation", trace_meta(parser) do - args = - for token <- tokens do - case token do - token when is_binary(token) -> - token - - {{line, col, _}, {cline, ccol, _}, tokens} -> - meta = [line: line, column: col] - - # construct a new parser - ast = - if tokens == [] do - {:__block__, [], []} - else - parser = - %{ - tokens: tokens ++ [:eof], - current_token: nil, - errors: [], - peek_token: nil, - nesting: 0, - fuel: 150, - literal_encoder: parser.literal_encoder - } - |> next_token() - |> next_token() + {args, parser} = + Enum.map_reduce(tokens, parser, fn + token, parser when is_binary(token) -> + {token, parser} + + {{line, col, _, _, _}, {cline, ccol, _, _, _}, tokens}, parser -> + meta = [line: line, column: col] + range = build_range(meta, end_line: cline, end_column: ccol + 1) + + {ast, parser} = parse_interpolation_ast(parser, tokens) + + {{:"::", put_range(meta, range), + [ + {{:., meta, [Kernel, :to_string]}, + [from_interpolation: true, closing: [line: cline, column: ccol], range: range] ++ meta, [ast]}, + {:binary, meta, nil} + ]}, parser} + end) - initial_meta = current_meta(parser) - parser = eat_eoe(parser) + {args, parser} + end + end - if current_token(parser) == :eof do - {:__block__, initial_meta, []} - else - {exprs, _parser} = - while2 current_token(parser) != :eof <- parser do - {ast, parser} = parse_expression(parser, @lowest, false, false, true) - - parser = - if peek_token(parser) in [:eol, :";", :eof] do - next_token(parser) - else - parser - end - - ast = push_eoe(ast, current_eoe(parser)) - {ast, eat_eoe(parser)} - end + defp parse_interpolation_ast(outer_parser, tokens) do + parser = + %{ + tokens: tokens ++ [:eof], + current_token: nil, + errors: [], + peek_token: nil, + nesting: 0, + fuel: 150, + literal_encoder: outer_parser.literal_encoder + } + |> next_token() + |> next_token() - build_block_nr(exprs) - end - end + initial_meta = current_meta(parser) + + parser = eat_eoe(parser) + + ast = + if current_token(parser) == :eof do + {:__block__, initial_meta, []} + else + {exprs, _parser} = + while2 current_token(parser) != :eof <- parser do + {ast, parser} = parse_expression(parser, @lowest, false, false, true) - {:"::", meta, - [ - {{:., meta, [Kernel, :to_string]}, - [from_interpolation: true, closing: [line: cline, column: ccol]] ++ meta, [ast]}, - {:binary, meta, nil} - ]} + parser = + if peek_token(parser) in [:eol, :";", :eof] do + next_token(parser) + else + parser + end + + ast = push_eoe(ast, current_eoe(parser)) + {ast, eat_eoe(parser)} end - end - {args, parser} - end + build_block_nr(exprs) + end + + {ast, outer_parser} end defp new(code, opts) do @@ -3994,15 +4234,15 @@ defmodule Spitfire do token end - defp current_meta(%{current_token: {:sigil, {line, col, _}, _token, _tokens, _mods, _, _delimiter}}) do + defp current_meta(%{current_token: {:sigil, {line, col, _, _, _}, _token, _tokens, _mods, _, _delimiter}}) do [line: line, column: col] end - defp current_meta(%{current_token: {:bin_heredoc, {line, col, _}, _indent, _tokens}}) do + defp current_meta(%{current_token: {:bin_heredoc, {line, col, _, _, _}, _indent, _tokens}}) do [line: line, column: col] end - defp current_meta(%{current_token: {:list_heredoc, {line, col, _}, _indent, _tokens}}) do + defp current_meta(%{current_token: {:list_heredoc, {line, col, _, _, _}, _indent, _tokens}}) do [line: line, column: col] end @@ -4011,11 +4251,11 @@ defmodule Spitfire do [] end - defp current_meta(%{current_token: {_token, {line, col, _}, _}}) do + defp current_meta(%{current_token: {_token, {line, col, _, _, _}, _}}) do [line: line, column: col] end - defp current_meta(%{current_token: {_token, {line, col, _}}}) do + defp current_meta(%{current_token: {_token, {line, col, _, _, _}}}) do [line: line, column: col] end @@ -4023,22 +4263,94 @@ defmodule Spitfire do [] end + defp current_end_meta(%{ + current_token: {:sigil, {_line, _col, end_line, end_col, _}, _token, _tokens, _mods, _, _delimiter} + }) do + [end_line: end_line, end_column: end_col] + end + + defp current_end_meta(%{current_token: {:bin_heredoc, {_line, _col, end_line, end_col, _}, _indent, _tokens}}) do + [end_line: end_line, end_column: end_col] + end + + defp current_end_meta(%{current_token: {:list_heredoc, {_line, _col, end_line, end_col, _}, _indent, _tokens}}) do + [end_line: end_line, end_column: end_col] + end + + defp current_end_meta(%{current_token: {token, _}}) + when token in [:fake_closing_brace, :fake_closing_bracket, :fake_closing_brackets] do + [] + end + + defp current_end_meta(%{current_token: {_token, {_line, _col, end_line, end_col, _}, _}}) do + [end_line: end_line, end_column: end_col] + end + + defp current_end_meta(%{current_token: {_token, {_line, _col, end_line, end_col, _}}}) do + [end_line: end_line, end_column: end_col] + end + + defp current_end_meta(_) do + [] + end + + defp put_range(meta, range) do + [{:range, range} | Keyword.delete(meta, :range)] + end + + defp ast_start_meta([first | _]), do: ast_start_meta(first) + defp ast_start_meta([]), do: [] + + defp ast_start_meta({_, meta, _}) when is_list(meta) do + case Keyword.get(meta, :range) do + %{start: {line, column}} when is_integer(line) and is_integer(column) -> + [line: line, column: column] + + _ -> + line = Keyword.get(meta, :line) + column = Keyword.get(meta, :column) + + if is_integer(line) and is_integer(column) do + [line: line, column: column] + else + [] + end + end + end + + defp ast_start_meta(_), do: [] + + defp expression_start(parser) do + Map.get(parser, :expression_start, current_meta(parser)) + end + + defp set_expression_start(parser, meta) do + Map.put(parser, :expression_start, meta) + end + + defp build_range(start_meta, end_meta) do + %{ + start: {Keyword.get(start_meta, :line), Keyword.get(start_meta, :column)}, + end: {Keyword.get(end_meta, :end_line), Keyword.get(end_meta, :end_column)} + } + end + if @trace? do defp trace_meta(parser) do [{:token, "'#{current_token(parser)}'"}, {:nesting, parser.nesting} | current_meta(parser)] end end - defp current_eoe(%{current_token: {token, {line, col, newlines}}}) + defp current_eoe(%{current_token: {token, {line, col, _, _, newlines}}}) when token in [:eol, :";"] and is_integer(newlines) do [newlines: newlines, line: line, column: col] end - defp current_eoe(%{current_token: {token, {line, col, _}, _}}) when token in [:eol, :";"] do + defp current_eoe(%{current_token: {token, {line, col, _, _, _}, _}}) when token in [:eol, :";"] do [line: line, column: col] end - defp current_eoe(%{current_token: {token, {line, col, _}}}) when token in [:eol, :";"] do + defp current_eoe(%{current_token: {token, {line, col, _, _, _}}}) when token in [:eol, :";"] do [line: line, column: col] end @@ -4046,15 +4358,16 @@ defmodule Spitfire do nil end - defp peek_eoe(%{peek_token: {token, {line, col, newlines}}}) when token in [:eol, :";"] and is_integer(newlines) do + defp peek_eoe(%{peek_token: {token, {line, col, _, _, newlines}}}) + when token in [:eol, :";"] and is_integer(newlines) do [newlines: newlines, line: line, column: col] end - defp peek_eoe(%{peek_token: {token, {line, col, _}, _}}) when token in [:eol, :";"] do + defp peek_eoe(%{peek_token: {token, {line, col, _, _, _}, _}}) when token in [:eol, :";"] do [line: line, column: col] end - defp peek_eoe(%{peek_token: {token, {line, col, _}}}) when token in [:eol, :";"] do + defp peek_eoe(%{peek_token: {token, {line, col, _, _, _}}}) when token in [:eol, :";"] do [line: line, column: col] end @@ -4085,12 +4398,12 @@ defmodule Spitfire do :in_op ] - defp current_newlines(%{current_token: {token, {_line, _col, newlines}, _}}) + defp current_newlines(%{current_token: {token, {_line, _col, _, _, newlines}, _}}) when token in @newline_carrying_tokens and is_integer(newlines) do newlines end - defp current_newlines(%{current_token: {token, {_line, _col, newlines}}}) + defp current_newlines(%{current_token: {token, {_line, _col, _, _, newlines}}}) when token in @newline_carrying_tokens and is_integer(newlines) do newlines end @@ -4099,7 +4412,7 @@ defmodule Spitfire do nil end - defp peek_newlines(%{peek_token: {:eol, {_line, _col, newlines}}}) when is_integer(newlines) do + defp peek_newlines(%{peek_token: {:eol, {_line, _col, _, _, newlines}}}) when is_integer(newlines) do newlines end @@ -4107,7 +4420,7 @@ defmodule Spitfire do nil end - defp peek_newlines(%{peek_token: {token, {_line, _col, newlines}}}, token) when is_integer(newlines) do + defp peek_newlines(%{peek_token: {token, {_line, _col, _, _, newlines}}}, token) when is_integer(newlines) do newlines end @@ -4131,10 +4444,23 @@ defmodule Spitfire do %{parser | nesting: nesting + 1} end - defp encode_literal(%{literal_encoder: encoder} = parser, literal, {line, col, _}) when is_function(encoder) do - meta = additional_meta(literal, parser) ++ [line: line, column: col] + defp encode_literal(%{literal_encoder: encoder} = parser, literal, {line, col, end_line, end_col, _}) + when is_function(encoder) do + start_meta = [line: line, column: col] + extra_meta = additional_meta(literal, parser) - case parser.literal_encoder.(literal, meta) do + end_meta = + literal + |> literal_end_meta(parser) + |> case do + [] -> [end_line: end_line, end_column: end_col] + meta -> meta + end + + range = build_range(start_meta, end_meta) + meta = extra_meta ++ [range: range, line: line, column: col] + + case encoder.(literal, meta) do {:ok, ast} -> ast @@ -4148,6 +4474,23 @@ defmodule Spitfire do literal end + defp literal_end_meta(literal, parser) when is_list(literal) do + cond do + current_token_type(parser) in [:list_string, :list_heredoc] -> + current_end_meta(parser) + + current_token(parser) == :"]" -> + current_end_meta(parser) + + true -> + parser + |> next_token() + |> current_end_meta() + end + end + + defp literal_end_meta(_literal, parser), do: current_end_meta(parser) + defp additional_meta(_literal, %{current_token: {:list_string, _, _}}) do [delimiter: "'"] end @@ -4161,7 +4504,7 @@ defmodule Spitfire do end defp additional_meta(literal, parser) when is_list(literal) do - parser = next_token(parser) + parser = if current_token(parser) == :"]", do: parser, else: next_token(parser) closing = current_meta(parser) [closing: closing] end @@ -4291,10 +4634,33 @@ defmodule Spitfire do expr _ -> - {:__block__, [], exprs} + {:__block__, block_range_meta(exprs), exprs} end end + defp block_range_meta([first | _] = exprs) do + with start_meta when start_meta != [] <- ast_start_meta(first), + end_meta when end_meta != [] <- ast_end_meta(List.last(exprs)) do + [range: build_range(start_meta, end_meta)] + else + _ -> [] + end + end + + defp block_range_meta([]), do: [] + + defp ast_end_meta({_, meta, _}) when is_list(meta) do + case Keyword.get(meta, :range) do + %{end: {line, column}} when is_integer(line) and is_integer(column) -> + [end_line: line, end_column: column] + + _ -> + [] + end + end + + defp ast_end_meta(_), do: [] + # Code taken from Code.string_to_quoted_with_comments in Elixir core # Check it out here: https://github.com/elixir-lang/elixir/blob/12f62e49ca2399a15976d2051a2d7743dae48449/lib/elixir/lib/code.ex#L1327 # Consult Elixir's license here: https://github.com/elixir-lang/elixir/blob/main/LICENSE @@ -4318,7 +4684,7 @@ defmodule Spitfire do defp next_eol_count([?\r, ?\n | rest], count), do: next_eol_count(rest, count + 1) defp next_eol_count(_, count), do: count - defp previous_eol_count([{token, {_, _, count}} | _]) when token in [:eol, :",", :";"] and count > 0 do + defp previous_eol_count([{token, {_, _, _, _, count}} | _]) when token in [:eol, :",", :";"] and count > 0 do count end @@ -4342,13 +4708,14 @@ defmodule Spitfire do Enum.map_reduce(terminators, column, fn {start, _, _}, column -> atom = :spitfire_tokenizer.terminator(start) - {{atom, {line, column, nil}}, column + length(Atom.to_charlist(atom))} + {{atom, {line, column, line, column + length(Atom.to_charlist(atom)), nil}}, + column + length(Atom.to_charlist(atom))} end) Enum.reverse(tokens, terminators) end - defp push_delimiter(meta, {_, _, delimiter}) when is_integer(delimiter) do + defp push_delimiter(meta, {_, _, _, _, delimiter}) when is_integer(delimiter) do [{:delimiter, "#{[delimiter]}"} | meta] end diff --git a/src/spitfire_interpolation.erl b/src/spitfire_interpolation.erl index d81679e..fd7838c 100644 --- a/src/spitfire_interpolation.erl +++ b/src/spitfire_interpolation.erl @@ -307,7 +307,7 @@ build_string([], Output) -> Output; build_string(Buffer, Output) -> [lists:reverse(Buffer) | Output]. build_interpol(Line, Column, EndLine, EndColumn, Buffer, Output) -> - [{{Line, Column, nil}, {EndLine, EndColumn, nil}, Buffer} | Output]. + [{{Line, Column, EndLine, EndColumn, nil}, {EndLine, EndColumn, EndLine, EndColumn, nil}, Buffer} | Output]. prepend_warning(Line, Column, Msg, #spitfire_tokenizer{warnings=Warnings} = Scope) -> Scope#spitfire_tokenizer{warnings = [{{Line, Column}, Msg} | Warnings]}. diff --git a/src/spitfire_tokenizer.erl b/src/spitfire_tokenizer.erl index 71a36d8..9543b91 100644 --- a/src/spitfire_tokenizer.erl +++ b/src/spitfire_tokenizer.erl @@ -180,17 +180,17 @@ tokenize(("<<<<<<<" ++ _) = Original, Line, 1, Scope, Tokens) -> tokenize([$0, $x, H | T], Line, Column, Scope, Tokens) when ?is_hex(H) -> {Rest, Number, OriginalRepresentation, Length} = tokenize_hex(T, [H], 1), - Token = {int, {Line, Column, Number}, OriginalRepresentation}, + Token = {int, {Line, Column, Line, Column + 2 + Length, Number}, OriginalRepresentation}, tokenize(Rest, Line, Column + 2 + Length, Scope, [Token | Tokens]); tokenize([$0, $b, H | T], Line, Column, Scope, Tokens) when ?is_bin(H) -> {Rest, Number, OriginalRepresentation, Length} = tokenize_bin(T, [H], 1), - Token = {int, {Line, Column, Number}, OriginalRepresentation}, + Token = {int, {Line, Column, Line, Column + 2 + Length, Number}, OriginalRepresentation}, tokenize(Rest, Line, Column + 2 + Length, Scope, [Token | Tokens]); tokenize([$0, $o, H | T], Line, Column, Scope, Tokens) when ?is_octal(H) -> {Rest, Number, OriginalRepresentation, Length} = tokenize_octal(T, [H], 1), - Token = {int, {Line, Column, Number}, OriginalRepresentation}, + Token = {int, {Line, Column, Line, Column + 2 + Length, Number}, OriginalRepresentation}, tokenize(Rest, Line, Column + 2 + Length, Scope, [Token | Tokens]); % Comments @@ -239,7 +239,7 @@ tokenize([$?, $\\, H | T], Line, Column, Scope, Tokens) -> Scope end, - Token = {char, {Line, Column, [$?, $\\, H]}, Char}, + Token = {char, {Line, Column, Line, Column + 3, [$?, $\\, H]}, Char}, tokenize(T, Line, Column + 3, NewScope, [Token | Tokens]); tokenize([$?, Char | T], Line, Column, Scope, Tokens) -> @@ -251,7 +251,7 @@ tokenize([$?, Char | T], Line, Column, Scope, Tokens) -> false -> Scope end, - Token = {char, {Line, Column, [$?, Char]}, Char}, + Token = {char, {Line, Column, Line, Column + 2, [$?, Char]}, Char}, tokenize(T, Line, Column + 2, NewScope, [Token | Tokens]); % Heredocs @@ -276,37 +276,37 @@ tokenize([$' | T], Line, Column, Scope, Tokens) -> % Operator atoms tokenize(".:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Column + 2, Scope, [{kw_identifier, {Line, Column, nil}, '.'} | Tokens]); + tokenize(Rest, Line, Column + 2, Scope, [{kw_identifier, {Line, Column, Line, Column + 2, nil}, '.'} | Tokens]); tokenize("<<>>:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Column + 5, Scope, [{kw_identifier, {Line, Column, nil}, '<<>>'} | Tokens]); + tokenize(Rest, Line, Column + 5, Scope, [{kw_identifier, {Line, Column, Line, Column + 5, nil}, '<<>>'} | Tokens]); tokenize("%{}:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Column + 4, Scope, [{kw_identifier, {Line, Column, nil}, '%{}'} | Tokens]); + tokenize(Rest, Line, Column + 4, Scope, [{kw_identifier, {Line, Column, Line, Column + 4, nil}, '%{}'} | Tokens]); tokenize("%:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Column + 2, Scope, [{kw_identifier, {Line, Column, nil}, '%'} | Tokens]); + tokenize(Rest, Line, Column + 2, Scope, [{kw_identifier, {Line, Column, Line, Column + 2, nil}, '%'} | Tokens]); tokenize("&:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Column + 2, Scope, [{kw_identifier, {Line, Column, nil}, '&'} | Tokens]); + tokenize(Rest, Line, Column + 2, Scope, [{kw_identifier, {Line, Column, Line, Column + 2, nil}, '&'} | Tokens]); tokenize("{}:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Column + 3, Scope, [{kw_identifier, {Line, Column, nil}, '{}'} | Tokens]); + tokenize(Rest, Line, Column + 3, Scope, [{kw_identifier, {Line, Column, Line, Column + 3, nil}, '{}'} | Tokens]); tokenize("..//:" ++ Rest, Line, Column, Scope, Tokens) when ?is_space(hd(Rest)) -> - tokenize(Rest, Line, Column + 5, Scope, [{kw_identifier, {Line, Column, nil}, '..//'} | Tokens]); + tokenize(Rest, Line, Column + 5, Scope, [{kw_identifier, {Line, Column, Line, Column + 5, nil}, '..//'} | Tokens]); tokenize(":<<>>" ++ Rest, Line, Column, Scope, Tokens) -> - tokenize(Rest, Line, Column + 5, Scope, [{atom, {Line, Column, nil}, '<<>>'} | Tokens]); + tokenize(Rest, Line, Column + 5, Scope, [{atom, {Line, Column, Line, Column + 5, nil}, '<<>>'} | Tokens]); tokenize(":%{}" ++ Rest, Line, Column, Scope, Tokens) -> - tokenize(Rest, Line, Column + 4, Scope, [{atom, {Line, Column, nil}, '%{}'} | Tokens]); + tokenize(Rest, Line, Column + 4, Scope, [{atom, {Line, Column, Line, Column + 4, nil}, '%{}'} | Tokens]); tokenize(":%" ++ Rest, Line, Column, Scope, Tokens) -> - tokenize(Rest, Line, Column + 2, Scope, [{atom, {Line, Column, nil}, '%'} | Tokens]); + tokenize(Rest, Line, Column + 2, Scope, [{atom, {Line, Column, Line, Column + 2, nil}, '%'} | Tokens]); tokenize(":{}" ++ Rest, Line, Column, Scope, Tokens) -> - tokenize(Rest, Line, Column + 3, Scope, [{atom, {Line, Column, nil}, '{}'} | Tokens]); + tokenize(Rest, Line, Column + 3, Scope, [{atom, {Line, Column, Line, Column + 3, nil}, '{}'} | Tokens]); tokenize(":..//" ++ Rest, Line, Column, Scope, Tokens) -> - tokenize(Rest, Line, Column + 5, Scope, [{atom, {Line, Column, nil}, '..//'} | Tokens]); + tokenize(Rest, Line, Column + 5, Scope, [{atom, {Line, Column, Line, Column + 5, nil}, '..//'} | Tokens]); % ## Three Token Operators tokenize([$:, T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?unary_op3(T1, T2, T3); ?comp_op3(T1, T2, T3); ?and_op3(T1, T2, T3); ?or_op3(T1, T2, T3); ?arrow_op3(T1, T2, T3); ?xor_op3(T1, T2, T3); ?concat_op3(T1, T2, T3); ?ellipsis_op3(T1, T2, T3) -> - Token = {atom, {Line, Column, nil}, list_to_atom([T1, T2, T3])}, + Token = {atom, {Line, Column, Line, Column + 4, nil}, list_to_atom([T1, T2, T3])}, tokenize(Rest, Line, Column + 4, Scope, [Token | Tokens]); % ## Two Token Operators @@ -314,33 +314,33 @@ tokenize([$:, T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when tokenize([$:, $:, $: | Rest], Line, Column, Scope, Tokens) -> Message = "atom ::: must be written between quotes, as in :\"::\", to avoid ambiguity", NewScope = prepend_warning(Line, Column, Message, Scope), - Token = {atom, {Line, Column, nil}, '::'}, + Token = {atom, {Line, Column, Line, Column + 3, nil}, '::'}, tokenize(Rest, Line, Column + 3, NewScope, [Token | Tokens]); tokenize([$:, T1, T2 | Rest], Line, Column, Scope, Tokens) when ?comp_op2(T1, T2); ?rel_op2(T1, T2); ?and_op(T1, T2); ?or_op(T1, T2); ?arrow_op(T1, T2); ?in_match_op(T1, T2); ?concat_op(T1, T2); ?power_op(T1, T2); ?stab_op(T1, T2); ?range_op(T1, T2) -> - Token = {atom, {Line, Column, nil}, list_to_atom([T1, T2])}, + Token = {atom, {Line, Column, Line, Column + 3, nil}, list_to_atom([T1, T2])}, tokenize(Rest, Line, Column + 3, Scope, [Token | Tokens]); % ## Single Token Operators tokenize([$:, T | Rest], Line, Column, Scope, Tokens) when ?at_op(T); ?unary_op(T); ?capture_op(T); ?dual_op(T); ?mult_op(T); ?rel_op(T); ?match_op(T); ?pipe_op(T); T =:= $. -> - Token = {atom, {Line, Column, nil}, list_to_atom([T])}, + Token = {atom, {Line, Column, Line, Column + 2, nil}, list_to_atom([T])}, tokenize(Rest, Line, Column + 2, Scope, [Token | Tokens]); % ## Stand-alone tokens tokenize("=>" ++ Rest, Line, Column, Scope, Tokens) -> - Token = {assoc_op, {Line, Column, previous_was_eol(Tokens)}, '=>'}, + Token = {assoc_op, {Line, Column, Line, Column + 2, previous_was_eol(Tokens)}, '=>'}, tokenize(Rest, Line, Column + 2, Scope, add_token_with_eol(Token, Tokens)); tokenize("..//" ++ Rest = String, Line, Column, Scope, Tokens) -> case strip_horizontal_space(Rest, 0) of {[$/ | _] = Remaining, Extra} -> - Token = {identifier, {Line, Column, nil}, '..//'}, + Token = {identifier, {Line, Column, Line, Column + 4 + Extra, nil}, '..//'}, tokenize(Remaining, Line, Column + 4 + Extra, Scope, [Token | Tokens]); {_, _} -> unexpected_token(String, Line, Column, Scope, Tokens) @@ -379,15 +379,15 @@ tokenize([T1, T2, T3 | Rest], Line, Column, Scope, Tokens) when ?arrow_op3(T1, T % ## Containers + punctuation tokens tokenize([$, | Rest], Line, Column, Scope, Tokens) -> - Token = {',', {Line, Column, 0}}, + Token = {',', {Line, Column, Line, Column + 1, 0}}, tokenize(Rest, Line, Column + 1, Scope, [Token | Tokens]); tokenize([$<, $< | Rest], Line, Column, Scope, Tokens) -> - Token = {'<<', {Line, Column, nil}}, + Token = {'<<', {Line, Column, Line, Column + 2, nil}}, handle_terminator(Rest, Line, Column + 2, Scope, Token, Tokens); tokenize([$>, $> | Rest], Line, Column, Scope, Tokens) -> - Token = {'>>', {Line, Column, previous_was_eol(Tokens)}}, + Token = {'>>', {Line, Column, Line, Column + 2, previous_was_eol(Tokens)}}, handle_terminator(Rest, Line, Column + 2, Scope, Token, Tokens); tokenize([${ | Rest], Line, Column, Scope, [{'%', _} | _] = Tokens) -> @@ -399,17 +399,17 @@ tokenize([${ | Rest], Line, Column, Scope, [{'%', _} | _] = Tokens) -> error({?LOC(Line, Column), Message, [${]}, Rest, Scope, Tokens); tokenize([T | Rest], Line, Column, Scope, Tokens) when T =:= $(; T =:= ${; T =:= $[ -> - Token = {list_to_atom([T]), {Line, Column, nil}}, + Token = {list_to_atom([T]), {Line, Column, Line, Column + 1, nil}}, handle_terminator(Rest, Line, Column + 1, Scope, Token, Tokens); tokenize([T | Rest], Line, Column, Scope, Tokens) when T =:= $); T =:= $}; T =:= $] -> - Token = {list_to_atom([T]), {Line, Column, previous_was_eol(Tokens)}}, + Token = {list_to_atom([T]), {Line, Column, Line, Column + 1, previous_was_eol(Tokens)}}, handle_terminator(Rest, Line, Column + 1, Scope, Token, Tokens); % ## Two Token Operators tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?ternary_op(T1, T2) -> Op = list_to_atom([T1, T2]), - Token = {ternary_op, {Line, Column, previous_was_eol(Tokens)}, Op}, + Token = {ternary_op, {Line, Column, Line, Column + 2, previous_was_eol(Tokens)}, Op}, tokenize(Rest, Line, Column + 2, Scope, add_token_with_eol(Token, Tokens)); tokenize([T1, T2 | Rest], Line, Column, Scope, Tokens) when ?power_op(T1, T2) -> @@ -463,7 +463,7 @@ tokenize([$& | Rest], Line, Column, Scope, Tokens) -> capture_op end, - Token = {Kind, {Line, Column, nil}, '&'}, + Token = {Kind, {Line, Column, Line, Column + 1, nil}, '&'}, tokenize(Rest, Line, Column + 1, Scope, [Token | Tokens]); tokenize([T | Rest], Line, Column, Scope, Tokens) when ?at_op(T) -> @@ -519,7 +519,7 @@ tokenize([$:, H | T] = Original, Line, Column, BaseScope, Tokens) when ?is_quote {ok, [Part]} when is_binary(Part) -> case unsafe_to_atom(Part, Line, Column, Scope) of {ok, Atom} -> - Token = {atom_quoted, {Line, Column, H}, Atom}, + Token = {atom_quoted, {Line, Column, NewLine, NewColumn, H}, Atom}, tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]); {error, Reason} -> @@ -531,7 +531,7 @@ tokenize([$:, H | T] = Original, Line, Column, BaseScope, Tokens) when ?is_quote true -> atom_safe; false -> atom_unsafe end, - Token = {Key, {Line, Column, H}, Unescaped}, + Token = {Key, {Line, Column, NewLine, NewColumn, H}, Unescaped}, tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]); {error, Reason} -> @@ -548,7 +548,7 @@ tokenize([$: | String] = Original, Line, Column, Scope, Tokens) -> {_Kind, Unencoded, Atom, Rest, Length, Ascii, _Special} -> NewScope = maybe_warn_for_ambiguous_bang_before_equals(atom, Unencoded, Rest, Line, Column, Scope), TrackedScope = track_ascii(Ascii, NewScope), - Token = {atom, {Line, Column, Unencoded}, Atom}, + Token = {atom, {Line, Column, Line, Column + 1 + Length, Unencoded}, Atom}, tokenize(Rest, Line, Column + 1 + Length, TrackedScope, [Token | Tokens]); empty when Scope#spitfire_tokenizer.cursor_completion == false -> unexpected_token(Original, Line, Column, Scope, Tokens); @@ -587,10 +587,10 @@ tokenize([H | T], Line, Column, Scope, Tokens) when ?is_digit(H) -> error({?LOC(Line, Column), Msg, [I]}, T, Scope, Tokens) end; {Rest, Number, Original, Length} when is_integer(Number) -> - Token = {int, {Line, Column, Number}, Original}, + Token = {int, {Line, Column, Line, Column + Length, Number}, Original}, tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]); {Rest, Number, Original, Length} -> - Token = {flt, {Line, Column, Number}, Original}, + Token = {flt, {Line, Column, Line, Column + Length, Number}, Original}, tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]) end; @@ -603,10 +603,10 @@ tokenize([T | Rest], Line, Column, Scope, Tokens) when ?is_horizontal_space(T) - % End of line tokenize(";" ++ Rest, Line, Column, Scope, []) -> - tokenize(Rest, Line, Column + 1, Scope, [{';', {Line, Column, 0}}]); + tokenize(Rest, Line, Column + 1, Scope, [{';', {Line, Column, Line, Column + 1, 0}}]); tokenize(";" ++ Rest, Line, Column, Scope, [Top | _] = Tokens) when element(1, Top) /= ';' -> - tokenize(Rest, Line, Column + 1, Scope, [{';', {Line, Column, 0}} | Tokens]); + tokenize(Rest, Line, Column + 1, Scope, [{';', {Line, Column, Line, Column + 1, 0}} | Tokens]); tokenize("\\" = Original, Line, Column, Scope, Tokens) -> error({?LOC(Line, Column), "invalid escape \\ at end of file", []}, Original, Scope, Tokens); @@ -640,14 +640,14 @@ tokenize([$%, $[ | Rest], Line, Column, Scope, Tokens) -> error(Reason, Rest, Scope, Tokens); tokenize([$%, ${ | T], Line, Column, Scope, Tokens) -> - Token = {'{', {Line, Column, nil}}, - handle_terminator(T, Line, Column + 2, Scope, Token, [{'%{}', {Line, Column, nil}} | Tokens]); + Token = {'{', {Line, Column, Line, Column + 1, nil}}, + handle_terminator(T, Line, Column + 2, Scope, Token, [{'%{}', {Line, Column, Line, Column + 2, nil}} | Tokens]); tokenize([$% | T], Line, Column, Scope, Tokens) -> - tokenize(T, Line, Column + 1, Scope, [{'%', {Line, Column, nil}} | Tokens]); + tokenize(T, Line, Column + 1, Scope, [{'%', {Line, Column, Line, Column + 1, nil}} | Tokens]); tokenize([$. | T], Line, Column, Scope, Tokens) -> - tokenize_dot(T, Line, Column + 1, {Line, Column, nil}, Scope, Tokens); + tokenize_dot(T, Line, Column + 1, {Line, Column, Line, Column + 1, nil}, Scope, Tokens); % Identifiers @@ -659,7 +659,7 @@ tokenize(String, Line, Column, OriginalScope, Tokens) -> case Rest of [$: | T] when ?is_space(hd(T)) -> - Token = {kw_identifier, {Line, Column, Unencoded}, Atom}, + Token = {kw_identifier, {Line, Column, Line, Column + Length, Unencoded}, Atom}, tokenize(T, Line, Column + Length + 1, Scope, [Token | Tokens]); [$: | T] when hd(T) =/= $: -> @@ -679,7 +679,7 @@ tokenize(String, Line, Column, OriginalScope, Tokens) -> _ when Kind == identifier -> NewScope = maybe_warn_for_ambiguous_bang_before_equals(identifier, Unencoded, Rest, Line, Column, Scope), - Token = check_call_identifier(Line, Column, Unencoded, Atom, Rest), + Token = check_call_identifier(Line, Column, Unencoded, Atom, Rest, Length), tokenize(Rest, Line, Column + Length, NewScope, [Token | Tokens]); _ -> @@ -768,7 +768,7 @@ handle_heredocs(T, Line, Column, H, Scope, Tokens) -> {ok, NewLine, NewColumn, Parts, Rest, NewScope} -> case unescape_tokens(Parts, Line, Column, NewScope) of {ok, Unescaped} -> - Token = {heredoc_type(H), {Line, Column, nil}, NewColumn - 4, Unescaped}, + Token = {heredoc_type(H), {Line, Column, NewLine, NewColumn, nil}, NewColumn - 4, Unescaped}, tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]); {error, Reason} -> @@ -809,7 +809,7 @@ handle_strings(T, Line, Column, H, Scope, Tokens) -> {ok, [Part]} when is_binary(Part) -> case unsafe_to_atom(Part, Line, Column - 1, Scope) of {ok, Atom} -> - Token = {kw_identifier, {Line, Column - 1, H}, Atom}, + Token = {kw_identifier, {Line, Column - 1, NewLine, NewColumn + 1, H}, Atom}, tokenize(Rest, NewLine, NewColumn + 1, NewScope, [Token | Tokens]); {error, Reason} -> error(Reason, Rest, NewScope, Tokens) @@ -820,7 +820,7 @@ handle_strings(T, Line, Column, H, Scope, Tokens) -> true -> kw_identifier_safe; false -> kw_identifier_unsafe end, - Token = {Key, {Line, Column - 1, H}, Unescaped}, + Token = {Key, {Line, Column - 1, NewLine, NewColumn + 1, H}, Unescaped}, tokenize(Rest, NewLine, NewColumn + 1, NewScope, [Token | Tokens]); {error, Reason} -> @@ -843,7 +843,7 @@ handle_strings(T, Line, Column, H, Scope, Tokens) -> case unescape_tokens(Parts, Line, Column, NewScope) of {ok, Unescaped} -> - Token = {string_type(H), {Line, Column - 1, nil}, Unescaped}, + Token = {string_type(H), {Line, Column - 1, NewLine, NewColumn, nil}, Unescaped}, tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]); {error, Reason} -> @@ -852,27 +852,27 @@ handle_strings(T, Line, Column, H, Scope, Tokens) -> end. handle_unary_op([$: | Rest], Line, Column, _Kind, Length, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> - Token = {kw_identifier, {Line, Column, nil}, Op}, + Token = {kw_identifier, {Line, Column, Line, Column + Length + 1, nil}, Op}, tokenize(Rest, Line, Column + Length + 1, Scope, [Token | Tokens]); handle_unary_op(Rest, Line, Column, Kind, Length, Op, Scope, Tokens) -> case strip_horizontal_space(Rest, 0) of {[$/ | _] = Remaining, Extra} -> - Token = {identifier, {Line, Column, nil}, Op}, + Token = {identifier, {Line, Column, Line, Column + Length + Extra, nil}, Op}, tokenize(Remaining, Line, Column + Length + Extra, Scope, [Token | Tokens]); {Remaining, Extra} -> - Token = {Kind, {Line, Column, nil}, Op}, + Token = {Kind, {Line, Column, Line, Column + Length + Extra, nil}, Op}, tokenize(Remaining, Line, Column + Length + Extra, Scope, [Token | Tokens]) end. handle_op([$: | Rest], Line, Column, _Kind, Length, Op, Scope, Tokens) when ?is_space(hd(Rest)) -> - Token = {kw_identifier, {Line, Column, nil}, Op}, + Token = {kw_identifier, {Line, Column, Line, Column + Length + 1, nil}, Op}, tokenize(Rest, Line, Column + Length + 1, Scope, [Token | Tokens]); handle_op(Rest, Line, Column, Kind, Length, Op, Scope, Tokens) -> case strip_horizontal_space(Rest, 0) of {[$/ | _] = Remaining, Extra} -> - Token = {identifier, {Line, Column, nil}, Op}, + Token = {identifier, {Line, Column, Line, Column + Length + Extra, nil}, Op}, tokenize(Remaining, Line, Column + Length + Extra, Scope, [Token | Tokens]); {Remaining, Extra} -> NewScope = @@ -894,7 +894,7 @@ handle_op(Rest, Line, Column, Kind, Length, Op, Scope, Tokens) -> Scope end, - Token = {Kind, {Line, Column, previous_was_eol(Tokens)}, Op}, + Token = {Kind, {Line, Column, Line, Column + Length + Extra, previous_was_eol(Tokens)}, Op}, tokenize(Remaining, Line, Column + Length + Extra, NewScope, add_token_with_eol(Token, Tokens)) end. @@ -951,7 +951,11 @@ handle_dot([$., H | T] = Original, Line, Column, DotInfo, BaseScope, Tokens) whe case unsafe_to_atom(UnescapedPart, Line, Column, NewScope) of {ok, Atom} -> - Token = check_call_identifier(Line, Column, H, Atom, Rest), + Token = case Rest of + [$( | _] -> {paren_identifier, {Line, Column, NewLine, NewColumn, H}, Atom}; + [$[ | _] -> {bracket_identifier, {Line, Column, NewLine, NewColumn, H}, Atom}; + _ -> {identifier, {Line, Column, NewLine, NewColumn, H}, Atom} + end, TokensSoFar = add_token_with_eol({'.', DotInfo}, Tokens), tokenize(Rest, NewLine, NewColumn, NewScope, [Token | TokensSoFar]); @@ -970,7 +974,7 @@ handle_dot([$. | Rest], Line, Column, DotInfo, Scope, Tokens) -> tokenize(Rest, Line, Column, Scope, TokensSoFar). handle_call_identifier(Rest, Line, Column, DotInfo, Length, UnencodedOp, Scope, Tokens) -> - Token = check_call_identifier(Line, Column, UnencodedOp, list_to_atom(UnencodedOp), Rest), + Token = check_call_identifier(Line, Column, UnencodedOp, list_to_atom(UnencodedOp), Rest, Length), TokensSoFar = add_token_with_eol({'.', DotInfo}, Tokens), tokenize(Rest, Line, Column + Length, Scope, [Token | TokensSoFar]). @@ -983,7 +987,7 @@ handle_space_sensitive_tokens([Sign, $:, Space | _] = String, Line, Column, Scop handle_space_sensitive_tokens([Sign, NotMarker | T], Line, Column, Scope, [{identifier, _, _} = H | Tokens]) when ?dual_op(Sign), not(?is_space(NotMarker)), NotMarker =/= Sign, NotMarker =/= $/, NotMarker =/= $> -> Rest = [NotMarker | T], - DualOpToken = {dual_op, {Line, Column, nil}, list_to_atom([Sign])}, + DualOpToken = {dual_op, {Line, Column, Line, Column + 1, nil}, list_to_atom([Sign])}, tokenize(Rest, Line, Column + 1, Scope, [DualOpToken, setelement(1, H, op_identifier) | Tokens]); % Handle cursor completion @@ -997,14 +1001,14 @@ handle_space_sensitive_tokens(String, Line, Column, Scope, Tokens) -> %% Helpers -eol(_Line, _Column, [{',', {Line, Column, Count}} | Tokens]) -> - [{',', {Line, Column, Count + 1}} | Tokens]; -eol(_Line, _Column, [{';', {Line, Column, Count}} | Tokens]) -> - [{';', {Line, Column, Count + 1}} | Tokens]; -eol(_Line, _Column, [{eol, {Line, Column, Count}} | Tokens]) -> - [{eol, {Line, Column, Count + 1}} | Tokens]; +eol(_Line, _Column, [{',', {Line, Column, _, _, Count}} | Tokens]) -> + [{',', {Line, Column, Line, Column + 1, Count + 1}} | Tokens]; +eol(_Line, _Column, [{';', {Line, Column, _, _, Count}} | Tokens]) -> + [{';', {Line, Column, Line, Column + 1, Count + 1}} | Tokens]; +eol(_Line, _Column, [{eol, {Line, Column, _, _, Count}} | Tokens]) -> + [{eol, {Line, Column, Line, Column + 1, Count + 1}} | Tokens]; eol(Line, Column, Tokens) -> - [{eol, {Line, Column, 1}} | Tokens]. + [{eol, {Line, Column, Line, Column + 1, 1}} | Tokens]. is_unnecessary_quote([Part], Scope) when is_list(Part) -> case (Scope#spitfire_tokenizer.identifier_tokenizer):tokenize(Part) of @@ -1087,7 +1091,7 @@ extract_heredoc_header(_) -> extract_heredoc_indent(Part, {Warned, Line}, Indent) when is_list(Part) -> extract_heredoc_indent(Part, [], Warned, Line, Indent); -extract_heredoc_indent({_, {EndLine, _, _}, _} = Part, {Warned, _Line}, _Indent) -> +extract_heredoc_indent({_, {EndLine, _, _, _, _}, _} = Part, {Warned, _Line}, _Indent) -> {Part, {Warned, EndLine}}. extract_heredoc_indent([$\n | Rest], Acc, Warned, Line, Indent) -> @@ -1215,7 +1219,7 @@ reverse_number([], Number, Original) -> %% Comments -reset_eol([{eol, {Line, Column, _}} | Rest]) -> [{eol, {Line, Column, 0}} | Rest]; +reset_eol([{eol, {Line, Column, _, _, _}} | Rest]) -> [{eol, {Line, Column, Line, Column + 1, 0}} | Rest]; reset_eol(Rest) -> Rest. tokenize_comment("\r\n" ++ _ = Rest, Acc) -> @@ -1357,26 +1361,26 @@ tokenize_alias(Rest, Line, Column, Unencoded, Atom, Length, Ascii, Special, Scop error(Reason, Unencoded ++ Rest, Scope, Tokens); true -> - AliasesToken = {alias, {Line, Column, Unencoded}, Atom}, + AliasesToken = {alias, {Line, Column, Line, Column + Length, Unencoded}, Atom}, tokenize(Rest, Line, Column + Length, Scope, [AliasesToken | Tokens]) end. %% Check if it is a call identifier (paren | bracket | do) -check_call_identifier(Line, Column, Info, Atom, [$( | _]) -> - {paren_identifier, {Line, Column, Info}, Atom}; -check_call_identifier(Line, Column, Info, Atom, [$[ | _]) -> - {bracket_identifier, {Line, Column, Info}, Atom}; -check_call_identifier(Line, Column, Info, Atom, _Rest) -> - {identifier, {Line, Column, Info}, Atom}. +check_call_identifier(Line, Column, Info, Atom, [$( | _], Length) -> + {paren_identifier, {Line, Column, Line, Column + Length, Info}, Atom}; +check_call_identifier(Line, Column, Info, Atom, [$[ | _], Length) -> + {bracket_identifier, {Line, Column, Line, Column + Length, Info}, Atom}; +check_call_identifier(Line, Column, Info, Atom, _Rest, Length) -> + {identifier, {Line, Column, Line, Column + Length, Info}, Atom}. add_token_with_eol({unary_op, _, _} = Left, T) -> [Left | T]; add_token_with_eol(Left, [{eol, _} | T]) -> [Left | T]; add_token_with_eol(Left, T) -> [Left | T]. -previous_was_eol([{',', {_, _, Count}} | _]) when Count > 0 -> Count; -previous_was_eol([{';', {_, _, Count}} | _]) when Count > 0 -> Count; -previous_was_eol([{eol, {_, _, Count}} | _]) when Count > 0 -> Count; +previous_was_eol([{',', {_, _, _, _, Count}} | _]) when Count > 0 -> Count; +previous_was_eol([{';', {_, _, _, _, Count}} | _]) when Count > 0 -> Count; +previous_was_eol([{eol, {_, _, _, _, Count}} | _]) when Count > 0 -> Count; previous_was_eol(_) -> nil. %% Error handling @@ -1399,7 +1403,7 @@ interpolation_format({_, _, _} = Reason, _Extension, _Args, _Line, _Column, _Ope %% Terminators -handle_terminator(Rest, _, _, Scope, {'(', {Line, Column, _}}, [{alias, _, Alias} | Tokens]) when is_atom(Alias) -> +handle_terminator(Rest, _, _, Scope, {'(', {Line, Column, _, _, _}}, [{alias, _, Alias} | Tokens]) when is_atom(Alias) -> Reason = io_lib:format( "unexpected ( after alias ~ts. Function names and identifiers in Elixir " @@ -1446,7 +1450,7 @@ check_terminator({Start, Meta}, Terminators, Scope) when Start == 'fn'; Start == {ok, NewScope#spitfire_tokenizer{terminators=[{Start, Meta, Indentation} | Terminators]}}; -check_terminator({'end', {EndLine, _, _}}, [{'do', _, Indentation} | Terminators], Scope) -> +check_terminator({'end', {EndLine, _, _, _, _}}, [{'do', _, Indentation} | Terminators], Scope) -> NewScope = %% If the end is more indented than the do, it may be a missing do error! case Scope#spitfire_tokenizer.indentation > Indentation of @@ -1460,7 +1464,7 @@ check_terminator({'end', {EndLine, _, _}}, [{'do', _, Indentation} | Terminators {ok, NewScope#spitfire_tokenizer{terminators=Terminators}}; -check_terminator({End, {EndLine, EndColumn, _}}, [{Start, {StartLine, StartColumn, _}, _} | Terminators], Scope) +check_terminator({End, {EndLine, EndColumn, _, _, _}}, [{Start, {StartLine, StartColumn, _, _, _}, _} | Terminators], Scope) when End == 'end'; End == ')'; End == ']'; End == '}'; End == '>>' -> case terminator(Start) of End -> @@ -1480,7 +1484,7 @@ check_terminator({End, {EndLine, EndColumn, _}}, [{Start, {StartLine, StartColum {error, {Meta, unexpected_token_or_reserved(End), [atom_to_list(End)]}} end; -check_terminator({'end', {Line, Column, _}}, [], #spitfire_tokenizer{mismatch_hints=Hints}) -> +check_terminator({'end', {Line, Column, _, _, _}}, [], #spitfire_tokenizer{mismatch_hints=Hints}) -> Suffix = case lists:keyfind('end', 1, Hints) of {'end', HintLine, _Indentation} -> @@ -1492,7 +1496,7 @@ check_terminator({'end', {Line, Column, _}}, [], #spitfire_tokenizer{mismatch_hi {error, {?LOC(Line, Column), {"unexpected reserved word: ", Suffix}, "end"}}; -check_terminator({End, {Line, Column, _}}, [], _Scope) +check_terminator({End, {Line, Column, _, _, _}}, [], _Scope) when End == ')'; End == ']'; End == '}'; End == '>>' -> {error, {?LOC(Line, Column), "unexpected token: ", atom_to_list(End)}}; @@ -1504,7 +1508,7 @@ unexpected_token_or_reserved(_) -> "unexpected token: ". missing_terminator_hint(Start, End, #spitfire_tokenizer{mismatch_hints=Hints}) -> case lists:keyfind(Start, 1, Hints) of - {Start, {HintLine, _, _}, _} -> + {Start, {HintLine, _, _, _, _}, _} -> io_lib:format("\n~ts it looks like the \"~ts\" on line ~B does not have a matching \"~ts\"", [elixir_errors:prefix(hint), Start, HintLine, End]); false -> @@ -1562,18 +1566,18 @@ tokenize_keyword(terminator, Rest, Line, Column, Atom, Length, Scope, Tokens) -> end; tokenize_keyword(token, Rest, Line, Column, Atom, Length, Scope, Tokens) -> - Token = {Atom, {Line, Column, nil}}, + Token = {Atom, {Line, Column, Line, Column + Length, nil}}, tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]); tokenize_keyword(block, Rest, Line, Column, Atom, Length, Scope, Tokens) -> - Token = {block_identifier, {Line, Column, nil}, Atom}, + Token = {block_identifier, {Line, Column, Line, Column + Length, nil}, Atom}, tokenize(Rest, Line, Column + Length, Scope, [Token | Tokens]); tokenize_keyword(Kind, Rest, Line, Column, Atom, Length, Scope, Tokens) -> NewTokens = case strip_horizontal_space(Rest, 0) of {[$/ | _], _} -> - [{identifier, {Line, Column, nil}, Atom} | Tokens]; + [{identifier, {Line, Column, Line, Column + Length, nil}, Atom} | Tokens]; _ -> case {Kind, Tokens} of @@ -1581,7 +1585,7 @@ tokenize_keyword(Kind, Rest, Line, Column, Atom, Length, Scope, Tokens) -> add_token_with_eol({in_op, NotInfo, 'not in'}, T); {_, _} -> - add_token_with_eol({Kind, {Line, Column, previous_was_eol(Tokens)}, Atom}, Tokens) + add_token_with_eol({Kind, {Line, Column, Line, Column + Length, previous_was_eol(Tokens)}, Atom}, Tokens) end end, @@ -1671,8 +1675,8 @@ add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, Scope, case MaybeEncoded of {ok, Atom} -> {Final, Modifiers} = collect_modifiers(Rest, []), - Token = {sigil, {Line, TokenColumn, nil}, Atom, Parts, Modifiers, Indentation, Delimiter}, NewColumnWithModifiers = NewColumn + length(Modifiers), + Token = {sigil, {Line, TokenColumn, NewLine, NewColumnWithModifiers, nil}, Atom, Parts, Modifiers, Indentation, Delimiter}, tokenize(Final, NewLine, NewColumnWithModifiers, Scope, [Token | Tokens]); {error, Reason} -> @@ -1681,18 +1685,18 @@ add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, Scope, %% Fail early on invalid do syntax. For example, after %% most keywords, after comma and so on. -tokenize_keyword_terminator(DoLine, DoColumn, do, [{identifier, {Line, Column, Meta}, Atom} | T]) -> - {ok, add_token_with_eol({do, {DoLine, DoColumn, nil}}, - [{do_identifier, {Line, Column, Meta}, Atom} | T])}; +tokenize_keyword_terminator(DoLine, DoColumn, do, [{identifier, {Line, Column, _, _, Meta}, Atom} | T]) -> + {ok, add_token_with_eol({do, {DoLine, DoColumn, DoLine, DoColumn + 2, nil}}, + [{do_identifier, {Line, Column, Line, Column + length(atom_to_list(Atom)), Meta}, Atom} | T])}; tokenize_keyword_terminator(_Line, _Column, do, [{'fn', _} | _]) -> {error, invalid_do_with_fn_error("unexpected reserved word: "), "do"}; tokenize_keyword_terminator(Line, Column, do, Tokens) -> case is_valid_do(Tokens) of - true -> {ok, add_token_with_eol({do, {Line, Column, nil}}, Tokens)}; + true -> {ok, add_token_with_eol({do, {Line, Column, Line, Column + 2, nil}}, Tokens)}; false -> {error, invalid_do_error("unexpected reserved word: "), "do"} end; tokenize_keyword_terminator(Line, Column, Atom, Tokens) -> - {ok, [{Atom, {Line, Column, nil}} | Tokens]}. + {ok, [{Atom, {Line, Column, Line, Column + length(atom_to_list(Atom)), nil}} | Tokens]}. is_valid_do([{Atom, _} | _]) -> case Atom of @@ -1786,9 +1790,9 @@ add_cursor(Line, Column, prune_and_cursor, Terminators, Tokens) -> PrePrunedTokens = prune_identifier(Tokens), PrunedTokens = prune_tokens(PrePrunedTokens, []), CursorTokens = [ - {')', {Line, Column + 11, nil}}, - {'(', {Line, Column + 10, nil}}, - {paren_identifier, {Line, Column, nil}, '__cursor__'} + {')', {Line, Column + 11, Line, Column + 12, nil}}, + {'(', {Line, Column + 10, Line, Column + 11, nil}}, + {paren_identifier, {Line, Column, Line, Column + 10, nil}, '__cursor__'} | PrunedTokens ], {Column + 12, Terminators, CursorTokens}. @@ -1862,4 +1866,3 @@ prune_tokens([_ | Tokens], Opener) -> prune_tokens(Tokens, Opener); prune_tokens([], _Opener) -> []. - diff --git a/test/char_property_test.exs b/test/char_property_test.exs index 7586856..3c41cec 100644 --- a/test/char_property_test.exs +++ b/test/char_property_test.exs @@ -1659,7 +1659,8 @@ defmodule Spitfire.CharPropertyTest do """ end - assert elixir_ast == spitfire_ast, msg + assert Spitfire.TestHelpers.strip_range_metadata(elixir_ast) == + Spitfire.TestHelpers.strip_range_metadata(spitfire_ast), msg {:error, _spitfire_ast, _errors} -> msg = diff --git a/test/conformance_test.exs b/test/conformance_test.exs index c3568e3..e33b480 100644 --- a/test/conformance_test.exs +++ b/test/conformance_test.exs @@ -2935,7 +2935,7 @@ defmodule Spitfire.ConformanceTest do defp spitfire_parse(code) do case Spitfire.parse(code) do - {:ok, ast} -> {:ok, ast} + {:ok, ast} -> {:ok, Spitfire.TestHelpers.strip_range_metadata(ast)} {:error, _ast, _errors} -> {:error, :parse_error} {:error, :no_fuel_remaining} -> {:error, :no_fuel_remaining} end diff --git a/test/operators_test.exs b/test/operators_test.exs index 6592a5e..95d3a5a 100644 --- a/test/operators_test.exs +++ b/test/operators_test.exs @@ -2316,7 +2316,7 @@ defmodule Spitfire.OperatorsTest do defp spitfire_parse(code, _options \\ []) do case Spitfire.parse(code) do - {:ok, ast} -> {:ok, ast} + {:ok, ast} -> {:ok, Spitfire.TestHelpers.strip_range_metadata(ast)} {:error, _ast, _errors} -> {:error, :parse_error} {:error, :no_fuel_remaining} -> {:error, :no_fuel_remaining} end diff --git a/test/range_metadata_test.exs b/test/range_metadata_test.exs new file mode 100644 index 0000000..50f681d --- /dev/null +++ b/test/range_metadata_test.exs @@ -0,0 +1,542 @@ +defmodule Spitfire.RangeMetadataTest do + use ExUnit.Case, async: true + + test "binary operators span the whole expression" do + assert {:ok, {:+, meta, [1, {:*, inner_meta, [2, 3]}]}} = Spitfire.parse("1 + 2 * 3") + + assert meta[:range] == %{start: {1, 1}, end: {1, 10}} + assert inner_meta[:range] == %{start: {1, 5}, end: {1, 10}} + end + + test "dot identifiers and dot calls keep a single wrapper range" do + assert {:ok, {{:., dot_meta, [_lhs, :bar]}, outer_meta, []}} = Spitfire.parse("foo.bar()") + + assert [dot_range] = Keyword.get_values(dot_meta, :range) + assert [outer_range] = Keyword.get_values(outer_meta, :range) + assert dot_range == %{start: {1, 1}, end: {1, 10}} + assert outer_range == %{start: {1, 1}, end: {1, 10}} + end + + test "dot calls used in access match stdlib after removing ranges" do + for code <- ["foo.bar[0]", ~S(foo."bar"[:key])] do + assert normalize(Spitfire.parse!(code)) == stdlib_parse!(code) + end + end + + test "dot call with do block keeps a single wrapper range" do + code = ~S''' + foo.() do + :ok + end + ''' + + assert {:ok, {{:., _dot_meta, [_lhs]}, outer_meta, [[do: :ok]]}} = Spitfire.parse(code) + + assert [outer_range] = Keyword.get_values(outer_meta, :range) + assert outer_range == %{start: {1, 1}, end: {3, 4}} + end + + test "dot identifier with do block spans the full receiver" do + code = ~S''' + foo.bar do + :ok + end + ''' + + assert {:ok, {{:., _dot_meta, [_lhs, :bar]}, outer_meta, [[do: :ok]]}} = Spitfire.parse(code) + assert [outer_range] = Keyword.get_values(outer_meta, :range) + assert outer_range == %{start: {1, 1}, end: {3, 4}} + assert normalize(Spitfire.parse!(code)) == stdlib_parse!(code) + end + + test "empty grouped expressions include the closing paren in range" do + assert {:ok, {:__block__, meta, []}} = Spitfire.parse("()") + + assert meta[:range] == %{start: {1, 1}, end: {1, 3}} + end + + test "anonymous functions include the closing end keyword in range" do + assert {:ok, {:fn, meta, clauses}} = Spitfire.parse("fn :a -> 1; :b -> 2 end") + + assert meta[:range] == %{start: {1, 1}, end: {1, 24}} + assert [_, _] = clauses + end + + test "struct literal keeps a narrower range for the inner synthetic map" do + assert {:ok, {:%, outer_meta, [{:__aliases__, _, [:MyStruct]}, {:%{}, inner_meta, []}]}} = + Spitfire.parse("%MyStruct{}") + + assert outer_meta[:range] == %{start: {1, 1}, end: {1, 12}} + assert inner_meta[:range] == %{start: {1, 10}, end: {1, 12}} + end + + test "range keys in user data are preserved while metadata ranges are removed" do + for code <- ["[range: 1]", "%{range: 1}", "[nested: %{range: 1}]", ~S|{[range: 1], "x"}|] do + assert normalize(Spitfire.parse!(code)) == stdlib_parse!(code) + end + end + + describe "range corpus" do + test "comments do not change expression ranges" do + for {code, range} <- [ + {~S''' + # Foo + :bar + ''', %{start: {2, 1}, end: {2, 5}}}, + {~S''' + # Foo + # Bar + :baz + ''', %{start: {3, 1}, end: {3, 5}}}, + {":baz # Foo", %{start: {1, 1}, end: {1, 5}}}, + {~S''' + # Foo + :baz # Bar + ''', %{start: {2, 1}, end: {2, 5}}} + ] do + assert {_token, meta, _args} = parse_with_literal_ranges(code) + assert meta[:range] == range + end + end + + test "literal roots use literal_encoder range metadata" do + for {code, range} <- [ + {~S|1|, %{start: {1, 1}, end: {1, 2}}}, + {~S|100|, %{start: {1, 1}, end: {1, 4}}}, + {~S|1_000|, %{start: {1, 1}, end: {1, 6}}}, + {~S|1.0|, %{start: {1, 1}, end: {1, 4}}}, + {~S|1.00|, %{start: {1, 1}, end: {1, 5}}}, + {~S|1_000.0|, %{start: {1, 1}, end: {1, 8}}}, + {~S|"foo"|, %{start: {1, 1}, end: {1, 6}}}, + {~S''' + "fo + o" + ''', %{start: {1, 1}, end: {2, 3}}}, + {~S|"key: \"value\""|, %{start: {1, 1}, end: {1, 17}}}, + {~S|'foo'|, %{start: {1, 1}, end: {1, 6}}}, + {~S''' + 'fo + o' + ''', %{start: {1, 1}, end: {2, 3}}}, + {~S|:foo|, %{start: {1, 1}, end: {1, 5}}}, + {~S|:"foo"|, %{start: {1, 1}, end: {1, 7}}}, + {~S|:'foo'|, %{start: {1, 1}, end: {1, 7}}}, + {~S|:"::"|, %{start: {1, 1}, end: {1, 6}}}, + {~S|foo|, %{start: {1, 1}, end: {1, 4}}}, + {~S|{1, 2, 3}|, %{start: {1, 1}, end: {1, 10}}}, + {~S|[1, 2, 3]|, %{start: {1, 1}, end: {1, 10}}} + ] do + assert {_token, meta, _args} = parse_with_literal_ranges(code) + assert meta[:range] == range + end + end + + test "multiline literal roots use exact closing delimiter ranges" do + for {code, range} <- [ + {~S''' + """ + foo + + bar + """ + ''', %{start: {1, 1}, end: {5, 4}}}, + {~S''' + """ + foo + bar + """ + ''', %{start: {1, 3}, end: {4, 6}}}, + {~S""" + ''' + foo + + bar + ''' + """, %{start: {1, 1}, end: {5, 4}}}, + {~S""" + ''' + foo + bar + ''' + """, %{start: {1, 3}, end: {4, 6}}}, + {~S''' + :"foo + + bar" + ''', %{start: {1, 1}, end: {3, 5}}} + ] do + assert {_token, meta, _args} = parse_with_literal_ranges(code) + assert meta[:range] == range + end + end + + test "interpolated strings, charlists, atoms, and sigils span delimiters" do + for {code, range} <- [ + {~S|"foo#{2}bar"|, %{start: {1, 1}, end: {1, 13}}}, + {~S''' + "foo#{ + 2 + }bar" + ''', %{start: {1, 1}, end: {3, 8}}}, + {~S''' + "foo#{ + 2 + } + bar" + ''', %{start: {1, 1}, end: {4, 7}}}, + {~S''' + "foo#{ + 2 + } + bar + " + ''', %{start: {1, 1}, end: {5, 2}}}, + {~S|'foo#{2}bar'|, %{start: {1, 1}, end: {1, 13}}}, + {~S''' + 'foo#{ + 2 + }bar' + ''', %{start: {1, 1}, end: {3, 8}}}, + {~S''' + 'foo#{ + 2 + } + bar' + ''', %{start: {1, 1}, end: {4, 7}}}, + {~S''' + 'foo#{ + 2 + } + bar + ' + ''', %{start: {1, 1}, end: {5, 2}}}, + {~S|:"foo#{2}bar"|, %{start: {1, 1}, end: {1, 14}}}, + {~S''' + :"foo#{ + 2 + }bar" + ''', %{start: {1, 1}, end: {3, 8}}}, + {~S''' + :"foo#{ + 2 + } + bar" + ''', %{start: {1, 1}, end: {4, 5}}}, + {~S|~s[foo#{2}bar]|, %{start: {1, 1}, end: {1, 15}}}, + {~S|~s[foo#{2}bar]abc|, %{start: {1, 1}, end: {1, 18}}}, + {~S''' + ~s""" + foo#{10 + } + bar + """ + ''', %{start: {1, 1}, end: {5, 4}}}, + {~S''' + ~s""" + foo#{10 + }bar + """abc + ''', %{start: {1, 1}, end: {4, 7}}} + ] do + assert {_token, meta, _args} = parse_with_literal_ranges(code) + assert meta[:range] == range + end + end + + test "interpolation child ranges cover interpolation delimiters" do + code = ~S''' + "foo#{ + 2 + }bar" + ''' + + assert {:<<>>, _string_meta, ["foo", {:"::", interpolation_meta, _args}, "bar"]} = + parse_with_literal_ranges(code) + + assert interpolation_meta[:range] == %{start: {1, 5}, end: {3, 4}} + + code = ~S''' + 'foo#{ + 2 + }bar' + ''' + + assert {{:., _charlist_dot_meta, [List, :to_charlist]}, _charlist_meta, + [["foo", {{:., _interpolation_dot_meta, [Kernel, :to_string]}, interpolation_meta, _args}, "bar"]]} = + parse_with_literal_ranges(code) + + assert interpolation_meta[:range] == %{start: {1, 5}, end: {3, 4}} + end + + test "containers and grouped blocks span their delimiters" do + for {code, range} <- [ + {~S''' + { + 1, + 2, + 3 + } + ''', %{start: {1, 1}, end: {5, 2}}}, + {~S''' + {1, + 2, + 3} + ''', %{start: {1, 1}, end: {3, 6}}}, + {~S''' + [ + 1, + 2, + 3 + ] + ''', %{start: {1, 1}, end: {5, 2}}}, + {~S''' + [1, + 2, + 3] + ''', %{start: {1, 1}, end: {3, 6}}}, + {~S|(1; 2; 3)|, %{start: {1, 1}, end: {1, 10}}}, + {~S''' + (1; + 2; + 3) + ''', %{start: {1, 1}, end: {3, 5}}}, + {~S''' + (1; + 2; + 3 + ) + ''', %{start: {1, 1}, end: {4, 2}}} + ] do + assert {_token, meta, _args} = parse_with_literal_ranges(code) + assert meta[:range] == range + end + end + + test "anonymous functions and stabs span their clauses" do + for {code, range} <- [ + {~S|fn -> :ok end|, %{start: {1, 1}, end: {1, 14}}}, + {~S''' + fn -> end + ''', %{start: {1, 1}, end: {1, 10}}}, + {~S''' + fn -> + end + ''', %{start: {1, 1}, end: {2, 4}}}, + {~S''' + fn -> + :ok + end + ''', %{start: {1, 1}, end: {3, 4}}} + ] do + assert {_token, meta, _args} = parse_with_literal_ranges(code) + assert meta[:range] == range + end + + assert {:fn, _fn_meta, [{:->, stab_meta, _args}]} = parse_with_literal_ranges(~S|fn -> :ok end|) + assert stab_meta[:range] == %{start: {1, 4}, end: {1, 10}} + + code = ~S''' + fn -> + :ok + end + ''' + + assert {:fn, _fn_meta, [{:->, stab_meta, _args}]} = parse_with_literal_ranges(code) + assert stab_meta[:range] == %{start: {1, 4}, end: {2, 6}} + + assert {:fn, _fn_meta, [{:->, stab_meta, _args}]} = parse_with_literal_ranges(~S|fn -> end|) + assert stab_meta[:range] == %{start: {1, 4}, end: {1, 6}} + + code = ~S''' + fn a -> + end + ''' + + assert {:fn, _fn_meta, [{:->, stab_meta, _args}]} = parse_with_literal_ranges(code) + assert stab_meta[:range] == %{start: {1, 4}, end: {1, 8}} + end + + test "aliases, calls, and blocks span receivers and terminators" do + for {code, range} <- [ + {~S|Foo|, %{start: {1, 1}, end: {1, 4}}}, + {~S|Foo.Bar|, %{start: {1, 1}, end: {1, 8}}}, + {~S''' + Foo. + Bar + ''', %{start: {1, 1}, end: {2, 6}}}, + {~S|__MODULE__|, %{start: {1, 1}, end: {1, 11}}}, + {~S|__MODULE__.Bar|, %{start: {1, 1}, end: {1, 15}}}, + {~S|@foo.Bar|, %{start: {1, 1}, end: {1, 9}}}, + {~S|foo().Bar|, %{start: {1, 1}, end: {1, 10}}}, + {~S|foo.bar.().Baz|, %{start: {1, 1}, end: {1, 15}}}, + {~S|Foo.{Bar, Baz}|, %{start: {1, 1}, end: {1, 15}}}, + {~S''' + Foo.{ + Bar, + Bar, + Qux + } + ''', %{start: {1, 1}, end: {5, 2}}}, + {~S''' + Foo.{Bar, + Baz, + Qux} + ''', %{start: {1, 1}, end: {3, 8}}}, + {~S|foo do :ok end|, %{start: {1, 1}, end: {1, 15}}}, + {~S''' + foo do + :ok + end + ''', %{start: {1, 1}, end: {3, 4}}}, + {~S|foo.bar|, %{start: {1, 1}, end: {1, 8}}}, + {~S|foo.bar()|, %{start: {1, 1}, end: {1, 10}}}, + {~S|foo.()|, %{start: {1, 1}, end: {1, 7}}}, + {~S|foo.bar.()|, %{start: {1, 1}, end: {1, 11}}}, + {~S''' + foo.bar( + ) + ''', %{start: {1, 1}, end: {2, 2}}}, + {~S|a.b.c|, %{start: {1, 1}, end: {1, 6}}}, + {~S|foo.bar(baz)|, %{start: {1, 1}, end: {1, 13}}}, + {~S|foo.bar.(baz)|, %{start: {1, 1}, end: {1, 14}}}, + {~S''' + foo.bar.( + baz) + ''', %{start: {1, 1}, end: {2, 5}}}, + {~S|foo.bar("baz#{2}qux")|, %{start: {1, 1}, end: {1, 22}}}, + {~S|foo.bar("baz#{2}qux", [])|, %{start: {1, 1}, end: {1, 26}}}, + {~S|foo."b-a-r"|, %{start: {1, 1}, end: {1, 12}}}, + {~S|foo."b-a-r"()|, %{start: {1, 1}, end: {1, 14}}}, + {~S|foo."b-a-r"(1)|, %{start: {1, 1}, end: {1, 15}}}, + {~S|Mod.unquote(foo)(bar)|, %{start: {1, 1}, end: {1, 22}}}, + {~S|foo.bar baz|, %{start: {1, 1}, end: {1, 12}}}, + {~S|foo.bar baz, qux|, %{start: {1, 1}, end: {1, 17}}}, + {~S|foo."b-a-r" baz|, %{start: {1, 1}, end: {1, 16}}}, + {~S|foo(bar)|, %{start: {1, 1}, end: {1, 9}}}, + {~S''' + foo( + bar + ) + ''', %{start: {1, 1}, end: {3, 4}}}, + {~S|foo bar|, %{start: {1, 1}, end: {1, 8}}}, + {~S|foo bar baz|, %{start: {1, 1}, end: {1, 12}}}, + {~S''' + foo + bar + ''', %{start: {1, 1}, end: {2, 6}}}, + {~S|Foo.bar|, %{start: {1, 1}, end: {1, 8}}}, + {~S''' + Foo. + bar + ''', %{start: {1, 1}, end: {2, 6}}}, + {~S|unquote(foo)()|, %{start: {1, 1}, end: {1, 15}}} + ] do + assert {_token, meta, _args} = parse_with_literal_ranges(code) + assert meta[:range] == range + end + end + + test "operators, ranges, bitstrings, sigils, captures, and access span operands" do + for {code, range} <- [ + {~S|!foo|, %{start: {1, 1}, end: {1, 5}}}, + {~S|! foo|, %{start: {1, 1}, end: {1, 8}}}, + {~S|not foo|, %{start: {1, 1}, end: {1, 9}}}, + {~S|@foo|, %{start: {1, 1}, end: {1, 5}}}, + {~S|@ foo|, %{start: {1, 1}, end: {1, 8}}}, + {~S|1 + 1|, %{start: {1, 1}, end: {1, 6}}}, + {~S|foo when bar|, %{start: {1, 1}, end: {1, 13}}}, + {~S''' + 5 + + 10 + ''', %{start: {1, 2}, end: {2, 7}}}, + {~S"foo |> bar", %{start: {1, 1}, end: {1, 11}}}, + {~S''' + foo + |> bar + ''', %{start: {1, 1}, end: {2, 7}}}, + {~S''' + foo + |> + bar + ''', %{start: {1, 1}, end: {3, 4}}}, + {~S|1..2|, %{start: {1, 1}, end: {1, 5}}}, + {~S|1..2//3|, %{start: {1, 1}, end: {1, 8}}}, + {~S|foo..bar//baz|, %{start: {1, 1}, end: {1, 14}}}, + {~S|<<1, 2, foo>>|, %{start: {1, 1}, end: {1, 14}}}, + {~S''' + <<1, 2, + + foo>> + ''', %{start: {1, 1}, end: {3, 7}}}, + {~S|~s[foo bar]|, %{start: {1, 1}, end: {1, 12}}}, + {~S''' + ~s""" + foo + bar + """ + ''', %{start: {1, 1}, end: {4, 4}}}, + {~S|&foo/1|, %{start: {1, 1}, end: {1, 7}}}, + {~S|&Foo.bar/1|, %{start: {1, 1}, end: {1, 11}}}, + {~S|&__MODULE__.Foo.bar/1|, %{start: {1, 1}, end: {1, 22}}}, + {~S|&foo(&1, :bar)|, %{start: {1, 1}, end: {1, 15}}}, + {~S|& &1.foo|, %{start: {1, 1}, end: {1, 9}}}, + {~S|& &1|, %{start: {1, 1}, end: {1, 5}}}, + {~S|foo[bar]|, %{start: {1, 1}, end: {1, 9}}} + ] do + assert {_token, meta, _args} = parse_with_literal_ranges(code) + assert meta[:range] == range + end + end + + test "regression ranges" do + code = ~S''' + def rpc_call(pid, call = %Call{method: unquote(method_name)}), + do: GenServer.unquote(genserver_method)(pid, call) + ''' + + assert {_token, meta, _args} = parse_with_literal_ranges(code) + assert meta[:range] == %{start: {1, 1}, end: {2, 53}} + + code = ~S''' + defmodule Foo do + def foo(arg) when (arg.valid? == true) do + arg + end + end + ''' + + assert {:defmodule, _module_meta, + [ + _module, + [ + {{:__literal__, _do_meta, [:do]}, + {:def, _def_meta, [{:when, _when_meta, [_call, {:==, meta, _args}]}, _body]}} + ] + ]} = parse_with_literal_ranges(code) + + assert meta[:range] == %{start: {2, 21}, end: {2, 41}} + + code = ~S''' + fn + 1 -> File.read!(arg1) + arg1 -> File.read!(arg1) + end + ''' + + assert {:fn, meta, _stabs} = parse_with_literal_ranges(code) + assert meta[:range] == %{start: {1, 21}, end: {4, 4}} + end + end + + defp normalize(ast) do + Spitfire.TestHelpers.strip_range_metadata(ast) + end + + defp parse_with_literal_ranges(code) do + Spitfire.parse!(code, literal_encoder: fn literal, meta -> {:ok, {:__literal__, meta, [literal]}} end) + end + + defp stdlib_parse!(code) do + Code.string_to_quoted!(code, columns: true, token_metadata: true, emit_warnings: false) + end +end diff --git a/test/spitfire_test.exs b/test/spitfire_test.exs index cb6f749..9e64c25 100644 --- a/test/spitfire_test.exs +++ b/test/spitfire_test.exs @@ -7,32 +7,32 @@ defmodule SpitfireTest do test "semicolons" do code = "res = Foo.Bar.run(1, 2, 3); IO.inspect(res)" - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' res = Foo.Bar.run(1, 2, 3); IO.inspect(res) ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' fn one -> IO.inspect(one); one end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' def foo, do: IO.inspect("bob"); "bob" ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' foo do: IO.inspect("bob"); "bob" ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parses valid elixir" do @@ -48,7 +48,7 @@ defmodule SpitfireTest do end """ - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "access syntax" do @@ -70,7 +70,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -81,7 +81,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' foo do @@ -91,7 +91,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "type syntax" do @@ -122,7 +122,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -131,14 +131,14 @@ defmodule SpitfireTest do ^foo ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' ^ foo ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parses numbers" do @@ -146,13 +146,13 @@ defmodule SpitfireTest do 111_111 """ - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = """ 1.4 """ - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parses strings" do @@ -160,7 +160,7 @@ defmodule SpitfireTest do "foobar" ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' """ @@ -168,7 +168,7 @@ defmodule SpitfireTest do """ ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parses charlists" do @@ -176,7 +176,7 @@ defmodule SpitfireTest do 'foobar' ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S""" ''' @@ -184,13 +184,13 @@ defmodule SpitfireTest do ''' """ - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' 'foo#{alice}bar' ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' 'foo#{ @@ -198,7 +198,7 @@ defmodule SpitfireTest do }bar' ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S""" ''' @@ -206,7 +206,7 @@ defmodule SpitfireTest do ''' """ - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S""" ''' @@ -216,7 +216,7 @@ defmodule SpitfireTest do ''' """ - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parses atoms" do @@ -224,25 +224,25 @@ defmodule SpitfireTest do :foobar ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~s''' :"," ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' :"foo#{}" ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' :"foo#{bar}" ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parses left stab" do @@ -250,7 +250,7 @@ defmodule SpitfireTest do apple <- apples """ - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end # these tests do not test against Code.string_to_quoted because these code fragments parse as errors @@ -260,7 +260,7 @@ defmodule SpitfireTest do # -> bar # """ - # assert Spitfire.parse!(code) == [ + # assert spitfire_parse!(code) == [ # {:->, [line: 1, column: 1], # [[], {:bar, [{:end_of_expression, [newlines: 1, line: 1, column: 7]}, line: 1, column: 4], nil}]} # ] @@ -269,13 +269,13 @@ defmodule SpitfireTest do # -> :ok # """ - # assert Spitfire.parse!(code) == [{:->, [line: 1, column: 1], [[], :ok]}] + # assert spitfire_parse!(code) == [{:->, [line: 1, column: 1], [[], :ok]}] # code = """ # foo -> bar # """ - # assert Spitfire.parse!(code) == [ + # assert spitfire_parse!(code) == [ # {:->, [line: 1, column: 5], # [ # [{:foo, [line: 1, column: 1], nil}], @@ -287,7 +287,7 @@ defmodule SpitfireTest do # foo, bar, baz -> bar # """ - # assert Spitfire.parse!(code) == [ + # assert spitfire_parse!(code) == [ # {:->, [line: 1, column: 15], # [ # [ @@ -308,7 +308,7 @@ defmodule SpitfireTest do # # if we get a prefix comma operator, that means we might need to backtrack and then # # parse a comma list. if we hit the operator, it means that we are not actually in an # # existing comma list, like a list or a map - # assert Spitfire.parse!(code) == [ + # assert spitfire_parse!(code) == [ # {:->, [newlines: 1, line: 1, column: 19], # [ # [ @@ -331,7 +331,7 @@ defmodule SpitfireTest do bar """ - assert Spitfire.parse!(code) == + assert spitfire_parse!(code) == [ {:->, [newlines: 1, line: 1, column: 5], [ @@ -372,7 +372,7 @@ defmodule SpitfireTest do :ok ''' - assert Spitfire.parse!(code) == [ + assert spitfire_parse!(code) == [ {:->, [newlines: 1, line: 1, column: 6], [[{:^, [line: 1, column: 1], [{:foo, [line: 1, column: 2], nil}]}], :ok]} ] @@ -382,7 +382,7 @@ defmodule SpitfireTest do :ok ''' - assert Spitfire.parse!(code) == [ + assert spitfire_parse!(code) == [ {:->, [newlines: 1, line: 1, column: 6], [[{:@, [line: 1, column: 1], [{:foo, [line: 1, column: 2], nil}]}], :ok]} ] @@ -418,7 +418,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -432,7 +432,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -448,7 +448,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -459,7 +459,7 @@ defmodule SpitfireTest do bob ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parses lists" do @@ -488,7 +488,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -523,7 +523,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -534,7 +534,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parses pattern matching in list" do @@ -548,7 +548,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -587,7 +587,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -619,7 +619,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -683,18 +683,18 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end test "parses ambiguous map update" do code = ~S'%{a do :ok end | b c, d => e}' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S'%{a do :ok end | b c, d => e, f => g}' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parses structs" do @@ -734,7 +734,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -886,7 +886,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -894,7 +894,7 @@ defmodule SpitfireTest do bad_code = ~S'!(left) in right' good_code = ~S'!(left in right)' - assert Spitfire.parse!(bad_code) == + assert spitfire_parse!(bad_code) == { :!, [line: 1, column: 9], @@ -918,7 +918,7 @@ defmodule SpitfireTest do ] } - assert Spitfire.parse!(good_code) == + assert spitfire_parse!(good_code) == { :!, [line: 1, column: 1], @@ -954,7 +954,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -1034,7 +1034,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -1047,7 +1047,7 @@ defmodule SpitfireTest do ) ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' (min_line = line(meta) @@ -1055,14 +1055,14 @@ defmodule SpitfireTest do Enum.any?(comments, fn %{line: line} -> line > min_line and line < max_line end)) ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' (foo -> bar baz -> boo) ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "when syntax inside normal expression" do @@ -1070,19 +1070,19 @@ defmodule SpitfireTest do match?(x when is_nil(x), x) ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "when syntax inside keyword list" do code = ~S'[a: b when c]' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "in_match_op inside keyword list" do code = ~S'[a: b <- c]' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "case expr" do @@ -1123,7 +1123,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -1139,7 +1139,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -1227,7 +1227,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -1264,22 +1264,22 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end test "parses nested stab expressions" do - assert Spitfire.parse(~s'fn :a -> 1; :b -> 2 end') == s2q(~s'fn :a -> 1; :b -> 2 end') - assert Spitfire.parse(~s'fn x -> fn y -> x + y end end') == s2q(~s'fn x -> fn y -> x + y end end') - assert Spitfire.parse(~s'fn x, y -> x + y end') == s2q(~s'fn x, y -> x + y end') + assert spitfire_parse(~s'fn :a -> 1; :b -> 2 end') == s2q(~s'fn :a -> 1; :b -> 2 end') + assert spitfire_parse(~s'fn x -> fn y -> x + y end end') == s2q(~s'fn x -> fn y -> x + y end end') + assert spitfire_parse(~s'fn x, y -> x + y end') == s2q(~s'fn x, y -> x + y end') - assert Spitfire.parse(~s'fn {:ok, x} -> x; {:error, _} -> nil end') == + assert spitfire_parse(~s'fn {:ok, x} -> x; {:error, _} -> nil end') == s2q(~s'fn {:ok, x} -> x; {:error, _} -> nil end') - assert Spitfire.parse(~s'fn a -> fn b -> fn c -> a + b + c end end end') == + assert spitfire_parse(~s'fn a -> fn b -> fn c -> a + b + c end end end') == s2q(~s'fn a -> fn b -> fn c -> a + b + c end end end') - assert Spitfire.parse(~s'fn x -> Enum.map(x, fn y -> y * 2 end) end') == + assert spitfire_parse(~s'fn x -> Enum.map(x, fn y -> y * 2 end) end') == s2q(~s'fn x -> Enum.map(x, fn y -> y * 2 end) end') end @@ -1292,7 +1292,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' fn @@ -1303,7 +1303,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' case x do @@ -1316,7 +1316,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' cond do @@ -1329,7 +1329,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' receive do @@ -1343,7 +1343,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' try do @@ -1356,7 +1356,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parses match operator" do @@ -1367,27 +1367,27 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end test "parses nil" do code = "nil" - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parses booleans" do code = "false" - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = "true" - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parses right stab argument with parens" do code = "if true do (x, y) -> x end" - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parses cond expression" do @@ -1403,7 +1403,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -1416,7 +1416,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "when operator" do @@ -1479,7 +1479,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -1501,7 +1501,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -1525,7 +1525,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -1548,7 +1548,7 @@ defmodule SpitfireTest do end) ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "big test" do @@ -1631,7 +1631,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "function def with case expression with anon function inside" do @@ -1648,7 +1648,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "case expression with anon function inside" do @@ -1663,7 +1663,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "not really sure" do @@ -1737,7 +1737,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "lonely parens" do @@ -1749,7 +1749,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "multi line for clause" do @@ -1764,17 +1764,17 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "ambiguous op" do code = "@all_info -1" - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "range op" do code = ".." - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "range operator precedence and nested ranges" do @@ -1789,7 +1789,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -1806,7 +1806,7 @@ defmodule SpitfireTest do ] ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "from ecto repo" do @@ -1816,7 +1816,7 @@ defmodule SpitfireTest do ] ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "big with" do @@ -1830,13 +1830,13 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "bitstrings" do code = ~S'<>' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' << @@ -1846,7 +1846,7 @@ defmodule SpitfireTest do >> ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "anonymous function typespecs" do @@ -1854,13 +1854,13 @@ defmodule SpitfireTest do @spec start_link((-> term), GenServer.options()) :: on_start ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' @spec get(agent, (state -> a), timeout) :: a when a: var ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "rescue with def" do @@ -1870,7 +1870,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "multiple block identifiers" do @@ -1886,7 +1886,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "starts with a comment" do @@ -1895,7 +1895,7 @@ defmodule SpitfireTest do some_code = :foo """ - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "default args" do @@ -1905,7 +1905,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "literal encoder" do @@ -1950,7 +1950,7 @@ defmodule SpitfireTest do encoder = fn l, m -> {:ok, {:__literal__, m, [l]}} end for code <- codes do - assert Spitfire.parse(code, literal_encoder: encoder) == + assert spitfire_parse(code, literal_encoder: encoder) == Code.string_to_quoted(code, literal_encoder: encoder, columns: true, @@ -1987,7 +1987,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -1996,7 +1996,7 @@ defmodule SpitfireTest do "foo#{alice}bar" ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' "foo#{ @@ -2004,13 +2004,13 @@ defmodule SpitfireTest do }bar" ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' "foo#{}bar" ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' """ @@ -2018,7 +2018,7 @@ defmodule SpitfireTest do """ ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' """ @@ -2028,13 +2028,13 @@ defmodule SpitfireTest do """ ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' "#{foo}" ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "end of expression metadata" do @@ -2063,7 +2063,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -2078,7 +2078,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -2091,7 +2091,7 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -2115,7 +2115,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "unquote_splicing" do @@ -2140,14 +2140,14 @@ defmodule SpitfireTest do ] for code <- codes do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end test "line and column opt" do code = "foo" - assert Spitfire.parse(code, line: 12, column: 7) == s2q(code, line: 12, column: 7) + assert spitfire_parse(code, line: 12, column: 7) == s2q(code, line: 12, column: 7) end test "ellipsis_op ..." do @@ -2158,19 +2158,19 @@ defmodule SpitfireTest do ] ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' ... + 1 * 2 ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) code = ~S''' @type fun :: (... -> any()) ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "blocks inside an anon function as a parameter" do @@ -2184,7 +2184,7 @@ defmodule SpitfireTest do end """ - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "parens on a macro with a do block on the right side of a match operator" do @@ -2202,8 +2202,8 @@ defmodule SpitfireTest do end """ ] do - # code |> Spitfire.parse!() |> Macro.to_string() |> IO.puts() - assert Spitfire.parse(code) == s2q(code) + # code |> spitfire_parse!() |> Macro.to_string() |> IO.puts() + assert spitfire_parse(code) == s2q(code) end end @@ -2215,7 +2215,7 @@ defmodule SpitfireTest do ~S|['➡️': x]|, ~S|foo.'➡️'| ] do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -2228,7 +2228,7 @@ defmodule SpitfireTest do :erlang.'=<'(left, right) """ ] do - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end end @@ -2239,87 +2239,87 @@ defmodule SpitfireTest do end |> c ** d >>> e ''' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end test "when in bracketless kw list" do code = ~S'with a <- b, do: c when a' - assert Spitfire.parse(code) == s2q(code) + assert spitfire_parse(code) == s2q(code) end # These were found by property tests but were not triggering reliably # We have them here to make sure they don't regress test "property test regression cases" do # Prefix operators in struct types - assert Spitfire.parse("%?0{}") == s2q("%?0{}") - assert Spitfire.parse("%^@_{}") == s2q("%^@_{}") - assert Spitfire.parse("%-..{}") == s2q("%-..{}") - assert Spitfire.parse("%!:c{}") == s2q("%!:c{}") - assert Spitfire.parse("%~a<>{}") == s2q("%~a<>{}") - assert Spitfire.parse("%!A{}") == s2q("%!A{}") - assert Spitfire.parse("%@A{}") == s2q("%@A{}") - assert Spitfire.parse("%@:rd{}") == s2q("%@:rd{}") - assert Spitfire.parse("%!0{}") == s2q("%!0{}") + assert spitfire_parse("%?0{}") == s2q("%?0{}") + assert spitfire_parse("%^@_{}") == s2q("%^@_{}") + assert spitfire_parse("%-..{}") == s2q("%-..{}") + assert spitfire_parse("%!:c{}") == s2q("%!:c{}") + assert spitfire_parse("%~a<>{}") == s2q("%~a<>{}") + assert spitfire_parse("%!A{}") == s2q("%!A{}") + assert spitfire_parse("%@A{}") == s2q("%@A{}") + assert spitfire_parse("%@:rd{}") == s2q("%@:rd{}") + assert spitfire_parse("%!0{}") == s2q("%!0{}") # Nested module attributes - assert Spitfire.parse("%@@u{}") == s2q("%@@u{}") + assert spitfire_parse("%@@u{}") == s2q("%@@u{}") # Quoted atoms - assert Spitfire.parse(~s(%:""{})) == s2q(~s(%:""{})) + assert spitfire_parse(~s(%:""{})) == s2q(~s(%:""{})) # Range operator in struct types - assert Spitfire.parse("%..{}") == s2q("%..{}") + assert spitfire_parse("%..{}") == s2q("%..{}") # Char tokens after module attributes in struct types - assert Spitfire.parse("%@?w{}") == s2q("%@?w{}") + assert spitfire_parse("%@?w{}") == s2q("%@?w{}") # Empty char list after module attributes in struct types - assert Spitfire.parse("%@''{}") == s2q(~s(%@''{})) + assert spitfire_parse("%@''{}") == s2q(~s(%@''{})) # Float in struct types - assert Spitfire.parse("%0.0{}") == s2q("%0.0{}") + assert spitfire_parse("%0.0{}") == s2q("%0.0{}") # Bin strings after module attributes in struct types - assert Spitfire.parse(~s(%@"foo"{})) == s2q(~s(%@"foo"{})) + assert spitfire_parse(~s(%@"foo"{})) == s2q(~s(%@"foo"{})) # Capture operator in struct types - assert Spitfire.parse("%&0{}") == s2q("%&0{}") + assert spitfire_parse("%&0{}") == s2q("%&0{}") # Boolean literals in struct types - assert Spitfire.parse("%false{}") == s2q("%false{}") - assert Spitfire.parse("%true{}") == s2q("%true{}") + assert spitfire_parse("%false{}") == s2q("%false{}") + assert spitfire_parse("%true{}") == s2q("%true{}") # Struct arg inside struct arg - assert Spitfire.parse("%%{}{}") == s2q("%%{}{}") - assert Spitfire.parse("%+[]{}") == s2q("%+[]{}") + assert spitfire_parse("%%{}{}") == s2q("%%{}{}") + assert spitfire_parse("%+[]{}") == s2q("%+[]{}") # In-match operator (<-) in map keys - should be part of key, not wrap it - assert Spitfire.parse("%{s\\\\r => 1}") == s2q("%{s\\\\r => 1}") + assert spitfire_parse("%{s\\\\r => 1}") == s2q("%{s\\\\r => 1}") # Fn args with semicolon/newline trivia - assert Spitfire.parse("fn ;\n -> :ok end") == s2q("fn ;\n -> :ok end") - assert Spitfire.parse("fn ; -> :ok end") == s2q("fn ; -> :ok end") + assert spitfire_parse("fn ;\n -> :ok end") == s2q("fn ;\n -> :ok end") + assert spitfire_parse("fn ; -> :ok end") == s2q("fn ; -> :ok end") # Struct type with dot-call target - assert Spitfire.parse("%e.(){}") == s2q("%e.(){}") - assert Spitfire.parse("%e.(1){}") == s2q("%e.(1){}") - assert Spitfire.parse("%e.(a, b){}") == s2q("%e.(a, b){}") + assert spitfire_parse("%e.(){}") == s2q("%e.(){}") + assert spitfire_parse("%e.(1){}") == s2q("%e.(1){}") + assert spitfire_parse("%e.(a, b){}") == s2q("%e.(a, b){}") # with/else stab body with leading semicolon after newline - assert Spitfire.parse("with x <- 1 do :ok else _ -> \n;a end") == + assert spitfire_parse("with x <- 1 do :ok else _ -> \n;a end") == s2q("with x <- 1 do :ok else _ -> \n;a end") # Ellipsis + ternary edge cases (newline and semicolon-separated) - assert Spitfire.parse("x...\n//y") == s2q("x...\n//y") - assert Spitfire.parse("x...;//y") == s2q("x...;//y") - assert Spitfire.parse("x...\n;//y") == s2q("x...\n;//y") - assert Spitfire.parse("x...\n;\n//y") == s2q("x...\n;\n//y") - assert Spitfire.parse("x...\n;\n# comment\n//y") == s2q("x...\n;\n# comment\n//y") + assert spitfire_parse("x...\n//y") == s2q("x...\n//y") + assert spitfire_parse("x...;//y") == s2q("x...;//y") + assert spitfire_parse("x...\n;//y") == s2q("x...\n;//y") + assert spitfire_parse("x...\n;\n//y") == s2q("x...\n;\n//y") + assert spitfire_parse("x...\n;\n# comment\n//y") == s2q("x...\n;\n# comment\n//y") # Ellipsis followed by infix operators that should not be consumed as RHS - assert Spitfire.parse("x...<-y") == s2q("x...<-y") - assert Spitfire.parse("x...::y") == s2q("x...::y") - assert Spitfire.parse("x... when y") == s2q("x... when y") + assert spitfire_parse("x...<-y") == s2q("x...<-y") + assert spitfire_parse("x...::y") == s2q("x...::y") + assert spitfire_parse("x... when y") == s2q("x... when y") end end @@ -2329,7 +2329,7 @@ defmodule SpitfireTest do test "unknown prefix operator" do code = "foo $bar, baz" - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, {:foo, [line: 1, column: 1], [{:__block__, [error: true, line: 1, column: 5], []}]}, [{[line: 1, column: 5], "unknown token: %"}]} end @@ -2338,7 +2338,7 @@ defmodule SpitfireTest do code = "x...//y" assert {:error, _} = s2q(code) - assert {:error, _ast, errors} = Spitfire.parse(code) + assert {:error, _ast, errors} = spitfire_parse(code) assert Enum.any?(errors, fn {_meta, message} -> String.contains?( @@ -2350,10 +2350,10 @@ defmodule SpitfireTest do # https://github.com/elixir-lang/expert/issues/461 test "fn -> followed by closing delimiter does not hang" do - assert {:error, _ast, _errors} = Spitfire.parse("fn ->)") - assert {:error, _ast, _errors} = Spitfire.parse("fn ->") - assert {:error, _ast, _errors} = Spitfire.parse("Enum.map(fn ->)") - assert {:error, _ast, _errors} = Spitfire.parse("fn ->\n)") + assert {:error, _ast, _errors} = spitfire_parse("fn ->)") + assert {:error, _ast, _errors} = spitfire_parse("fn ->") + assert {:error, _ast, _errors} = spitfire_parse("Enum.map(fn ->)") + assert {:error, _ast, _errors} = spitfire_parse("fn ->\n)") end test "missing bitstring brackets" do @@ -2362,7 +2362,7 @@ defmodule SpitfireTest do :ok """ - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, {:<<>>, [ @@ -2379,7 +2379,7 @@ defmodule SpitfireTest do test "missing closing parentheses" do code = "1 * (2 + 3" - assert Spitfire.parse(code) == + assert spitfire_parse(code) == { :error, {{:*, [line: 1, column: 3], [1, {:__block__, [error: true, line: 1, column: 3], []}]}, @@ -2394,7 +2394,7 @@ defmodule SpitfireTest do test "missing closing list bracket" do code = "([1, 2 ++ [4])" - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, [1, {:++, [line: 1, column: 8], [2, [4]]}], [{[line: 1, column: 2], "missing closing bracket for list"}]} @@ -2403,21 +2403,21 @@ defmodule SpitfireTest do :ok """ - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, {:__block__, [], [[1], :ok]}, [{[line: 1, column: 1], "missing closing bracket for list"}]} code = """ [1, 2, 3,, """ - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, [1, 2, 3], [{[line: 1, column: 1], "missing closing bracket for list"}]} end test "missing closing tuple brace" do code = "({1, 2 ++ [4])" - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, {1, {:++, [line: 1, column: 8], [2, [4]]}}, [{[line: 1, column: 2], "missing closing brace for tuple"}]} @@ -2426,7 +2426,7 @@ defmodule SpitfireTest do :ok """ - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, {:__block__, [], [ @@ -2442,14 +2442,14 @@ defmodule SpitfireTest do code = "fn x -> %{a: {1," - assert {:error, _ast, errors} = Spitfire.parse(code) + assert {:error, _ast, errors} = spitfire_parse(code) assert {[line: 1, column: 14], "missing closing brace for tuple"} in errors end test "missing closing map brace" do code = ~S'foo(%{alice: "bob")' - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, {:foo, [{:closing, [line: 1, column: 19]}, line: 1, column: 1], [{:%{}, [{:closing, [line: 1, column: 14]}, line: 1, column: 5], [alice: "bob"]}]}, @@ -2459,14 +2459,14 @@ defmodule SpitfireTest do test "missing comma in list" do code = ~S'[:foo :bar, :baz]' - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, [:foo, :baz], [{[line: 1, column: 7], "syntax error"}]} end test "missing comma in map" do code = ~S'%{foo: :bar baz: :boo}' - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, {:%{}, [{:closing, [line: 1, column: 22]}, line: 1, column: 1], [foo: :bar]}, [ {[line: 1, column: 13], "syntax error"}, @@ -2477,7 +2477,7 @@ defmodule SpitfireTest do test "missing comma in tuple" do code = ~S'{:foo :bar, :baz}' - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, {:foo, :baz}, [{[line: 1, column: 7], "syntax error"}]} end @@ -2488,7 +2488,7 @@ defmodule SpitfireTest do :ok ''' - assert Spitfire.parse(code) == { + assert spitfire_parse(code) == { :error, { :foo, @@ -2541,7 +2541,7 @@ defmodule SpitfireTest do end ''' - assert Spitfire.parse(code) == { + assert spitfire_parse(code) == { :error, { :bar, @@ -2598,7 +2598,7 @@ defmodule SpitfireTest do bar(two) ''' - assert Spitfire.parse(code) == { + assert spitfire_parse(code) == { :error, { :__block__, @@ -2634,7 +2634,7 @@ defmodule SpitfireTest do bar(two) ''' - assert Spitfire.parse(code) == { + assert spitfire_parse(code) == { :error, {:foo, [{:line, 1}, {:column, 1}], [ @@ -2657,7 +2657,7 @@ defmodule SpitfireTest do send(pid, new_list) ''' - assert Spitfire.parse(code) == { + assert spitfire_parse(code) == { :error, { :=, @@ -2724,7 +2724,7 @@ defmodule SpitfireTest do end ''' - assert {:error, _ast, _} = result = Spitfire.parse(code) + assert {:error, _ast, _} = result = spitfire_parse(code) assert result == { @@ -2822,7 +2822,7 @@ defmodule SpitfireTest do end ''' - assert {:error, _ast, _} = result = Spitfire.parse(code) + assert {:error, _ast, _} = result = spitfire_parse(code) assert result == { @@ -2926,7 +2926,7 @@ defmodule SpitfireTest do <% end %> ''' - assert {:error, _ast, _errors} = Spitfire.parse(code) + assert {:error, _ast, _errors} = spitfire_parse(code) end test "doesn't drop the cursor node" do @@ -2939,7 +2939,7 @@ defmodule SpitfireTest do bar: Foo.Bar.load(state.foo, state.baz)} ''' - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, {:__block__, [], [ @@ -2996,7 +2996,7 @@ defmodule SpitfireTest do end ''' - assert {:error, _ast, _} = result = Spitfire.parse(code) + assert {:error, _ast, _} = result = spitfire_parse(code) assert result == { :error, @@ -3080,7 +3080,7 @@ defmodule SpitfireTest do end """ - assert {:error, _ast, _} = result = Spitfire.parse(code) + assert {:error, _ast, _} = result = spitfire_parse(code) assert result == { :error, @@ -3120,36 +3120,36 @@ defmodule SpitfireTest do foo, do: IO.inspect("bob"); "bob" ''' - assert {:error, _ast, errors} = Spitfire.parse(code) + assert {:error, _ast, errors} = spitfire_parse(code) assert Enum.any?(errors, fn {_, msg} -> String.contains?(msg, ",") end) end test "semicolon in list/tuple/map" do - assert {:error, _, errors} = Spitfire.parse("[;]") + assert {:error, _, errors} = spitfire_parse("[;]") assert Enum.any?(errors, fn {_, msg} -> msg == "unexpected token: ;" end) - assert {:error, _, errors} = Spitfire.parse("{;}") + assert {:error, _, errors} = spitfire_parse("{;}") assert Enum.any?(errors, fn {_, msg} -> msg == "unexpected token: ;" end) - assert {:error, _, errors} = Spitfire.parse("%{;}") + assert {:error, _, errors} = spitfire_parse("%{;}") assert Enum.any?(errors, fn {_, msg} -> msg == "unexpected token: ;" end) end test "semicolon in parentheses is valid empty block" do - assert {:ok, {:__block__, _, []}} = Spitfire.parse("(;)") + assert {:ok, {:__block__, _, []}} = spitfire_parse("(;)") end test "leading semicolon is skipped" do - assert {:ok, {:foo, _, nil}} = Spitfire.parse("; foo") + assert {:ok, {:foo, _, nil}} = spitfire_parse("; foo") end test "semicolon as statement separator is valid" do - assert {:ok, _} = Spitfire.parse("foo\n; bar") - assert {:ok, _} = Spitfire.parse("foo\n\n; bar") + assert {:ok, _} = spitfire_parse("foo\n; bar") + assert {:ok, _} = spitfire_parse("foo\n\n; bar") - assert {:ok, 1} = Spitfire.parse("(1;)") - assert {:ok, 1} = Spitfire.parse("(1;\n)") - assert {:ok, 1} = Spitfire.parse("(1\n;)") + assert {:ok, 1} = spitfire_parse("(1;)") + assert {:ok, 1} = spitfire_parse("(1;\n)") + assert {:ok, 1} = spitfire_parse("(1\n;)") code = ~S""" defmodule MyModule do @@ -3158,23 +3158,23 @@ defmodule SpitfireTest do end """ - assert {:ok, _} = Spitfire.parse(code) + assert {:ok, _} = spitfire_parse(code) end test "unexpected expression after keyword list" do - assert {:error, _ast, errors} = Spitfire.parse(~S|foo(a: 1, b)|) + assert {:error, _ast, errors} = spitfire_parse(~S|foo(a: 1, b)|) assert Enum.any?(errors, fn {_, msg} -> String.contains?(msg, "unexpected expression after keyword list") end) - assert {:error, _ast, errors} = Spitfire.parse(~S|foo(a: 1, :bar)|) + assert {:error, _ast, errors} = spitfire_parse(~S|foo(a: 1, :bar)|) assert Enum.any?(errors, fn {_, msg} -> String.contains?(msg, "unexpected expression after keyword list") end) - assert {:error, _ast, errors} = Spitfire.parse(~S|@tag foo: bar, baz|) + assert {:error, _ast, errors} = spitfire_parse(~S|@tag foo: bar, baz|) assert Enum.any?(errors, fn {_, msg} -> String.contains?(msg, "unexpected expression after keyword list") @@ -3183,7 +3183,7 @@ defmodule SpitfireTest do test "__cursor__ after keyword list does not crash" do code = ~S|foo(a: 1, __cursor__())| - assert {:ok, {:foo, _, [[{:a, 1}, {:__cursor__, _, []}]]}} = Spitfire.parse(code) + assert {:ok, {:foo, _, [[{:a, 1}, {:__cursor__, _, []}]]}} = spitfire_parse(code) end test "weird characters" do @@ -3191,25 +3191,25 @@ defmodule SpitfireTest do [«] """ - assert {:error, _ast, [{[line: 1, column: 1], "missing closing bracket for list"}]} = Spitfire.parse(code) + assert {:error, _ast, [{[line: 1, column: 1], "missing closing bracket for list"}]} = spitfire_parse(code) code = """ {«} """ - assert {:error, _ast, [{[line: 1, column: 1], "missing closing brace for tuple"}]} = Spitfire.parse(code) + assert {:error, _ast, [{[line: 1, column: 1], "missing closing brace for tuple"}]} = spitfire_parse(code) code = """ %{«} """ - assert {:error, _ast, [{[line: 1, column: 1], "missing closing brace for map"}]} = Spitfire.parse(code) + assert {:error, _ast, [{[line: 1, column: 1], "missing closing brace for map"}]} = spitfire_parse(code) code = """ («) """ - assert {:error, _ast, [{[line: 1, column: 1], "missing closing parentheses"}]} = Spitfire.parse(code) + assert {:error, _ast, [{[line: 1, column: 1], "missing closing parentheses"}]} = spitfire_parse(code) code = """ defp foo(«) do @@ -3217,13 +3217,13 @@ defmodule SpitfireTest do end """ - assert {:error, _ast, [{[line: 1, column: 9], "missing closing parentheses"}]} = Spitfire.parse(code) + assert {:error, _ast, [{[line: 1, column: 9], "missing closing parentheses"}]} = spitfire_parse(code) code = """ foo(«) """ - assert {:error, _ast, [{[line: 1, column: 4], "missing closing parentheses"}]} = Spitfire.parse(code) + assert {:error, _ast, [{[line: 1, column: 4], "missing closing parentheses"}]} = spitfire_parse(code) end test "missing braces for struct" do @@ -3238,7 +3238,7 @@ defmodule SpitfireTest do {:x, [line: 1, column: 8], nil} ]}, [{[line: 1, column: 2], "missing opening brace for struct %Foo"}]} == - Spitfire.parse("%Foo = x") + spitfire_parse("%Foo = x") assert {:error, {:%, [line: 1, column: 1], @@ -3247,7 +3247,7 @@ defmodule SpitfireTest do {:%{}, [], []} ]}, [{[line: 1, column: 2], "missing opening brace for struct %Foo"}]} == - Spitfire.parse("%Foo") + spitfire_parse("%Foo") assert {:error, {:=, [line: 1, column: 10], @@ -3260,7 +3260,7 @@ defmodule SpitfireTest do {:x, [line: 1, column: 12], nil} ]}, [{[line: 1, column: 6], "missing opening brace for struct %Foo.Bar"}]} == - Spitfire.parse("%Foo.Bar = x") + spitfire_parse("%Foo.Bar = x") assert {:error, {:=, [line: 1, column: 13], @@ -3273,7 +3273,7 @@ defmodule SpitfireTest do {:x, [line: 1, column: 15], nil} ]}, [{[line: 1, column: 2], "missing opening brace for struct %Foo"}]} == - Spitfire.parse("%Foo a: 42} = x") + spitfire_parse("%Foo a: 42} = x") assert {:error, {:%, [line: 1, column: 1], @@ -3284,7 +3284,7 @@ defmodule SpitfireTest do [ {[line: 1, column: 2], "missing opening brace for struct %Foo"}, {[line: 1, column: 9], "missing closing brace for struct %Foo"} - ]} == Spitfire.parse("%Foo a: 1") + ]} == spitfire_parse("%Foo a: 1") assert {:error, {:%, [line: 1, column: 1], @@ -3293,7 +3293,7 @@ defmodule SpitfireTest do {:%{}, [closing: [line: 1, column: 16], line: 1, column: 6], [a: 1, b: 2]} ]}, [{[line: 1, column: 2], "missing opening brace for struct %Foo"}]} == - Spitfire.parse("%Foo a: 1, b: 2}") + spitfire_parse("%Foo a: 1, b: 2}") assert {:error, {:%, [line: 1, column: 1], @@ -3303,7 +3303,7 @@ defmodule SpitfireTest do [{:|, [line: 1, column: 8], [{:x, [line: 1, column: 6], nil}, [a: 1]]}]} ]}, [{[line: 1, column: 2], "missing opening brace for struct %Foo"}]} == - Spitfire.parse("%Foo x | a: 1}") + spitfire_parse("%Foo x | a: 1}") assert {:error, {:%, [line: 1, column: 1], @@ -3314,7 +3314,7 @@ defmodule SpitfireTest do [ {[line: 1, column: 2], "missing opening brace for struct %Foo"}, {[line: 1, column: 13], "missing closing brace for struct %Foo"} - ]} == Spitfire.parse("%Foo x | a: 1") + ]} == spitfire_parse("%Foo x | a: 1") assert {:error, {:foo, [closing: [line: 1, column: 15], line: 1, column: 1], @@ -3326,7 +3326,7 @@ defmodule SpitfireTest do ]} ]}, [{[line: 1, column: 6], "missing opening brace for struct %Bar"}]} == - Spitfire.parse("foo(%Bar a: 1})") + spitfire_parse("foo(%Bar a: 1})") assert {:error, {:|>, [line: 1, column: 3], @@ -3339,7 +3339,7 @@ defmodule SpitfireTest do ]} ]}, [{[line: 1, column: 7], "missing opening brace for struct %Foo"}]} == - Spitfire.parse("x |> %Foo a: 1}") + spitfire_parse("x |> %Foo a: 1}") # Nested structs assert {:error, @@ -3359,7 +3359,7 @@ defmodule SpitfireTest do [ {[line: 1, column: 16], "missing opening brace for struct %Inner"}, {[line: 1, column: 26], "missing closing brace for struct %Outer"} - ]} == Spitfire.parse("%Outer{inner: %Inner a: 1}") + ]} == spitfire_parse("%Outer{inner: %Inner a: 1}") assert {:error, {:%, [line: 1, column: 1], @@ -3376,7 +3376,7 @@ defmodule SpitfireTest do ]} ]}, [{[line: 1, column: 2], "missing opening brace for struct %Outer"}]} == - Spitfire.parse("%Outer inner: %Inner{a: 1}}") + spitfire_parse("%Outer inner: %Inner{a: 1}}") assert {:error, {:%, [line: 1, column: 1], @@ -3395,7 +3395,7 @@ defmodule SpitfireTest do [ {[line: 1, column: 2], "missing opening brace for struct %Outer"}, {[line: 1, column: 16], "missing opening brace for struct %Inner"} - ]} == Spitfire.parse("%Outer inner: %Inner a: 1}}") + ]} == spitfire_parse("%Outer inner: %Inner a: 1}}") # Module attribute struct assert {:error, @@ -3407,7 +3407,7 @@ defmodule SpitfireTest do [ {[line: 1, column: 3], "missing opening brace for struct %@foo"}, {[line: 1, column: 10], "missing closing brace for struct %@foo"} - ]} == Spitfire.parse("%@foo a: 1") + ]} == spitfire_parse("%@foo a: 1") # __MODULE__ struct assert {:error, @@ -3419,27 +3419,27 @@ defmodule SpitfireTest do [ {[line: 1, column: 2], "missing opening brace for struct %__MODULE__"}, {[line: 1, column: 16], "missing closing brace for struct %__MODULE__"} - ]} == Spitfire.parse("%__MODULE__ a: 1") + ]} == spitfire_parse("%__MODULE__ a: 1") end test "stray closing delimiter after identifier" do assert {:error, {:__block__, [], [{:x, _, nil}, {:__block__, [error: true, line: 1, column: 3], []}]}, - [{[line: 1, column: 3], "unknown token: ]"}]} = Spitfire.parse("x ]") + [{[line: 1, column: 3], "unknown token: ]"}]} = spitfire_parse("x ]") assert {:error, {:__block__, [], [{:x, _, nil}, {:__block__, [error: true, line: 1, column: 3], []}]}, - [{[line: 1, column: 3], "unknown token: }"}]} = Spitfire.parse("x }") + [{[line: 1, column: 3], "unknown token: }"}]} = spitfire_parse("x }") assert {:error, {:__block__, [], [{:x, _, nil}, {:__block__, [error: true, line: 1, column: 3], []}]}, - [{[line: 1, column: 3], "unknown token: )"}]} = Spitfire.parse("x )") + [{[line: 1, column: 3], "unknown token: )"}]} = spitfire_parse("x )") assert {:error, {:__block__, [], [{:@, _, [{:x, _, nil}]}, {:__block__, [error: true, line: 1, column: 4], []}]}, - [{[line: 1, column: 4], "unknown token: ]"}]} = Spitfire.parse("@x ]") + [{[line: 1, column: 4], "unknown token: ]"}]} = spitfire_parse("@x ]") assert {:error, {:__block__, [], [1, {:__block__, [error: true, line: 1, column: 3], []}]}, - [{[line: 1, column: 3], "unknown token: ]"}]} = Spitfire.parse("1 ]") + [{[line: 1, column: 3], "unknown token: ]"}]} = spitfire_parse("1 ]") assert {:error, _ast, [{[line: 1, column: 3], "unknown token: ]"}, {[line: 1, column: 5], "unknown token: }"}]} = - Spitfire.parse("x ] }") + spitfire_parse("x ] }") code = """ x] @@ -3453,7 +3453,7 @@ defmodule SpitfireTest do {:x, _, nil}, {:__block__, [end_of_expression: _, error: true, line: 1, column: 2], []}, {:=, _, [{:foo, _, nil}, {{:., _, [{:__aliases__, _, [:Foo]}, :bar]}, _, [42]}]} - ]}, [{[line: 1, column: 2], "unknown token: ]"}]} = Spitfire.parse(code) + ]}, [{[line: 1, column: 2], "unknown token: ]"}]} = spitfire_parse(code) end test "stray closing delimiter after complete expression" do @@ -3462,23 +3462,23 @@ defmodule SpitfireTest do [ {:foo, [closing: [line: 1, column: 5], line: 1, column: 1], []}, {:__block__, [error: true, line: 1, column: 6], []} - ]}, [{[line: 1, column: 6], "unknown token: }"}]} = Spitfire.parse("foo()}") + ]}, [{[line: 1, column: 6], "unknown token: }"}]} = spitfire_parse("foo()}") assert {:error, {:__block__, [], [[1, 2], {:__block__, [error: true, line: 1, column: 7], []}]}, - [{[line: 1, column: 7], "unknown token: }"}]} = Spitfire.parse("[1, 2]}") + [{[line: 1, column: 7], "unknown token: }"}]} = spitfire_parse("[1, 2]}") assert {:error, {:__block__, [], [ {:%{}, [closing: [line: 1, column: 7], line: 1, column: 1], [a: 1]}, {:__block__, [error: true, line: 1, column: 8], []} - ]}, [{[line: 1, column: 8], "unknown token: )"}]} = Spitfire.parse("%{a: 1})") + ]}, [{[line: 1, column: 8], "unknown token: )"}]} = spitfire_parse("%{a: 1})") end test "stab expressions with capture between followed by fn" do # This case is very contrived and was found by fuzzing tokens, trying to # reproduce a real world crash for which we didn't have a repro case. - assert Spitfire.parse("a -> b &0 c -> d fn e f -> g end end") == + assert spitfire_parse("a -> b &0 c -> d fn e f -> g end end") == {:error, [ {:->, [line: 1, column: 3], [[{:a, [line: 1, column: 1], nil}], nil]}, @@ -3511,7 +3511,7 @@ defmodule SpitfireTest do code = ~S'(a, b -> c, d)' # Comma expressions are not allowed in stab bodies - this is a syntax error - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, [ {:->, [line: 1, column: 7], @@ -3525,7 +3525,7 @@ defmodule SpitfireTest do test "fn stab cannot have comma expression in body" do code = ~S'fn x -> a, b end' - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, {:fn, [closing: [line: 1, column: 14], line: 1, column: 1], [ @@ -3540,7 +3540,7 @@ defmodule SpitfireTest do test "errors on invalid map update" do code = ~S'%{a do :ok end | b c, d => e, f => g, h i, j => k}' - assert Spitfire.parse(code) == + assert spitfire_parse(code) == {:error, {:%{}, [closing: [line: 1, column: 50], line: 1, column: 1], [ @@ -3557,7 +3557,7 @@ defmodule SpitfireTest do end test "stab expression with fn and stray closing delimiter does not exhaust fuel" do - assert Spitfire.parse("a -> b -> fn} -> c") == + assert spitfire_parse("a -> b -> fn} -> c") == {:error, [ {:->, [line: 1, column: 3], [[{:a, [line: 1, column: 1], nil}], nil]}, @@ -3586,9 +3586,9 @@ defmodule SpitfireTest do :foo ''' - assert {:ok, _ast, comments} = Spitfire.parse_with_comments(code) + assert {:ok, _ast, comments} = spitfire_parse_with_comments(code) assert [%{line: 1, text: "# hello"}, %{line: 2, text: "# world"}] = comments - assert Spitfire.parse_with_comments(code) == s2qwc(code) + assert spitfire_parse_with_comments(code) == s2qwc(code) end test "returns the same comments as string_to_quoted_with_comments" do @@ -3604,7 +3604,7 @@ defmodule SpitfireTest do # Some more comments! ''' - assert Spitfire.parse_with_comments(code) == s2qwc(code) + assert spitfire_parse_with_comments(code) == s2qwc(code) end end @@ -3620,7 +3620,7 @@ defmodule SpitfireTest do [ {:some_value, [line: 1, column: 5], nil}, {:__cursor__, [closing: [line: 2, column: 12], line: 2, column: 1], []} - ]}} = Spitfire.container_cursor_to_quoted(code) + ]}} = spitfire_container_cursor_to_quoted(code) end test "more complex example" do @@ -3650,7 +3650,7 @@ defmodule SpitfireTest do ] ]} ] - ]}} = Spitfire.container_cursor_to_quoted(code) + ]}} = spitfire_container_cursor_to_quoted(code) end test "ending on kw list" do @@ -3683,7 +3683,7 @@ defmodule SpitfireTest do ] ]} ] - ]}} = Spitfire.container_cursor_to_quoted(code) + ]}} = spitfire_container_cursor_to_quoted(code) end test "ending inside a -> expression" do @@ -3769,7 +3769,7 @@ defmodule SpitfireTest do } ] ] - }} = Spitfire.container_cursor_to_quoted(code) + }} = spitfire_container_cursor_to_quoted(code) end end @@ -3781,27 +3781,27 @@ defmodule SpitfireTest do code = ~S|:"abc#{foo}"| # atom_unsafe -> :binary_to_atom - assert {:ok, {{:., _, [:erlang, :binary_to_atom]}, _, _}} = Spitfire.parse(code) + assert {:ok, {{:., _, [:erlang, :binary_to_atom]}, _, _}} = spitfire_parse(code) # atom_safe -> :binary_to_existing_atom assert {:ok, {{:., _, [:erlang, :binary_to_existing_atom]}, _, _}} = - Spitfire.parse(code, existing_atoms_only: true) + spitfire_parse(code, existing_atoms_only: true) end test "kw_identifier_safe token is parsed to :binary_to_existing_atom" do # in list code = ~S|["abc#{foo}": 1]| - assert {:ok, [{{{:., _, [:erlang, :binary_to_atom]}, _, _}, 1}]} = Spitfire.parse(code) + assert {:ok, [{{{:., _, [:erlang, :binary_to_atom]}, _, _}, 1}]} = spitfire_parse(code) assert {:ok, [{{{:., _, [:erlang, :binary_to_existing_atom]}, _, _}, 1}]} = - Spitfire.parse(code, existing_atoms_only: true) + spitfire_parse(code, existing_atoms_only: true) # in map code = ~S|%{"abc#{foo}": 1}| - assert {:ok, {:%{}, _, [{{{:., _, [:erlang, :binary_to_atom]}, _, _}, 1}]}} = Spitfire.parse(code) + assert {:ok, {:%{}, _, [{{{:., _, [:erlang, :binary_to_atom]}, _, _}, 1}]}} = spitfire_parse(code) assert {:ok, {:%{}, _, [{{{:., _, [:erlang, :binary_to_existing_atom]}, _, _}, 1}]}} = - Spitfire.parse(code, existing_atoms_only: true) + spitfire_parse(code, existing_atoms_only: true) # multiple items in bracketless kw list code = ~S|foo("a#{b}": 1, "c#{d}": 2)| @@ -3814,7 +3814,7 @@ defmodule SpitfireTest do {{{:., _, [:erlang, :binary_to_existing_atom]}, _, _}, 2} ] ]}} = - Spitfire.parse(code, existing_atoms_only: true) + spitfire_parse(code, existing_atoms_only: true) end end @@ -3832,6 +3832,30 @@ defmodule SpitfireTest do ) end + defp spitfire_parse(code, opts \\ []) do + code + |> Spitfire.parse(opts) + |> Spitfire.TestHelpers.strip_parse_result_range_metadata() + end + + defp spitfire_parse!(code, opts \\ []) do + code + |> Spitfire.parse!(opts) + |> Spitfire.TestHelpers.strip_range_metadata() + end + + defp spitfire_parse_with_comments(code, opts \\ []) do + code + |> Spitfire.parse_with_comments(opts) + |> Spitfire.TestHelpers.strip_parse_result_range_metadata() + end + + defp spitfire_container_cursor_to_quoted(code, opts \\ []) do + code + |> Spitfire.container_cursor_to_quoted(opts) + |> Spitfire.TestHelpers.strip_parse_result_range_metadata() + end + def print(ast) do ast |> Macro.to_string() |> IO.puts() ast diff --git a/test/systematic_operators_test.exs b/test/systematic_operators_test.exs index 68a9bae..3ae4c90 100644 --- a/test/systematic_operators_test.exs +++ b/test/systematic_operators_test.exs @@ -582,6 +582,9 @@ defmodule Spitfire.SystematicOperatorsTest do {:ok, expected} -> case Spitfire.parse(code) do {:ok, actual} -> + actual = Spitfire.TestHelpers.strip_range_metadata(actual) + expected = Spitfire.TestHelpers.strip_range_metadata(expected) + if actual != expected, do: {code, expected, actual} {:error, _} -> diff --git a/test/test_helper.exs b/test/test_helper.exs index e022c9a..1d1e7b3 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,5 +1,6 @@ defmodule Spitfire.TestHelpers do @moduledoc false + defmacro lhs == rhs do quote do lhs = @@ -28,6 +29,50 @@ defmodule Spitfire.TestHelpers do end end end + + def strip_range_metadata(term) do + Macro.postwalk(term, fn + {token, meta, args} when is_list(meta) -> + {token, Keyword.delete(meta, :range), args} + + other -> + other + end) + end + + def strip_parse_result_range_metadata({:ok, ast, comments}) do + {:ok, strip_range_metadata(ast), comments} + end + + def strip_parse_result_range_metadata({:error, ast, errors}) do + {:error, strip_range_metadata(ast), strip_error_metadata(errors)} + end + + def strip_parse_result_range_metadata({:error, ast, comments, errors}) do + {:error, strip_range_metadata(ast), comments, strip_error_metadata(errors)} + end + + def strip_parse_result_range_metadata({status, ast}) when status in [:ok, :error] do + {status, strip_range_metadata(ast)} + end + + def strip_parse_result_range_metadata(other), do: other + + defp strip_metadata_range(meta) do + Keyword.delete(meta, :range) + end + + defp strip_error_metadata(errors) when is_list(errors) do + Enum.map(errors, fn + {meta, message} when is_list(meta) and is_binary(message) -> + {strip_metadata_range(meta), message} + + other -> + other + end) + end + + defp strip_error_metadata(errors), do: errors end ExUnit.start(exclude: [:skip])