Allow to prune metadata while escaping (#7949)

We use this option in ExUnit as we are escaping
the code to eventually convert it to a string
representation and therefore the metadata is not
relevant.

Signed-off-by: José Valim <jose.valim@plataformatec.com.br>

Author: josevalim

lib/elixir/lib/kernel.ex

@@ -2776,7 +2776,7 @@ defmodule Kernel do
           end
 
         try do
-          :elixir_quote.escape(value, false)
+          :elixir_quote.escape(value, :default, false)
         rescue
           ex in [ArgumentError] ->
             raise ArgumentError,
@@ -3698,7 +3698,7 @@ defmodule Kernel do
           quote(do: Kernel.LexicalTracker.read_cache(unquote(pid), unquote(integer)))
 
         %{} ->
-          {escaped, _} = :elixir_quote.escape(block, false)
+          {escaped, _} = :elixir_quote.escape(block, :default, false)
           escaped
       end
 
@@ -3972,8 +3972,8 @@ defmodule Kernel do
     module = assert_module_scope(env, kind, 2)
     assert_no_function_scope(env, kind, 2)
 
-    {escaped_call, unquoted_call} = :elixir_quote.escape(call, true)
-    {escaped_expr, unquoted_expr} = :elixir_quote.escape(expr, true)
+    {escaped_call, unquoted_call} = :elixir_quote.escape(call, :default, true)
+    {escaped_expr, unquoted_expr} = :elixir_quote.escape(expr, :default, true)
 
     escaped_expr =
       case unquoted_expr do

lib/elixir/lib/macro.ex

@@ -356,12 +356,7 @@ defmodule Macro do
   def decompose_call(_), do: :error
 
   @doc """
-  Recursively escapes a value so it can be inserted
-  into a syntax tree.
-
-  One may pass `unquote: true` to `escape/2`
-  which leaves `unquote/1` statements unescaped, effectively
-  unquoting the contents on escape.
+  Recursively escapes a value so it can be inserted into a syntax tree.
 
   ## Examples
 
@@ -374,6 +369,26 @@ defmodule Macro do
       iex> Macro.escape({:unquote, [], [1]}, unquote: true)
       1
 
+  ## Options
+
+    * `:unquote` - when true, this function leaves `unquote/1` and
+      `unquote_splicing/1` statements unescaped, effectively unquoting
+      the contents on escape. This option is useful only when escaping
+      ASTs which may have quoted fragments in them. Defaults to false.
+
+    * `:prune_metadata` - when true, removes metadata from escaped AST
+      nodes. Note this option changes the semantics of escaped code and
+      it should only be used when escaping ASTs, never values. Defaults
+      to false.
+      
+      As an example, `ExUnit` stores the AST of every assertion, so when
+      an assertion fails we can show code snippets to users. Without this
+      option, each time the test module is compiled, we get a different
+      MD5 of the module byte code, because the AST contains metadata,
+      such as counters, specific to the compilation environment. By pruning
+      the metadata, we ensure that the module is deterministic and reduce
+      the amount of data `ExUnit` needs to keep around.
+
   ## Comparison to `Kernel.quote/2`
 
   The `escape/2` function is sometimes confused with `Kernel.SpecialForms.quote/2`,
@@ -402,7 +417,9 @@ defmodule Macro do
   """
   @spec escape(term, keyword) :: Macro.t()
   def escape(expr, opts \\ []) do
-    elem(:elixir_quote.escape(expr, Keyword.get(opts, :unquote, false)), 0)
+    unquote = Keyword.get(opts, :unquote, false)
+    kind = if Keyword.get(opts, :prune_metadata, false), do: :prune_metadata, else: :default
+    elem(:elixir_quote.escape(expr, kind, unquote), 0)
   end
 
   @doc """

lib/elixir/src/elixir.hrl

@@ -24,7 +24,6 @@
   imports_hygiene=true,
   unquote=true,
   unquoted=false,
-  escape=false,
   generated=false
 }).
 

lib/elixir/src/elixir_bootstrap.erl

@@ -19,7 +19,7 @@
 'MACRO-defmacrop'(Caller, Call, Expr) -> define(Caller, defmacrop, Call, Expr).
 
 'MACRO-defmodule'(_Caller, Alias, [{do, Block}]) ->
-  {Escaped, _} = elixir_quote:escape(Block, false),
+  {Escaped, _} = elixir_quote:escape(Block, default, false),
   Args = [Alias, Escaped, [], env()],
   {{'.', [], [elixir_module, compile]}, [], Args}.
 
@@ -36,8 +36,8 @@
    {defp, 2}].
 
 define({Line, E}, Kind, Call, Expr) ->
-  {EscapedCall, UC} = elixir_quote:escape(Call, true),
-  {EscapedExpr, UE} = elixir_quote:escape(Expr, true),
+  {EscapedCall, UC} = elixir_quote:escape(Call, default, true),
+  {EscapedExpr, UE} = elixir_quote:escape(Expr, default, true),
   Args = [Kind, not(UC or UE), EscapedCall, EscapedExpr, elixir_locals:cache_env(E#{line := Line})],
   {{'.', [], [elixir_def, store_definition]}, [], Args}.
 

lib/elixir/src/elixir_map.erl

@@ -33,7 +33,8 @@ expand_struct(Meta, Left, {'%{}', MapMeta, MapArgs}, #{context := Context} = E)
           Struct = load_struct(Meta, ELeft, [Assocs], InContext, EE),
           assert_struct_keys(Meta, ELeft, Struct, Assocs, EE),
           Keys = ['__struct__'] ++ [K || {K, _} <- Assocs],
-          {StructAssocs, _} = elixir_quote:escape(maps:to_list(maps:without(Keys, Struct)), false),
+          WithoutKeys = maps:to_list(maps:without(Keys, Struct)),
+          {StructAssocs, _} = elixir_quote:escape(WithoutKeys, default, false),
           {{'%', Meta, [ELeft, {'%{}', MapMeta, StructAssocs ++ Assocs}]}, EE};
 
         {_, _, Assocs} -> %% Update or match

lib/elixir/src/elixir_quote.erl

@@ -1,5 +1,5 @@
 -module(elixir_quote).
--export([escape/2, linify/2, linify/3, linify_with_context_counter/3, quote/4]).
+-export([escape/3, linify/2, linify/3, linify_with_context_counter/3, quote/4]).
 -export([dot/5, tail_list/3, list/2]). %% Quote callbacks
 
 -include("elixir.hrl").
@@ -132,16 +132,15 @@ annotate(Tree, _Context) -> Tree.
 
 %% Escapes the given expression. It is similar to quote, but
 %% lines are kept and hygiene mechanisms are disabled.
-escape(Expr, Unquote) ->
+escape(Expr, Kind, Unquote) ->
   {Res, Q} = quote(Expr, nil, #elixir_quote{
     line=true,
     file=nil,
     vars_hygiene=false,
     aliases_hygiene=false,
     imports_hygiene=false,
-    unquote=Unquote,
-    escape=true
- }, nil),
+    unquote=Unquote
+  }, Kind),
   {Res, Q#elixir_quote.unquoted}.
 
 %% Quotes an expression and return its quoted Elixir AST.
@@ -161,7 +160,7 @@ quote(Expr, Binding, Q, E) ->
       {'{}', [], [K, [], Context]},
       V
     ] ]
- } || {K, V} <- Binding],
+  } || {K, V} <- Binding],
 
   {TExprs, TQ} = do_quote(Expr, Q, E),
   {{'{}', [], ['__block__', [], Vars ++ [TExprs] ]}, TQ}.
@@ -211,11 +210,7 @@ do_quote({'&', Meta, [{'/', _, [{F, _, C}, A]}] = Args},
 do_quote({Name, Meta, ArgsOrAtom}, #elixir_quote{imports_hygiene=true} = Q, E) when is_atom(Name) ->
   do_quote_import(Name, Meta, ArgsOrAtom, Q, E);
 
-do_quote({_, _, _} = Tuple, #elixir_quote{escape=false} = Q, E) ->
-  Annotated = annotate(Tuple, Q#elixir_quote.context),
-  do_quote_tuple(Annotated, Q, E);
-
-%% Literals
+%% Two element tuples
 
 do_quote({Left, Right}, #elixir_quote{unquote=true} = Q, E) when
     is_tuple(Left)  andalso (element(1, Left) == unquote_splicing);
@@ -227,7 +222,33 @@ do_quote({Left, Right}, Q, E) ->
   {TRight, RQ} = do_quote(Right, LQ, E),
   {{TLeft, TRight}, RQ};
 
-do_quote(BitString, #elixir_quote{escape=true} = Q, _) when is_bitstring(BitString) ->
+%% Everything else
+
+do_quote(Other, Q, E) when is_atom(E) ->
+  do_escape(Other, Q, E);
+
+do_quote({_, _, _} = Tuple, Q, E) ->
+  Annotated = annotate(Tuple, Q#elixir_quote.context),
+  do_quote_tuple(Annotated, Q, E);
+
+do_quote(List, Q, E) when is_list(List) ->
+  do_quote_splice(lists:reverse(List), Q, E);
+
+do_quote(Other, Q, _) ->
+  {Other, Q}.
+
+%% do_escape
+
+do_escape({Left, _Meta, Right}, Q, E = prune_metadata) ->
+  {TL, QL} = do_quote(Left, Q, E),
+  {TR, QR} = do_quote(Right, QL, E),
+  {{'{}', [], [TL, [], TR]}, QR};
+
+do_escape(Tuple, Q, E) when is_tuple(Tuple) ->
+  {TT, TQ} = do_quote(tuple_to_list(Tuple), Q, E),
+  {{'{}', [], TT}, TQ};
+
+do_escape(BitString, Q, _) when is_bitstring(BitString) ->
   case bit_size(BitString) rem 8 of
     0 ->
       {BitString, Q};
@@ -236,51 +257,41 @@ do_quote(BitString, #elixir_quote{escape=true} = Q, _) when is_bitstring(BitStri
       {{'<<>>', [], [{'::', [], [Bits, {size, [], [Size]}]}, {'::', [], [Bytes, {binary, [], []}]}]}, Q}
   end;
 
-do_quote(Map, #elixir_quote{escape=true} = Q, E) when is_map(Map) ->
+do_escape(Map, Q, E) when is_map(Map) ->
   {TT, TQ} = do_quote(maps:to_list(Map), Q, E),
   {{'%{}', [], TT}, TQ};
 
-do_quote(Tuple, #elixir_quote{escape=true} = Q, E) when is_tuple(Tuple) ->
-  {TT, TQ} = do_quote(tuple_to_list(Tuple), Q, E),
-  {{'{}', [], TT}, TQ};
-
-do_quote(List, #elixir_quote{escape=true} = Q, E) when is_list(List) ->
+do_escape(List, Q, E) when is_list(List) ->
   %% The improper case is a bit inefficient, but improper lists are rare.
   case reverse_improper(List) of
-    {L}       -> do_splice(L, Q, E);
-    {L, R}    ->
-      {TL, QL} = do_splice(L, Q, E, [], []),
+    {L} -> do_quote_splice(L, Q, E);
+    {L, R} ->
+      {TL, QL} = do_quote_splice(L, Q, E, [], []),
       {TR, QR} = do_quote(R, QL, E),
       {update_last(TL, fun(X) -> {'|', [], [X, TR]} end), QR}
   end;
 
-do_quote(Other, #elixir_quote{escape=true} = Q, _)
+do_escape(Other, Q, _)
     when is_number(Other); is_pid(Other); is_atom(Other) ->
   {Other, Q};
 
-do_quote(Fun, #elixir_quote{escape=true} = Q, _) when is_function(Fun) ->
+do_escape(Fun, Q, _) when is_function(Fun) ->
   case (erlang:fun_info(Fun, env) == {env, []}) andalso
        (erlang:fun_info(Fun, type) == {type, external}) of
     true  -> {Fun, Q};
     false -> bad_escape(Fun)
   end;
 
-do_quote(Other, #elixir_quote{escape=true}, _) ->
-  bad_escape(Other);
-
-do_quote(List, Q, E) when is_list(List) ->
-  do_splice(lists:reverse(List), Q, E);
-
-do_quote(Other, Q, _) ->
-  {Other, Q}.
-
-%% Quote helpers
+do_escape(Other, _, _) ->
+  bad_escape(Other).
 
 bad_escape(Arg) ->
   argument_error(<<"cannot escape ", ('Elixir.Kernel':inspect(Arg, []))/binary, ". ",
                    "The supported values are: lists, tuples, maps, atoms, numbers, bitstrings, ",
                    "PIDs and remote functions in the format &Mod.fun/arity">>).
 
+%% do_quote_*
+
 do_quote_import(Name, Meta, ArgsOrAtom, #elixir_quote{imports_hygiene=true} = Q, E) ->
   Arity = case is_atom(ArgsOrAtom) of
     true  -> 0;
@@ -332,10 +343,41 @@ do_quote_tuple(Left, Meta, [{{unquote, _, _}, _, _}, _] = Right, Q, E) when ?def
   {{'{}', [], [TLeft, meta(Meta, Q), [NewHead, Body]]}, RQ};
 
 do_quote_tuple(Left, Meta, Right, Q, E) ->
-  {TLeft, LQ}  = do_quote(Left, Q, E),
+  {TLeft, LQ} = do_quote(Left, Q, E),
   {TRight, RQ} = do_quote(Right, LQ, E),
   {{'{}', [], [TLeft, meta(Meta, Q), TRight]}, RQ}.
 
+do_quote_splice([{'|', Meta, [{unquote_splicing, _, [Left]}, Right]} | T], #elixir_quote{unquote=true} = Q, E) ->
+  %% Process the remaining entries on the list.
+  %% For [1, 2, 3, unquote_splicing(arg) | tail], this will quote
+  %% 1, 2 and 3, which could even be unquotes.
+  {TT, QT} = do_quote_splice(T, Q, E, [], []),
+  {TR, QR} = do_quote(Right, QT, E),
+  {do_runtime_list(Meta, tail_list, [Left, TR, TT]), QR#elixir_quote{unquoted=true}};
+
+do_quote_splice(List, Q, E) ->
+  do_quote_splice(List, Q, E, [], []).
+
+do_quote_splice([{unquote_splicing, Meta, [Expr]} | T], #elixir_quote{unquote=true} = Q, E, Buffer, Acc) ->
+  Runtime = do_runtime_list(Meta, list, [Expr, do_list_concat(Buffer, Acc)]),
+  do_quote_splice(T, Q#elixir_quote{unquoted=true}, E, [], Runtime);
+
+do_quote_splice([H | T], Q, E, Buffer, Acc) ->
+  {TH, TQ} = do_quote(H, Q, E),
+  do_quote_splice(T, TQ, E, [TH | Buffer], Acc);
+
+do_quote_splice([], Q, _E, Buffer, Acc) ->
+  {do_list_concat(Buffer, Acc), Q}.
+
+do_list_concat(Left, []) -> Left;
+do_list_concat([], Right) -> Right;
+do_list_concat(Left, Right) -> {{'.', [], [erlang, '++']}, [], [Left, Right]}.
+
+do_runtime_list(Meta, Fun, Args) ->
+  {{'.', Meta, [elixir_quote, Fun]}, Meta, Args}.
+
+%% Helpers
+
 meta(Meta, Q) ->
   generated(keep(Meta, Q), Q).
 
@@ -381,33 +423,3 @@ keynew(Key, Meta, Value) ->
     {Key, _} -> Meta;
     _ -> keystore(Key, Meta, Value)
   end.
-
-%% Quote splicing
-
-do_splice([{'|', Meta, [{unquote_splicing, _, [Left]}, Right]} | T], #elixir_quote{unquote=true} = Q, E) ->
-  %% Process the remaining entries on the list.
-  %% For [1, 2, 3, unquote_splicing(arg) | tail], this will quote
-  %% 1, 2 and 3, which could even be unquotes.
-  {TT, QT} = do_splice(T, Q, E, [], []),
-  {TR, QR} = do_quote(Right, QT, E),
-  {do_runtime_list(Meta, tail_list, [Left, TR, TT]), QR#elixir_quote{unquoted=true}};
-
-do_splice(List, Q, E) ->
-  do_splice(List, Q, E, [], []).
-
-do_splice([{unquote_splicing, Meta, [Expr]} | T], #elixir_quote{unquote=true} = Q, E, Buffer, Acc) ->
-  do_splice(T, Q#elixir_quote{unquoted=true}, E, [], do_runtime_list(Meta, list, [Expr, do_join(Buffer, Acc)]));
-
-do_splice([H | T], Q, E, Buffer, Acc) ->
-  {TH, TQ} = do_quote(H, Q, E),
-  do_splice(T, TQ, E, [TH | Buffer], Acc);
-
-do_splice([], Q, _E, Buffer, Acc) ->
-  {do_join(Buffer, Acc), Q}.
-
-do_join(Left, [])    -> Left;
-do_join([], Right)   -> Right;
-do_join(Left, Right) -> {{'.', [], [erlang, '++']}, [], [Left, Right]}.
-
-do_runtime_list(Meta, Fun, Args) ->
-  {{'.', Meta, [elixir_quote, Fun]}, Meta, Args}.

lib/elixir/test/elixir/macro_test.exs

@@ -24,38 +24,44 @@ defmodule MacroTest do
   import Macro.ExternalTest
 
   describe "escape/2" do
-    test "handles tuples with size different than two" do
-      assert Macro.escape({:a}) == {:{}, [], [:a]}
-      assert Macro.escape({:a, :b, :c}) == {:{}, [], [:a, :b, :c]}
-      assert Macro.escape({:a, {1, 2, 3}, :c}) == {:{}, [], [:a, {:{}, [], [1, 2, 3]}, :c]}
-    end
-
-    test "simply returns tuples with size equal to two" do
+    test "returns tuples with size equal to two" do
       assert Macro.escape({:a, :b}) == {:a, :b}
     end
 
-    test "simply returns any other structure" do
+    test "returns lists" do
       assert Macro.escape([1, 2, 3]) == [1, 2, 3]
     end
 
-    test "handles maps" do
+    test "escapes tuples with size different than two" do
+      assert Macro.escape({:a}) == {:{}, [], [:a]}
+      assert Macro.escape({:a, :b, :c}) == {:{}, [], [:a, :b, :c]}
+      assert Macro.escape({:a, {1, 2, 3}, :c}) == {:{}, [], [:a, {:{}, [], [1, 2, 3]}, :c]}
+    end
+
+    test "escapes maps" do
       assert Macro.escape(%{a: 1}) == {:%{}, [], [a: 1]}
     end
 
-    test "handles bitstring" do
+    test "escapes bitstring" do
       assert {:<<>>, [], args} = Macro.escape(<<300::12>>)
       assert [{:::, [], [1, {:size, [], [4]}]}, {:::, [], [",", {:binary, [], []}]}] = args
     end
 
-    test "works recursively" do
+    test "escapes recursively" do
       assert Macro.escape([1, {:a, :b, :c}, 3]) == [1, {:{}, [], [:a, :b, :c]}, 3]
     end
 
-    test "with improper lists" do
+    test "escapes improper lists" do
       assert Macro.escape([1 | 2]) == [{:|, [], [1, 2]}]
       assert Macro.escape([1, 2 | 3]) == [1, {:|, [], [2, 3]}]
     end
 
+    test "prunes metadata" do
+      meta = [nothing: :important, counter: 1]
+      assert Macro.escape({:foo, meta, []}) == {:{}, [], [:foo, meta, []]}
+      assert Macro.escape({:foo, meta, []}, prune_metadata: true) == {:{}, [], [:foo, [], []]}
+    end
+
     test "with unquote" do
       contents = quote(unquote: false, do: unquote(1))
       assert Macro.escape(contents, unquote: true) == 1