mix xref graph - add json support and related changes (#14327)

This patch adds support for output the xref graph as a two level Map of
Maps which can be trivially loaded by anything that can process JSON
data. The top level Map contains keys which specify source files whose
values are maps contain sink files as keys, and dependency type data as
values. Source files with no dependencies have empty maps as values. For
example a if "lib/foo.ex" has a compile time dependency on "lib/bar.ex"
which had no dependencies at all, then the output would look like this:

   {
     "lib/foo.ex": { "lib/bar.ex": "compile" },
     "lib/bar.ex": {},
   }

At the same time it adds support for renaming existing xref_graph.dot
and xref_graph.json files to have a .bak extension instead of
overwriting them.

This patch also includes logic to fix some misleading verbiage output by
the 'dot' format when the output file is not in the current working
directory. This change is included with the JSON changes because I
refactored the logic for writing to a file so that both the 'json' and
'dot' formats would use the same code.

Closes #14324.

Author: demerphq

lib/mix/lib/mix/tasks/xref.ex

@@ -252,16 +252,31 @@ defmodule Mix.Tasks.Xref do
 
       * `cycles` - prints all strongly connected cycles in the graph;
 
-      * `dot` - produces a DOT graph description in `xref_graph.dot` in the
-        current directory. Warning: this will override any previously generated file
+      * `dot` - produces a DOT graph description, by default written to `xref_graph.dot`
+        in the current directory.  See the documentation for the `--output` option to
+        learn how to control where the file is written and other related details.
 
-    * `--output` *(since v1.15.0)* - can be set to one of
+      * `json` *(since v1.19.0)* - produces a JSON file, by default written to
+        `xref_graph.json` in the current directory.  See the documentation for the
+        `--output` option to learn how to control where the file is written and other
+        related details.
+
+        The JSON format is always a two level map of maps. The top level keys
+        specify source files, with their values containing maps whose keys specify
+        sink files and whose values specify the type of relationship, which will
+        be one of `compile`, `export` or `runtime`. Files which have no dependencies
+        will be present in the top level map, and will have empty maps for values.
+
+    * `--output` *(since v1.15.0)* - can be used to override the location of
+      the files created by the `dot` and `json` formats. It can be set to
 
       * `-` - prints the output to standard output;
 
       * a path - writes the output graph to the given path
 
-      Defaults to `xref_graph.dot` in the current directory.
+      If the output file already exists then it will be renamed in place
+      to have a `.bak` suffix, possibly overwriting any existing `.bak` file.
+      If this rename fails a fatal exception will be thrown.
 
   The `--source` and `--sink` options are particularly useful when trying to understand
   how the modules in a particular file interact with the whole system. You can combine
@@ -949,17 +964,22 @@ defmodule Mix.Tasks.Xref do
           {roots, callback, count} =
             roots_and_callback(file_references, filter, sources, sinks, opts)
 
-          path = Keyword.get(opts, :output, "xref_graph.dot")
-
-          Mix.Utils.write_dot_graph!(path, "xref graph", Enum.sort(roots), callback, opts)
+          file_spec =
+            Mix.Utils.write_dot_graph!(
+              "xref_graph.dot",
+              "xref graph",
+              Enum.sort(roots),
+              callback,
+              opts
+            )
 
-          if path != "-" do
-            png_path = (path |> Path.rootname() |> Path.basename()) <> ".png"
+          if file_spec != "-" do
+            png_file_spec = (file_spec |> Path.rootname() |> Path.basename()) <> ".png"
 
             """
-            Generated #{inspect(path)} in the current directory. To generate a PNG:
+            Generated "#{Path.relative_to_cwd(file_spec)}". To generate a PNG:
 
-               dot -Tpng #{inspect(path)} -o #{inspect(png_path)}
+               dot -Tpng #{inspect(file_spec)} -o #{inspect(png_file_spec)}
 
             For more options see http://www.graphviz.org/.
             """
@@ -976,6 +996,19 @@ defmodule Mix.Tasks.Xref do
         "cycles" ->
           {:cycles, print_cycles(file_references, filter, opts)}
 
+        "json" ->
+          {roots, callback, count} =
+            roots_and_callback(file_references, filter, sources, sinks, opts)
+
+          file_spec =
+            Mix.Utils.write_json_tree!("xref_graph.json", Enum.sort(roots), callback, opts)
+
+          if file_spec != "-" do
+            Mix.shell().info("Generated \"#{file_spec}\".")
+          end
+
+          {:references, count}
+
         other when other in [nil, "plain", "pretty"] ->
           {roots, callback, count} =
             roots_and_callback(file_references, filter, sources, sinks, opts)

lib/mix/lib/mix/utils.ex

@@ -2,6 +2,9 @@
 # SPDX-FileCopyrightText: 2021 The Elixir Team
 # SPDX-FileCopyrightText: 2012 Plataformatec
 
+# NOTE: As this is a utils file it should not contain hard coded files
+# everything should be parameterized.
+
 defmodule Mix.Utils do
   @moduledoc false
 
@@ -261,6 +264,75 @@ defmodule Mix.Utils do
     |> Enum.uniq()
   end
 
+  @doc """
+  Handles writing the contents to either STDOUT or to a file, as specified
+  by the :output keyword in opts, defaulting to the provided default_file_spec.
+
+  If the resolved file specification is "-" then the contents is written to STDOUT,
+  otherwise if the file already exists it is renamed with a ".bak" suffix before
+  the contents is written.  The underlying IO operations will throw an exception
+  if there is an error.
+
+  Returns the name of the file written to, or "-" if the output was to STDOUT.
+  This function is made public mostly for testing.
+  """
+  @spec write_according_to_opts!(Path.t(), iodata(), keyword) :: Path.t()
+  def write_according_to_opts!(default_file_spec, contents, opts) do
+    file_spec = Keyword.get(opts, :output, default_file_spec)
+
+    if file_spec == "-" do
+      IO.write(contents)
+    else
+      if File.exists?(file_spec) do
+        new_file_spec = "#{file_spec}.bak"
+        File.rename!(file_spec, new_file_spec)
+      end
+
+      File.write!(file_spec, contents)
+    end
+
+    # return the file_spec just in case the caller has a use for it.
+    file_spec
+  end
+
+  @doc """
+  Outputs the given tree according to the callback as a two level
+  map of maps in JSON format.
+
+  The callback will be invoked for each node and it
+  must return a `{printed, children}` tuple.
+
+  If the `:output` option is `-` then prints to standard output,
+  see write_according_to_opts!/3 for details.
+  """
+  @spec write_json_tree!(Path.t(), [node], (node -> {formatted_node, [node]}), keyword) :: :ok
+        when node: term()
+  def write_json_tree!(default_file_spec, nodes, callback, opts \\ []) do
+    src_map = build_json_tree(_src_map = %{}, nodes, callback)
+    write_according_to_opts!(default_file_spec, JSON.encode_to_iodata!(src_map), opts)
+  end
+
+  defp build_json_tree(src_map, [], _callback), do: src_map
+
+  defp build_json_tree(src_map, nodes, callback) do
+    Enum.reduce(nodes, src_map, fn node, src_map ->
+      {{name, _}, children} = callback.(node)
+
+      if Map.has_key?(src_map, name) do
+        src_map
+      else
+        sink_map =
+          Enum.reduce(children, %{}, fn {name, info}, sink_map ->
+            info = if info == nil, do: "runtime", else: Atom.to_string(info)
+            Map.put(sink_map, name, info)
+          end)
+
+        Map.put(src_map, name, sink_map)
+        |> build_json_tree(children, callback)
+      end
+    end)
+  end
+
   @type formatted_node :: {name :: String.Chars.t(), edge_info :: String.Chars.t()}
 
   @doc """
@@ -333,25 +405,21 @@ defmodule Mix.Utils do
   The callback will be invoked for each node and it
   must return a `{printed, children}` tuple.
 
-  If `path` is `-`, prints the output to standard output.
+  If the `:output` option is `-` then prints to standard output,
+  see write_according_to_opts!/3 for details.
   """
   @spec write_dot_graph!(
           Path.t(),
           String.t(),
           [node],
           (node -> {formatted_node, [node]}),
           keyword
-        ) :: :ok
+        ) :: Path.t()
         when node: term()
-  def write_dot_graph!(path, title, nodes, callback, _opts \\ []) do
+  def write_dot_graph!(default_file_spec, title, nodes, callback, opts \\ []) do
     {dot, _} = build_dot_graph(make_ref(), nodes, MapSet.new(), callback)
     contents = ["digraph ", quoted(title), " {\n", dot, "}\n"]
-
-    if path == "-" do
-      IO.write(contents)
-    else
-      File.write!(path, contents)
-    end
+    write_according_to_opts!(default_file_spec, contents, opts)
   end
 
   defp build_dot_graph(_parent, [], seen, _callback), do: {[], seen}

lib/mix/test/mix/tasks/xref_test.exs

@@ -863,6 +863,14 @@ defmodule Mix.Tasks.XrefTest do
                  "lib/b.ex"
                }
                """
+
+        assert Mix.Task.run("xref", ["graph", "--format", "json", "--output", "xref_graph.json"]) ==
+                 :ok
+
+        assert File.read!("xref_graph.json") ===
+                 String.trim_trailing("""
+                 {"lib/a.ex":{"lib/b.ex":"compile"},"lib/b.ex":{}}
+                 """)
       end)
     end
 
@@ -891,6 +899,14 @@ defmodule Mix.Tasks.XrefTest do
                  "lib/b.ex"
                }
                """
+
+        assert Mix.Task.run("xref", ["graph", "--format", "json", "--output", "xref_graph.json"]) ==
+                 :ok
+
+        assert File.read!("xref_graph.json") ===
+                 String.trim_trailing("""
+                 {"lib/a.ex":{"lib/b.ex":"export"},"lib/b.ex":{}}
+                 """)
       end)
     end
 
@@ -933,6 +949,16 @@ defmodule Mix.Tasks.XrefTest do
         end
         """)
 
+        output =
+          capture_io(fn ->
+            assert Mix.Task.run("xref", ["graph", "--format", "json", "--output", "-"]) == :ok
+          end)
+
+        assert output ===
+                 String.trim_trailing("""
+                 {"lib/a.ex":{},"lib/b.ex":{}}
+                 """)
+
         output =
           capture_io(fn ->
             assert Mix.Task.run("xref", ["graph", "--format", "dot", "--output", "-"]) == :ok
@@ -984,6 +1010,14 @@ defmodule Mix.Tasks.XrefTest do
                  "lib/b.ex"
                }
                """
+
+        assert Mix.Task.run("xref", ["graph", "--format", "json"]) ==
+                 :ok
+
+        assert File.read!("xref_graph.json") ===
+                 String.trim_trailing("""
+                 {"lib/a.ex":{"lib/b.ex":"compile"},"lib/b.ex":{"lib/a.ex":"compile"}}
+                 """)
       end)
     end
 
@@ -1127,6 +1161,44 @@ defmodule Mix.Tasks.XrefTest do
       """)
     end
 
+    test "dot with cycle" do
+      assert_graph_dot(
+        ~w[],
+        """
+        digraph "xref graph" {
+          "lib/a.ex"
+          "lib/a.ex" -> "lib/b.ex" [label="(compile)"]
+          "lib/b.ex" -> "lib/a.ex"
+          "lib/b.ex" -> "lib/c.ex"
+          "lib/c.ex" -> "lib/d.ex" [label="(compile)"]
+          "lib/d.ex" -> "lib/e.ex"
+          "lib/b.ex" -> "lib/e.ex" [label="(compile)"]
+          "lib/b.ex"
+          "lib/c.ex"
+          "lib/d.ex"
+          "lib/e.ex"
+        }
+        """
+      )
+    end
+
+    test "json with cycle" do
+      assert_graph_json(
+        ~w[],
+        """
+        { "lib/a.ex": { "lib/b.ex": "compile" },
+          "lib/b.ex": { "lib/a.ex": "runtime",
+                        "lib/c.ex": "runtime",
+                        "lib/e.ex": "compile" },
+          "lib/c.ex": { "lib/d.ex": "compile" },
+          "lib/d.ex": { "lib/e.ex": "runtime" },
+          "lib/e.ex": { } }
+        """,
+        # make it easier to read the expected output
+        strip_ws: true
+      )
+    end
+
     @default_files %{
       "lib/a.ex" => """
       defmodule A do
@@ -1160,6 +1232,43 @@ defmodule Mix.Tasks.XrefTest do
       """
     }
 
+    defp assert_graph_json(args, expected, opts) do
+      assert_graph_io("json", args, expected, opts)
+    end
+
+    defp assert_graph_dot(args, expected, opts \\ []) do
+      assert_graph_io("dot", args, expected, opts)
+    end
+
+    # note that we trim_trailing on expected and output
+    # we also support :strip_ws in opts, which removes all
+    # whitespace from expected (but not output!) before performing
+    # the test.
+    defp assert_graph_io(format, args, expected, opts) do
+      in_fixture("no_mixfile", fn ->
+        Enum.each(opts[:files] || @default_files, fn {path, content} ->
+          File.write!(path, content)
+        end)
+
+        output =
+          String.trim_trailing(
+            capture_io(fn ->
+              assert Mix.Task.run(
+                       "xref",
+                       args ++ ["graph", "--format", format, "--output", "-"]
+                     ) == :ok
+            end)
+          )
+
+        expected =
+          if opts[:strip_ws],
+            do: Regex.replace(~r/\s+/, expected, "", global: true),
+            else: String.trim_trailing(expected)
+
+        assert output === expected
+      end)
+    end
+
     defp assert_graph(args \\ [], expected, opts \\ []) do
       in_fixture("no_mixfile", fn ->
         nb_files =