add git sparse checkout (#5063)

Signed-off-by: José Valim <jose.valim@plataformatec.com.br>

Author: mobileoverlord

lib/mix/lib/mix/scm/git.ex

@@ -25,6 +25,7 @@ defmodule Mix.SCM.Git do
   end
 
   def accepts_options(_app, opts) do
+    opts = sparse_opts(opts)
     cond do
       gh = opts[:github] ->
         opts
@@ -41,7 +42,9 @@ defmodule Mix.SCM.Git do
 
   def checked_out?(opts) do
     # Are we inside a Git repository?
-    File.regular?(Path.join(opts[:dest], ".git/HEAD"))
+    git_dest(opts)
+    |> Path.join(".git/HEAD")
+    |> File.regular?
   end
 
   def lock_status(opts) do
@@ -50,7 +53,7 @@ defmodule Mix.SCM.Git do
 
     cond do
       lock_rev = get_lock_rev(lock, opts) ->
-        File.cd!(opts[:dest], fn ->
+        File.cd!(git_dest(opts), fn ->
           %{origin: origin, rev: rev} = get_rev_info()
           if get_lock_repo(lock) == origin and lock_rev == rev do
             :ok
@@ -77,19 +80,30 @@ defmodule Mix.SCM.Git do
   def checkout(opts) do
     assert_git!()
 
-    path     = opts[:dest]
+    path     = git_dest(opts)
     location = opts[:git]
 
     _ = File.rm_rf!(path)
-    git!(~s(clone --no-checkout --progress "#{location}" "#{path}"))
 
-    File.cd! path, fn -> do_checkout(opts) end
+    fun =
+      if opts[:sparse] do
+        sparse_check(git_version())
+        File.mkdir_p!(path)
+        fn -> init_sparse(opts) end
+      else
+        git!(~s(clone --no-checkout --progress "#{location}" "#{path}"))
+        fn -> do_checkout(opts) end
+      end
+
+    File.cd! path, fun
   end
 
   def update(opts) do
     assert_git!()
 
-    File.cd! opts[:dest], fn ->
+    File.cd! git_dest(opts), fn ->
+      sparse_toggle(opts)
+
       location = opts[:git]
       update_origin(location)
 
@@ -102,24 +116,81 @@ defmodule Mix.SCM.Git do
     end
   end
 
+  defp sparse_opts(opts) do
+    if opts[:sparse] do
+      dest = Path.join(opts[:dest], opts[:sparse])
+      opts
+      |> Keyword.put(:git_dest, opts[:dest])
+      |> Keyword.put(:dest, dest)
+    else
+      opts
+    end
+  end
+
+  defp sparse_check(version) do
+    unless {1, 7, 0} <= version do
+      version =
+        version
+        |> Tuple.to_list
+        |> Enum.join(".")
+      Mix.raise "Git >= 1.7.0 is required to use sparse checkout. " <>
+                 "You are running version #{version}"
+    end
+  end
+
+  defp sparse_toggle(opts) do
+    git!("config core.sparsecheckout #{opts[:sparse] != nil}")
+  end
+
+
   defp progress_switch(version) when {1, 7, 1} <= version, do: " --progress"
   defp progress_switch(_),                                 do: ""
 
   defp tags_switch(nil), do: ""
   defp tags_switch(_), do: " --tags"
 
+  defp git_dest(opts) do
+    if opts[:git_dest] do
+      opts[:git_dest]
+    else
+      opts[:dest]
+    end
+  end
+
   ## Helpers
 
   defp validate_git_options(opts) do
-    case Keyword.take(opts, [:branch, :ref, :tag]) do
-      []  -> opts
+    err = "You should specify only one of branch, ref or tag, and only once. " <>
+          "Error on Git dependency: #{opts[:git]}"
+    validate_single_uniq(opts, [:branch, :ref, :tag], err)
+
+    err = "You should specify only one sparse path. " <>
+          "Error on Git dependency: #{opts[:git]}"
+    validate_single_uniq(opts, [:sparse], err)
+  end
+
+  defp validate_single_uniq(opts, take, error) do
+    case Keyword.take(opts, take) do
+      [] -> opts
       [_] -> opts
-      _   ->
-        Mix.raise "You should specify only one of branch, ref or tag, and only once. " <>
-                  "Error on Git dependency: #{opts[:git]}"
+      _   -> Mix.raise error
     end
   end
 
+  defp init_sparse(opts) do
+    git!("init --quiet")
+    git!("remote add origin #{opts[:git]} --fetch")
+    sparse_toggle(opts)
+
+    sparse_info =
+      File.cwd!
+      |> Path.join(".git/info/sparse-checkout")
+
+    File.write(sparse_info, opts[:sparse])
+
+    do_checkout(opts)
+  end
+
   defp do_checkout(opts) do
     rev = get_lock_rev(opts[:lock], opts) || get_opts_rev(opts)
     git!("--git-dir=.git checkout --quiet #{rev}")
@@ -147,6 +218,13 @@ defmodule Mix.SCM.Git do
 
   defp get_lock_opts(opts) do
     lock_opts = Keyword.take(opts, [:branch, :ref, :tag])
+    lock_opts =
+      if opts[:sparse] do
+        lock_opts ++ [sparse: opts[:sparse]]
+      else
+        lock_opts
+      end
+
     if opts[:submodules] do
       lock_opts ++ [submodules: true]
     else

lib/mix/test/fixtures/.gitignore

@@ -1,3 +1,4 @@
 git_repo
+git_sparse_repo
 deps_on_git_repo
 git_rebar

lib/mix/test/mix/scm/git_test.exs

@@ -30,6 +30,10 @@ defmodule Mix.SCM.GitTest do
     assert_raise Mix.Error, ~r/You should specify only one of branch, ref or tag/, fn ->
       Mix.SCM.Git.accepts_options(nil, [git: "/repo", branch: "master", branch: "develop"])
     end
+
+    assert_raise Mix.Error, ~r/You should specify only one sparse path/, fn ->
+      Mix.SCM.Git.accepts_options(nil, [git: "/repo", sparse: "/a", sparse: "/b", dest: "/repo"])
+    end
   end
 
   defp lock(opts \\ []) do

lib/mix/test/mix/tasks/deps.git_test.exs

@@ -34,6 +34,16 @@ defmodule Mix.Tasks.DepsGitTest do
     end
   end
 
+  defmodule GitSparseApp do
+    def project do
+      [app: :git_app,
+       version: "0.1.0",
+       deps: [
+         {:git_sparse_repo, "0.1.0", git: fixture_path("git_sparse_repo"), sparse: "sparse_dir"}
+       ]]
+    end
+  end
+
   defmodule GitErrorApp do
     def project do
       [deps: [
@@ -69,6 +79,17 @@ defmodule Mix.Tasks.DepsGitTest do
     end
   end
 
+  test "gets and updates Git repos with sparse checkout" do
+    Mix.Project.push GitSparseApp
+
+    in_fixture "no_mixfile", fn ->
+      Mix.Tasks.Deps.Get.run []
+      message = "* Getting git_sparse_repo (#{fixture_path("git_sparse_repo")})"
+      assert_received {:mix_shell, :info, [^message]}
+      assert File.read!("mix.lock") =~ "sparse: \"sparse_dir\""
+    end
+  end
+
   test "handles invalid .git directory" do
     Mix.Project.push GitApp
 
@@ -265,6 +286,41 @@ defmodule Mix.Tasks.DepsGitTest do
     purge [GitRepo, GitRepo.Mixfile]
   end
 
+  # sparse
+  test "updates the repo when sparse changes" do
+    Mix.Project.push GitSparseApp
+    [ref | _] = get_git_repo_revs("git_sparse_repo")
+
+    in_fixture "no_mixfile", fn ->
+      Mix.Dep.Lock.write %{git_sparse_repo: {:git, fixture_path("git_sparse_repo"), ref, [sparse: "sparse_dir"]}}
+
+      Mix.Tasks.Deps.Get.run []
+
+      # Update the lock and now we should get an error
+      Mix.Dep.Lock.write %{git_sparse_repo: {:git, fixture_path("git_sparse_repo"), ref, []}}
+      assert_raise Mix.Error, fn ->
+        Mix.Tasks.Deps.Loadpaths.run []
+      end
+
+      # # Flush the errors we got, move to a clean slate
+      # Mix.shell.flush
+      # Mix.Task.clear
+      #
+      # # Calling get should update the dependency
+      # Mix.Tasks.Deps.Get.run []
+      # assert File.exists?("deps/git_repo/lib/git_repo.ex")
+      # assert File.read!("mix.lock") =~ last
+      #
+      # message = "* Updating git_repo (#{fixture_path("git_repo")})"
+      # assert_received {:mix_shell, :info, [^message]}
+      #
+      # # Check we got no error
+      # refute_received {:mix_shell, :error, _}
+    end
+  after
+    purge [GitSparseRepo, GitSparseRepo.Mixfile]
+  end
+
   test "updates the repo and the lock when the mixfile updates" do
     Mix.Project.push GitApp
     [last, first | _] = get_git_repo_revs
@@ -364,8 +420,8 @@ defmodule Mix.Tasks.DepsGitTest do
     Mix.Project.push(name, file)
   end
 
-  defp get_git_repo_revs do
-    File.cd! fixture_path("git_repo"), fn ->
+  defp get_git_repo_revs(fixture \\ "git_repo") do
+    File.cd! fixture_path(fixture), fn ->
       Regex.split ~r(\r?\n), System.cmd("git", ["log", "--format=%H"]) |> elem(0)
     end
   end

lib/mix/test/test_helper.exs

@@ -237,6 +237,60 @@ unless File.dir?(target) do
   end
 end
 
+# Git Sparse
+target = Path.expand("fixtures/git_sparse_repo", __DIR__)
+
+unless File.dir?(target) do
+  subdir = Path.join(target, "sparse_dir")
+
+  File.mkdir_p!(Path.join(subdir, "lib"))
+
+  File.write! Path.join(subdir, "mix.exs"), """
+  ## Auto-generated fixture
+  raise "I was not supposed to be loaded"
+  """
+
+  File.cd! target, fn ->
+    System.cmd("git", ~w[init])
+    System.cmd("git", ~w[config user.email "mix@example.com"])
+    System.cmd("git", ~w[config user.name "mix-repo"])
+    System.cmd("git", ~w[add .])
+    System.cmd("git", ~w[commit -m "bad"])
+  end
+
+  File.write! Path.join(subdir, "mix.exs"), """
+  ## Auto-generated fixture
+  defmodule GitSparseRepo.Mixfile do
+    use Mix.Project
+
+    def project do
+      [app: :git_sparse_repo, version: "0.1.0"]
+    end
+  end
+  """
+
+  File.cd! target, fn ->
+    System.cmd("git", ~w[add .])
+    System.cmd("git", ~w[commit -m "ok"])
+    System.cmd("git", ~w[tag without_module])
+  end
+
+  File.write! Path.join(subdir, "lib/git_sparse_repo.ex"), """
+  ## Auto-generated fixture
+  defmodule GitSparseRepo do
+    def hello do
+      "World"
+    end
+  end
+  """
+
+  File.cd! target, fn ->
+    System.cmd("git", ~w[add .])
+    System.cmd("git", ~w[commit -m "lib"])
+    System.cmd("git", ~w[tag with_module])
+  end
+end
+
 # Deps on Git repo
 target = Path.expand("fixtures/deps_on_git_repo", __DIR__)