Parallelize the Git-free backend by default

README.md

@@ -6,8 +6,8 @@ removed, and when `SPDX-License-Identifier` declarations appear or disappear,
 with the commit and path behind each change.
 It scans large histories in seconds. That speed makes it practical to trace
 license evidence through every revision, even in a large repository.
-On an 8-core M1 Pro, the default native Git backend scanned 1.3 GB of blobs
-across Cargo's 23,190 commits in an average of 4.05 seconds.
+On an 18-core M5 Max, the default native Git backend scanned 1.42 GB of blobs
+across Cargo's 23,683 commits in an average of 1.61 seconds.
 
 The `licenses` command reports what the current checkout contains. `git-spdx`
 adds the timeline, so it can answer questions such as:
@@ -131,19 +131,25 @@ git spdx log --backend=gogit --details
 git spdx scan --backend=gogit
 ```
 
-Run either command with `--help` to see cache, mmap, object reader, and history
-worker controls for the Git-free backend.
+By default it loads pack indexes into memory, memory-maps pack files where the
+platform supports it, and runs `GOMAXPROCS` blob readers and up to eight
+history workers. Run either command with `--help` to see cache, mmap, object
+reader, and history worker controls for the Git-free backend.
 
 ## Performance
 
-On an 8-core M1 Pro with 16 GB of memory, a full clone of `rust-lang/cargo` at
-`a07c49a` contained 23,190 commits and 1.3 GB of historical blobs. Two-run
-averages were:
+On an 18-core M5 Max (6 performance, 12 efficiency) with 64 GB of memory, a
+clone of `rust-lang/cargo` with history up to `a07c49a` contained 23,683
+commits and 1.42 GB of historical blobs. Five-run averages of `scan` with
+default flags were:
 
 | Backend | Wall time | CPU time | Peak process-tree RSS |
 | --- | ---: | ---: | ---: |
-| Native Git | 4.05 s | 20.03 s | 2.12 GiB |
-| Git-free | 3.57 s | 20.14 s | 614 MiB |
+| Native Git | 1.61 s | 14.50 s | 3.81 GiB |
+| Git-free | 1.49 s | 13.29 s | 896 MiB |
+
+The native backend starts one `git cat-file` process per reader, so its memory
+grows with core count. Pass `--readers` to limit it.
 
 Both backends produced the same normalized output. The benchmark includes all
 stored history rather than only the files in the checked-out tree.

gogit_history_parallel.go

@@ -1,9 +1,22 @@
 package main
 
-var historyWorkers = newOption(1)
+import "runtime"
+
+var historyWorkers = newOption(0)
 
 const historyLookahead = 2
 
+// maxAutoHistoryWorkers caps --history-workers=0. On cargo's history the walk
+// stopped getting faster beyond 4-8 workers and slowed slightly at 18.
+const maxAutoHistoryWorkers = 8
+
+func resolvedHistoryWorkers() int {
+	if *historyWorkers > 0 {
+		return *historyWorkers
+	}
+	return min(runtime.GOMAXPROCS(0), maxAutoHistoryWorkers)
+}
+
 type historyResult struct {
 	changes []change
 	err     error

gogit_history_parallel_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 	"time"
@@ -30,7 +31,7 @@ func TestGitFreeParallelHistory(t *testing.T) {
 				commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: "+license+"\n", fmt.Sprintf("Change %d", n))
 			}
 			git(t, repo, "gc", "--quiet")
-			for _, workers := range []string{"2", "4"} {
+			for _, workers := range []string{"0", "1", "2", "4"} {
 				args := []string{"--history-workers=" + workers, testMmapFlag, "--readers=4"}
 				for _, mode := range []string{"--details", "--monthly"} {
 					commandArgs := append([]string{"log", repo}, args...)
@@ -45,6 +46,32 @@ func TestGitFreeParallelHistory(t *testing.T) {
 	}
 }
 
+func TestAutoWorkerCounts(t *testing.T) {
+	oldProcs := runtime.GOMAXPROCS(0)
+	oldWorkers, oldReaders := *historyWorkers, *readers
+	t.Cleanup(func() {
+		runtime.GOMAXPROCS(oldProcs)
+		*historyWorkers, *readers = oldWorkers, oldReaders
+	})
+	for _, test := range []struct {
+		procs, flag, wantHistory, wantReaders int
+	}{
+		{procs: 2, flag: 0, wantHistory: 2, wantReaders: 2},
+		{procs: 16, flag: 0, wantHistory: maxAutoHistoryWorkers, wantReaders: 16},
+		{procs: 2, flag: 12, wantHistory: 12, wantReaders: 12},
+		{procs: 16, flag: 1, wantHistory: 1, wantReaders: 1},
+	} {
+		runtime.GOMAXPROCS(test.procs)
+		*historyWorkers, *readers = test.flag, test.flag
+		if got := resolvedHistoryWorkers(); got != test.wantHistory {
+			t.Errorf("GOMAXPROCS=%d --history-workers=%d: got %d, want %d", test.procs, test.flag, got, test.wantHistory)
+		}
+		if got := blobReaders(); got != test.wantReaders {
+			t.Errorf("GOMAXPROCS=%d --readers=%d: got %d, want %d", test.procs, test.flag, got, test.wantReaders)
+		}
+	}
+}
+
 func TestGitFreeParallelHistoryMissingTree(t *testing.T) {
 	repo := repository(t)
 	for n := range 12 {

gogit_history_stream.go

@@ -180,8 +180,8 @@ func walkChangesGoGitStream(r *gogit.Repository, merges bool, visit func(change)
 		return err
 	}
 	cache := newHistoryTreeCache()
-	if *historyWorkers > 1 {
-		return visitHistoryStreamParallel(iter, cache, merges, visit, *historyWorkers)
+	if workers := resolvedHistoryWorkers(); workers > 1 {
+		return visitHistoryStreamParallel(iter, cache, merges, visit, workers)
 	}
 	for {
 		node, err := iter.Next()
@@ -398,13 +398,14 @@ func legalBlobsGoGit(path string) (map[string]bool, error) {
 		boundary[hash] = true
 	}
 	collector := newLegalTreeCollector()
-	if *historyWorkers <= 1 {
+	workers := resolvedHistoryWorkers()
+	if workers <= 1 {
 		err = visitCommitTrees(r, roots, boundary, func(hash plumbing.Hash) error {
 			return collectLegalTreeRaw(r, hash, false, collector)
 		})
 		return collector.legal, err
 	}
-	err = collectLegalTreesParallel(r, roots, boundary, collector, *historyWorkers-1)
+	err = collectLegalTreesParallel(r, roots, boundary, collector, workers-1)
 	return collector.legal, err
 }
 

gogit_history_test.go

@@ -16,7 +16,10 @@ import (
 func cliWithoutGit(t *testing.T, args ...string) string {
 	t.Helper()
 	commandArgs := append([]string{}, args...)
-	commandArgs = append(commandArgs, "--backend=gogit", "--history-workers=4")
+	commandArgs = append(commandArgs, "--backend=gogit")
+	if !slices.ContainsFunc(args, func(arg string) bool { return strings.HasPrefix(arg, "--history-workers") }) {
+		commandArgs = append(commandArgs, "--history-workers=4")
+	}
 	cmd := exec.Command(os.Args[0], commandArgs...)
 	cmd.Env = append(os.Environ(), "GIT_SPDX_TEST_CLI=1", "GOMAXPROCS=2", "PATH="+t.TempDir())
 	out, err := cmd.CombinedOutput()

gogit_tuning.go

@@ -21,8 +21,8 @@ const (
 )
 
 var (
-	goGitMemoryIndex  = newOption(false)
-	goGitMmap         = newOption(false)
+	goGitMemoryIndex  = newOption(true)
+	goGitMmap         = newOption(true)
 	goGitCacheBytes   = newOption(uint64(cache.DefaultMaxSize))
 	goGitCacheShards  = newOption(1)
 	goGitObjectInfos  = newOption(true)

gogit_tuning_test.go

@@ -16,6 +16,13 @@ var (
 	benchmarkZlibErr  error
 )
 
+var goGitIndexModes = [][]string{
+	{"--gogit-memory-index=true", "--gogit-mmap=true"},
+	{"--gogit-memory-index=true", "--gogit-mmap=false"},
+	{"--gogit-memory-index=false", "--gogit-mmap=true"},
+	{"--gogit-memory-index=false", "--gogit-mmap=false"},
+}
+
 func TestReadGoGitBlobReusesMemoryObjectContent(t *testing.T) {
 	obj := plumbing.NewMemoryObject(nil)
 	obj.SetType(plumbing.BlobObject)
@@ -66,10 +73,10 @@ func TestGitFreeTunedPackedScan(t *testing.T) {
 			if metric(t, want, "blobs seen") != 301 {
 				t.Fatal(want)
 			}
-			for _, mode := range []string{"--gogit-memory-index", "--gogit-mmap"} {
-				for _, readers := range []string{"1", "2", "4"} {
-					args := []string{"scan", repo, "--readers=" + readers, mode, "--gogit-cache-bytes=536870912", "--gogit-cache-shards=8", "--gogit-object-buffer=64", "--gogit-object-batch=64"}
-					got := cliWithoutGit(t, args...)
+			for _, mode := range goGitIndexModes {
+				for _, readers := range []string{"0", "1", "2", "4"} {
+					args := []string{"scan", repo, "--readers=" + readers, "--gogit-cache-bytes=536870912", "--gogit-cache-shards=8", "--gogit-object-buffer=64", "--gogit-object-batch=64"}
+					got := cliWithoutGit(t, append(args, mode...)...)
 					assertScanMetrics(t, got, want)
 				}
 			}
@@ -101,8 +108,9 @@ func TestGitFreeTunedHistoryLayouts(t *testing.T) {
 	shallow := filepath.Join(t.TempDir(), "shallow")
 	git(t, repo, "clone", "--depth=1", "file://"+repo, shallow)
 	for _, path := range []string{repo, bare, worktree, shallow} {
-		for _, mode := range []string{"--gogit-memory-index", "--gogit-mmap"} {
-			got := cliWithoutGit(t, "log", path, "--readers=4", mode, "--gogit-cache-bytes=536870912", "--details")
+		for _, mode := range goGitIndexModes {
+			args := []string{"log", path, "--readers=4", "--gogit-cache-bytes=536870912", "--details"}
+			got := cliWithoutGit(t, append(args, mode...)...)
 			if want := cli(t, "log", path, "--details"); got != want {
 				t.Fatalf("history differs for %s:\n%s\n%s", path, got, want)
 			}
@@ -123,7 +131,7 @@ func BenchmarkGoGitTuning(b *testing.B) {
 			cache  uint64
 			mmap   bool
 		}{
-			{"default", false, 96 << 20, false},
+			{"lazy", false, 96 << 20, false},
 			{"memory", true, 96 << 20, false},
 			{"mmap", false, 96 << 20, true},
 		} {

main.go

@@ -134,17 +134,17 @@ func addSharedFlags(cmd *cobra.Command) {
 	flags := cmd.Flags()
 	flags.StringVar(backend, "backend", "git", "backend: git (subprocesses) or gogit (Git-free)")
 	flags.Int64Var(maxBlobSize, "max-blob-size", defaultMaxBlobSize, "skip blobs larger than this many bytes")
-	flags.IntVar(readers, "readers", 0, "blob readers (0 = GOMAXPROCS for git, 1 for gogit)")
+	flags.IntVar(readers, "readers", 0, "blob readers (0 = GOMAXPROCS)")
 	flags.Int64Var(legalBlobSize, "max-legal-blob-size", defaultLegalBlobSize, "size limit for blobs used at legal paths")
-	flags.BoolVar(goGitMemoryIndex, "gogit-memory-index", false, "load pack indexes into memory")
-	flags.BoolVar(goGitMmap, "gogit-mmap", false, "memory-map read-only pack and index files")
+	flags.BoolVar(goGitMemoryIndex, "gogit-memory-index", true, "load pack indexes into memory")
+	flags.BoolVar(goGitMmap, "gogit-mmap", true, "memory-map read-only pack and index files where supported")
 	flags.Uint64Var(goGitCacheBytes, "gogit-cache-bytes", uint64(cache.DefaultMaxSize), "go-git object cache capacity in bytes")
 	flags.IntVar(goGitCacheShards, "gogit-cache-shards", 1, "go-git object cache shards")
 	flags.BoolVar(goGitObjectInfos, "gogit-object-infos", true, "enumerate go-git object metadata before loading blobs")
 	flags.IntVar(goGitObjectBuffer, "gogit-object-buffer", defaultGoGitObjectBuffer, "buffered go-git object metadata entries")
 	flags.IntVar(goGitObjectBatch, "gogit-object-batch", defaultGoGitObjectBatch, "adjacent go-git object metadata entries per reader task")
 	flags.BoolVar(goGitKlauspostZlib, "gogit-klauspost-zlib", false, "use klauspost zlib for go-git object decompression")
-	flags.IntVar(historyWorkers, "history-workers", 1, "go-git history workers")
+	flags.IntVar(historyWorkers, "history-workers", 0, "go-git history workers (0 = GOMAXPROCS, at most 8)")
 }
 
 func validateOptions(history bool) error {
@@ -455,11 +455,7 @@ func feederFor(name string) feeder {
 func feedBlobsCatFile(repo string, jobs chan<- job, skip func(string), limit func(string) int64) (int, error) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	shards := *readers
-	if shards <= 0 {
-		shards = runtime.GOMAXPROCS(0)
-	}
-	shards = max(1, shards)
+	shards := blobReaders()
 	blobs := make(chan listedBlob, jobQueueSize)
 	var wg sync.WaitGroup
 	errs := make(chan error, shards)
@@ -516,9 +512,16 @@ func feedBlobsCatFile(repo string, jobs chan<- job, skip func(string), limit fun
 	return total, err
 }
 
+func blobReaders() int {
+	if *readers > 0 {
+		return *readers
+	}
+	return runtime.GOMAXPROCS(0)
+}
+
 func feedBlobsGoGit(repo string, jobs chan<- job, skip func(string), limit func(string) int64) (int, error) {
-	if *readers > 1 {
-		return feedBlobsGoGitParallel(repo, jobs, skip, limit, *readers)
+	if workers := blobReaders(); workers > 1 {
+		return feedBlobsGoGitParallel(repo, jobs, skip, limit, workers)
 	}
 	return feedBlobsGoGitSerial(repo, jobs, skip, limit)
 }