From 7e02613b26636eb54781aef2c8e61f5fdc0c3089 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 9 Nov 2026 21:50:55 +0000 Subject: [PATCH 1/3] Parallelize the Git-free backend by default Load pack indexes into memory and memory-map pack files by default, and size blob readers (GOMAXPROCS) and history workers (GOMAXPROCS, at most 8) from the machine instead of running one of each. Without the in-memory index, extra workers contend on index reads and get slower, which is why the old single-worker defaults were safe but left most cores idle. On an 18-core M5 Max scanning cargo's history, the gogit backend drops from 6.31 s to 1.50 s with identical output. Co-Authored-By: Claude Opus 5.5 --- README.md | 6 ++++-- gogit_history_parallel.go | 17 ++++++++++++++++- gogit_history_stream.go | 9 +++++---- gogit_tuning.go | 4 ++-- main.go | 27 ++++++++++++++++----------- 5 files changed, 43 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index eb95e20..6a77883 100644 --- a/README.md +++ b/README.md @@ -131,8 +131,10 @@ git spdx log --backend=gogit --details git spdx scan --backend=gogit ``` -Run either command with `--help` to see cache, mmap, object reader, and history -worker controls for the Git-free backend. +By default it loads pack indexes into memory, memory-maps pack files where the +platform supports it, and runs `GOMAXPROCS` blob readers and up to eight +history workers. Run either command with `--help` to see cache, mmap, object +reader, and history worker controls for the Git-free backend. ## Performance diff --git a/gogit_history_parallel.go b/gogit_history_parallel.go index 9c2e4bb..a3aaaab 100644 --- a/gogit_history_parallel.go +++ b/gogit_history_parallel.go @@ -1,9 +1,24 @@ package main -var historyWorkers = newOption(1) +import "runtime" + +var historyWorkers = newOption(0) const historyLookahead = 2 +// maxAutoHistoryWorkers caps --history-workers=0. On cargo's history the walk +// stopped getting faster beyond 4-8 workers and slowed slightly at 18. +const maxAutoHistoryWorkers = 8 + +// resolvedHistoryWorkers returns the go-git history worker count, choosing one +// from GOMAXPROCS when --history-workers is 0. +func resolvedHistoryWorkers() int { + if *historyWorkers > 0 { + return *historyWorkers + } + return min(runtime.GOMAXPROCS(0), maxAutoHistoryWorkers) +} + type historyResult struct { changes []change err error diff --git a/gogit_history_stream.go b/gogit_history_stream.go index 11604ad..c988669 100644 --- a/gogit_history_stream.go +++ b/gogit_history_stream.go @@ -180,8 +180,8 @@ func walkChangesGoGitStream(r *gogit.Repository, merges bool, visit func(change) return err } cache := newHistoryTreeCache() - if *historyWorkers > 1 { - return visitHistoryStreamParallel(iter, cache, merges, visit, *historyWorkers) + if workers := resolvedHistoryWorkers(); workers > 1 { + return visitHistoryStreamParallel(iter, cache, merges, visit, workers) } for { node, err := iter.Next() @@ -398,13 +398,14 @@ func legalBlobsGoGit(path string) (map[string]bool, error) { boundary[hash] = true } collector := newLegalTreeCollector() - if *historyWorkers <= 1 { + workers := resolvedHistoryWorkers() + if workers <= 1 { err = visitCommitTrees(r, roots, boundary, func(hash plumbing.Hash) error { return collectLegalTreeRaw(r, hash, false, collector) }) return collector.legal, err } - err = collectLegalTreesParallel(r, roots, boundary, collector, *historyWorkers-1) + err = collectLegalTreesParallel(r, roots, boundary, collector, workers-1) return collector.legal, err } diff --git a/gogit_tuning.go b/gogit_tuning.go index f5f4474..fc24024 100644 --- a/gogit_tuning.go +++ b/gogit_tuning.go @@ -21,8 +21,8 @@ const ( ) var ( - goGitMemoryIndex = newOption(false) - goGitMmap = newOption(false) + goGitMemoryIndex = newOption(true) + goGitMmap = newOption(true) goGitCacheBytes = newOption(uint64(cache.DefaultMaxSize)) goGitCacheShards = newOption(1) goGitObjectInfos = newOption(true) diff --git a/main.go b/main.go index 3cd5ada..102167a 100644 --- a/main.go +++ b/main.go @@ -134,17 +134,17 @@ func addSharedFlags(cmd *cobra.Command) { flags := cmd.Flags() flags.StringVar(backend, "backend", "git", "backend: git (subprocesses) or gogit (Git-free)") flags.Int64Var(maxBlobSize, "max-blob-size", defaultMaxBlobSize, "skip blobs larger than this many bytes") - flags.IntVar(readers, "readers", 0, "blob readers (0 = GOMAXPROCS for git, 1 for gogit)") + flags.IntVar(readers, "readers", 0, "blob readers (0 = GOMAXPROCS)") flags.Int64Var(legalBlobSize, "max-legal-blob-size", defaultLegalBlobSize, "size limit for blobs used at legal paths") - flags.BoolVar(goGitMemoryIndex, "gogit-memory-index", false, "load pack indexes into memory") - flags.BoolVar(goGitMmap, "gogit-mmap", false, "memory-map read-only pack and index files") + flags.BoolVar(goGitMemoryIndex, "gogit-memory-index", true, "load pack indexes into memory") + flags.BoolVar(goGitMmap, "gogit-mmap", true, "memory-map read-only pack and index files where supported") flags.Uint64Var(goGitCacheBytes, "gogit-cache-bytes", uint64(cache.DefaultMaxSize), "go-git object cache capacity in bytes") flags.IntVar(goGitCacheShards, "gogit-cache-shards", 1, "go-git object cache shards") flags.BoolVar(goGitObjectInfos, "gogit-object-infos", true, "enumerate go-git object metadata before loading blobs") flags.IntVar(goGitObjectBuffer, "gogit-object-buffer", defaultGoGitObjectBuffer, "buffered go-git object metadata entries") flags.IntVar(goGitObjectBatch, "gogit-object-batch", defaultGoGitObjectBatch, "adjacent go-git object metadata entries per reader task") flags.BoolVar(goGitKlauspostZlib, "gogit-klauspost-zlib", false, "use klauspost zlib for go-git object decompression") - flags.IntVar(historyWorkers, "history-workers", 1, "go-git history workers") + flags.IntVar(historyWorkers, "history-workers", 0, "go-git history workers (0 = GOMAXPROCS, at most 8)") } func validateOptions(history bool) error { @@ -455,11 +455,7 @@ func feederFor(name string) feeder { func feedBlobsCatFile(repo string, jobs chan<- job, skip func(string), limit func(string) int64) (int, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - shards := *readers - if shards <= 0 { - shards = runtime.GOMAXPROCS(0) - } - shards = max(1, shards) + shards := blobReaders() blobs := make(chan listedBlob, jobQueueSize) var wg sync.WaitGroup errs := make(chan error, shards) @@ -516,9 +512,18 @@ func feedBlobsCatFile(repo string, jobs chan<- job, skip func(string), limit fun return total, err } +// blobReaders returns the blob reader count for either backend, using +// GOMAXPROCS when --readers is 0. +func blobReaders() int { + if *readers > 0 { + return *readers + } + return runtime.GOMAXPROCS(0) +} + func feedBlobsGoGit(repo string, jobs chan<- job, skip func(string), limit func(string) int64) (int, error) { - if *readers > 1 { - return feedBlobsGoGitParallel(repo, jobs, skip, limit, *readers) + if workers := blobReaders(); workers > 1 { + return feedBlobsGoGitParallel(repo, jobs, skip, limit, workers) } return feedBlobsGoGitSerial(repo, jobs, skip, limit) } From a01367f0869d4af335bf1e559825c229ac905f0e Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 9 Nov 2026 22:16:32 +0000 Subject: [PATCH 2/3] Update README benchmarks for the new gogit defaults Re-measured on an 18-core M5 Max: five-run averages of scan with default flags, with peak RSS summed across the process tree. Co-Authored-By: Claude Opus 5.5 --- README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 6a77883..2642d17 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ removed, and when `SPDX-License-Identifier` declarations appear or disappear, with the commit and path behind each change. It scans large histories in seconds. That speed makes it practical to trace license evidence through every revision, even in a large repository. -On an 8-core M1 Pro, the default native Git backend scanned 1.3 GB of blobs -across Cargo's 23,190 commits in an average of 4.05 seconds. +On an 18-core M5 Max, the default native Git backend scanned 1.42 GB of blobs +across Cargo's 23,683 commits in an average of 1.61 seconds. The `licenses` command reports what the current checkout contains. `git-spdx` adds the timeline, so it can answer questions such as: @@ -138,14 +138,18 @@ reader, and history worker controls for the Git-free backend. ## Performance -On an 8-core M1 Pro with 16 GB of memory, a full clone of `rust-lang/cargo` at -`a07c49a` contained 23,190 commits and 1.3 GB of historical blobs. Two-run -averages were: +On an 18-core M5 Max (6 performance, 12 efficiency) with 64 GB of memory, a +clone of `rust-lang/cargo` with history up to `a07c49a` contained 23,683 +commits and 1.42 GB of historical blobs. Five-run averages of `scan` with +default flags were: | Backend | Wall time | CPU time | Peak process-tree RSS | | --- | ---: | ---: | ---: | -| Native Git | 4.05 s | 20.03 s | 2.12 GiB | -| Git-free | 3.57 s | 20.14 s | 614 MiB | +| Native Git | 1.61 s | 14.50 s | 3.81 GiB | +| Git-free | 1.49 s | 13.29 s | 896 MiB | + +The native backend starts one `git cat-file` process per reader, so its memory +grows with core count. Pass `--readers` to limit it. Both backends produced the same normalized output. The benchmark includes all stored history rather than only the files in the checked-out tree. From 2ccd40610c45fe697e41d6f8fd28f008367c4c5b Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Tue, 10 Nov 2026 14:59:08 +0000 Subject: [PATCH 3/3] Test every gogit index mode and the auto worker counts With memory index and mmap on by default, the bare flags no longer selected distinct configurations. The CLI helper also overrode any --history-workers a test passed. --- gogit_history_parallel.go | 2 -- gogit_history_parallel_test.go | 29 ++++++++++++++++++++++++++++- gogit_history_test.go | 5 ++++- gogit_tuning_test.go | 22 +++++++++++++++------- main.go | 2 -- 5 files changed, 47 insertions(+), 13 deletions(-) diff --git a/gogit_history_parallel.go b/gogit_history_parallel.go index a3aaaab..bb753c1 100644 --- a/gogit_history_parallel.go +++ b/gogit_history_parallel.go @@ -10,8 +10,6 @@ const historyLookahead = 2 // stopped getting faster beyond 4-8 workers and slowed slightly at 18. const maxAutoHistoryWorkers = 8 -// resolvedHistoryWorkers returns the go-git history worker count, choosing one -// from GOMAXPROCS when --history-workers is 0. func resolvedHistoryWorkers() int { if *historyWorkers > 0 { return *historyWorkers diff --git a/gogit_history_parallel_test.go b/gogit_history_parallel_test.go index 0499327..7ac57ce 100644 --- a/gogit_history_parallel_test.go +++ b/gogit_history_parallel_test.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" "time" @@ -30,7 +31,7 @@ func TestGitFreeParallelHistory(t *testing.T) { commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: "+license+"\n", fmt.Sprintf("Change %d", n)) } git(t, repo, "gc", "--quiet") - for _, workers := range []string{"2", "4"} { + for _, workers := range []string{"0", "1", "2", "4"} { args := []string{"--history-workers=" + workers, testMmapFlag, "--readers=4"} for _, mode := range []string{"--details", "--monthly"} { commandArgs := append([]string{"log", repo}, args...) @@ -45,6 +46,32 @@ func TestGitFreeParallelHistory(t *testing.T) { } } +func TestAutoWorkerCounts(t *testing.T) { + oldProcs := runtime.GOMAXPROCS(0) + oldWorkers, oldReaders := *historyWorkers, *readers + t.Cleanup(func() { + runtime.GOMAXPROCS(oldProcs) + *historyWorkers, *readers = oldWorkers, oldReaders + }) + for _, test := range []struct { + procs, flag, wantHistory, wantReaders int + }{ + {procs: 2, flag: 0, wantHistory: 2, wantReaders: 2}, + {procs: 16, flag: 0, wantHistory: maxAutoHistoryWorkers, wantReaders: 16}, + {procs: 2, flag: 12, wantHistory: 12, wantReaders: 12}, + {procs: 16, flag: 1, wantHistory: 1, wantReaders: 1}, + } { + runtime.GOMAXPROCS(test.procs) + *historyWorkers, *readers = test.flag, test.flag + if got := resolvedHistoryWorkers(); got != test.wantHistory { + t.Errorf("GOMAXPROCS=%d --history-workers=%d: got %d, want %d", test.procs, test.flag, got, test.wantHistory) + } + if got := blobReaders(); got != test.wantReaders { + t.Errorf("GOMAXPROCS=%d --readers=%d: got %d, want %d", test.procs, test.flag, got, test.wantReaders) + } + } +} + func TestGitFreeParallelHistoryMissingTree(t *testing.T) { repo := repository(t) for n := range 12 { diff --git a/gogit_history_test.go b/gogit_history_test.go index b0a1a77..7f9e7ff 100644 --- a/gogit_history_test.go +++ b/gogit_history_test.go @@ -16,7 +16,10 @@ import ( func cliWithoutGit(t *testing.T, args ...string) string { t.Helper() commandArgs := append([]string{}, args...) - commandArgs = append(commandArgs, "--backend=gogit", "--history-workers=4") + commandArgs = append(commandArgs, "--backend=gogit") + if !slices.ContainsFunc(args, func(arg string) bool { return strings.HasPrefix(arg, "--history-workers") }) { + commandArgs = append(commandArgs, "--history-workers=4") + } cmd := exec.Command(os.Args[0], commandArgs...) cmd.Env = append(os.Environ(), "GIT_SPDX_TEST_CLI=1", "GOMAXPROCS=2", "PATH="+t.TempDir()) out, err := cmd.CombinedOutput() diff --git a/gogit_tuning_test.go b/gogit_tuning_test.go index 2d41b10..e29cbc2 100644 --- a/gogit_tuning_test.go +++ b/gogit_tuning_test.go @@ -16,6 +16,13 @@ var ( benchmarkZlibErr error ) +var goGitIndexModes = [][]string{ + {"--gogit-memory-index=true", "--gogit-mmap=true"}, + {"--gogit-memory-index=true", "--gogit-mmap=false"}, + {"--gogit-memory-index=false", "--gogit-mmap=true"}, + {"--gogit-memory-index=false", "--gogit-mmap=false"}, +} + func TestReadGoGitBlobReusesMemoryObjectContent(t *testing.T) { obj := plumbing.NewMemoryObject(nil) obj.SetType(plumbing.BlobObject) @@ -66,10 +73,10 @@ func TestGitFreeTunedPackedScan(t *testing.T) { if metric(t, want, "blobs seen") != 301 { t.Fatal(want) } - for _, mode := range []string{"--gogit-memory-index", "--gogit-mmap"} { - for _, readers := range []string{"1", "2", "4"} { - args := []string{"scan", repo, "--readers=" + readers, mode, "--gogit-cache-bytes=536870912", "--gogit-cache-shards=8", "--gogit-object-buffer=64", "--gogit-object-batch=64"} - got := cliWithoutGit(t, args...) + for _, mode := range goGitIndexModes { + for _, readers := range []string{"0", "1", "2", "4"} { + args := []string{"scan", repo, "--readers=" + readers, "--gogit-cache-bytes=536870912", "--gogit-cache-shards=8", "--gogit-object-buffer=64", "--gogit-object-batch=64"} + got := cliWithoutGit(t, append(args, mode...)...) assertScanMetrics(t, got, want) } } @@ -101,8 +108,9 @@ func TestGitFreeTunedHistoryLayouts(t *testing.T) { shallow := filepath.Join(t.TempDir(), "shallow") git(t, repo, "clone", "--depth=1", "file://"+repo, shallow) for _, path := range []string{repo, bare, worktree, shallow} { - for _, mode := range []string{"--gogit-memory-index", "--gogit-mmap"} { - got := cliWithoutGit(t, "log", path, "--readers=4", mode, "--gogit-cache-bytes=536870912", "--details") + for _, mode := range goGitIndexModes { + args := []string{"log", path, "--readers=4", "--gogit-cache-bytes=536870912", "--details"} + got := cliWithoutGit(t, append(args, mode...)...) if want := cli(t, "log", path, "--details"); got != want { t.Fatalf("history differs for %s:\n%s\n%s", path, got, want) } @@ -123,7 +131,7 @@ func BenchmarkGoGitTuning(b *testing.B) { cache uint64 mmap bool }{ - {"default", false, 96 << 20, false}, + {"lazy", false, 96 << 20, false}, {"memory", true, 96 << 20, false}, {"mmap", false, 96 << 20, true}, } { diff --git a/main.go b/main.go index 102167a..073c340 100644 --- a/main.go +++ b/main.go @@ -512,8 +512,6 @@ func feedBlobsCatFile(repo string, jobs chan<- job, skip func(string), limit fun return total, err } -// blobReaders returns the blob reader count for either backend, using -// GOMAXPROCS when --readers is 0. func blobReaders() int { if *readers > 0 { return *readers