diff --git a/README.md b/README.md index eb95e20..2642d17 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ removed, and when `SPDX-License-Identifier` declarations appear or disappear, with the commit and path behind each change. It scans large histories in seconds. That speed makes it practical to trace license evidence through every revision, even in a large repository. -On an 8-core M1 Pro, the default native Git backend scanned 1.3 GB of blobs -across Cargo's 23,190 commits in an average of 4.05 seconds. +On an 18-core M5 Max, the default native Git backend scanned 1.42 GB of blobs +across Cargo's 23,683 commits in an average of 1.61 seconds. The `licenses` command reports what the current checkout contains. `git-spdx` adds the timeline, so it can answer questions such as: @@ -131,19 +131,25 @@ git spdx log --backend=gogit --details git spdx scan --backend=gogit ``` -Run either command with `--help` to see cache, mmap, object reader, and history -worker controls for the Git-free backend. +By default it loads pack indexes into memory, memory-maps pack files where the +platform supports it, and runs `GOMAXPROCS` blob readers and up to eight +history workers. Run either command with `--help` to see cache, mmap, object +reader, and history worker controls for the Git-free backend. ## Performance -On an 8-core M1 Pro with 16 GB of memory, a full clone of `rust-lang/cargo` at -`a07c49a` contained 23,190 commits and 1.3 GB of historical blobs. Two-run -averages were: +On an 18-core M5 Max (6 performance, 12 efficiency) with 64 GB of memory, a +clone of `rust-lang/cargo` with history up to `a07c49a` contained 23,683 +commits and 1.42 GB of historical blobs. Five-run averages of `scan` with +default flags were: | Backend | Wall time | CPU time | Peak process-tree RSS | | --- | ---: | ---: | ---: | -| Native Git | 4.05 s | 20.03 s | 2.12 GiB | -| Git-free | 3.57 s | 20.14 s | 614 MiB | +| Native Git | 1.61 s | 14.50 s | 3.81 GiB | +| Git-free | 1.49 s | 13.29 s | 896 MiB | + +The native backend starts one `git cat-file` process per reader, so its memory +grows with core count. Pass `--readers` to limit it. Both backends produced the same normalized output. The benchmark includes all stored history rather than only the files in the checked-out tree. diff --git a/gogit_history_parallel.go b/gogit_history_parallel.go index 9c2e4bb..bb753c1 100644 --- a/gogit_history_parallel.go +++ b/gogit_history_parallel.go @@ -1,9 +1,22 @@ package main -var historyWorkers = newOption(1) +import "runtime" + +var historyWorkers = newOption(0) const historyLookahead = 2 +// maxAutoHistoryWorkers caps --history-workers=0. On cargo's history the walk +// stopped getting faster beyond 4-8 workers and slowed slightly at 18. +const maxAutoHistoryWorkers = 8 + +func resolvedHistoryWorkers() int { + if *historyWorkers > 0 { + return *historyWorkers + } + return min(runtime.GOMAXPROCS(0), maxAutoHistoryWorkers) +} + type historyResult struct { changes []change err error diff --git a/gogit_history_parallel_test.go b/gogit_history_parallel_test.go index 0499327..7ac57ce 100644 --- a/gogit_history_parallel_test.go +++ b/gogit_history_parallel_test.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" "time" @@ -30,7 +31,7 @@ func TestGitFreeParallelHistory(t *testing.T) { commitFile(t, repo, "LICENSE", "SPDX-License-Identifier: "+license+"\n", fmt.Sprintf("Change %d", n)) } git(t, repo, "gc", "--quiet") - for _, workers := range []string{"2", "4"} { + for _, workers := range []string{"0", "1", "2", "4"} { args := []string{"--history-workers=" + workers, testMmapFlag, "--readers=4"} for _, mode := range []string{"--details", "--monthly"} { commandArgs := append([]string{"log", repo}, args...) @@ -45,6 +46,32 @@ func TestGitFreeParallelHistory(t *testing.T) { } } +func TestAutoWorkerCounts(t *testing.T) { + oldProcs := runtime.GOMAXPROCS(0) + oldWorkers, oldReaders := *historyWorkers, *readers + t.Cleanup(func() { + runtime.GOMAXPROCS(oldProcs) + *historyWorkers, *readers = oldWorkers, oldReaders + }) + for _, test := range []struct { + procs, flag, wantHistory, wantReaders int + }{ + {procs: 2, flag: 0, wantHistory: 2, wantReaders: 2}, + {procs: 16, flag: 0, wantHistory: maxAutoHistoryWorkers, wantReaders: 16}, + {procs: 2, flag: 12, wantHistory: 12, wantReaders: 12}, + {procs: 16, flag: 1, wantHistory: 1, wantReaders: 1}, + } { + runtime.GOMAXPROCS(test.procs) + *historyWorkers, *readers = test.flag, test.flag + if got := resolvedHistoryWorkers(); got != test.wantHistory { + t.Errorf("GOMAXPROCS=%d --history-workers=%d: got %d, want %d", test.procs, test.flag, got, test.wantHistory) + } + if got := blobReaders(); got != test.wantReaders { + t.Errorf("GOMAXPROCS=%d --readers=%d: got %d, want %d", test.procs, test.flag, got, test.wantReaders) + } + } +} + func TestGitFreeParallelHistoryMissingTree(t *testing.T) { repo := repository(t) for n := range 12 { diff --git a/gogit_history_stream.go b/gogit_history_stream.go index 11604ad..c988669 100644 --- a/gogit_history_stream.go +++ b/gogit_history_stream.go @@ -180,8 +180,8 @@ func walkChangesGoGitStream(r *gogit.Repository, merges bool, visit func(change) return err } cache := newHistoryTreeCache() - if *historyWorkers > 1 { - return visitHistoryStreamParallel(iter, cache, merges, visit, *historyWorkers) + if workers := resolvedHistoryWorkers(); workers > 1 { + return visitHistoryStreamParallel(iter, cache, merges, visit, workers) } for { node, err := iter.Next() @@ -398,13 +398,14 @@ func legalBlobsGoGit(path string) (map[string]bool, error) { boundary[hash] = true } collector := newLegalTreeCollector() - if *historyWorkers <= 1 { + workers := resolvedHistoryWorkers() + if workers <= 1 { err = visitCommitTrees(r, roots, boundary, func(hash plumbing.Hash) error { return collectLegalTreeRaw(r, hash, false, collector) }) return collector.legal, err } - err = collectLegalTreesParallel(r, roots, boundary, collector, *historyWorkers-1) + err = collectLegalTreesParallel(r, roots, boundary, collector, workers-1) return collector.legal, err } diff --git a/gogit_history_test.go b/gogit_history_test.go index b0a1a77..7f9e7ff 100644 --- a/gogit_history_test.go +++ b/gogit_history_test.go @@ -16,7 +16,10 @@ import ( func cliWithoutGit(t *testing.T, args ...string) string { t.Helper() commandArgs := append([]string{}, args...) - commandArgs = append(commandArgs, "--backend=gogit", "--history-workers=4") + commandArgs = append(commandArgs, "--backend=gogit") + if !slices.ContainsFunc(args, func(arg string) bool { return strings.HasPrefix(arg, "--history-workers") }) { + commandArgs = append(commandArgs, "--history-workers=4") + } cmd := exec.Command(os.Args[0], commandArgs...) cmd.Env = append(os.Environ(), "GIT_SPDX_TEST_CLI=1", "GOMAXPROCS=2", "PATH="+t.TempDir()) out, err := cmd.CombinedOutput() diff --git a/gogit_tuning.go b/gogit_tuning.go index f5f4474..fc24024 100644 --- a/gogit_tuning.go +++ b/gogit_tuning.go @@ -21,8 +21,8 @@ const ( ) var ( - goGitMemoryIndex = newOption(false) - goGitMmap = newOption(false) + goGitMemoryIndex = newOption(true) + goGitMmap = newOption(true) goGitCacheBytes = newOption(uint64(cache.DefaultMaxSize)) goGitCacheShards = newOption(1) goGitObjectInfos = newOption(true) diff --git a/gogit_tuning_test.go b/gogit_tuning_test.go index 2d41b10..e29cbc2 100644 --- a/gogit_tuning_test.go +++ b/gogit_tuning_test.go @@ -16,6 +16,13 @@ var ( benchmarkZlibErr error ) +var goGitIndexModes = [][]string{ + {"--gogit-memory-index=true", "--gogit-mmap=true"}, + {"--gogit-memory-index=true", "--gogit-mmap=false"}, + {"--gogit-memory-index=false", "--gogit-mmap=true"}, + {"--gogit-memory-index=false", "--gogit-mmap=false"}, +} + func TestReadGoGitBlobReusesMemoryObjectContent(t *testing.T) { obj := plumbing.NewMemoryObject(nil) obj.SetType(plumbing.BlobObject) @@ -66,10 +73,10 @@ func TestGitFreeTunedPackedScan(t *testing.T) { if metric(t, want, "blobs seen") != 301 { t.Fatal(want) } - for _, mode := range []string{"--gogit-memory-index", "--gogit-mmap"} { - for _, readers := range []string{"1", "2", "4"} { - args := []string{"scan", repo, "--readers=" + readers, mode, "--gogit-cache-bytes=536870912", "--gogit-cache-shards=8", "--gogit-object-buffer=64", "--gogit-object-batch=64"} - got := cliWithoutGit(t, args...) + for _, mode := range goGitIndexModes { + for _, readers := range []string{"0", "1", "2", "4"} { + args := []string{"scan", repo, "--readers=" + readers, "--gogit-cache-bytes=536870912", "--gogit-cache-shards=8", "--gogit-object-buffer=64", "--gogit-object-batch=64"} + got := cliWithoutGit(t, append(args, mode...)...) assertScanMetrics(t, got, want) } } @@ -101,8 +108,9 @@ func TestGitFreeTunedHistoryLayouts(t *testing.T) { shallow := filepath.Join(t.TempDir(), "shallow") git(t, repo, "clone", "--depth=1", "file://"+repo, shallow) for _, path := range []string{repo, bare, worktree, shallow} { - for _, mode := range []string{"--gogit-memory-index", "--gogit-mmap"} { - got := cliWithoutGit(t, "log", path, "--readers=4", mode, "--gogit-cache-bytes=536870912", "--details") + for _, mode := range goGitIndexModes { + args := []string{"log", path, "--readers=4", "--gogit-cache-bytes=536870912", "--details"} + got := cliWithoutGit(t, append(args, mode...)...) if want := cli(t, "log", path, "--details"); got != want { t.Fatalf("history differs for %s:\n%s\n%s", path, got, want) } @@ -123,7 +131,7 @@ func BenchmarkGoGitTuning(b *testing.B) { cache uint64 mmap bool }{ - {"default", false, 96 << 20, false}, + {"lazy", false, 96 << 20, false}, {"memory", true, 96 << 20, false}, {"mmap", false, 96 << 20, true}, } { diff --git a/main.go b/main.go index 3cd5ada..073c340 100644 --- a/main.go +++ b/main.go @@ -134,17 +134,17 @@ func addSharedFlags(cmd *cobra.Command) { flags := cmd.Flags() flags.StringVar(backend, "backend", "git", "backend: git (subprocesses) or gogit (Git-free)") flags.Int64Var(maxBlobSize, "max-blob-size", defaultMaxBlobSize, "skip blobs larger than this many bytes") - flags.IntVar(readers, "readers", 0, "blob readers (0 = GOMAXPROCS for git, 1 for gogit)") + flags.IntVar(readers, "readers", 0, "blob readers (0 = GOMAXPROCS)") flags.Int64Var(legalBlobSize, "max-legal-blob-size", defaultLegalBlobSize, "size limit for blobs used at legal paths") - flags.BoolVar(goGitMemoryIndex, "gogit-memory-index", false, "load pack indexes into memory") - flags.BoolVar(goGitMmap, "gogit-mmap", false, "memory-map read-only pack and index files") + flags.BoolVar(goGitMemoryIndex, "gogit-memory-index", true, "load pack indexes into memory") + flags.BoolVar(goGitMmap, "gogit-mmap", true, "memory-map read-only pack and index files where supported") flags.Uint64Var(goGitCacheBytes, "gogit-cache-bytes", uint64(cache.DefaultMaxSize), "go-git object cache capacity in bytes") flags.IntVar(goGitCacheShards, "gogit-cache-shards", 1, "go-git object cache shards") flags.BoolVar(goGitObjectInfos, "gogit-object-infos", true, "enumerate go-git object metadata before loading blobs") flags.IntVar(goGitObjectBuffer, "gogit-object-buffer", defaultGoGitObjectBuffer, "buffered go-git object metadata entries") flags.IntVar(goGitObjectBatch, "gogit-object-batch", defaultGoGitObjectBatch, "adjacent go-git object metadata entries per reader task") flags.BoolVar(goGitKlauspostZlib, "gogit-klauspost-zlib", false, "use klauspost zlib for go-git object decompression") - flags.IntVar(historyWorkers, "history-workers", 1, "go-git history workers") + flags.IntVar(historyWorkers, "history-workers", 0, "go-git history workers (0 = GOMAXPROCS, at most 8)") } func validateOptions(history bool) error { @@ -455,11 +455,7 @@ func feederFor(name string) feeder { func feedBlobsCatFile(repo string, jobs chan<- job, skip func(string), limit func(string) int64) (int, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - shards := *readers - if shards <= 0 { - shards = runtime.GOMAXPROCS(0) - } - shards = max(1, shards) + shards := blobReaders() blobs := make(chan listedBlob, jobQueueSize) var wg sync.WaitGroup errs := make(chan error, shards) @@ -516,9 +512,16 @@ func feedBlobsCatFile(repo string, jobs chan<- job, skip func(string), limit fun return total, err } +func blobReaders() int { + if *readers > 0 { + return *readers + } + return runtime.GOMAXPROCS(0) +} + func feedBlobsGoGit(repo string, jobs chan<- job, skip func(string), limit func(string) int64) (int, error) { - if *readers > 1 { - return feedBlobsGoGitParallel(repo, jobs, skip, limit, *readers) + if workers := blobReaders(); workers > 1 { + return feedBlobsGoGitParallel(repo, jobs, skip, limit, workers) } return feedBlobsGoGitSerial(repo, jobs, skip, limit) }