From 51c6b8ac1cffbb0ef6068bd17b4db7e5e5b6d046 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Tue, 29 Sep 2026 09:55:15 -0300 Subject: [PATCH 1/5] mutate: the scheduler is the package's, and held by tests What runs a change's mutations - spread over its functions, built into one binary per package or alone, put to the tests that reach each line and to the importers', counted - was 330 lines of cmd/mutate, which has no tests, so the gate never broke its own concurrent half. It is RunAll and Spread in internal/mutate now; main keeps the flags, the plan and the reports. The lines it said as each outcome came go to the writer it is given, and an edit it could not build a way to ask is RunAll's error rather than an exit from inside a goroutine. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/mutate/main.go | 300 ++------------------------------- internal/mutate/runner.go | 297 ++++++++++++++++++++++++++++++++ internal/mutate/runner_test.go | 92 ++++++++++ 3 files changed, 400 insertions(+), 289 deletions(-) create mode 100644 internal/mutate/runner.go create mode 100644 internal/mutate/runner_test.go diff --git a/cmd/mutate/main.go b/cmd/mutate/main.go index b3b0582..0f5d1e5 100644 --- a/cmd/mutate/main.go +++ b/cmd/mutate/main.go @@ -11,12 +11,9 @@ import ( "fmt" "os" "os/exec" - "path/filepath" "runtime" "sort" - "strconv" "strings" - "sync" "text/tabwriter" "time" @@ -71,7 +68,7 @@ func main() { // Spread over the changed functions rather than taking the first of each file: a cap that // always lands in the same function asks one question many times. - chosen := spread(planned, *maxRuns) + chosen := mutate.Spread(planned, *maxRuns) for i := range chosen { chosen[i].ID = i + 1 } @@ -85,16 +82,20 @@ func main() { mutate.Share(*jobs) started := time.Now() - t := runAll(chosen, importers, *timeout, *jobs) + t, err := mutate.RunAll(chosen, importers, *timeout, *jobs, os.Stdout) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(2) + } reportCosts(time.Since(started)) - survived := t.survived + survived := t.Survived fmt.Printf("mutate: %d refused, %d survived, %d did not build, %d in packages with no tests\n", - t.killed, len(survived), t.unbuildable, t.untested) - if len(t.untestedPackages) > 0 { + t.Killed, len(survived), t.Unbuildable, t.Untested) + if len(t.UntestedPackages) > 0 { // Said rather than counted as a survivor: a package with no tests is a decision // somebody made, and this gate is about the tests that exist. - packages := make([]string, 0, len(t.untestedPackages)) - for p := range t.untestedPackages { + packages := make([]string, 0, len(t.UntestedPackages)) + for p := range t.UntestedPackages { packages = append(packages, p) } sort.Strings(packages) @@ -115,254 +116,6 @@ func main() { os.Exit(1) } -// tally is what the tests did about every mutation. -type tally struct { - survived []mutate.Mutation - killed, unbuildable, untested int - untestedPackages map[string]bool -} - -// result is what the tests did about one mutation. -type result struct { - m mutate.Mutation - outcome mutate.Outcome -} - -// runAll puts every mutation to the tests, jobs at a time, and says each outcome as it comes. -// Each package's mutations are built into one binary (mutate.Schemata), run once per mutation; -// an edit the schema could not build is built alone, through an overlay of its own. Each -// package's coverage is measured once, for both. -func runAll(chosen []mutate.Mutation, importers map[string][]string, timeout string, jobs int) tally { - work, err := os.MkdirTemp("", "mutate-") - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(2) - } - defer func() { _ = os.RemoveAll(work) }() - - // One budget of test processes for the whole run, which measuring coverage draws on too: a - // mutation's run and each package's coverage each had one of their own, and ten mutations - // measuring ten tests each were a hundred test processes and as many PostgreSQL containers. - slots := make(chan struct{}, max(jobs, 1)) // mutate-exempt: how many run at once, not what they say - // Builds are what cost: go parallelises one across the machine already, so they go one at a - // time, beside the test processes. - b := &builds{sem: make(chan struct{}, 1), work: work, bins: map[string]*importerBinary{}} - covers := &coverages{by: map[string]*coverage{}, timeout: timeout, slots: slots} - byDir := map[string][]mutate.Mutation{} - var dirs []string - for _, m := range chosen { - dir := filepath.Dir(m.File) - if byDir[dir] == nil { - dirs = append(dirs, dir) - } - byDir[dir] = append(byDir[dir], m) - } - results := make(chan result) - var wg sync.WaitGroup - for i, dir := range dirs { - wg.Add(1) - go func() { - defer wg.Done() - p := pkgRun{dir: dir, work: filepath.Join(work, strconv.Itoa(i)), timeout: timeout, slots: slots, - builds: b, importers: importers[filepath.ToSlash(dir)], results: results} - p.run(byDir[dir], covers) - }() - } - go func() { wg.Wait(); close(results) }() - - t := tally{untestedPackages: map[string]bool{}} - for r := range results { - m := r.m - switch r.outcome { - case mutate.Killed: - t.killed++ - fmt.Printf(" refused %s\n", m) - case mutate.Unbuildable: - t.unbuildable++ - fmt.Printf(" no edit %s (does not build)\n", m) - case mutate.Untested: - t.untested++ - t.untestedPackages[filepath.Dir(m.File)] = true - case mutate.Survived: - t.survived = append(t.survived, m) - fmt.Printf(" SURVIVED %s\n", m) - } - } - sort.Slice(t.survived, func(i, j int) bool { return t.survived[i].String() < t.survived[j].String() }) - return t -} - -// pkgRun is one package's mutations put to the tests. -type pkgRun struct { - dir, work, timeout string - slots chan struct{} - builds *builds - importers []string - results chan<- result -} - -// run builds the package's schema and puts each mutation in it to the binary, and each it could -// not build to the tests alone, all side by side within the slots. -func (p pkgRun) run(ms []mutate.Mutation, covers *coverages) { - p.builds.sem <- struct{}{} - s, err := mutate.Schemata(p.dir, ms, filepath.Join(p.work, "schema")) - <-p.builds.sem - if err != nil { - fmt.Fprintf(os.Stderr, " (%s: no schema, each edit is built alone: %v)\n", p.dir, err) - s = &mutate.Schema{Alone: ms} - } - fmt.Printf(" %s: %d edit(s) in one binary, %d built alone\n", p.dir, len(s.Built), len(s.Alone)) - // The package as it is says which tests reach a line: the schema moves none. - var c *mutate.Coverage - if len(s.Built) > 0 && s.Binary != "" { - c = covers.of(p.dir) - } - var wg sync.WaitGroup - for _, m := range s.Built { - wg.Add(1) - go func() { - defer wg.Done() - p.results <- result{m, p.built(m, s, c)} - }() - } - for i, m := range s.Alone { - wg.Add(1) - go func() { - defer wg.Done() - // Measured before a slot is taken: the measuring takes slots of its own, and holding - // one while waiting on the others is how a budget of one never finishes. - reaching := covers.of(p.dir) - p.slots <- struct{}{} - defer func() { <-p.slots }() - p.results <- result{m, run(m, filepath.Join(p.work, "alone-"+strconv.Itoa(i)), p.timeout, reaching, p.importers)} - }() - } - wg.Wait() -} - -// built puts m, which is in s's binary, to the package's tests that reach its line, then to each -// importer's tests, built once with the same schema: a survivor pays for the second. -func (p pkgRun) built(m mutate.Mutation, s *mutate.Schema, c *mutate.Coverage) mutate.Outcome { - if s.Binary == "" && len(p.importers) == 0 { - return mutate.Untested - } - ask := func(bin, dir string, only []string) mutate.Outcome { - p.slots <- struct{}{} - defer func() { <-p.slots }() - outcome, _ := mutate.Ask(bin, dir, m.ID, p.timeout, only) - return outcome - } - if s.Binary != "" { - var reaching []string - if c != nil { - reaching = c.Reaching(m.File, m.Line) - } - if (reaching == nil || len(reaching) > 0) && ask(s.Binary, p.dir, reaching) == mutate.Killed { - return mutate.Killed - } - } - for _, importer := range p.importers { - bin, dir := p.builds.importer(p.dir, importer, s.Overlay) - if bin != "" && ask(bin, dir, nil) == mutate.Killed { - return mutate.Killed - } - } - return mutate.Survived -} - -// builds is the run's one build at a time, and the importers' binaries it made, each once. -type builds struct { - sem chan struct{} - work string - mu sync.Mutex - bins map[string]*importerBinary -} - -type importerBinary struct { - once sync.Once - bin, dir string -} - -// importer is importer's test binary with the schema of the package in dir, and the directory it -// runs in; an empty binary where it has no tests or would not build, which asks it nothing. -func (b *builds) importer(dir, importer, overlay string) (string, string) { - key := dir + " " + importer - b.mu.Lock() - ib := b.bins[key] - if ib == nil { - ib = &importerBinary{} - b.bins[key] = ib - } - n := len(b.bins) - b.mu.Unlock() - ib.once.Do(func() { - b.sem <- struct{}{} - defer func() { <-b.sem }() - var err error - ib.bin, ib.dir, err = mutate.ImporterBinary(importer, overlay, filepath.Join(b.work, "importer-"+strconv.Itoa(n)+".test")) - if err != nil { - fmt.Fprintf(os.Stderr, " (%v)\n", err) - } - }) - return ib.bin, ib.dir -} - -// run builds the mutation in through an overlay under dir, and runs the package's tests that -// reach its line and then its importers'. The file itself is never written. -func run(m mutate.Mutation, dir, timeout string, c *mutate.Coverage, importers []string) mutate.Outcome { - if err := os.MkdirAll(dir, 0o700); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(2) - } - overlay, err := mutate.Overlay(m, dir) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(2) - } - asked := mutate.Asked{Overlay: overlay, Timeout: timeout, Importers: importers} - if c != nil { - asked.Reaching = c.Reaching(m.File, m.Line) - } - started := time.Now() - outcome, _ := mutate.Run(m, asked) - if took := time.Since(started); took > time.Minute { - fmt.Printf(" (%s took %s)\n", filepath.Dir(m.File), took.Round(time.Second)) - } - return outcome -} - -// coverages is each package's Coverage, measured once and shared by its mutations. -type coverages struct { - mu sync.Mutex - by map[string]*coverage - timeout string - slots chan struct{} -} - -type coverage struct { - once sync.Once - c *mutate.Coverage -} - -// of is dir's coverage, or nil where it could not be measured: every test is asked then. -func (cs *coverages) of(dir string) *mutate.Coverage { - cs.mu.Lock() - c, ok := cs.by[dir] - if !ok { - c = &coverage{} - cs.by[dir] = c - } - cs.mu.Unlock() - c.once.Do(func() { - measured, err := mutate.Cover(dir, cs.timeout, cs.slots) - if err != nil { - fmt.Fprintf(os.Stderr, " (%s: no coverage, every test is asked: %v)\n", dir, err) - } - c.c = measured - }) - return c.c -} - // reportCosts says where the run's time went, by what each process was for: the processes of // one purpose run side by side, so their wall is summed and can pass the run's own. func reportCosts(took time.Duration) { @@ -419,34 +172,3 @@ func reportStale() int { fmt.Printf("OK: every mutate-exempt reason in %d file(s) covers an edit this tool makes.\n", len(files)) return 0 } - -// spread takes mutations in rounds, one per function each time round, so a cap covers as many -// of the changed functions as it can rather than the first one exhaustively. -func spread(all []mutate.Mutation, max int) []mutate.Mutation { - byFunc := map[string][]mutate.Mutation{} - var order []string - for _, m := range all { - key := m.File + ":" + m.Func - if _, seen := byFunc[key]; !seen { - order = append(order, key) - } - byFunc[key] = append(byFunc[key], m) - } - var out []mutate.Mutation - for round := 0; len(out) < max; round++ { - took := false - for _, key := range order { - if round < len(byFunc[key]) { - out = append(out, byFunc[key][round]) - took = true - if len(out) == max { - return out - } - } - } - if !took { - break - } - } - return out -} diff --git a/internal/mutate/runner.go b/internal/mutate/runner.go new file mode 100644 index 0000000..8c17875 --- /dev/null +++ b/internal/mutate/runner.go @@ -0,0 +1,297 @@ +package mutate + +import ( + "cmp" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "sync" + "time" +) + +// Tally is what the tests did about every mutation. +type Tally struct { + Survived []Mutation + Killed, Unbuildable, Untested int + UntestedPackages map[string]bool +} + +// result is what the tests did about one mutation. +type result struct { + m Mutation + outcome Outcome + err error // the run could not ask the tests at all +} + +// RunAll puts every mutation to the tests, jobs at a time, and says each outcome as it comes. +// Each package's mutations are built into one binary (Schemata), run once per mutation; +// an edit the schema could not build is built alone, through an overlay of its own. Each +// package's coverage is measured once, for both. +func RunAll(chosen []Mutation, importers map[string][]string, timeout string, jobs int, w io.Writer) (Tally, error) { + work, err := os.MkdirTemp("", "mutate-") + if err != nil { + return Tally{}, err + } + defer func() { _ = os.RemoveAll(work) }() + + // One budget of test processes for the whole run, which measuring coverage draws on too: a + // mutation's run and each package's coverage each had one of their own, and ten mutations + // measuring ten tests each were a hundred test processes and as many PostgreSQL containers. + slots := make(chan struct{}, max(jobs, 1)) // mutate-exempt: how many run at once, not what they say + // Builds are what cost: go parallelises one across the machine already, so they go one at a + // time, beside the test processes. + b := &builds{sem: make(chan struct{}, 1), work: work, bins: map[string]*importerBinary{}} + covers := &coverages{by: map[string]*coverage{}, timeout: timeout, slots: slots} + byDir := map[string][]Mutation{} + var dirs []string + for _, m := range chosen { + dir := filepath.Dir(m.File) + if byDir[dir] == nil { + dirs = append(dirs, dir) + } + byDir[dir] = append(byDir[dir], m) + } + results := make(chan result) + var wg sync.WaitGroup + for i, dir := range dirs { + wg.Add(1) + go func() { + defer wg.Done() + p := pkgRun{dir: dir, work: filepath.Join(work, strconv.Itoa(i)), timeout: timeout, slots: slots, + builds: b, importers: importers[filepath.ToSlash(dir)], results: results, out: w} + p.run(byDir[dir], covers) + }() + } + go func() { wg.Wait(); close(results) }() + + t := Tally{UntestedPackages: map[string]bool{}} + var firstErr error + for r := range results { + m := r.m + if r.err != nil { + firstErr = cmp.Or(firstErr, r.err) + continue + } + switch r.outcome { + case Killed: + t.Killed++ + _, _ = fmt.Fprintf(w, " refused %s\n", m) + case Unbuildable: + t.Unbuildable++ + _, _ = fmt.Fprintf(w, " no edit %s (does not build)\n", m) + case Untested: + t.Untested++ + t.UntestedPackages[filepath.Dir(m.File)] = true + case Survived: + t.Survived = append(t.Survived, m) + _, _ = fmt.Fprintf(w, " SURVIVED %s\n", m) + } + } + sort.Slice(t.Survived, func(i, j int) bool { return t.Survived[i].String() < t.Survived[j].String() }) + return t, firstErr +} + +// pkgRun is one package's mutations put to the tests. +type pkgRun struct { + dir, work, timeout string + slots chan struct{} + builds *builds + importers []string + results chan<- result + out io.Writer +} + +// run builds the package's schema and puts each mutation in it to the binary, and each it could +// not build to the tests alone, all side by side within the slots. +func (p pkgRun) run(ms []Mutation, covers *coverages) { + p.builds.sem <- struct{}{} + s, err := Schemata(p.dir, ms, filepath.Join(p.work, "schema")) + <-p.builds.sem + if err != nil { + fmt.Fprintf(os.Stderr, " (%s: no schema, each edit is built alone: %v)\n", p.dir, err) + s = &Schema{Alone: ms} + } + _, _ = fmt.Fprintf(p.out, " %s: %d edit(s) in one binary, %d built alone\n", p.dir, len(s.Built), len(s.Alone)) + // The package as it is says which tests reach a line: the schema moves none. + var c *Coverage + if len(s.Built) > 0 && s.Binary != "" { + c = covers.of(p.dir) + } + var wg sync.WaitGroup + for _, m := range s.Built { + wg.Add(1) + go func() { + defer wg.Done() + p.results <- result{m: m, outcome: p.built(m, s, c)} + }() + } + for i, m := range s.Alone { + wg.Add(1) + go func() { + defer wg.Done() + // Measured before a slot is taken: the measuring takes slots of its own, and holding + // one while waiting on the others is how a budget of one never finishes. + reaching := covers.of(p.dir) + p.slots <- struct{}{} + defer func() { <-p.slots }() + outcome, err := runAlone(m, filepath.Join(p.work, "alone-"+strconv.Itoa(i)), p.timeout, reaching, p.importers, p.out) + p.results <- result{m: m, outcome: outcome, err: err} + }() + } + wg.Wait() +} + +// built puts m, which is in s's binary, to the package's tests that reach its line, then to each +// importer's tests, built once with the same schema: a survivor pays for the second. +func (p pkgRun) built(m Mutation, s *Schema, c *Coverage) Outcome { + if s.Binary == "" && len(p.importers) == 0 { + return Untested + } + ask := func(bin, dir string, only []string) Outcome { + p.slots <- struct{}{} + defer func() { <-p.slots }() + outcome, _ := Ask(bin, dir, m.ID, p.timeout, only) + return outcome + } + if s.Binary != "" { + var reaching []string + if c != nil { + reaching = c.Reaching(m.File, m.Line) + } + if (reaching == nil || len(reaching) > 0) && ask(s.Binary, p.dir, reaching) == Killed { + return Killed + } + } + for _, importer := range p.importers { + bin, dir := p.builds.importer(p.dir, importer, s.Overlay) + if bin != "" && ask(bin, dir, nil) == Killed { + return Killed + } + } + return Survived +} + +// builds is the run's one build at a time, and the importers' binaries it made, each once. +type builds struct { + sem chan struct{} + work string + mu sync.Mutex + bins map[string]*importerBinary +} + +type importerBinary struct { + once sync.Once + bin, dir string +} + +// importer is importer's test binary with the schema of the package in dir, and the directory it +// runs in; an empty binary where it has no tests or would not build, which asks it nothing. +func (b *builds) importer(dir, importer, overlay string) (string, string) { + key := dir + " " + importer + b.mu.Lock() + ib := b.bins[key] + if ib == nil { + ib = &importerBinary{} + b.bins[key] = ib + } + n := len(b.bins) + b.mu.Unlock() + ib.once.Do(func() { + b.sem <- struct{}{} + defer func() { <-b.sem }() + var err error + ib.bin, ib.dir, err = ImporterBinary(importer, overlay, filepath.Join(b.work, "importer-"+strconv.Itoa(n)+".test")) + if err != nil { + fmt.Fprintf(os.Stderr, " (%v)\n", err) + } + }) + return ib.bin, ib.dir +} + +// runAlone builds the mutation in through an overlay under dir, and runs the package's tests that +// reach its line and then its importers'. The file itself is never written. +func runAlone(m Mutation, dir, timeout string, c *Coverage, importers []string, out io.Writer) (Outcome, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return 0, err + } + overlay, err := Overlay(m, dir) + if err != nil { + return 0, err + } + asked := Asked{Overlay: overlay, Timeout: timeout, Importers: importers} + if c != nil { + asked.Reaching = c.Reaching(m.File, m.Line) + } + started := time.Now() + outcome, _ := Run(m, asked) + if took := time.Since(started); took > time.Minute { + _, _ = fmt.Fprintf(out, " (%s took %s)\n", filepath.Dir(m.File), took.Round(time.Second)) + } + return outcome, nil +} + +// coverages is each package's Coverage, measured once and shared by its mutations. +type coverages struct { + mu sync.Mutex + by map[string]*coverage + timeout string + slots chan struct{} +} + +type coverage struct { + once sync.Once + c *Coverage +} + +// of is dir's coverage, or nil where it could not be measured: every test is asked then. +func (cs *coverages) of(dir string) *Coverage { + cs.mu.Lock() + c, ok := cs.by[dir] + if !ok { + c = &coverage{} + cs.by[dir] = c + } + cs.mu.Unlock() + c.once.Do(func() { + measured, err := Cover(dir, cs.timeout, cs.slots) + if err != nil { + fmt.Fprintf(os.Stderr, " (%s: no coverage, every test is asked: %v)\n", dir, err) + } + c.c = measured + }) + return c.c +} + +// Spread takes mutations in rounds, one per function each time round, so a cap covers as many +// of the changed functions as it can rather than the first one exhaustively. +func Spread(all []Mutation, max int) []Mutation { + byFunc := map[string][]Mutation{} + var order []string + for _, m := range all { + key := m.File + ":" + m.Func + if _, seen := byFunc[key]; !seen { + order = append(order, key) + } + byFunc[key] = append(byFunc[key], m) + } + var out []Mutation + for round := 0; len(out) < max; round++ { + took := false + for _, key := range order { + if round < len(byFunc[key]) { + out = append(out, byFunc[key][round]) + took = true + if len(out) == max { + return out + } + } + } + if !took { + break + } + } + return out +} diff --git a/internal/mutate/runner_test.go b/internal/mutate/runner_test.go new file mode 100644 index 0000000..12301a8 --- /dev/null +++ b/internal/mutate/runner_test.go @@ -0,0 +1,92 @@ +package mutate_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/spin-stack/go-tools/internal/mutate" +) + +// A cap is spread over the changed functions, one edit of each a round, so it covers as many of +// them as it can rather than the first one exhaustively; under the cap, every edit is taken. +func TestACapIsSpreadOverTheChangedFunctions(t *testing.T) { + m := func(file, fn, what string) mutate.Mutation { return mutate.Mutation{File: file, Func: fn, What: what} } + all := []mutate.Mutation{ + m("a.go", "F", "1"), m("a.go", "F", "2"), m("a.go", "F", "3"), + m("a.go", "G", "1"), + m("b.go", "F", "1"), m("b.go", "F", "2"), + } + key := func(ms []mutate.Mutation) []string { + var out []string + for _, m := range ms { + out = append(out, m.File+":"+m.Func+":"+m.What) + } + return out + } + assert.Equal(t, []string{"a.go:F:1", "a.go:G:1", "b.go:F:1", "a.go:F:2"}, key(mutate.Spread(all, 4)), + "a function of the same name in another file is another function, and each is taken once a round") + assert.Equal(t, []string{"a.go:F:1", "a.go:G:1", "b.go:F:1", "a.go:F:2", "b.go:F:2", "a.go:F:3"}, key(mutate.Spread(all, 10))) + assert.Equal(t, []string{"a.go:F:1"}, key(mutate.Spread(all, 1))) + assert.Empty(t, mutate.Spread(all, 0)) + assert.Empty(t, mutate.Spread(nil, 3)) +} + +// Every edit is put to the tests and counted once, as the tests answered it, and each is said as +// it comes: Double's refused by TestDouble, Half's let through by a test that checks nothing, and +// a package with no tests named rather than counted as a survivor. +func TestEveryEditIsCountedAsTheTestsAnswered(t *testing.T) { + dir := refuses(t) + plan, err := mutate.Plan(filepath.Join(dir, "code.go"), map[int]bool{4: true, 8: true}) + require.NoError(t, err) + + bare := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(bare, "go.mod"), []byte("module example.com/bare\n\ngo 1.24\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(bare, "code.go"), []byte("package bare\n\nfunc Double(n int) int {\n\treturn n + n\n}\n"), 0o600)) + untested, err := mutate.Plan(filepath.Join(bare, "code.go"), map[int]bool{4: true}) + require.NoError(t, err) + require.NotEmpty(t, untested) + + chosen := append(plan, untested...) + for i := range chosen { + chosen[i].ID = i + 1 + } + var said strings.Builder + tally, err := mutate.RunAll(chosen, nil, "2m", 2, &said) + require.NoError(t, err) + + var double, half int + for _, m := range plan { + switch m.Func { + case "Double": + double++ + case "Half": + half++ + } + } + require.Positive(t, double) + require.Positive(t, half) + assert.Equal(t, double, tally.Killed, "Double's edits refused: %s", said.String()) + assert.Len(t, tally.Survived, half, "Half's edits let through: %s", said.String()) + for _, m := range tally.Survived { + assert.Equal(t, "Half", m.Func) + } + assert.Equal(t, len(untested), tally.Untested) + assert.Equal(t, map[string]bool{bare: true}, tally.UntestedPackages) + assert.Zero(t, tally.Unbuildable) + assert.Equal(t, double, strings.Count(said.String(), " refused ")) + assert.Equal(t, half, strings.Count(said.String(), " SURVIVED ")) + assert.Contains(t, said.String(), dir+": ") +} + +// An edit the run cannot even build a way to ask is an error, not an outcome: a file that is not +// there has no overlay. +func TestAnEditThatCannotBeAskedIsAnError(t *testing.T) { + gone := mutate.Mutation{ID: 1, File: filepath.Join(t.TempDir(), "gone.go"), Func: "F", What: "+ becomes -"} + _, err := mutate.RunAll([]mutate.Mutation{gone}, nil, "1m", 1, &strings.Builder{}) + assert.Error(t, err) +} From c0036dc03e6bf6898c4232d91aceb81b053fa4ca Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Tue, 29 Sep 2026 09:58:47 -0300 Subject: [PATCH 2/5] mutate: tests for the scheduler's importers, order and missing coverage Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/mutate/runner_internal_test.go | 40 +++++++++++++++++++ internal/mutate/runner_test.go | 51 +++++++++++++++++++++---- 2 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 internal/mutate/runner_internal_test.go diff --git a/internal/mutate/runner_internal_test.go b/internal/mutate/runner_internal_test.go new file mode 100644 index 0000000..18ee99e --- /dev/null +++ b/internal/mutate/runner_internal_test.go @@ -0,0 +1,40 @@ +package mutate + +import ( + "os" + "path/filepath" + "testing" +) + +// An edit in a package whose coverage could not be measured is asked of every test, not of none: +// no coverage is not knowing which tests reach the line, and a test that reaches nothing is only +// what a measurement says. +func TestAnEditWithNoCoverageIsAskedOfEveryTest(t *testing.T) { + dir := t.TempDir() + for name, body := range map[string]string{ + "go.mod": "module example.com/nocover\n\ngo 1.24\n", + "code.go": "package nocover\n\nfunc Double(n int) int {\n\treturn n + n\n}\n", + "code_test.go": "package nocover\n\nimport \"testing\"\n\nfunc TestDouble(t *testing.T) {\n\tif Double(2) != 4 {\n\t\tt.Fatal(\"not doubled\")\n\t}\n}\n", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + plan, err := Plan(filepath.Join(dir, "code.go"), map[int]bool{4: true}) + if err != nil || len(plan) == 0 { + t.Fatalf("plan: %v, %v", plan, err) + } + for i := range plan { + plan[i].ID = i + 1 + } + s, err := Schemata(dir, plan, t.TempDir()) + if err != nil || s.Binary == "" || len(s.Built) == 0 { + t.Fatalf("schema: %+v, %v", s, err) + } + p := pkgRun{dir: dir, timeout: "2m", slots: make(chan struct{}, 1)} + for _, m := range s.Built { + if got := p.built(m, s, nil); got != Killed { + t.Errorf("%s: %v with no coverage, want refused by the test nobody measured", m, got) + } + } +} diff --git a/internal/mutate/runner_test.go b/internal/mutate/runner_test.go index 12301a8..444921f 100644 --- a/internal/mutate/runner_test.go +++ b/internal/mutate/runner_test.go @@ -1,8 +1,10 @@ package mutate_test import ( + "fmt" "os" "path/filepath" + "slices" "strings" "testing" @@ -51,7 +53,7 @@ func TestEveryEditIsCountedAsTheTestsAnswered(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, untested) - chosen := append(plan, untested...) + chosen := slices.Concat(plan, untested) for i := range chosen { chosen[i].ID = i + 1 } @@ -75,18 +77,51 @@ func TestEveryEditIsCountedAsTheTestsAnswered(t *testing.T) { for _, m := range tally.Survived { assert.Equal(t, "Half", m.Func) } + assert.True(t, slices.IsSortedFunc(tally.Survived, func(a, b mutate.Mutation) int { return strings.Compare(a.String(), b.String()) }), + "the survivors are not in order: %v", tally.Survived) assert.Equal(t, len(untested), tally.Untested) assert.Equal(t, map[string]bool{bare: true}, tally.UntestedPackages) assert.Zero(t, tally.Unbuildable) assert.Equal(t, double, strings.Count(said.String(), " refused ")) assert.Equal(t, half, strings.Count(said.String(), " SURVIVED ")) - assert.Contains(t, said.String(), dir+": ") + assert.Contains(t, said.String(), fmt.Sprintf("%s: %d edit(s) in one binary, 0 built alone", dir, len(plan)), + "the package's edits were not built into its schema") + assert.NotContains(t, said.String(), " took ", "a run under a minute was said to be slow") } -// An edit the run cannot even build a way to ask is an error, not an outcome: a file that is not -// there has no overlay. -func TestAnEditThatCannotBeAskedIsAnError(t *testing.T) { - gone := mutate.Mutation{ID: 1, File: filepath.Join(t.TempDir(), "gone.go"), Func: "F", What: "+ becomes -"} - _, err := mutate.RunAll([]mutate.Mutation{gone}, nil, "1m", 1, &strings.Builder{}) - assert.Error(t, err) +// A package with no tests of its own is asked through the packages that import it, each built +// once: Double's edits are refused by the importer's test and Half's, which it never calls, are not. +func TestAPackageWithNoTestsIsAskedThroughItsImporters(t *testing.T) { + dir := t.TempDir() + for name, body := range map[string]string{ + "go.mod": "module example.com/imp\n\ngo 1.24\n", + "lib/lib.go": "package lib\n\nfunc Double(n int) int {\n\treturn n + n\n}\n\nfunc Half(n int) int {\n\treturn n / 2\n}\n", + "use/use.go": "package use\n\nimport \"example.com/imp/lib\"\n\nfunc Four() int { return lib.Double(2) }\n", + "use/use_test.go": "package use\n\nimport \"testing\"\n\nfunc TestFour(t *testing.T) {\n\tif Four() != 4 {\n\t\tt.Fatal(\"not four\")\n\t}\n}\n", + "also/also.go": "package also\n\nimport \"example.com/imp/lib\"\n\nvar _ = lib.Half\n", + } { + require.NoError(t, os.MkdirAll(filepath.Join(dir, filepath.Dir(name)), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600)) + } + t.Chdir(dir) + plan, err := mutate.Plan(filepath.Join("lib", "lib.go"), map[int]bool{4: true, 8: true}) + require.NoError(t, err) + var double, half int + for i := range plan { + plan[i].ID = i + 1 + if plan[i].Func == "Double" { + double++ + } else { + half++ + } + } + require.Positive(t, double) + require.Positive(t, half) + + var said strings.Builder + tally, err := mutate.RunAll(plan, map[string][]string{"lib": {"./use", "./also"}}, "2m", 2, &said) + require.NoError(t, err) + assert.Equal(t, double, tally.Killed, "Double's edits refused by the importer: %s", said.String()) + assert.Len(t, tally.Survived, half, "Half's edits, which no importer's test reaches: %s", said.String()) + assert.Zero(t, tally.Untested, "a package whose importers have tests was said untested") } From 10736670d9f6b0081601329815d81ef2f3e65944 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Tue, 29 Sep 2026 10:00:43 -0300 Subject: [PATCH 3/5] mutate: the survivors' order and the error path, held Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/mutate/runner_test.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/internal/mutate/runner_test.go b/internal/mutate/runner_test.go index 444921f..5fe4479 100644 --- a/internal/mutate/runner_test.go +++ b/internal/mutate/runner_test.go @@ -39,11 +39,11 @@ func TestACapIsSpreadOverTheChangedFunctions(t *testing.T) { } // Every edit is put to the tests and counted once, as the tests answered it, and each is said as -// it comes: Double's refused by TestDouble, Half's let through by a test that checks nothing, and +// it comes: Double's refused by TestDouble, Half's and Sign's let through by tests that check neither, and // a package with no tests named rather than counted as a survivor. func TestEveryEditIsCountedAsTheTestsAnswered(t *testing.T) { dir := refuses(t) - plan, err := mutate.Plan(filepath.Join(dir, "code.go"), map[int]bool{4: true, 8: true}) + plan, err := mutate.Plan(filepath.Join(dir, "code.go"), map[int]bool{4: true, 8: true, 13: true, 14: true}) require.NoError(t, err) bare := t.TempDir() @@ -61,21 +61,20 @@ func TestEveryEditIsCountedAsTheTestsAnswered(t *testing.T) { tally, err := mutate.RunAll(chosen, nil, "2m", 2, &said) require.NoError(t, err) - var double, half int + var double, unchecked int for _, m := range plan { - switch m.Func { - case "Double": + if m.Func == "Double" { double++ - case "Half": - half++ + } else { + unchecked++ } } require.Positive(t, double) - require.Positive(t, half) + require.GreaterOrEqual(t, unchecked, 2, "the order of the survivors needs two of them") assert.Equal(t, double, tally.Killed, "Double's edits refused: %s", said.String()) - assert.Len(t, tally.Survived, half, "Half's edits let through: %s", said.String()) + assert.Len(t, tally.Survived, unchecked, "Half's and Sign's edits let through: %s", said.String()) for _, m := range tally.Survived { - assert.Equal(t, "Half", m.Func) + assert.NotEqual(t, "Double", m.Func) } assert.True(t, slices.IsSortedFunc(tally.Survived, func(a, b mutate.Mutation) int { return strings.Compare(a.String(), b.String()) }), "the survivors are not in order: %v", tally.Survived) @@ -83,7 +82,7 @@ func TestEveryEditIsCountedAsTheTestsAnswered(t *testing.T) { assert.Equal(t, map[string]bool{bare: true}, tally.UntestedPackages) assert.Zero(t, tally.Unbuildable) assert.Equal(t, double, strings.Count(said.String(), " refused ")) - assert.Equal(t, half, strings.Count(said.String(), " SURVIVED ")) + assert.Equal(t, unchecked, strings.Count(said.String(), " SURVIVED ")) assert.Contains(t, said.String(), fmt.Sprintf("%s: %d edit(s) in one binary, 0 built alone", dir, len(plan)), "the package's edits were not built into its schema") assert.NotContains(t, said.String(), " took ", "a run under a minute was said to be slow") @@ -125,3 +124,11 @@ func TestAPackageWithNoTestsIsAskedThroughItsImporters(t *testing.T) { assert.Len(t, tally.Survived, half, "Half's edits, which no importer's test reaches: %s", said.String()) assert.Zero(t, tally.Untested, "a package whose importers have tests was said untested") } + +// An edit the run cannot even build a way to ask is an error, not an outcome: a file that is not +// there has no overlay. +func TestAnEditThatCannotBeAskedIsAnError(t *testing.T) { + gone := mutate.Mutation{ID: 1, File: filepath.Join(t.TempDir(), "gone.go"), Func: "F", What: "+ becomes -"} + _, err := mutate.RunAll([]mutate.Mutation{gone}, nil, "1m", 1, &strings.Builder{}) + assert.Error(t, err) +} From f66bb7cad9af318998ce698d3e0388083de25c89 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Tue, 29 Sep 2026 10:02:44 -0300 Subject: [PATCH 4/5] mutate: a test broken regardless refuses nothing, held; stderr notes exempt Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/mutate/runner.go | 4 ++-- internal/mutate/runner_test.go | 31 +++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/internal/mutate/runner.go b/internal/mutate/runner.go index 8c17875..9a53549 100644 --- a/internal/mutate/runner.go +++ b/internal/mutate/runner.go @@ -204,7 +204,7 @@ func (b *builds) importer(dir, importer, overlay string) (string, string) { defer func() { <-b.sem }() var err error ib.bin, ib.dir, err = ImporterBinary(importer, overlay, filepath.Join(b.work, "importer-"+strconv.Itoa(n)+".test")) - if err != nil { + if err != nil { // mutate-exempt: what is said on stderr; an importer with no binary asks nothing either way fmt.Fprintf(os.Stderr, " (%v)\n", err) } }) @@ -257,7 +257,7 @@ func (cs *coverages) of(dir string) *Coverage { cs.mu.Unlock() c.once.Do(func() { measured, err := Cover(dir, cs.timeout, cs.slots) - if err != nil { + if err != nil { // mutate-exempt: what is said on stderr; the coverage is nil either way fmt.Fprintf(os.Stderr, " (%s: no coverage, every test is asked: %v)\n", dir, err) } c.c = measured diff --git a/internal/mutate/runner_test.go b/internal/mutate/runner_test.go index 5fe4479..3d2810f 100644 --- a/internal/mutate/runner_test.go +++ b/internal/mutate/runner_test.go @@ -48,8 +48,8 @@ func TestEveryEditIsCountedAsTheTestsAnswered(t *testing.T) { bare := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(bare, "go.mod"), []byte("module example.com/bare\n\ngo 1.24\n"), 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(bare, "code.go"), []byte("package bare\n\nfunc Double(n int) int {\n\treturn n + n\n}\n"), 0o600)) - untested, err := mutate.Plan(filepath.Join(bare, "code.go"), map[int]bool{4: true}) + require.NoError(t, os.WriteFile(filepath.Join(bare, "code.go"), []byte("package bare\n\nfunc Double(n int) int {\n\treturn n + n\n}\n\nfunc Wide() int64 {\n\treturn 5\n}\n"), 0o600)) + untested, err := mutate.Plan(filepath.Join(bare, "code.go"), map[int]bool{4: true, 8: true}) require.NoError(t, err) require.NotEmpty(t, untested) @@ -85,9 +85,36 @@ func TestEveryEditIsCountedAsTheTestsAnswered(t *testing.T) { assert.Equal(t, unchecked, strings.Count(said.String(), " SURVIVED ")) assert.Contains(t, said.String(), fmt.Sprintf("%s: %d edit(s) in one binary, 0 built alone", dir, len(plan)), "the package's edits were not built into its schema") + assert.Contains(t, said.String(), bare+": 2 edit(s) in one binary, 1 built alone", + "Wide's int64 literal is built alone, and a run of one is what is timed") assert.NotContains(t, said.String(), " took ", "a run under a minute was said to be slow") } +// A test that fails with nothing broken refuses no edit: the coverage the run measures first says +// so, for an edit in the package's binary and for one built alone. Asked anyway, it would refuse +// every edit, and a change would pass the gate on a broken test. +func TestATestThatAlwaysFailsRefusesNothing(t *testing.T) { + dir := t.TempDir() + for name, body := range map[string]string{ + "go.mod": "module example.com/broken\n\ngo 1.24\n", + "code.go": "package broken\n\nfunc One() int {\n\treturn 1\n}\n\nfunc Wide() int64 {\n\treturn 5\n}\n", + "code_test.go": "package broken\n\nimport \"testing\"\n\nfunc TestBroken(t *testing.T) {\n\t_, _ = One(), Wide()\n\tt.Fatal(\"broken regardless\")\n}\n", + } { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600)) + } + plan, err := mutate.Plan(filepath.Join(dir, "code.go"), map[int]bool{4: true, 8: true}) + require.NoError(t, err) + for i := range plan { + plan[i].ID = i + 1 + } + var said strings.Builder + tally, err := mutate.RunAll(plan, nil, "2m", 2, &said) + require.NoError(t, err) + assert.Contains(t, said.String(), dir+": 1 edit(s) in one binary, 1 built alone", "One's literal in the binary, Wide's alone") + assert.Zero(t, tally.Killed, "a test broken regardless refused an edit: %s", said.String()) + assert.Len(t, tally.Survived, 2, "%s", said.String()) +} + // A package with no tests of its own is asked through the packages that import it, each built // once: Double's edits are refused by the importer's test and Half's, which it never calls, are not. func TestAPackageWithNoTestsIsAskedThroughItsImporters(t *testing.T) { From 300109538087a959ee894a556bc69cc5c5e9bccb Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Tue, 29 Sep 2026 10:03:59 -0300 Subject: [PATCH 5/5] mutate: the measuring of a package with nothing built is a cost, exempt Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/mutate/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mutate/runner.go b/internal/mutate/runner.go index 9a53549..5bcd7d9 100644 --- a/internal/mutate/runner.go +++ b/internal/mutate/runner.go @@ -117,7 +117,7 @@ func (p pkgRun) run(ms []Mutation, covers *coverages) { _, _ = fmt.Fprintf(p.out, " %s: %d edit(s) in one binary, %d built alone\n", p.dir, len(s.Built), len(s.Alone)) // The package as it is says which tests reach a line: the schema moves none. var c *Coverage - if len(s.Built) > 0 && s.Binary != "" { + if len(s.Built) > 0 && s.Binary != "" { // mutate-exempt: with no edit built, only one built alone reads it, and measures it itself c = covers.of(p.dir) } var wg sync.WaitGroup