diff --git a/cmd/mutate/main.go b/cmd/mutate/main.go index b3b0582..0f5d1e5 100644 --- a/cmd/mutate/main.go +++ b/cmd/mutate/main.go @@ -11,12 +11,9 @@ import ( "fmt" "os" "os/exec" - "path/filepath" "runtime" "sort" - "strconv" "strings" - "sync" "text/tabwriter" "time" @@ -71,7 +68,7 @@ func main() { // Spread over the changed functions rather than taking the first of each file: a cap that // always lands in the same function asks one question many times. - chosen := spread(planned, *maxRuns) + chosen := mutate.Spread(planned, *maxRuns) for i := range chosen { chosen[i].ID = i + 1 } @@ -85,16 +82,20 @@ func main() { mutate.Share(*jobs) started := time.Now() - t := runAll(chosen, importers, *timeout, *jobs) + t, err := mutate.RunAll(chosen, importers, *timeout, *jobs, os.Stdout) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(2) + } reportCosts(time.Since(started)) - survived := t.survived + survived := t.Survived fmt.Printf("mutate: %d refused, %d survived, %d did not build, %d in packages with no tests\n", - t.killed, len(survived), t.unbuildable, t.untested) - if len(t.untestedPackages) > 0 { + t.Killed, len(survived), t.Unbuildable, t.Untested) + if len(t.UntestedPackages) > 0 { // Said rather than counted as a survivor: a package with no tests is a decision // somebody made, and this gate is about the tests that exist. - packages := make([]string, 0, len(t.untestedPackages)) - for p := range t.untestedPackages { + packages := make([]string, 0, len(t.UntestedPackages)) + for p := range t.UntestedPackages { packages = append(packages, p) } sort.Strings(packages) @@ -115,254 +116,6 @@ func main() { os.Exit(1) } -// tally is what the tests did about every mutation. -type tally struct { - survived []mutate.Mutation - killed, unbuildable, untested int - untestedPackages map[string]bool -} - -// result is what the tests did about one mutation. -type result struct { - m mutate.Mutation - outcome mutate.Outcome -} - -// runAll puts every mutation to the tests, jobs at a time, and says each outcome as it comes. -// Each package's mutations are built into one binary (mutate.Schemata), run once per mutation; -// an edit the schema could not build is built alone, through an overlay of its own. Each -// package's coverage is measured once, for both. -func runAll(chosen []mutate.Mutation, importers map[string][]string, timeout string, jobs int) tally { - work, err := os.MkdirTemp("", "mutate-") - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(2) - } - defer func() { _ = os.RemoveAll(work) }() - - // One budget of test processes for the whole run, which measuring coverage draws on too: a - // mutation's run and each package's coverage each had one of their own, and ten mutations - // measuring ten tests each were a hundred test processes and as many PostgreSQL containers. - slots := make(chan struct{}, max(jobs, 1)) // mutate-exempt: how many run at once, not what they say - // Builds are what cost: go parallelises one across the machine already, so they go one at a - // time, beside the test processes. - b := &builds{sem: make(chan struct{}, 1), work: work, bins: map[string]*importerBinary{}} - covers := &coverages{by: map[string]*coverage{}, timeout: timeout, slots: slots} - byDir := map[string][]mutate.Mutation{} - var dirs []string - for _, m := range chosen { - dir := filepath.Dir(m.File) - if byDir[dir] == nil { - dirs = append(dirs, dir) - } - byDir[dir] = append(byDir[dir], m) - } - results := make(chan result) - var wg sync.WaitGroup - for i, dir := range dirs { - wg.Add(1) - go func() { - defer wg.Done() - p := pkgRun{dir: dir, work: filepath.Join(work, strconv.Itoa(i)), timeout: timeout, slots: slots, - builds: b, importers: importers[filepath.ToSlash(dir)], results: results} - p.run(byDir[dir], covers) - }() - } - go func() { wg.Wait(); close(results) }() - - t := tally{untestedPackages: map[string]bool{}} - for r := range results { - m := r.m - switch r.outcome { - case mutate.Killed: - t.killed++ - fmt.Printf(" refused %s\n", m) - case mutate.Unbuildable: - t.unbuildable++ - fmt.Printf(" no edit %s (does not build)\n", m) - case mutate.Untested: - t.untested++ - t.untestedPackages[filepath.Dir(m.File)] = true - case mutate.Survived: - t.survived = append(t.survived, m) - fmt.Printf(" SURVIVED %s\n", m) - } - } - sort.Slice(t.survived, func(i, j int) bool { return t.survived[i].String() < t.survived[j].String() }) - return t -} - -// pkgRun is one package's mutations put to the tests. -type pkgRun struct { - dir, work, timeout string - slots chan struct{} - builds *builds - importers []string - results chan<- result -} - -// run builds the package's schema and puts each mutation in it to the binary, and each it could -// not build to the tests alone, all side by side within the slots. -func (p pkgRun) run(ms []mutate.Mutation, covers *coverages) { - p.builds.sem <- struct{}{} - s, err := mutate.Schemata(p.dir, ms, filepath.Join(p.work, "schema")) - <-p.builds.sem - if err != nil { - fmt.Fprintf(os.Stderr, " (%s: no schema, each edit is built alone: %v)\n", p.dir, err) - s = &mutate.Schema{Alone: ms} - } - fmt.Printf(" %s: %d edit(s) in one binary, %d built alone\n", p.dir, len(s.Built), len(s.Alone)) - // The package as it is says which tests reach a line: the schema moves none. - var c *mutate.Coverage - if len(s.Built) > 0 && s.Binary != "" { - c = covers.of(p.dir) - } - var wg sync.WaitGroup - for _, m := range s.Built { - wg.Add(1) - go func() { - defer wg.Done() - p.results <- result{m, p.built(m, s, c)} - }() - } - for i, m := range s.Alone { - wg.Add(1) - go func() { - defer wg.Done() - // Measured before a slot is taken: the measuring takes slots of its own, and holding - // one while waiting on the others is how a budget of one never finishes. - reaching := covers.of(p.dir) - p.slots <- struct{}{} - defer func() { <-p.slots }() - p.results <- result{m, run(m, filepath.Join(p.work, "alone-"+strconv.Itoa(i)), p.timeout, reaching, p.importers)} - }() - } - wg.Wait() -} - -// built puts m, which is in s's binary, to the package's tests that reach its line, then to each -// importer's tests, built once with the same schema: a survivor pays for the second. -func (p pkgRun) built(m mutate.Mutation, s *mutate.Schema, c *mutate.Coverage) mutate.Outcome { - if s.Binary == "" && len(p.importers) == 0 { - return mutate.Untested - } - ask := func(bin, dir string, only []string) mutate.Outcome { - p.slots <- struct{}{} - defer func() { <-p.slots }() - outcome, _ := mutate.Ask(bin, dir, m.ID, p.timeout, only) - return outcome - } - if s.Binary != "" { - var reaching []string - if c != nil { - reaching = c.Reaching(m.File, m.Line) - } - if (reaching == nil || len(reaching) > 0) && ask(s.Binary, p.dir, reaching) == mutate.Killed { - return mutate.Killed - } - } - for _, importer := range p.importers { - bin, dir := p.builds.importer(p.dir, importer, s.Overlay) - if bin != "" && ask(bin, dir, nil) == mutate.Killed { - return mutate.Killed - } - } - return mutate.Survived -} - -// builds is the run's one build at a time, and the importers' binaries it made, each once. -type builds struct { - sem chan struct{} - work string - mu sync.Mutex - bins map[string]*importerBinary -} - -type importerBinary struct { - once sync.Once - bin, dir string -} - -// importer is importer's test binary with the schema of the package in dir, and the directory it -// runs in; an empty binary where it has no tests or would not build, which asks it nothing. -func (b *builds) importer(dir, importer, overlay string) (string, string) { - key := dir + " " + importer - b.mu.Lock() - ib := b.bins[key] - if ib == nil { - ib = &importerBinary{} - b.bins[key] = ib - } - n := len(b.bins) - b.mu.Unlock() - ib.once.Do(func() { - b.sem <- struct{}{} - defer func() { <-b.sem }() - var err error - ib.bin, ib.dir, err = mutate.ImporterBinary(importer, overlay, filepath.Join(b.work, "importer-"+strconv.Itoa(n)+".test")) - if err != nil { - fmt.Fprintf(os.Stderr, " (%v)\n", err) - } - }) - return ib.bin, ib.dir -} - -// run builds the mutation in through an overlay under dir, and runs the package's tests that -// reach its line and then its importers'. The file itself is never written. -func run(m mutate.Mutation, dir, timeout string, c *mutate.Coverage, importers []string) mutate.Outcome { - if err := os.MkdirAll(dir, 0o700); err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(2) - } - overlay, err := mutate.Overlay(m, dir) - if err != nil { - fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(2) - } - asked := mutate.Asked{Overlay: overlay, Timeout: timeout, Importers: importers} - if c != nil { - asked.Reaching = c.Reaching(m.File, m.Line) - } - started := time.Now() - outcome, _ := mutate.Run(m, asked) - if took := time.Since(started); took > time.Minute { - fmt.Printf(" (%s took %s)\n", filepath.Dir(m.File), took.Round(time.Second)) - } - return outcome -} - -// coverages is each package's Coverage, measured once and shared by its mutations. -type coverages struct { - mu sync.Mutex - by map[string]*coverage - timeout string - slots chan struct{} -} - -type coverage struct { - once sync.Once - c *mutate.Coverage -} - -// of is dir's coverage, or nil where it could not be measured: every test is asked then. -func (cs *coverages) of(dir string) *mutate.Coverage { - cs.mu.Lock() - c, ok := cs.by[dir] - if !ok { - c = &coverage{} - cs.by[dir] = c - } - cs.mu.Unlock() - c.once.Do(func() { - measured, err := mutate.Cover(dir, cs.timeout, cs.slots) - if err != nil { - fmt.Fprintf(os.Stderr, " (%s: no coverage, every test is asked: %v)\n", dir, err) - } - c.c = measured - }) - return c.c -} - // reportCosts says where the run's time went, by what each process was for: the processes of // one purpose run side by side, so their wall is summed and can pass the run's own. func reportCosts(took time.Duration) { @@ -419,34 +172,3 @@ func reportStale() int { fmt.Printf("OK: every mutate-exempt reason in %d file(s) covers an edit this tool makes.\n", len(files)) return 0 } - -// spread takes mutations in rounds, one per function each time round, so a cap covers as many -// of the changed functions as it can rather than the first one exhaustively. -func spread(all []mutate.Mutation, max int) []mutate.Mutation { - byFunc := map[string][]mutate.Mutation{} - var order []string - for _, m := range all { - key := m.File + ":" + m.Func - if _, seen := byFunc[key]; !seen { - order = append(order, key) - } - byFunc[key] = append(byFunc[key], m) - } - var out []mutate.Mutation - for round := 0; len(out) < max; round++ { - took := false - for _, key := range order { - if round < len(byFunc[key]) { - out = append(out, byFunc[key][round]) - took = true - if len(out) == max { - return out - } - } - } - if !took { - break - } - } - return out -} diff --git a/internal/mutate/runner.go b/internal/mutate/runner.go new file mode 100644 index 0000000..5bcd7d9 --- /dev/null +++ b/internal/mutate/runner.go @@ -0,0 +1,297 @@ +package mutate + +import ( + "cmp" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "sync" + "time" +) + +// Tally is what the tests did about every mutation. +type Tally struct { + Survived []Mutation + Killed, Unbuildable, Untested int + UntestedPackages map[string]bool +} + +// result is what the tests did about one mutation. +type result struct { + m Mutation + outcome Outcome + err error // the run could not ask the tests at all +} + +// RunAll puts every mutation to the tests, jobs at a time, and says each outcome as it comes. +// Each package's mutations are built into one binary (Schemata), run once per mutation; +// an edit the schema could not build is built alone, through an overlay of its own. Each +// package's coverage is measured once, for both. +func RunAll(chosen []Mutation, importers map[string][]string, timeout string, jobs int, w io.Writer) (Tally, error) { + work, err := os.MkdirTemp("", "mutate-") + if err != nil { + return Tally{}, err + } + defer func() { _ = os.RemoveAll(work) }() + + // One budget of test processes for the whole run, which measuring coverage draws on too: a + // mutation's run and each package's coverage each had one of their own, and ten mutations + // measuring ten tests each were a hundred test processes and as many PostgreSQL containers. + slots := make(chan struct{}, max(jobs, 1)) // mutate-exempt: how many run at once, not what they say + // Builds are what cost: go parallelises one across the machine already, so they go one at a + // time, beside the test processes. + b := &builds{sem: make(chan struct{}, 1), work: work, bins: map[string]*importerBinary{}} + covers := &coverages{by: map[string]*coverage{}, timeout: timeout, slots: slots} + byDir := map[string][]Mutation{} + var dirs []string + for _, m := range chosen { + dir := filepath.Dir(m.File) + if byDir[dir] == nil { + dirs = append(dirs, dir) + } + byDir[dir] = append(byDir[dir], m) + } + results := make(chan result) + var wg sync.WaitGroup + for i, dir := range dirs { + wg.Add(1) + go func() { + defer wg.Done() + p := pkgRun{dir: dir, work: filepath.Join(work, strconv.Itoa(i)), timeout: timeout, slots: slots, + builds: b, importers: importers[filepath.ToSlash(dir)], results: results, out: w} + p.run(byDir[dir], covers) + }() + } + go func() { wg.Wait(); close(results) }() + + t := Tally{UntestedPackages: map[string]bool{}} + var firstErr error + for r := range results { + m := r.m + if r.err != nil { + firstErr = cmp.Or(firstErr, r.err) + continue + } + switch r.outcome { + case Killed: + t.Killed++ + _, _ = fmt.Fprintf(w, " refused %s\n", m) + case Unbuildable: + t.Unbuildable++ + _, _ = fmt.Fprintf(w, " no edit %s (does not build)\n", m) + case Untested: + t.Untested++ + t.UntestedPackages[filepath.Dir(m.File)] = true + case Survived: + t.Survived = append(t.Survived, m) + _, _ = fmt.Fprintf(w, " SURVIVED %s\n", m) + } + } + sort.Slice(t.Survived, func(i, j int) bool { return t.Survived[i].String() < t.Survived[j].String() }) + return t, firstErr +} + +// pkgRun is one package's mutations put to the tests. +type pkgRun struct { + dir, work, timeout string + slots chan struct{} + builds *builds + importers []string + results chan<- result + out io.Writer +} + +// run builds the package's schema and puts each mutation in it to the binary, and each it could +// not build to the tests alone, all side by side within the slots. +func (p pkgRun) run(ms []Mutation, covers *coverages) { + p.builds.sem <- struct{}{} + s, err := Schemata(p.dir, ms, filepath.Join(p.work, "schema")) + <-p.builds.sem + if err != nil { + fmt.Fprintf(os.Stderr, " (%s: no schema, each edit is built alone: %v)\n", p.dir, err) + s = &Schema{Alone: ms} + } + _, _ = fmt.Fprintf(p.out, " %s: %d edit(s) in one binary, %d built alone\n", p.dir, len(s.Built), len(s.Alone)) + // The package as it is says which tests reach a line: the schema moves none. + var c *Coverage + if len(s.Built) > 0 && s.Binary != "" { // mutate-exempt: with no edit built, only one built alone reads it, and measures it itself + c = covers.of(p.dir) + } + var wg sync.WaitGroup + for _, m := range s.Built { + wg.Add(1) + go func() { + defer wg.Done() + p.results <- result{m: m, outcome: p.built(m, s, c)} + }() + } + for i, m := range s.Alone { + wg.Add(1) + go func() { + defer wg.Done() + // Measured before a slot is taken: the measuring takes slots of its own, and holding + // one while waiting on the others is how a budget of one never finishes. + reaching := covers.of(p.dir) + p.slots <- struct{}{} + defer func() { <-p.slots }() + outcome, err := runAlone(m, filepath.Join(p.work, "alone-"+strconv.Itoa(i)), p.timeout, reaching, p.importers, p.out) + p.results <- result{m: m, outcome: outcome, err: err} + }() + } + wg.Wait() +} + +// built puts m, which is in s's binary, to the package's tests that reach its line, then to each +// importer's tests, built once with the same schema: a survivor pays for the second. +func (p pkgRun) built(m Mutation, s *Schema, c *Coverage) Outcome { + if s.Binary == "" && len(p.importers) == 0 { + return Untested + } + ask := func(bin, dir string, only []string) Outcome { + p.slots <- struct{}{} + defer func() { <-p.slots }() + outcome, _ := Ask(bin, dir, m.ID, p.timeout, only) + return outcome + } + if s.Binary != "" { + var reaching []string + if c != nil { + reaching = c.Reaching(m.File, m.Line) + } + if (reaching == nil || len(reaching) > 0) && ask(s.Binary, p.dir, reaching) == Killed { + return Killed + } + } + for _, importer := range p.importers { + bin, dir := p.builds.importer(p.dir, importer, s.Overlay) + if bin != "" && ask(bin, dir, nil) == Killed { + return Killed + } + } + return Survived +} + +// builds is the run's one build at a time, and the importers' binaries it made, each once. +type builds struct { + sem chan struct{} + work string + mu sync.Mutex + bins map[string]*importerBinary +} + +type importerBinary struct { + once sync.Once + bin, dir string +} + +// importer is importer's test binary with the schema of the package in dir, and the directory it +// runs in; an empty binary where it has no tests or would not build, which asks it nothing. +func (b *builds) importer(dir, importer, overlay string) (string, string) { + key := dir + " " + importer + b.mu.Lock() + ib := b.bins[key] + if ib == nil { + ib = &importerBinary{} + b.bins[key] = ib + } + n := len(b.bins) + b.mu.Unlock() + ib.once.Do(func() { + b.sem <- struct{}{} + defer func() { <-b.sem }() + var err error + ib.bin, ib.dir, err = ImporterBinary(importer, overlay, filepath.Join(b.work, "importer-"+strconv.Itoa(n)+".test")) + if err != nil { // mutate-exempt: what is said on stderr; an importer with no binary asks nothing either way + fmt.Fprintf(os.Stderr, " (%v)\n", err) + } + }) + return ib.bin, ib.dir +} + +// runAlone builds the mutation in through an overlay under dir, and runs the package's tests that +// reach its line and then its importers'. The file itself is never written. +func runAlone(m Mutation, dir, timeout string, c *Coverage, importers []string, out io.Writer) (Outcome, error) { + if err := os.MkdirAll(dir, 0o700); err != nil { + return 0, err + } + overlay, err := Overlay(m, dir) + if err != nil { + return 0, err + } + asked := Asked{Overlay: overlay, Timeout: timeout, Importers: importers} + if c != nil { + asked.Reaching = c.Reaching(m.File, m.Line) + } + started := time.Now() + outcome, _ := Run(m, asked) + if took := time.Since(started); took > time.Minute { + _, _ = fmt.Fprintf(out, " (%s took %s)\n", filepath.Dir(m.File), took.Round(time.Second)) + } + return outcome, nil +} + +// coverages is each package's Coverage, measured once and shared by its mutations. +type coverages struct { + mu sync.Mutex + by map[string]*coverage + timeout string + slots chan struct{} +} + +type coverage struct { + once sync.Once + c *Coverage +} + +// of is dir's coverage, or nil where it could not be measured: every test is asked then. +func (cs *coverages) of(dir string) *Coverage { + cs.mu.Lock() + c, ok := cs.by[dir] + if !ok { + c = &coverage{} + cs.by[dir] = c + } + cs.mu.Unlock() + c.once.Do(func() { + measured, err := Cover(dir, cs.timeout, cs.slots) + if err != nil { // mutate-exempt: what is said on stderr; the coverage is nil either way + fmt.Fprintf(os.Stderr, " (%s: no coverage, every test is asked: %v)\n", dir, err) + } + c.c = measured + }) + return c.c +} + +// Spread takes mutations in rounds, one per function each time round, so a cap covers as many +// of the changed functions as it can rather than the first one exhaustively. +func Spread(all []Mutation, max int) []Mutation { + byFunc := map[string][]Mutation{} + var order []string + for _, m := range all { + key := m.File + ":" + m.Func + if _, seen := byFunc[key]; !seen { + order = append(order, key) + } + byFunc[key] = append(byFunc[key], m) + } + var out []Mutation + for round := 0; len(out) < max; round++ { + took := false + for _, key := range order { + if round < len(byFunc[key]) { + out = append(out, byFunc[key][round]) + took = true + if len(out) == max { + return out + } + } + } + if !took { + break + } + } + return out +} diff --git a/internal/mutate/runner_internal_test.go b/internal/mutate/runner_internal_test.go new file mode 100644 index 0000000..18ee99e --- /dev/null +++ b/internal/mutate/runner_internal_test.go @@ -0,0 +1,40 @@ +package mutate + +import ( + "os" + "path/filepath" + "testing" +) + +// An edit in a package whose coverage could not be measured is asked of every test, not of none: +// no coverage is not knowing which tests reach the line, and a test that reaches nothing is only +// what a measurement says. +func TestAnEditWithNoCoverageIsAskedOfEveryTest(t *testing.T) { + dir := t.TempDir() + for name, body := range map[string]string{ + "go.mod": "module example.com/nocover\n\ngo 1.24\n", + "code.go": "package nocover\n\nfunc Double(n int) int {\n\treturn n + n\n}\n", + "code_test.go": "package nocover\n\nimport \"testing\"\n\nfunc TestDouble(t *testing.T) {\n\tif Double(2) != 4 {\n\t\tt.Fatal(\"not doubled\")\n\t}\n}\n", + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + plan, err := Plan(filepath.Join(dir, "code.go"), map[int]bool{4: true}) + if err != nil || len(plan) == 0 { + t.Fatalf("plan: %v, %v", plan, err) + } + for i := range plan { + plan[i].ID = i + 1 + } + s, err := Schemata(dir, plan, t.TempDir()) + if err != nil || s.Binary == "" || len(s.Built) == 0 { + t.Fatalf("schema: %+v, %v", s, err) + } + p := pkgRun{dir: dir, timeout: "2m", slots: make(chan struct{}, 1)} + for _, m := range s.Built { + if got := p.built(m, s, nil); got != Killed { + t.Errorf("%s: %v with no coverage, want refused by the test nobody measured", m, got) + } + } +} diff --git a/internal/mutate/runner_test.go b/internal/mutate/runner_test.go new file mode 100644 index 0000000..3d2810f --- /dev/null +++ b/internal/mutate/runner_test.go @@ -0,0 +1,161 @@ +package mutate_test + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/spin-stack/go-tools/internal/mutate" +) + +// A cap is spread over the changed functions, one edit of each a round, so it covers as many of +// them as it can rather than the first one exhaustively; under the cap, every edit is taken. +func TestACapIsSpreadOverTheChangedFunctions(t *testing.T) { + m := func(file, fn, what string) mutate.Mutation { return mutate.Mutation{File: file, Func: fn, What: what} } + all := []mutate.Mutation{ + m("a.go", "F", "1"), m("a.go", "F", "2"), m("a.go", "F", "3"), + m("a.go", "G", "1"), + m("b.go", "F", "1"), m("b.go", "F", "2"), + } + key := func(ms []mutate.Mutation) []string { + var out []string + for _, m := range ms { + out = append(out, m.File+":"+m.Func+":"+m.What) + } + return out + } + assert.Equal(t, []string{"a.go:F:1", "a.go:G:1", "b.go:F:1", "a.go:F:2"}, key(mutate.Spread(all, 4)), + "a function of the same name in another file is another function, and each is taken once a round") + assert.Equal(t, []string{"a.go:F:1", "a.go:G:1", "b.go:F:1", "a.go:F:2", "b.go:F:2", "a.go:F:3"}, key(mutate.Spread(all, 10))) + assert.Equal(t, []string{"a.go:F:1"}, key(mutate.Spread(all, 1))) + assert.Empty(t, mutate.Spread(all, 0)) + assert.Empty(t, mutate.Spread(nil, 3)) +} + +// Every edit is put to the tests and counted once, as the tests answered it, and each is said as +// it comes: Double's refused by TestDouble, Half's and Sign's let through by tests that check neither, and +// a package with no tests named rather than counted as a survivor. +func TestEveryEditIsCountedAsTheTestsAnswered(t *testing.T) { + dir := refuses(t) + plan, err := mutate.Plan(filepath.Join(dir, "code.go"), map[int]bool{4: true, 8: true, 13: true, 14: true}) + require.NoError(t, err) + + bare := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(bare, "go.mod"), []byte("module example.com/bare\n\ngo 1.24\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(bare, "code.go"), []byte("package bare\n\nfunc Double(n int) int {\n\treturn n + n\n}\n\nfunc Wide() int64 {\n\treturn 5\n}\n"), 0o600)) + untested, err := mutate.Plan(filepath.Join(bare, "code.go"), map[int]bool{4: true, 8: true}) + require.NoError(t, err) + require.NotEmpty(t, untested) + + chosen := slices.Concat(plan, untested) + for i := range chosen { + chosen[i].ID = i + 1 + } + var said strings.Builder + tally, err := mutate.RunAll(chosen, nil, "2m", 2, &said) + require.NoError(t, err) + + var double, unchecked int + for _, m := range plan { + if m.Func == "Double" { + double++ + } else { + unchecked++ + } + } + require.Positive(t, double) + require.GreaterOrEqual(t, unchecked, 2, "the order of the survivors needs two of them") + assert.Equal(t, double, tally.Killed, "Double's edits refused: %s", said.String()) + assert.Len(t, tally.Survived, unchecked, "Half's and Sign's edits let through: %s", said.String()) + for _, m := range tally.Survived { + assert.NotEqual(t, "Double", m.Func) + } + assert.True(t, slices.IsSortedFunc(tally.Survived, func(a, b mutate.Mutation) int { return strings.Compare(a.String(), b.String()) }), + "the survivors are not in order: %v", tally.Survived) + assert.Equal(t, len(untested), tally.Untested) + assert.Equal(t, map[string]bool{bare: true}, tally.UntestedPackages) + assert.Zero(t, tally.Unbuildable) + assert.Equal(t, double, strings.Count(said.String(), " refused ")) + assert.Equal(t, unchecked, strings.Count(said.String(), " SURVIVED ")) + assert.Contains(t, said.String(), fmt.Sprintf("%s: %d edit(s) in one binary, 0 built alone", dir, len(plan)), + "the package's edits were not built into its schema") + assert.Contains(t, said.String(), bare+": 2 edit(s) in one binary, 1 built alone", + "Wide's int64 literal is built alone, and a run of one is what is timed") + assert.NotContains(t, said.String(), " took ", "a run under a minute was said to be slow") +} + +// A test that fails with nothing broken refuses no edit: the coverage the run measures first says +// so, for an edit in the package's binary and for one built alone. Asked anyway, it would refuse +// every edit, and a change would pass the gate on a broken test. +func TestATestThatAlwaysFailsRefusesNothing(t *testing.T) { + dir := t.TempDir() + for name, body := range map[string]string{ + "go.mod": "module example.com/broken\n\ngo 1.24\n", + "code.go": "package broken\n\nfunc One() int {\n\treturn 1\n}\n\nfunc Wide() int64 {\n\treturn 5\n}\n", + "code_test.go": "package broken\n\nimport \"testing\"\n\nfunc TestBroken(t *testing.T) {\n\t_, _ = One(), Wide()\n\tt.Fatal(\"broken regardless\")\n}\n", + } { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600)) + } + plan, err := mutate.Plan(filepath.Join(dir, "code.go"), map[int]bool{4: true, 8: true}) + require.NoError(t, err) + for i := range plan { + plan[i].ID = i + 1 + } + var said strings.Builder + tally, err := mutate.RunAll(plan, nil, "2m", 2, &said) + require.NoError(t, err) + assert.Contains(t, said.String(), dir+": 1 edit(s) in one binary, 1 built alone", "One's literal in the binary, Wide's alone") + assert.Zero(t, tally.Killed, "a test broken regardless refused an edit: %s", said.String()) + assert.Len(t, tally.Survived, 2, "%s", said.String()) +} + +// A package with no tests of its own is asked through the packages that import it, each built +// once: Double's edits are refused by the importer's test and Half's, which it never calls, are not. +func TestAPackageWithNoTestsIsAskedThroughItsImporters(t *testing.T) { + dir := t.TempDir() + for name, body := range map[string]string{ + "go.mod": "module example.com/imp\n\ngo 1.24\n", + "lib/lib.go": "package lib\n\nfunc Double(n int) int {\n\treturn n + n\n}\n\nfunc Half(n int) int {\n\treturn n / 2\n}\n", + "use/use.go": "package use\n\nimport \"example.com/imp/lib\"\n\nfunc Four() int { return lib.Double(2) }\n", + "use/use_test.go": "package use\n\nimport \"testing\"\n\nfunc TestFour(t *testing.T) {\n\tif Four() != 4 {\n\t\tt.Fatal(\"not four\")\n\t}\n}\n", + "also/also.go": "package also\n\nimport \"example.com/imp/lib\"\n\nvar _ = lib.Half\n", + } { + require.NoError(t, os.MkdirAll(filepath.Join(dir, filepath.Dir(name)), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(body), 0o600)) + } + t.Chdir(dir) + plan, err := mutate.Plan(filepath.Join("lib", "lib.go"), map[int]bool{4: true, 8: true}) + require.NoError(t, err) + var double, half int + for i := range plan { + plan[i].ID = i + 1 + if plan[i].Func == "Double" { + double++ + } else { + half++ + } + } + require.Positive(t, double) + require.Positive(t, half) + + var said strings.Builder + tally, err := mutate.RunAll(plan, map[string][]string{"lib": {"./use", "./also"}}, "2m", 2, &said) + require.NoError(t, err) + assert.Equal(t, double, tally.Killed, "Double's edits refused by the importer: %s", said.String()) + assert.Len(t, tally.Survived, half, "Half's edits, which no importer's test reaches: %s", said.String()) + assert.Zero(t, tally.Untested, "a package whose importers have tests was said untested") +} + +// An edit the run cannot even build a way to ask is an error, not an outcome: a file that is not +// there has no overlay. +func TestAnEditThatCannotBeAskedIsAnError(t *testing.T) { + gone := mutate.Mutation{ID: 1, File: filepath.Join(t.TempDir(), "gone.go"), Func: "F", What: "+ becomes -"} + _, err := mutate.RunAll([]mutate.Mutation{gone}, nil, "1m", 1, &strings.Builder{}) + assert.Error(t, err) +}