Skip to content

mutate: the scheduler is the package's, and held by tests #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aledbf merged 5 commits into from
Sep 29, 2026
Merged

mutate: the scheduler is the package's, and held by tests #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
51c6b8a
mutate: the scheduler is the package's, and held by tests
aledbf Sep 29, 2026
c0036dc
mutate: tests for the scheduler's importers, order and missing coverage
aledbf Sep 29, 2026
1073667
mutate: the survivors' order and the error path, held
aledbf Sep 29, 2026
f66bb7c
mutate: a test broken regardless refuses nothing, held; stderr notes …
aledbf Sep 29, 2026
3001095
mutate: the measuring of a package with nothing built is a cost, exempt
aledbf Sep 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
300 changes: 11 additions & 289 deletions cmd/mutate/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,9 @@ import (
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"text/tabwriter"
"time"

Expand Down Expand Up @@ -71,7 +68,7 @@ func main() {

// Spread over the changed functions rather than taking the first of each file: a cap that
// always lands in the same function asks one question many times.
chosen := spread(planned, *maxRuns)
chosen := mutate.Spread(planned, *maxRuns)
for i := range chosen {
chosen[i].ID = i + 1
}
Expand All @@ -85,16 +82,20 @@ func main() {

mutate.Share(*jobs)
started := time.Now()
t := runAll(chosen, importers, *timeout, *jobs)
t, err := mutate.RunAll(chosen, importers, *timeout, *jobs, os.Stdout)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(2)
}
reportCosts(time.Since(started))
survived := t.survived
survived := t.Survived
fmt.Printf("mutate: %d refused, %d survived, %d did not build, %d in packages with no tests\n",
t.killed, len(survived), t.unbuildable, t.untested)
if len(t.untestedPackages) > 0 {
t.Killed, len(survived), t.Unbuildable, t.Untested)
if len(t.UntestedPackages) > 0 {
// Said rather than counted as a survivor: a package with no tests is a decision
// somebody made, and this gate is about the tests that exist.
packages := make([]string, 0, len(t.untestedPackages))
for p := range t.untestedPackages {
packages := make([]string, 0, len(t.UntestedPackages))
for p := range t.UntestedPackages {
packages = append(packages, p)
}
sort.Strings(packages)
Expand All @@ -115,254 +116,6 @@ func main() {
os.Exit(1)
}

// tally is what the tests did about every mutation.
type tally struct {
survived []mutate.Mutation
killed, unbuildable, untested int
untestedPackages map[string]bool
}

// result is what the tests did about one mutation.
type result struct {
m mutate.Mutation
outcome mutate.Outcome
}

// runAll puts every mutation to the tests, jobs at a time, and says each outcome as it comes.
// Each package's mutations are built into one binary (mutate.Schemata), run once per mutation;
// an edit the schema could not build is built alone, through an overlay of its own. Each
// package's coverage is measured once, for both.
func runAll(chosen []mutate.Mutation, importers map[string][]string, timeout string, jobs int) tally {
work, err := os.MkdirTemp("", "mutate-")
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(2)
}
defer func() { _ = os.RemoveAll(work) }()

// One budget of test processes for the whole run, which measuring coverage draws on too: a
// mutation's run and each package's coverage each had one of their own, and ten mutations
// measuring ten tests each were a hundred test processes and as many PostgreSQL containers.
slots := make(chan struct{}, max(jobs, 1)) // mutate-exempt: how many run at once, not what they say
// Builds are what cost: go parallelises one across the machine already, so they go one at a
// time, beside the test processes.
b := &builds{sem: make(chan struct{}, 1), work: work, bins: map[string]*importerBinary{}}
covers := &coverages{by: map[string]*coverage{}, timeout: timeout, slots: slots}
byDir := map[string][]mutate.Mutation{}
var dirs []string
for _, m := range chosen {
dir := filepath.Dir(m.File)
if byDir[dir] == nil {
dirs = append(dirs, dir)
}
byDir[dir] = append(byDir[dir], m)
}
results := make(chan result)
var wg sync.WaitGroup
for i, dir := range dirs {
wg.Add(1)
go func() {
defer wg.Done()
p := pkgRun{dir: dir, work: filepath.Join(work, strconv.Itoa(i)), timeout: timeout, slots: slots,
builds: b, importers: importers[filepath.ToSlash(dir)], results: results}
p.run(byDir[dir], covers)
}()
}
go func() { wg.Wait(); close(results) }()

t := tally{untestedPackages: map[string]bool{}}
for r := range results {
m := r.m
switch r.outcome {
case mutate.Killed:
t.killed++
fmt.Printf(" refused %s\n", m)
case mutate.Unbuildable:
t.unbuildable++
fmt.Printf(" no edit %s (does not build)\n", m)
case mutate.Untested:
t.untested++
t.untestedPackages[filepath.Dir(m.File)] = true
case mutate.Survived:
t.survived = append(t.survived, m)
fmt.Printf(" SURVIVED %s\n", m)
}
}
sort.Slice(t.survived, func(i, j int) bool { return t.survived[i].String() < t.survived[j].String() })
return t
}

// pkgRun is one package's mutations put to the tests.
type pkgRun struct {
dir, work, timeout string
slots chan struct{}
builds *builds
importers []string
results chan<- result
}

// run builds the package's schema and puts each mutation in it to the binary, and each it could
// not build to the tests alone, all side by side within the slots.
func (p pkgRun) run(ms []mutate.Mutation, covers *coverages) {
p.builds.sem <- struct{}{}
s, err := mutate.Schemata(p.dir, ms, filepath.Join(p.work, "schema"))
<-p.builds.sem
if err != nil {
fmt.Fprintf(os.Stderr, " (%s: no schema, each edit is built alone: %v)\n", p.dir, err)
s = &mutate.Schema{Alone: ms}
}
fmt.Printf(" %s: %d edit(s) in one binary, %d built alone\n", p.dir, len(s.Built), len(s.Alone))
// The package as it is says which tests reach a line: the schema moves none.
var c *mutate.Coverage
if len(s.Built) > 0 && s.Binary != "" {
c = covers.of(p.dir)
}
var wg sync.WaitGroup
for _, m := range s.Built {
wg.Add(1)
go func() {
defer wg.Done()
p.results <- result{m, p.built(m, s, c)}
}()
}
for i, m := range s.Alone {
wg.Add(1)
go func() {
defer wg.Done()
// Measured before a slot is taken: the measuring takes slots of its own, and holding
// one while waiting on the others is how a budget of one never finishes.
reaching := covers.of(p.dir)
p.slots <- struct{}{}
defer func() { <-p.slots }()
p.results <- result{m, run(m, filepath.Join(p.work, "alone-"+strconv.Itoa(i)), p.timeout, reaching, p.importers)}
}()
}
wg.Wait()
}

// built puts m, which is in s's binary, to the package's tests that reach its line, then to each
// importer's tests, built once with the same schema: a survivor pays for the second.
func (p pkgRun) built(m mutate.Mutation, s *mutate.Schema, c *mutate.Coverage) mutate.Outcome {
if s.Binary == "" && len(p.importers) == 0 {
return mutate.Untested
}
ask := func(bin, dir string, only []string) mutate.Outcome {
p.slots <- struct{}{}
defer func() { <-p.slots }()
outcome, _ := mutate.Ask(bin, dir, m.ID, p.timeout, only)
return outcome
}
if s.Binary != "" {
var reaching []string
if c != nil {
reaching = c.Reaching(m.File, m.Line)
}
if (reaching == nil || len(reaching) > 0) && ask(s.Binary, p.dir, reaching) == mutate.Killed {
return mutate.Killed
}
}
for _, importer := range p.importers {
bin, dir := p.builds.importer(p.dir, importer, s.Overlay)
if bin != "" && ask(bin, dir, nil) == mutate.Killed {
return mutate.Killed
}
}
return mutate.Survived
}

// builds is the run's one build at a time, and the importers' binaries it made, each once.
type builds struct {
sem chan struct{}
work string
mu sync.Mutex
bins map[string]*importerBinary
}

type importerBinary struct {
once sync.Once
bin, dir string
}

// importer is importer's test binary with the schema of the package in dir, and the directory it
// runs in; an empty binary where it has no tests or would not build, which asks it nothing.
func (b *builds) importer(dir, importer, overlay string) (string, string) {
key := dir + " " + importer
b.mu.Lock()
ib := b.bins[key]
if ib == nil {
ib = &importerBinary{}
b.bins[key] = ib
}
n := len(b.bins)
b.mu.Unlock()
ib.once.Do(func() {
b.sem <- struct{}{}
defer func() { <-b.sem }()
var err error
ib.bin, ib.dir, err = mutate.ImporterBinary(importer, overlay, filepath.Join(b.work, "importer-"+strconv.Itoa(n)+".test"))
if err != nil {
fmt.Fprintf(os.Stderr, " (%v)\n", err)
}
})
return ib.bin, ib.dir
}

// run builds the mutation in through an overlay under dir, and runs the package's tests that
// reach its line and then its importers'. The file itself is never written.
func run(m mutate.Mutation, dir, timeout string, c *mutate.Coverage, importers []string) mutate.Outcome {
if err := os.MkdirAll(dir, 0o700); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(2)
}
overlay, err := mutate.Overlay(m, dir)
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(2)
}
asked := mutate.Asked{Overlay: overlay, Timeout: timeout, Importers: importers}
if c != nil {
asked.Reaching = c.Reaching(m.File, m.Line)
}
started := time.Now()
outcome, _ := mutate.Run(m, asked)
if took := time.Since(started); took > time.Minute {
fmt.Printf(" (%s took %s)\n", filepath.Dir(m.File), took.Round(time.Second))
}
return outcome
}

// coverages is each package's Coverage, measured once and shared by its mutations.
type coverages struct {
mu sync.Mutex
by map[string]*coverage
timeout string
slots chan struct{}
}

type coverage struct {
once sync.Once
c *mutate.Coverage
}

// of is dir's coverage, or nil where it could not be measured: every test is asked then.
func (cs *coverages) of(dir string) *mutate.Coverage {
cs.mu.Lock()
c, ok := cs.by[dir]
if !ok {
c = &coverage{}
cs.by[dir] = c
}
cs.mu.Unlock()
c.once.Do(func() {
measured, err := mutate.Cover(dir, cs.timeout, cs.slots)
if err != nil {
fmt.Fprintf(os.Stderr, " (%s: no coverage, every test is asked: %v)\n", dir, err)
}
c.c = measured
})
return c.c
}

// reportCosts says where the run's time went, by what each process was for: the processes of
// one purpose run side by side, so their wall is summed and can pass the run's own.
func reportCosts(took time.Duration) {
Expand Down Expand Up @@ -419,34 +172,3 @@ func reportStale() int {
fmt.Printf("OK: every mutate-exempt reason in %d file(s) covers an edit this tool makes.\n", len(files))
return 0
}

// spread takes mutations in rounds, one per function each time round, so a cap covers as many
// of the changed functions as it can rather than the first one exhaustively.
func spread(all []mutate.Mutation, max int) []mutate.Mutation {
byFunc := map[string][]mutate.Mutation{}
var order []string
for _, m := range all {
key := m.File + ":" + m.Func
if _, seen := byFunc[key]; !seen {
order = append(order, key)
}
byFunc[key] = append(byFunc[key], m)
}
var out []mutate.Mutation
for round := 0; len(out) < max; round++ {
took := false
for _, key := range order {
if round < len(byFunc[key]) {
out = append(out, byFunc[key][round])
took = true
if len(out) == max {
return out
}
}
}
if !took {
break
}
}
return out
}
Loading