diff --git a/compileopts/finalizer_coverage_test.go b/compileopts/finalizer_coverage_test.go new file mode 100644 index 0000000000..604d8b6de7 --- /dev/null +++ b/compileopts/finalizer_coverage_test.go @@ -0,0 +1,70 @@ +package compileopts + +import ( + "go/build/constraint" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestFinalizerRunnerSchedulerCoverage checks that the build constraints on the +// gc_finalizer_sched*.go files define spawnFinalizerRunner for exactly one file +// per scheduler. The three constraints must partition the scheduler space: every +// scheduler matches exactly one file, so none can be left with the symbol +// undefined or defined twice. It iterates validSchedulerOptions as the source of +// truth, so a newly added scheduler is covered by this check automatically. +func TestFinalizerRunnerSchedulerCoverage(t *testing.T) { + files := []string{ + "gc_finalizer_sched.go", + "gc_finalizer_sched_none.go", + "gc_finalizer_sched_other.go", + } + exprs := make([]constraint.Expr, len(files)) + for i, name := range files { + exprs[i] = readBuildConstraint(t, filepath.Join("..", "src", "runtime", name)) + } + + for _, sched := range validSchedulerOptions { + // The finalizer table exists under the block GCs; gc.conservative + // satisfies the "gc.conservative || gc.precise" half of every constraint. + tags := map[string]bool{ + "gc.conservative": true, + "scheduler." + sched: true, + } + var matched []string + for i, expr := range exprs { + if expr.Eval(func(tag string) bool { return tags[tag] }) { + matched = append(matched, files[i]) + } + } + if len(matched) != 1 { + t.Errorf("scheduler.%s: spawnFinalizerRunner defined in %d files %v, want exactly 1", + sched, len(matched), matched) + } + } +} + +// readBuildConstraint returns the parsed //go:build expression of a Go file. +func readBuildConstraint(t *testing.T, path string) constraint.Expr { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if constraint.IsGoBuild(line) { + expr, err := constraint.Parse(line) + if err != nil { + t.Fatalf("%s: %v", path, err) + } + return expr + } + if line != "" && !strings.HasPrefix(line, "//") { + break // reached code before any //go:build line + } + } + t.Fatalf("%s: no //go:build line found", path) + return nil +} diff --git a/main_test.go b/main_test.go index 2dfd1683f6..9dddc06912 100644 --- a/main_test.go +++ b/main_test.go @@ -60,6 +60,9 @@ func TestBuild(t *testing.T) { "channel.go", "embed/", "finalizer.go", + "finalizerbits.go", + "finalizeridle.go", + "finalizerinvariants.go", "float.go", "gc.go", "generics.go", @@ -364,16 +367,40 @@ func runPlatTests(options compileopts.Options, tests []string, t *testing.T) { continue } } - if name == "finalizer.go" && options.Target != "wasm" { - // runtime.SetFinalizer is implemented for the block GC, but the - // test asserts deterministic collection of a dropped object, which - // only holds on the GOOS=js wasm target. The host default GC is - // boehm (SetFinalizer is a no-op there); conservative stack scanning - // on the emulated targets can pin the object; and the wasip2 - // component entry lays out the stack differently, so collection is - // not deterministic on those. The feature still works on all of - // them, it just can't be golden-tested for firing. - continue + if options.Target != "wasm" { + switch name { + case "finalizer.go", "finalizerbits.go", "finalizeridle.go": + // runtime.SetFinalizer is implemented for the block GC, but these + // tests assert deterministic collection of a dropped object, + // which only holds on the GOOS=js wasm target. The host default + // GC is boehm (SetFinalizer is a no-op there); conservative stack + // scanning on the emulated targets can pin the object; and the + // wasip2 component entry lays out the stack differently, so + // collection is not deterministic on those. The feature still + // works on all of them, it just can't be golden-tested for + // firing, which is what finalizerinvariants.go covers instead. + continue + } + } + if options.Target == "" && options.GC == "" { + switch name { + case "finalizerinvariants.go": + // The default GC on these is boehm, where SetFinalizer is + // unimplemented, so there is nothing to assert. The explicit + // -gc=conservative variants below cover the host instead. + continue + } + } + if options.Target == "simavr" { + switch name { + case "finalizerinvariants.go": + // Finalizers are detected by the GC, and gc.go is already skipped + // on AVR for its high mark false positive rate (see the simavr + // switch above). Registering and clearing a finalizer works + // there, but a single runtime.GC() call does not return, so this + // test inherits that limitation rather than adding a new one. + continue + } } name := name // redefine to avoid race condition @@ -396,6 +423,33 @@ func runPlatTests(options compileopts.Options, tests []string, t *testing.T) { runTest("alias.go", options, t, nil, nil) }) } + buildGOOS := options.GOOS + if buildGOOS == "" { + buildGOOS = runtime.GOOS + } + if options.Target == "" && (buildGOOS == "linux" || buildGOOS == "darwin") { + // The host default GC is boehm, where SetFinalizer is unimplemented, so + // the plain host run of finalizerinvariants.go is skipped above. Run it + // on the block GC instead, which also covers the two schedulers no other + // target in this suite reaches: threads (the host default) and none. + // Together with cortex-m-qemu (tasks), riscv-qemu (cores) and the wasm + // targets (asyncify), that covers every scheduler variant. + // + // Restricted to linux and darwin: internal/task only defines threadID + // for those two, so scheduler.threads does not build anywhere else, and + // scheduler.none does not link on Windows either. Both predate this test + // (they reproduce with any testdata file), so this skips rather than + // works around them. Same reasoning as TestTimerStopResetRace above. + for _, scheduler := range []string{"threads", "none"} { + t.Run("finalizerinvariants.go-gc-conservative-scheduler-"+scheduler, func(t *testing.T) { + t.Parallel() + options := compileopts.Options(options) + options.GC = "conservative" + options.Scheduler = scheduler + runTest("finalizerinvariants.go", options, t, nil, nil) + }) + } + } if options.Target == "" || isWASI { t.Run("filesystem.go", func(t *testing.T) { t.Parallel() diff --git a/src/internal/task/task_asyncify.go b/src/internal/task/task_asyncify.go index d7d9a6de42..9423e40447 100644 --- a/src/internal/task/task_asyncify.go +++ b/src/internal/task/task_asyncify.go @@ -25,6 +25,12 @@ type state struct { stackState launched bool + + // finishing is set immediately before this goroutine, having run to + // completion, pauses for the last time. Resume observes it and clears the + // goroutine's stack. It lives on the task so each finishing goroutine owns + // its own flag, independent of scheduler timing. + finishing bool } // stackState is the saved state of a stack while unwound. @@ -42,6 +48,11 @@ type stackState struct { // overwritten. It can be checked from time to time to see whether a stack // overflow happened in the past. canaryPtr *uintptr + + // top is the first address past the end of the stack allocation (the + // initial C stack pointer). Kept so the whole stack buffer can be located + // again after the goroutine finishes. + top unsafe.Pointer } // start creates and starts a new goroutine with the given function and arguments. @@ -78,6 +89,31 @@ func (s *state) initialize(fn uintptr, args unsafe.Pointer, stackSize uintptr) { // Calculate stack base addresses. s.asyncifysp = unsafe.Add(stack, unsafe.Sizeof(uintptr(0))) s.csp = unsafe.Add(stack, stackSize) + s.top = unsafe.Add(stack, stackSize) +} + +//go:linkname memzero runtime.memzero +func memzero(ptr unsafe.Pointer, size uintptr) + +// MarkFinishing records that the current goroutine has finished and will not be +// resumed, so Resume may reclaim its stack once control returns to the scheduler. +// The flag lives on the task itself, so each finishing goroutine owns its own and +// the handoff to Resume does not depend on scheduler timing. +func MarkFinishing() { + currentTask.state.finishing = true +} + +// clearStack zeroes a finished goroutine's entire stack buffer. The buffer is a +// plain heap allocation scanned conservatively by the GC (it can hold arbitrary +// pointers), so any stale pointer left in it by the goroutine's now-returned +// call frames would keep unrelated objects reachable (and, transitively, other +// finished stacks reachable through them) until a later collection happens to +// break the chain. Zeroing the buffer the moment the goroutine finishes drops +// those stale references immediately, so the objects they pointed at (and the +// stack itself) become collectable at the next cycle. +func (t *Task) clearStack() { + base := unsafe.Pointer(t.state.canaryPtr) + memzero(base, uintptr(t.state.top)-uintptr(base)) } // currentTask is the current running task, or nil if currently in the scheduler. @@ -123,6 +159,16 @@ func (t *Task) Resume() { if uintptr(t.state.asyncifysp) > uintptr(t.state.csp) { runtimeFatal("stack overflow") } + if t.state.finishing { + // The goroutine just ran to completion and paused for the last time. It + // will never be resumed, so its stack can be cleared now to drop any + // pointers its returned frames left behind (see clearStack). The args + // bundle is likewise no longer needed, so drop that reference too, else + // any pointers in the arguments would keep their objects reachable. + t.state.finishing = false + t.clearStack() + t.state.args = nil + } } //go:linkname saveStackPointer runtime.saveStackPointer diff --git a/src/internal/task/task_finishing_tasks.go b/src/internal/task/task_finishing_tasks.go new file mode 100644 index 0000000000..5ba351a58d --- /dev/null +++ b/src/internal/task/task_finishing_tasks.go @@ -0,0 +1,10 @@ +//go:build scheduler.tasks + +package task + +// MarkFinishing is a no-op for the stack-based scheduler. Zeroing a finished +// goroutine's stack to drop the stale pointers its returned frames leave behind +// is only implemented for the asyncify scheduler, whose goroutine stacks are +// heap buffers scanned conservatively (see the asyncify MarkFinishing and +// Resume). deadlock and goexit in the cooperative scheduler call this for both. +func MarkFinishing() {} diff --git a/src/runtime/gc_finalizer.go b/src/runtime/gc_finalizer.go index 13dd6f2c7e..103b9208bb 100644 --- a/src/runtime/gc_finalizer.go +++ b/src/runtime/gc_finalizer.go @@ -33,10 +33,33 @@ type finalizerEntry struct { fn interface{} } +// finalizerGCThreshold bounds how many finalizers may be registered since the +// last collection before the scheduler proactively runs one at its idle point. +// A registered finalizer almost always guards an external resource, most +// importantly a syscall/js bridge-table slot (js.Value or js.Func), that costs +// only a few bytes of Go heap but pins a whole JS object and its slot. Without +// this, a long-lived instance with a large resident heap defers GC (and thus +// finalizer draining) until the Go heap itself fills, which for a bursty, +// mostly-idle workload may be never, so the external resources accumulate +// without bound. Coupling a GC to finalizer-registration pressure caps that +// accumulation at roughly this many entries regardless of heap size. +// +// This is a compile-time policy constant, in the spirit of Go's forcegcperiod. +// The trigger only fires at the scheduler's idle point (a drained run queue) and +// each firing resets the count (see scanFinalizers), so it is throttled to that +// point rather than firing once per this-many registrations: a setup phase that +// registers many long-lived finalizers pays at most one extra collection at the +// first idle point after it, not one per threshold, and that collection just +// marks still-live data during otherwise-idle time without freeing anything +// early. Keeping it a const also lets the compiler constant-fold the check and, +// with zero, drop the pressure path entirely. Zero disables the trigger. +const finalizerGCThreshold = 32 + var ( finalizers *finalizerEntry // registered finalizers; a GC root that keeps fn values alive finalizerPending *finalizerEntry // finalizers whose object died, waiting to run numFinalizers uintptr // number of registered finalizers; fast-path gate for scanFinalizers + finalizersSinceGC uintptr // finalizers registered since the last GC; drives the scheduler idle-point pressure trigger finalizersQueued bool // set when scanFinalizers queued at least one finalizer to run finalizerFutex task.Futex // wakes the finalizerRunner goroutine after a GC queues work finalizerDraining bool // guards against re-entrant inline draining (scheduler.none) @@ -50,6 +73,108 @@ var ( finalizerRunnerStarted bool ) +// finalizerGCDivisor scales the registration trigger with the size of the +// table: the next collection is due after roughly numFinalizers/this many new +// registrations, never fewer than finalizerGCThreshold. +const finalizerGCDivisor = 2 + +// finalizerGCTrigger returns how many registrations since the last collection +// are needed to run the next one. Each collection scans the whole table, which +// costs O(numFinalizers), so a trigger that stays constant while the table grows +// makes N registrations cost O(N^2) in scanning alone. Scaling the trigger with +// the table keeps the amortized scan cost per registration constant, the same +// reasoning behind Go's proportional GOGC pacing: collect when the tracked set +// has grown by a fraction of itself, not by a fixed count. +// +// The floor keeps the original behaviour for small tables, where a proportional +// trigger would fire too rarely to be useful. +func finalizerGCTrigger() uintptr { + if finalizerGCThreshold == 0 { + return 0 + } + if proportional := numFinalizers / finalizerGCDivisor; proportional > finalizerGCThreshold { + return proportional + } + return finalizerGCThreshold +} + +// finalizerBits records, one bit per heap block, whether the object starting at +// that block already has a registered finalizer. It answers the "is this object +// already registered?" question that SetFinalizer's replace semantics require +// without walking the table, so the common case (a fresh object, which is every +// syscall/js value) never scans anything. +// +// This mirrors what upstream Go gets from its per-span specials plus the +// arena-level "span has specials" bitmap: a constant-time way to skip objects +// that have nothing registered. +// +// The bitmap is allocated on the first registration and grown with the heap, so +// a program that never registers a finalizer keeps the whole feature dead. +// +// Every access goes through gcLock, including the reads. The slice header itself +// is replaced when the heap grows, so an unlocked reader on a parallel scheduler +// (cores, threads) could observe a stale bit, or tear the header and index the +// old, shorter buffer with the new length. +var finalizerBits []byte + +// finalizerBitsShortfall returns the bitmap length needed to cover the current +// heap, or zero if the current bitmap already covers it. It must be called under +// gcLock: that is what makes reading finalizerBits and endBlock safe against a +// concurrent adoptFinalizerBits on another core. The caller then allocates with +// the lock released (allocating takes gcLock) and installs the result with +// adoptFinalizerBits. +func finalizerBitsShortfall() uintptr { + need := (uintptr(endBlock) + 7) / 8 + if uintptr(len(finalizerBits)) >= need { + return 0 + } + return need +} + +// adoptFinalizerBits installs a wider bitmap under gcLock, carrying the old bits +// over. A nil or already-obsolete buffer is ignored, which is what makes it safe +// for the heap to have grown again (or another core to have installed its own +// wider bitmap) while the caller was allocating with the lock released. A buffer +// that covers less than the current heap is still an improvement: addresses past +// its end just keep answering conservatively in finalizerBitGet. +func adoptFinalizerBits(buf []byte) { + if len(buf) <= len(finalizerBits) { + return + } + copy(buf, finalizerBits) + finalizerBits = buf +} + +func finalizerBitIndex(addr uintptr) uintptr { return uintptr(blockFromAddr(addr)) } + +func finalizerBitGet(addr uintptr) bool { + i := finalizerBitIndex(addr) + if i/8 >= uintptr(len(finalizerBits)) { + // The bitmap does not describe this address yet (the heap grew since it + // was sized). Answer conservatively: a spurious "maybe" only costs one + // scan, while a wrong "no" would let a second entry be registered for an + // object that already has one, and its finalizer would run twice. + return true + } + return finalizerBits[i/8]&(1<<(i%8)) != 0 +} + +func finalizerBitSet(addr uintptr) { + i := finalizerBitIndex(addr) + if i/8 >= uintptr(len(finalizerBits)) { + return + } + finalizerBits[i/8] |= 1 << (i % 8) +} + +func finalizerBitClear(addr uintptr) { + i := finalizerBitIndex(addr) + if i/8 >= uintptr(len(finalizerBits)) { + return + } + finalizerBits[i/8] &^= 1 << (i % 8) +} + // The object address is stored bitwise-NOT so it never looks like a live heap // pointer to the conservative scanner. Otherwise the entry would pin every // finalizable object forever and the object could never be detected as dead. @@ -58,6 +183,38 @@ var ( func encodeFinalizerPtr(addr uintptr) uintptr { return ^addr } func decodeFinalizerPtr(enc uintptr) uintptr { return ^enc } +// finalizerRegistered reports whether the table holds an entry for enc. This is +// the linear answer the registration bitmap exists to avoid, so it is only used +// under gcAsserts, to check the bitmap against the table it summarizes. +func finalizerRegistered(enc uintptr) bool { + for n := finalizers; n != nil; n = n.next { + if n.obj == enc { + return true + } + } + return false +} + +// assertFinalizerTable verifies the bookkeeping that the table, the counter and +// the registration bitmap have to agree on. A registered entry without its bit +// is the dangerous direction: the clear and replace paths trust a clear bit to +// mean "nothing registered" and skip the table walk, so a missing bit turns +// SetFinalizer(obj, nil) into a silent no-op and lets a re-registration add a +// second entry, which runs the finalizer twice. Only called under gcAsserts. +func assertFinalizerTable() { + var count uintptr + for n := finalizers; n != nil; n = n.next { + count++ + addr := decodeFinalizerPtr(n.obj) + if isOnHeap(addr) && !finalizerBitGet(addr) { + runtimeFatal("gc: registered finalizer without its bitmap bit") + } + } + if count != numFinalizers { + runtimeFatal("gc: numFinalizers does not match the finalizer table") + } +} + // registerFinalizer records fn as the finalizer for the object at addr. A nil fn // removes any registration for the object. Growing the table (allocating a node) // is the only allocation and it happens here, on the caller, never during GC. @@ -67,8 +224,27 @@ func registerFinalizer(addr uintptr, fn interface{}) { enc := encodeFinalizerPtr(addr) if fn == nil { - // Clear: remove every registration for this object. + // Clear: remove every registration for this object. The bit proves in + // one test that there is nothing to remove, but only while gcLock is + // held: a registration on another core may be setting that same bit (and + // replacing the bitmap) right now, and a stale read of zero would skip + // the removal and leave the finalizer registered on a live object. + // Holding the lock for the check costs nothing extra, because the removal + // below needs it anyway; what the bit saves is the O(numFinalizers) walk. gcLock.Lock() + tracked := isOnHeap(addr) + if tracked && !finalizerBitGet(addr) { + // Taking this shortcut on a stale bit would silently skip the + // removal, so check the answer against the table it stands in for. + if gcAsserts && finalizerRegistered(enc) { + runtimeFatal("gc: finalizer bit clear but the object is registered") + } + gcLock.Unlock() + return + } + if tracked { + finalizerBitClear(addr) + } prev := &finalizers for n := *prev; n != nil; n = *prev { if n.obj == enc { @@ -82,11 +258,30 @@ func registerFinalizer(addr uintptr, fn interface{}) { return } - // Register or replace. The allocation happens before gcLock is taken, - // because alloc acquires gcLock itself. + // Register or replace. Allocating acquires gcLock, so the entry is allocated + // before the lock is taken and a wider bitmap is allocated by dropping the + // lock for just that call. Only a heap that outgrew the bitmap pays that + // round trip; the common case holds the lock once, and adoptFinalizerBits + // tolerates the heap having grown again (or another core having installed a + // wider bitmap) while this one was allocating. entry := &finalizerEntry{obj: enc, fn: fn} gcLock.Lock() - for n := finalizers; n != nil; n = n.next { + if shortfall := finalizerBitsShortfall(); shortfall != 0 { + gcLock.Unlock() + wider := make([]byte, shortfall) + gcLock.Lock() + adoptFinalizerBits(wider) + } + tracked := isOnHeap(addr) + // Skipping the scan on a stale bit would add a second entry for an object + // that already has one, and its finalizer would then run twice. + if gcAsserts && tracked && !finalizerBitGet(addr) && finalizerRegistered(enc) { + runtimeFatal("gc: finalizer bit clear but the object is registered") + } + // Only an object whose bit is set can already be in the table, so a fresh + // object skips the scan entirely. An address the bitmap cannot describe + // (not on the heap) always scans, as before. + for n := finalizers; (!tracked || finalizerBitGet(addr)) && n != nil; n = n.next { if n.obj == enc { // Replace the finalizer for an already-registered object, so it // still runs only once (Go SetFinalizer replace semantics). @@ -105,7 +300,11 @@ func registerFinalizer(addr uintptr, fn interface{}) { } entry.next = finalizers finalizers = entry + if tracked { + finalizerBitSet(addr) + } numFinalizers++ + finalizersSinceGC++ // pressure signal for the proactive GC trigger at the scheduler's idle point // A finalizer is registered, so make sure the runner exists. The flag is // serialized by gcLock; the spawn itself allocates, so it must run after the // lock is released. @@ -121,6 +320,11 @@ func registerFinalizer(addr uintptr, fn interface{}) { // current GC cycle and queues their finalizers. It must be called under gcLock, // after marking is complete and before sweep frees anything. func scanFinalizers() { + // A collection is running now, so reset the registration-pressure counter + // that drives the proactive idle-point trigger, regardless of whether any + // finalizer is registered or fires this cycle. + finalizersSinceGC = 0 + // Nothing registered and nothing waiting to run: fast path. if numFinalizers == 0 && finalizerPending == nil { return @@ -146,6 +350,9 @@ func scanFinalizers() { // and into the pending queue (alloc-free), so its finalizer runs once. *prev = n.next numFinalizers-- + // The object is gone; clear its bit so a later object reusing the + // address starts clean. + finalizerBitClear(addr) n.next = finalizerPending finalizerPending = n finalizersQueued = true @@ -155,18 +362,38 @@ func scanFinalizers() { // found above and any queued by an earlier cycle that the runner has not // drained yet. Otherwise the next GC would not mark them (their only // reference is the encoded, scanner-invisible pending entry) and sweep would - // free them out from under a finalizer that hasn't run — a use-after-free. + // free them out from under a finalizer that hasn't run, a use-after-free. // Walking the pending list is safe: scanFinalizers and dequeueFinalizer are // both serialized under gcLock. var resurrected bool for n := finalizerPending; n != nil; n = n.next { - markRoot(0, decodeFinalizerPtr(n.obj)) + addr := decodeFinalizerPtr(n.obj) + if gcAsserts && !isOnHeap(addr) { + runtimeFatal("gc: pending finalizer for an object off the heap") + } + markRoot(0, addr) resurrected = true } if resurrected { // Re-scan so objects reachable only from resurrected objects also // survive this sweep. finishMark() + if gcAsserts { + // Every pending object must have survived the resurrection above. + // One that did not is about to be swept while its finalizer is + // still queued, which is a use-after-free in callFinalizer. + for n := finalizerPending; n != nil; n = n.next { + // Inside the collection, so the resurrected object is expected + // to carry the mark state rather than plain head. + if blockFromAddr(decodeFinalizerPtr(n.obj)).state() != blockStateMark { + runtimeFatal("gc: pending finalizer object was not resurrected") + } + } + } + } + + if gcAsserts { + assertFinalizerTable() } } @@ -219,12 +446,55 @@ func dequeueFinalizer() (*finalizerEntry, unsafe.Pointer) { var objPtr unsafe.Pointer if n != nil { finalizerPending = n.next - objPtr = unsafe.Pointer(decodeFinalizerPtr(n.obj)) + addr := decodeFinalizerPtr(n.obj) + if gcAsserts { + if !isOnHeap(addr) { + runtimeFatal("gc: dequeued finalizer for an object off the heap") + } + // scanFinalizers resurrects everything still pending, so sweep must + // have left the object allocated. A freed block here means + // callFinalizer is about to run on memory that is back in the free + // list. The mark bit is not the thing to check: this runs outside a + // collection, where unmark has already turned mark back into head. + if blockFromAddr(addr).state() == blockStateFree { + runtimeFatal("gc: dequeued finalizer for a freed object") + } + } + objPtr = unsafe.Pointer(addr) } gcLock.Unlock() return n, objPtr } +// finalizerPressureGC collects when at least finalizerGCThreshold finalizers +// have been registered since the last GC, then hands any freshly-queued +// finalizers to the runner. It reports whether it collected. A registered +// finalizer almost always guards an external resource whose Go-heap cost is tiny +// (a few bytes) relative to what it pins, so the registration count is a proxy +// for external memory pressure that the heap-size GC trigger cannot see. +// +// It is installed as the cooperative scheduler's idle hook by the first +// SetFinalizer (see spawnFinalizerRunner) and called only from the scheduler's +// drained-runqueue point, where no goroutine is running on its own stack. That +// reclaims a completed run of goroutines' now-dead values in a single pass, +// rather than forcing a collection synchronously inside alloc while an +// operation's values are still live, which would scale GC frequency with +// allocation churn and waste most collections on still-live values. +func finalizerPressureGC() bool { + trigger := finalizerGCTrigger() + if trigger == 0 || finalizersSinceGC < trigger { + return false + } + gcLock.Lock() + runGC() + gcLock.Unlock() + if finalizersQueued { + finalizersQueued = false + wakeFinalizer() + } + return true +} + // wakeFinalizer is called after a GC (with gcLock already released) that queued // finalizers. On schedulers with goroutines it wakes the finalizerRunner; on // scheduler.none it drains inline. diff --git a/src/runtime/gc_finalizer_sched.go b/src/runtime/gc_finalizer_sched.go index d983decc7c..7c2dcb4554 100644 --- a/src/runtime/gc_finalizer_sched.go +++ b/src/runtime/gc_finalizer_sched.go @@ -1,8 +1,13 @@ -//go:build (gc.conservative || gc.precise) && !scheduler.none +//go:build (gc.conservative || gc.precise) && (scheduler.tasks || scheduler.asyncify) package runtime -// The go statement lives in this scheduler-gated file, not inline in -// registerFinalizer, so scheduler.none builds never reference internal/task.start -// and the runner is DCE'd when SetFinalizer is unused. -func spawnFinalizerRunner() { go finalizerRunner() } +// The go statement and the idle-hook install live in this scheduler-gated file, +// not inline in registerFinalizer, so a build that never calls SetFinalizer +// keeps internal/task.start and the whole finalizer collection path DCE'd. The +// cooperative scheduler additionally collects on finalizer-registration pressure +// at its idle point (see finalizerIdleGC in scheduler_cooperative.go). +func spawnFinalizerRunner() { + finalizerIdleGC = finalizerPressureGC + go finalizerRunner() +} diff --git a/src/runtime/gc_finalizer_sched_other.go b/src/runtime/gc_finalizer_sched_other.go new file mode 100644 index 0000000000..dbade9d323 --- /dev/null +++ b/src/runtime/gc_finalizer_sched_other.go @@ -0,0 +1,15 @@ +//go:build (gc.conservative || gc.precise) && !scheduler.none && !scheduler.tasks && !scheduler.asyncify + +package runtime + +// spawnFinalizerRunner is defined once per scheduler class, and the three build +// constraints partition the scheduler space exactly (exactly one scheduler.* tag +// is ever set): scheduler.none in gc_finalizer_sched_none.go, scheduler.tasks and +// scheduler.asyncify in gc_finalizer_sched.go, and every other variant here. This +// is the catch-all, so a new scheduler variant lands here and stays defined +// rather than falling through to an undefined reference. +// +// Non-cooperative schedulers (cores, threads) spawn the finalizer runner but have +// no cooperative idle point, so they do not install the idle-pressure collector; +// the runner drains finalizers as GCs queue them. +func spawnFinalizerRunner() { go finalizerRunner() } diff --git a/src/runtime/scheduler_cooperative.go b/src/runtime/scheduler_cooperative.go index 6f8d6b0dae..e31e0caa73 100644 --- a/src/runtime/scheduler_cooperative.go +++ b/src/runtime/scheduler_cooperative.go @@ -41,6 +41,13 @@ var ( sleepQueueBaseTime timeUnit ) +// finalizerIdleGC, when non-nil, is called at the scheduler's idle point to +// collect on finalizer-registration pressure (returning whether it did). It is +// installed lazily by the first SetFinalizer, so a program that never registers +// a finalizer never assigns it and the linker drops the whole collection path. +// It is nil under GCs without a finalizer table. +var finalizerIdleGC func() bool + // deadlock is called when a goroutine cannot proceed any more, but is in theory // not exited (so deferred calls won't run). This can happen for example in code // like this, that blocks forever: @@ -49,12 +56,18 @@ var ( // //go:noinline func deadlock() { + // A goroutine reaches deadlock when it can make no further progress. The + // common case by far is a goroutine that ran to completion: the compiler + // emits a deadlock call at the end of every goroutine wrapper. Flag it so + // the scheduler can reclaim the finished goroutine's stack. + task.MarkFinishing() // call yield without requesting a wakeup task.Pause() runtimeFatal("unreachable") } func goexit() { + task.MarkFinishing() task.Exit() } @@ -183,6 +196,20 @@ func scheduler(returnAtDeadlock bool) { t := runqueue.Pop() if t == nil { + // Idle point: the run queue is drained, so no goroutine is running on + // its own stack. This is the safe place to reclaim external resources + // whose finalizers have piled up since the last collection. Running it + // here, once per drained run queue and only at the top level + // (task.Current() == nil, so a re-entrant call from a suspended + // goroutine does not collect while that goroutine is mid-operation), + // reclaims a completed run of goroutines' now-dead values in one pass. + // Forcing the collection inside alloc instead would scale GC frequency + // with allocation churn: a goroutine that allocates hundreds of + // short-lived finalized objects would trigger dozens of collections + // mid-run, most of them wasted on values that are still live. + if task.Current() == nil && finalizerIdleGC != nil && finalizerIdleGC() { + continue + } if sleepQueue == nil && timerQueue == nil { if returnAtDeadlock { return diff --git a/testdata/finalizerbits.go b/testdata/finalizerbits.go new file mode 100644 index 0000000000..5ab10de32c --- /dev/null +++ b/testdata/finalizerbits.go @@ -0,0 +1,212 @@ +package main + +// Tests the registration bookkeeping behind runtime.SetFinalizer on the block +// GC: the per-block bit that records whether an object already has a finalizer. +// The bit is what lets a fresh object skip the registered-finalizer scan, so +// these cases pin the invariants that skipping must never break: +// +// - an object whose finalizer was cleared and then registered again still runs +// it exactly once, so clearing resets the bookkeeping; +// - registering twice replaces, it never leaves two registrations behind +// (which would run the finalizer twice); +// - churning register/clear on one object leaves no residue; +// - memory reused by a later object registers correctly, so a dead object's +// bookkeeping does not leak onto whatever lands at its address next; +// - a batch where only some objects keep a finalizer runs exactly those. +// +// Like finalizer.go, this is only run on the precise wasm target (see the tests +// slice and the skip in main_test.go): there a dropped object is deterministically +// collected, so the finalizers fire predictably. +// +// Each test calls its alloc helper and scrubStack at the same call depth, so the +// recursion reuses and clears the frame that just held the dropped pointers. + +import "runtime" + +type box struct{ x int } + +const batch = 32 + +var ( + reregisteredRan int + replacedOldRan int + replacedNewRan int + churnRan int + reuseFirstRan int + reuseSecondRan int + keptRan int + droppedRan int + sink int +) + +// scrubStack overwrites the stack region used by an alloc-and-drop helper with +// non-pointer words. It must be called at the same call depth as that helper so +// this recursion reuses (and clears) the frame that just held the dropped +// pointer; otherwise a stale copy keeps the object marked and it is never +// collected. The returned value derived from buf keeps the writes live. +// +//go:noinline +func scrubStack(depth int) int { + if depth <= 0 { + return sink + } + var buf [64]int + for i := range buf { + buf[i] = depth + i + } + sink += buf[depth&63] + return scrubStack(depth-1) + buf[0] +} + +//go:noinline +func allocClearThenRegister() { + p := &box{x: 1} + runtime.SetFinalizer(p, func(*box) { panic("cleared finalizer ran") }) + runtime.SetFinalizer(p, nil) + runtime.SetFinalizer(p, func(*box) { reregisteredRan++ }) +} + +// testClearThenRegister checks that clearing a finalizer and registering a new +// one leaves exactly the new one: clearing has to reset the bookkeeping, not +// just unlink the entry. +func testClearThenRegister() { + allocClearThenRegister() + for i := 0; i < 200 && reregisteredRan == 0; i++ { + sink += scrubStack(40) + runtime.GC() + runtime.Gosched() + } + if reregisteredRan != 1 { + panic("finalizerbits: re-registered finalizer did not run exactly once") + } +} + +//go:noinline +func allocRegisterTwice() { + for i := 0; i < batch; i++ { + p := &box{x: i} + runtime.SetFinalizer(p, func(*box) { replacedOldRan++ }) + runtime.SetFinalizer(p, func(*box) { replacedNewRan++ }) + } +} + +// testRegisterTwiceLeavesOne checks the replace path over a whole batch: the +// second registration must find the first one and take its place. A missed +// lookup would leave two registrations for the same object, and its finalizer +// would run twice. +func testRegisterTwiceLeavesOne() { + allocRegisterTwice() + for i := 0; i < 200 && replacedNewRan < batch; i++ { + sink += scrubStack(40) + runtime.GC() + runtime.Gosched() + } + if replacedOldRan != 0 { + panic("finalizerbits: replaced finalizer still ran") + } + if replacedNewRan != batch { + panic("finalizerbits: replacement did not run exactly once per object") + } +} + +//go:noinline +func allocChurn() { + p := &box{x: 3} + for i := 0; i < 64; i++ { + runtime.SetFinalizer(p, func(*box) { churnRan++ }) + runtime.SetFinalizer(p, nil) + } +} + +// testChurnLeavesNothing checks that many register/clear rounds on one object +// leave nothing behind: the object dies with no finalizer, so nothing runs. +func testChurnLeavesNothing() { + allocChurn() + for i := 0; i < 200; i++ { + sink += scrubStack(40) + runtime.GC() + runtime.Gosched() + } + if churnRan != 0 { + panic("finalizerbits: churned register/clear left a live registration") + } +} + +//go:noinline +func allocFirstRound() { + for i := 0; i < batch; i++ { + p := &box{x: i} + runtime.SetFinalizer(p, func(*box) { reuseFirstRan++ }) + } +} + +//go:noinline +func allocSecondRound() { + for i := 0; i < batch; i++ { + p := &box{x: i} + runtime.SetFinalizer(p, func(*box) { reuseSecondRan++ }) + } +} + +// testAddressReuse checks that objects allocated into memory freed by a previous +// finalized batch register correctly themselves. A dead object's bookkeeping must +// not survive onto whatever lands at its address next. +func testAddressReuse() { + allocFirstRound() + for i := 0; i < 200 && reuseFirstRan < batch; i++ { + sink += scrubStack(40) + runtime.GC() + runtime.Gosched() + } + if reuseFirstRan != batch { + panic("finalizerbits: first round did not run every finalizer") + } + allocSecondRound() + for i := 0; i < 200 && reuseSecondRan < batch; i++ { + sink += scrubStack(40) + runtime.GC() + runtime.Gosched() + } + if reuseSecondRan != batch { + panic("finalizerbits: second round into reused memory lost finalizers") + } +} + +//go:noinline +func allocMixedBatch() { + for i := 0; i < batch; i++ { + p := &box{x: i} + if i%2 == 0 { + runtime.SetFinalizer(p, func(*box) { droppedRan++ }) + runtime.SetFinalizer(p, nil) + } else { + runtime.SetFinalizer(p, func(*box) { keptRan++ }) + } + } +} + +// testMixedBatch checks that clearing some registrations inside a batch affects +// only those objects: the ones still registered run, the cleared ones do not. +func testMixedBatch() { + allocMixedBatch() + for i := 0; i < 200 && keptRan < batch/2; i++ { + sink += scrubStack(40) + runtime.GC() + runtime.Gosched() + } + if droppedRan != 0 { + panic("finalizerbits: a cleared finalizer inside the batch ran") + } + if keptRan != batch/2 { + panic("finalizerbits: kept finalizers did not all run exactly once") + } +} + +func main() { + testClearThenRegister() + testRegisterTwiceLeavesOne() + testChurnLeavesNothing() + testAddressReuse() + testMixedBatch() + println("ok") +} diff --git a/testdata/finalizerbits.txt b/testdata/finalizerbits.txt new file mode 100644 index 0000000000..9766475a41 --- /dev/null +++ b/testdata/finalizerbits.txt @@ -0,0 +1 @@ +ok diff --git a/testdata/finalizeridle.go b/testdata/finalizeridle.go new file mode 100644 index 0000000000..172b82bd60 --- /dev/null +++ b/testdata/finalizeridle.go @@ -0,0 +1,148 @@ +package main + +// Tests that the cooperative scheduler reclaims finalizer-guarded objects on its +// own, without an explicit runtime.GC(), once enough finalizers have been +// registered since the last collection. A registered finalizer usually guards an +// external resource whose Go-heap cost is tiny relative to what it pins, so the +// registration count drives a proactive collection at the scheduler's idle +// point. The second case additionally checks that a finished goroutine's stack +// no longer pins the objects its frames held. +// +// Like finalizer.go, this is only run on the precise wasm target (see the tests +// slice and the skip in main_test.go): there a dropped object is deterministically +// collected, so the finalizers fire predictably. It never calls runtime.GC(): the +// point is that the idle-point trigger collects on its own. + +import ( + "runtime" + "time" +) + +// batch must exceed the runtime's finalizer-registration threshold so the idle +// collection is guaranteed to trigger. +const batch = 64 + +var ( + ranDropped int + ranOnStack int + ranInArgs int + sink int +) + +// scrubStack overwrites the stack region used by an alloc-and-drop helper with +// non-pointer words. It is called at the same depth as that helper so this +// recursion reuses (and clears) the frame that just held the dropped pointers; +// otherwise a stale copy keeps an object marked and it is never collected. The +// returned value derived from buf keeps the writes live. +// +//go:noinline +func scrubStack(depth int) int { + if depth <= 0 { + return sink + } + var buf [64]int + for i := range buf { + buf[i] = depth + i + } + sink += buf[depth&63] + return scrubStack(depth-1) + buf[0] +} + +// registerAndDrop registers `batch` finalizers and returns without leaking any +// reference to the objects, so they become unreachable. The finalizer must not +// capture its object (that would pin it forever): it takes the pointer as its +// argument and touches only a package global. +// +//go:noinline +func registerAndDrop() { + for i := 0; i < batch; i++ { + p := new([2]int) + runtime.SetFinalizer(p, func(*[2]int) { ranDropped++ }) + } +} + +// testIdleCollect checks that registering many finalizers and then only parking +// the goroutine (time.Sleep, never runtime.GC()) is enough for the objects to be +// collected and their finalizers to run. +func testIdleCollect() { + registerAndDrop() + for i := 0; i < 500 && ranDropped < batch; i++ { + sink += scrubStack(40) + time.Sleep(time.Millisecond) + } + if ranDropped != batch { + panic("idle collection did not run every finalizer") + } +} + +// testFinishedGoroutineStacks checks that a goroutine which registers a finalizer +// on a stack-local object and then returns no longer pins that object: once the +// goroutine has finished, the idle collection reclaims the object. Without +// zeroing a finished goroutine's conservatively scanned stack, the stale pointer +// would keep the object alive. +func testFinishedGoroutineStacks() { + done := make(chan struct{}) + for i := 0; i < batch; i++ { + go func() { + p := new([2]int) + runtime.SetFinalizer(p, func(*[2]int) { ranOnStack++ }) + // p stays on this goroutine's stack until it returns just below. + done <- struct{}{} + }() + } + for i := 0; i < batch; i++ { + <-done + } + for i := 0; i < 500 && ranOnStack < batch; i++ { + sink += scrubStack(40) + time.Sleep(time.Millisecond) + } + if ranOnStack != batch { + panic("finished goroutine stack still pinned finalized objects") + } +} + +// launchArgGoroutine allocates a finalized object and launches a goroutine that +// receives it as an argument, then returns without leaving any reference behind. +// The object reaches the goroutine only through its argument bundle, and the +// only transient copies (of the pointer and the bundle) live in this frame, which +// returns immediately so the later scrubStack recursion reuses and clears it. +// +//go:noinline +func launchArgGoroutine(done chan struct{}) { + p := new([2]int) + runtime.SetFinalizer(p, func(*[2]int) { ranInArgs++ }) + go func(q *[2]int) { + sink += q[0] + done <- struct{}{} + }(p) +} + +// testFinishedGoroutineArgs checks that a goroutine which receives a finalized +// object as an argument no longer pins it once finished: the argument bundle the +// goroutine was launched with is dropped when it completes, so the idle collection +// reclaims the object. Without clearing a finished goroutine's args pointer the +// bundle would keep the object alive even after its stack has been zeroed. +func testFinishedGoroutineArgs() { + done := make(chan struct{}) + for i := 0; i < batch; i++ { + launchArgGoroutine(done) + } + for i := 0; i < batch; i++ { + <-done + } + for i := 0; i < 500 && ranInArgs < batch; i++ { + sink += scrubStack(40) + time.Sleep(time.Millisecond) + } + if ranInArgs != batch { + panic("finished goroutine args still pinned finalized objects") + } +} + +func main() { + testIdleCollect() + testFinishedGoroutineStacks() + testFinishedGoroutineArgs() + println("ok") +} diff --git a/testdata/finalizeridle.txt b/testdata/finalizeridle.txt new file mode 100644 index 0000000000..9766475a41 --- /dev/null +++ b/testdata/finalizeridle.txt @@ -0,0 +1 @@ +ok diff --git a/testdata/finalizerinvariants.go b/testdata/finalizerinvariants.go new file mode 100644 index 0000000000..fd31e2579f --- /dev/null +++ b/testdata/finalizerinvariants.go @@ -0,0 +1,185 @@ +package main + +// Invariants of runtime.SetFinalizer that hold on every target the block GC +// supports, not only the ones where a dropped object is deterministically +// collected. +// +// finalizer.go, finalizerbits.go and finalizeridle.go all assert that a +// finalizer fired, which needs the dropped object to actually be collected, so +// they only run on wasm (see the skip in main_test.go). Conservative stack +// scanning elsewhere can keep a dropped object alive and the finalizer then +// correctly does not run. +// +// The opposite direction is portable: a conservative collector only ever +// over-retains, never under-retains, so "this finalizer must never run" holds +// on every target. Those are exactly the invariants the per-block registration +// bitmap can break, because a wrong bit skips the table walk that the clear and +// replace semantics of SetFinalizer depend on. A stale or torn bit therefore +// shows up here as a finalizer that runs when it must not. + +import ( + "runtime" + "sync/atomic" + "time" +) + +type obj struct{ x int } + +const batch = 8 + +// The counters are written by finalizers and read by main. Under +// scheduler.threads and scheduler.cores those are different threads running at +// the same time, so every access goes through sync/atomic rather than a plain +// int. +var ( + clearedRan atomic.Int32 + replacedRan atomic.Int32 + reachedRan atomic.Int32 + ranTwice atomic.Int32 + seen [batch]atomic.Int32 + reachable []*obj + sink int +) + +// scrubStack overwrites the stack region used by an alloc-and-drop helper with +// non-pointer words, so a stale frame does not keep the dropped object marked. +// It must be called at the same call depth as those helpers. +// +//go:noinline +func scrubStack(depth int) int { + if depth <= 0 { + return sink + } + var buf [16]int + for i := range buf { + buf[i] = depth + i + } + sink += buf[depth&15] + return scrubStack(depth-1) + buf[0] +} + +// dropCleared registers a finalizer, clears it, then drops the object. Clearing +// must remove the registration, so this finalizer may never run. +// +//go:noinline +func dropCleared() { + p := &obj{} + runtime.SetFinalizer(p, func(*obj) { clearedRan.Add(1) }) + runtime.SetFinalizer(p, nil) +} + +// dropReplaced registers a finalizer and then replaces it. Registering twice +// must replace rather than accumulate, so the first func may never run and the +// second may run at most once. +// +//go:noinline +func dropReplaced(id int) { + p := &obj{} + runtime.SetFinalizer(p, func(*obj) { replacedRan.Add(1) }) + runtime.SetFinalizer(p, func(*obj) { + if seen[id].Add(1) > 1 { + ranTwice.Add(1) + } + }) +} + +// keepReachable registers a finalizer on an object held by a global. A +// reachable object must never be finalized. +// +//go:noinline +func keepReachable(id int) { + p := &obj{x: id} + runtime.SetFinalizer(p, func(*obj) { reachedRan.Add(1) }) + reachable = append(reachable, p) +} + +// replacementsRan is how many of the batch replacement finalizers have run. +func replacementsRan() int { + n := 0 + for i := range seen { + n += int(seen[i].Load()) + } + return n +} + +// maxRounds bounds waitForDrain. Measured, every target here reaches the full +// count within a handful of rounds, so this is headroom for a loaded machine +// rather than an expected cost. +const maxRounds = 500 + +// waitForDrain collects until every replacement finalizer has run, and reports +// whether it got there. +// +// It waits for a known count rather than for the queue to look idle. Idleness +// cannot be observed from here: runtime exposes no way to ask whether the +// finalizer queue is empty, and under scheduler.threads the runner is a +// separate thread, so a stretch with no new invocation is indistinguishable +// from a runner that has simply not been scheduled yet. Waiting for a specific +// number of invocations has no such ambiguity, and not reaching it is a test +// failure rather than a silently short wait. +// +// batch is the right target because these objects are allocated and dropped +// inside a //go:noinline helper whose frame scrubStack then overwrites, so +// nothing is left pointing at them for a conservative scan to find. +func waitForDrain() bool { + for i := 0; i < maxRounds; i++ { + runtime.GC() + runtime.Gosched() + time.Sleep(time.Millisecond) + if replacementsRan() == batch { + // The runner has worked through the queue these objects were in. Do + // one more pass so that a finalizer which must NOT run, but which a + // wrong bitmap bit left registered, is queued and drained here + // instead of after the counters are read. + runtime.GC() + runtime.Gosched() + time.Sleep(time.Millisecond) + return true + } + } + return false +} + +func main() { + for i := 0; i < batch; i++ { + dropCleared() + } + scrubStack(12) + + for i := 0; i < batch; i++ { + dropReplaced(i) + } + scrubStack(12) + + for i := 0; i < batch; i++ { + keepReachable(i) + } + + drained := waitForDrain() + + // Touch the reachable set after the collections so it stays a live root + // across all of them. + total := 0 + for _, p := range reachable { + total += p.x + } + if total != batch*(batch-1)/2 { + println("FAIL: reachable set corrupted:", total) + return + } + + switch { + case !drained: + println("FAIL: only", replacementsRan(), "of", batch, "replacement finalizers ran, the assertions below prove nothing") + case clearedRan.Load() != 0: + println("FAIL: cleared finalizer ran:", clearedRan.Load()) + case replacedRan.Load() != 0: + println("FAIL: replaced finalizer ran:", replacedRan.Load()) + case reachedRan.Load() != 0: + println("FAIL: reachable object was finalized:", reachedRan.Load()) + case ranTwice.Load() != 0: + println("FAIL: finalizer ran more than once:", ranTwice.Load()) + default: + println("ok") + } +} diff --git a/testdata/finalizerinvariants.txt b/testdata/finalizerinvariants.txt new file mode 100644 index 0000000000..9766475a41 --- /dev/null +++ b/testdata/finalizerinvariants.txt @@ -0,0 +1 @@ +ok