PGO: Run benchmarks for a shorter time, but force GC to run more often.

The PGO profiles don't need to run that long, but they do need to ensure
GC happens a few times to exercise GC-triggered codepaths (e.g. on
`sync.Pool` objects, network packet chunk pools, etc).

This change introduces a flag to force GC to run on a set interval during
CPU profiling, and enables it for PGO benchmark runs. It also reduces the
PGO benchmark time target, since GC is now guaranteed to run a couple of
times.

It also deletes all profiles, in order to force them to be refreshed
with this change in effect.

PiperOrigin-RevId: 797835198

Author: EtiennePerot

Makefile

@@ -548,7 +548,7 @@ run-benchmark: load-benchmarks ## Runs single benchmark and optionally sends dat
 # The arguments passed to benchmarks when run for PGO profile collection.
 # This should *not* include the `-profile` or `-profile-cpu` arguments, as
 # those are added automatically.
-BENCHMARKS_ARGS_PGO  := -test.v -test.bench=. -test.benchtime=30s
+BENCHMARKS_ARGS_PGO  := -test.v -test.bench=. -test.benchtime=20s
 # The threshold below which the `benchmark-refresh-pgo` rule will update the
 # profile.
 BENCHMARKS_PGO_REFRESH_THRESHOLD ?= 0.7
@@ -590,7 +590,7 @@ benchmark-refresh-pgo: load-benchmarks $(RUNTIME_BIN) ## Refresh profiles of all
 				fi; \
 				mkdir -p "$$(dirname "$${PGO_PROFILE_OLD}")"; \
 				mkdir -p "$${PLATFORM_TMPDIR}/$${PGO_BENCHMARK_BASENAME}"; \
-				$(call install_runtime,$${PLATFORM}_$${PGO_RUNTIME_KEY}_pgo_$${PGO_BENCHMARK_BASENAME},--platform $${PLATFORM} --profile --profile-cpu="$${PLATFORM_TMPDIR}/$${PGO_BENCHMARK_BASENAME}/$${PGO_BENCHMARK_BASENAME}.%YYYY%-%MM%-%DD%_%HH%-%II%-%SS%-%NN%.pgo.pprof.pb.gz"); \
+				$(call install_runtime,$${PLATFORM}_$${PGO_RUNTIME_KEY}_pgo_$${PGO_BENCHMARK_BASENAME},--platform $${PLATFORM} --profile --profile-gc-interval=3s --profile-cpu="$${PLATFORM_TMPDIR}/$${PGO_BENCHMARK_BASENAME}/$${PGO_BENCHMARK_BASENAME}.%YYYY%-%MM%-%DD%_%HH%-%II%-%SS%-%NN%.pgo.pprof.pb.gz"); \
 				$(call sudo,$${PGO_BENCHMARK_TARGET},-runtime=$${PLATFORM}_$${PGO_RUNTIME_KEY}_pgo_$${PGO_BENCHMARK_BASENAME} -pgo-benchmarks=true $(BENCHMARKS_ARGS_PGO)); \
 				$(call run,tools/profiletool,merge --out="$${PGO_PROFILE_NEW}" "$${PLATFORM_TMPDIR}/$${PGO_BENCHMARK_BASENAME}"); \
 				rm -rf --one-file-system "$${PLATFORM_TMPDIR}/$${PGO_BENCHMARK_BASENAME}"; \

runsc/cmd/boot.go

@@ -523,7 +523,7 @@ func (b *Boot) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomma
 		ProductName:         b.productName,
 		PodInitConfigFD:     b.podInitConfigFD,
 		SinkFDs:             b.sinkFDs.GetArray(),
-		ProfileOpts:         b.profileFDs.ToOpts(),
+		ProfileOpts:         profile.MakeOpts(&b.profileFDs, conf.ProfileGCInterval),
 		NvidiaDriverVersion: nvidiaDriverVersion,
 		HostTHP:             b.hostTHP,
 		SaveFDs:             b.saveFDs.GetFDs(),

runsc/cmd/gofer.go

@@ -201,7 +201,7 @@ func (g *Gofer) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomm
 	}
 
 	// Start profiling. This will be a noop if no profiling arguments were passed.
-	profileOpts := g.profileFDs.ToOpts()
+	profileOpts := profile.MakeOpts(&g.profileFDs, conf.ProfileGCInterval)
 	g.stopProfiling = profile.Start(profileOpts)
 
 	// At this point we won't re-execute, so it's safe to limit via rlimits. Any
@@ -278,7 +278,7 @@ func (g *Gofer) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcomm
 	opts := filter.Options{
 		UDSOpenEnabled:   conf.GetHostUDS().AllowOpen(),
 		UDSCreateEnabled: conf.GetHostUDS().AllowCreate(),
-		ProfileEnabled:   len(profileOpts) > 0,
+		ProfileEnabled:   profileOpts.Enabled(),
 		DirectFS:         conf.DirectFS,
 		CgoEnabled:       config.CgoEnabled,
 	}

runsc/config/config.go

@@ -24,6 +24,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 
 	"gvisor.dev/gvisor/pkg/log"
 	"gvisor.dev/gvisor/pkg/refs"
@@ -230,6 +231,15 @@ type Config struct {
 	// duration of the container execution. Requires ProfileEnabled.
 	ProfileCPU string `flag:"profile-cpu"`
 
+	// ProfileGCInterval forces a garbage-collection cycle at regular intervals.
+	// Useful for benchmark runs where the GC threshold may not be reached
+	// but for which it is useful to have GC (and the effects of GC) show
+	// up in the traces, while keeping the profile short.
+	// Requires `ProfileEnable`, and triggered when any other type of
+	// time-based profiling is enabled.
+	// If zero, GC happens per the Go runtime's default behavior.
+	ProfileGCInterval time.Duration `flag:"profile-gc-interval"`
+
 	// ProfileHeap collects a heap profile to the passed file for the
 	// duration of the container execution. Requires ProfileEnabled.
 	ProfileHeap string `flag:"profile-heap"`

runsc/config/flags.go

@@ -99,6 +99,7 @@ func RegisterFlags(flagSet *flag.FlagSet) {
 	flagSet.Bool("profile", false, "prepares the sandbox to use Golang profiler. Note that enabling profiler loosens the seccomp protection added to the sandbox (DO NOT USE IN PRODUCTION).")
 	flagSet.String("profile-block", "", "collects a block profile to this file path for the duration of the container execution. Requires -profile=true.")
 	flagSet.String("profile-cpu", "", "collects a CPU profile to this file path for the duration of the container execution. Requires -profile=true.")
+	flagSet.Duration("profile-gc-interval", 0, "forces a garbage collection cycle every this duration while another type of time-based profiling is enabled. Requires -profile=true.")
 	flagSet.String("profile-heap", "", "collects a heap profile to this file path for the duration of the container execution. Requires -profile=true.")
 	flagSet.String("profile-mutex", "", "collects a mutex profile to this file path for the duration of the container execution. Requires -profile=true.")
 	flagSet.String("trace", "", "collects a Go runtime execution trace to this file path for the duration of the container execution.")

runsc/profile/profile.go

@@ -77,26 +77,46 @@ func (fds *FDArgs) SetFromFlags(f *flag.FlagSet) {
 	f.IntVar(&fds.TraceFD, "trace-fd", -1, "file descriptor to write Go execution trace to. -1 disables tracing.")
 }
 
-// Opts is a map of profile Kind to FD.
-type Opts map[Kind]uintptr
+// Opts is a struct that holds the options for profiling.
+type Opts struct {
+	// FDs is a map of profile Kind to FD.
+	FDs map[Kind]uintptr
 
-// ToOpts turns FDArgs into an Opts struct which can be passed to Start.
-func (fds *FDArgs) ToOpts() Opts {
-	o := Opts{}
+	// GCInterval is the interval at which to force a garbage collection cycle.
+	// If zero, GC happens per the Go runtime's default behavior.
+	GCInterval time.Duration
+}
+
+// Enabled returns true if any profile type is enabled.
+func (opts Opts) Enabled() bool {
+	for _, fd := range opts.FDs {
+		if fd != 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// MakeOpts creates an Opts struct from the given FDArgs and GC interval.
+func MakeOpts(fds *FDArgs, gcInterval time.Duration) Opts {
+	o := Opts{
+		FDs:        make(map[Kind]uintptr),
+		GCInterval: gcInterval,
+	}
 	if fds.BlockFD >= 0 {
-		o[Block] = uintptr(fds.BlockFD)
+		o.FDs[Block] = uintptr(fds.BlockFD)
 	}
 	if fds.CPUFD >= 0 {
-		o[CPU] = uintptr(fds.CPUFD)
+		o.FDs[CPU] = uintptr(fds.CPUFD)
 	}
 	if fds.HeapFD >= 0 {
-		o[Heap] = uintptr(fds.HeapFD)
+		o.FDs[Heap] = uintptr(fds.HeapFD)
 	}
 	if fds.MutexFD >= 0 {
-		o[Mutex] = uintptr(fds.MutexFD)
+		o.FDs[Mutex] = uintptr(fds.MutexFD)
 	}
 	if fds.TraceFD >= 0 {
-		o[Trace] = uintptr(fds.TraceFD)
+		o.FDs[Trace] = uintptr(fds.TraceFD)
 	}
 	return o
 }
@@ -112,11 +132,41 @@ func Start(opts Opts) func() {
 		}
 	}
 
-	if fd, ok := opts[Block]; ok {
+	var stopPeriodicGC chan struct{}
+	maybeEnablePeriodicGC := func() {
+		if opts.GCInterval > 0 && stopPeriodicGC == nil {
+			log.Infof("Periodic garbage collection enabled during profiling; will run GC every %v.", opts.GCInterval)
+			stopPeriodicGC = make(chan struct{}, 1)
+			stoppedCh := make(chan struct{}, 1)
+			go func() {
+				ticker := time.NewTicker(opts.GCInterval)
+				defer ticker.Stop()
+				for {
+					select {
+					case <-stopPeriodicGC:
+						stoppedCh <- struct{}{}
+						close(stoppedCh)
+						return
+					case <-ticker.C:
+						log.Debugf("Forcing garbage collection per profile options")
+						runtime.GC()
+					}
+				}
+			}()
+			onStopProfiling = append(onStopProfiling, func() {
+				stopPeriodicGC <- struct{}{}
+				close(stopPeriodicGC)
+				<-stoppedCh // Wait for the goroutine to exit; this ensures periodic GC is not going to run further.
+			})
+		}
+	}
+
+	if fd, ok := opts.FDs[Block]; ok {
 		log.Infof("Block profiling enabled")
 		file := os.NewFile(fd, "profile-block")
 
 		runtime.SetBlockProfileRate(control.DefaultBlockProfileRate)
+		maybeEnablePeriodicGC()
 		onStopProfiling = append(onStopProfiling, func() {
 			if err := pprof.Lookup("block").WriteTo(file, 0); err != nil {
 				log.Warningf("Error writing block profile: %v", err)
@@ -127,19 +177,20 @@ func Start(opts Opts) func() {
 		})
 	}
 
-	if fd, ok := opts[CPU]; ok {
+	if fd, ok := opts.FDs[CPU]; ok {
 		log.Infof("CPU profiling enabled")
 		file := os.NewFile(fd, "profile-cpu")
 
 		pprof.StartCPUProfile(file)
+		maybeEnablePeriodicGC()
 		onStopProfiling = append(onStopProfiling, func() {
 			pprof.StopCPUProfile()
 			file.Close()
 			log.Infof("CPU profiling stopped")
 		})
 	}
 
-	if fd, ok := opts[Heap]; ok {
+	if fd, ok := opts.FDs[Heap]; ok {
 		log.Infof("Heap profiling enabled")
 		file := os.NewFile(fd, "profile-heap")
 
@@ -152,11 +203,12 @@ func Start(opts Opts) func() {
 		})
 	}
 
-	if fd, ok := opts[Mutex]; ok {
+	if fd, ok := opts.FDs[Mutex]; ok {
 		log.Infof("Mutex profiling enabled")
 		file := os.NewFile(fd, "profile-mutex")
 
 		prev := runtime.SetMutexProfileFraction(control.DefaultMutexProfileRate)
+		maybeEnablePeriodicGC()
 		onStopProfiling = append(onStopProfiling, func() {
 			if err := pprof.Lookup("mutex").WriteTo(file, 0); err != nil {
 				log.Warningf("Error writing mutex profile: %v", err)
@@ -167,7 +219,7 @@ func Start(opts Opts) func() {
 		})
 	}
 
-	if fd, ok := opts[Trace]; ok {
+	if fd, ok := opts.FDs[Trace]; ok {
 		log.Infof("Tracing enabled")
 		file := os.NewFile(fd, "trace")