diff --git a/cmd/ateom-microvm/internal/kata/agentclient.go b/cmd/ateom-microvm/internal/kata/agentclient.go index 12f7b0e6cd..ab04ebc77b 100644 --- a/cmd/ateom-microvm/internal/kata/agentclient.go +++ b/cmd/ateom-microvm/internal/kata/agentclient.go @@ -39,7 +39,7 @@ const debugConsoleVsockPort = 1026 // DebugConsoleDump connects to the guest's kata debug console (vsock 1026) and // runs cmd, returning its combined output. Diagnostic only (requires the guest to -// have booted with the WithDebugConsole kernel params). Best-effort: returns the +// have booted with the WithAgentDebug kernel params). Best-effort: returns the // error text on failure rather than failing the caller. func DebugConsoleDump(ctx context.Context, vsockPath, cmd string) string { d := net.Dialer{} diff --git a/cmd/ateom-microvm/internal/kata/config.go b/cmd/ateom-microvm/internal/kata/config.go index 9e605d7874..ba95222e43 100644 --- a/cmd/ateom-microvm/internal/kata/config.go +++ b/cmd/ateom-microvm/internal/kata/config.go @@ -26,28 +26,12 @@ const ( DefaultVCPUs = 1 ) -const ( - // baseKernelParams is the guest kernel command line parameters ateom boots with; - // there is no kata shim to inject them. - baseKernelParams = "cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1" - debugConsoleKernelParams = baseKernelParams + " agent.debug_console agent.debug_console_vport=1026" -) - -// WithDebugConsole returns the kernel params with the kata-agent debug console -// enabled, so the guest agent binds a root debug shell on vsock port 1026, which -// DebugConsoleDump connects to for in-guest diagnostics. Both params are required: -// agent.debug_console enables the console and agent.debug_console_vport=1026 makes -// the agent bind it on the vsock port (the agent only binds a vsock listener when -// the vport is > 0). -func WithDebugConsole() string { - return debugConsoleKernelParams -} +// BaseKernelParams is the guest kernel command line parameters ateom boots with +const BaseKernelParams = "cgroup_no_v1=all systemd.unified_cgroup_hierarchy=1" -// WithAgentDebug appends agent.log=debug so the guest kata-agent emits -// debug-level logs (including the failing path on errors) over its vsock log -// channel. Idempotent. +// WithAgentDebug enables guest agent debug logging and the debug console on vsock 1026 func WithAgentDebug(kernelParams string) string { - return appendKernelParams(kernelParams, "agent.log=", "agent.log=debug agent.debug_console") + return appendKernelParams(kernelParams, "agent.log=", "agent.log=debug agent.debug_console agent.debug_console_vport=1026") } // appendKernelParams appends add to a kernel_params string unless marker is diff --git a/cmd/ateom-microvm/internal/kata/config_test.go b/cmd/ateom-microvm/internal/kata/config_test.go index 4fdb19d86e..0d3c737aee 100644 --- a/cmd/ateom-microvm/internal/kata/config_test.go +++ b/cmd/ateom-microvm/internal/kata/config_test.go @@ -19,13 +19,26 @@ import ( "testing" ) +func TestBaseKernelParamsNoDebugConsole(t *testing.T) { + for _, forbidden := range []string{"agent.debug_console", "agent.debug_console_vport", "1026"} { + if strings.Contains(BaseKernelParams, forbidden) { + t.Errorf("expected BaseKernelParams %q not to contain %q", BaseKernelParams, forbidden) + } + } +} + func TestWithAgentDebug(t *testing.T) { - got := WithAgentDebug("root=/dev/vda1") - if !strings.Contains(got, "agent.log=debug") { - t.Errorf("WithAgentDebug did not append agent.log=debug: %q", got) + got := WithAgentDebug(BaseKernelParams) + for _, want := range []string{ + "agent.log=debug", + "agent.debug_console", + "agent.debug_console_vport=1026", + } { + if !strings.Contains(got, want) { + t.Errorf("expected WithAgentDebug(%q) to contain %q, but got %q", BaseKernelParams, want, got) + } } - // Idempotent: a second call must not append agent.log again. if again := WithAgentDebug(got); again != got { - t.Errorf("WithAgentDebug not idempotent:\n first = %q\nsecond = %q", got, again) + t.Errorf("expected WithAgentDebug to be idempotent, but got %q then %q", got, again) } } diff --git a/cmd/ateom-microvm/main.go b/cmd/ateom-microvm/main.go index 0a67fb66b3..3250b3dea6 100644 --- a/cmd/ateom-microvm/main.go +++ b/cmd/ateom-microvm/main.go @@ -61,7 +61,7 @@ import ( var ( podUID = pflag.String("pod-uid", "", "The UID of the current pod") chBinary = pflag.String("cloud-hypervisor-binary", "cloud-hypervisor", "Path to the cloud-hypervisor binary (used to relaunch on restore).") - kataDebug = pflag.Bool("kata-debug", false, "Verbose kata-agent debugging: raise the guest agent log level and forward the guest console (incl. agent logs) into the pod logs.") + guestDebug = pflag.Bool("guest-debug", false, "enable guest debugging (agent debug logs, debug console, and early serial console)") vmmMemReserve = pflag.Int("vmm-mem-reserve-mib", vmmMemReserveMiB, "Guest RAM (MiB) held back from the pod's memory limit for the cloud-hypervisor VMM + virtiofsd, which run as host processes in the pod cgroup alongside the guest RAM. Prevents the pod OOMing when the VM is sized to the pod's memory limit.") showVersion = pflag.Bool("version", false, "Print version and exit.") logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") @@ -220,7 +220,7 @@ func do(ctx context.Context) error { if err != nil { return err } - ateomService := NewService(*podUID, *chBinary, *kataDebug, *vmmMemReserve, *maxActors, tunnel, actorLogger) + ateomService := NewService(*podUID, *chBinary, *guestDebug, *vmmMemReserve, *maxActors, tunnel, actorLogger) ateomService.actorCgroups = actorCgroups svr := grpc.NewServer( @@ -329,9 +329,9 @@ type AteomService struct { // turned away instead of queueing behind it. shuttingDown atomic.Bool - podUID string - chBinary string - kataDebug bool + podUID string + chBinary string + guestDebug bool // memReserveMiB is guest RAM (MiB) held back from the pod's memory limit for // the cloud-hypervisor VMM + virtiofsd (host processes sharing the pod cgroup @@ -359,7 +359,7 @@ type AteomService struct { var _ ateompb.AteomServer = (*AteomService)(nil) // NewService creates a new AteomService. -func NewService(podUID, chBinary string, kataDebug bool, memReserveMiB, maxActors int, tunnel *ateomtunnel.Tunnel, actorLogger *actorlog.ActorLogger) *AteomService { +func NewService(podUID, chBinary string, guestDebug bool, memReserveMiB, maxActors int, tunnel *ateomtunnel.Tunnel, actorLogger *actorlog.ActorLogger) *AteomService { return &AteomService{ locks: actorlock.New(), inFlight: actorlock.NewInFlight(), @@ -367,7 +367,7 @@ func NewService(podUID, chBinary string, kataDebug bool, memReserveMiB, maxActor maxActors: maxActors, podUID: podUID, chBinary: chBinary, - kataDebug: kataDebug, + guestDebug: guestDebug, memReserveMiB: memReserveMiB, tunnel: tunnel, actorLogger: actorLogger, diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index 4b0b6873f8..1fcb0edf90 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -516,7 +516,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re // on a failed agent dial below, so keep it here. consoleLog := kata.ConsoleLogPath(actorUID) vmCfg := buildVMConfig(actorUID, kernel, image, kparams, consoleLog, memMiB, vcpus, - agentInit(ctx, client.Info()), s.kataDebug) + agentInit(ctx, client.Info()), s.guestDebug) if err := client.CreateVM(ctx, vmCfg); err != nil { return fmt.Errorf("while creating VM: %w", err) } @@ -713,12 +713,10 @@ func (s *AteomService) stageMergedRootfs(ctx context.Context, rr resolvedRuntime return vfsdCmd, nil } -// guestConfig returns the default guest sizing and the agent kernel params, enabling -// the debug console (vsock 1026) for in-guest diagnostics and, with kataDebug, raising -// the agent log level. +// guestConfig returns the default guest sizing and kernel params func (s *AteomService) guestConfig() (memMiB, vcpus int, kparams string) { - kparams = kata.WithDebugConsole() - if s.kataDebug { + kparams = kata.BaseKernelParams + if s.guestDebug { kparams = kata.WithAgentDebug(kparams) } return kata.DefaultMemoryMiB, kata.DefaultVCPUs, kparams @@ -802,7 +800,7 @@ func initParams(agentInit bool) string { // boot: measured host-launch to the agent's ttrpc accept, 1.24s -> 0.39s on a GKE // amd64 node and 21.6s -> 1.9s on a nested-virt arm64 kind node. What that costs is // the earliest messages: hvc0 only exists once virtio-console probes, so the memory -// map, CPU features and ACPI lines never reach the log. kataDebug adds the UART back +// map, CPU features and ACPI lines never reach the log. guestDebug adds the UART back // with earlycon (and pays the ~800ms) for diagnosing a guest that dies before then. func buildVMConfig(id, kernel, image, kparams, consoleLog string, memMiB, vcpus int, agentInit, debug bool) ch.VmConfig { cmdline := "root=/dev/vda1 rootflags=data=ordered,errors=remount-ro ro rootfstype=ext4 " + @@ -881,15 +879,17 @@ func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentC err = s.configureGuestNetwork(netCtx, ac, mtu) netCancel() if err != nil { - dump := kata.DebugConsoleDump(ctx, vsockPath, "ip addr 2>&1; echo '== route =='; ip route 2>&1; echo '== neigh =='; ip neigh 2>&1") - slog.ErrorContext(ctx, "guest network config failed; dump", slog.String("dump", dump)) + if s.guestDebug { + dump := kata.DebugConsoleDump(ctx, vsockPath, "ip addr 2>&1; echo '== route =='; ip route 2>&1; echo '== neigh =='; ip neigh 2>&1") + slog.ErrorContext(ctx, "guest network config failed; dump", slog.String("dump", dump)) + } return fmt.Errorf("while configuring guest network: %w", err) } tNetwork := time.Now() for _, c := range ctrs { - if err := startRootfsContainer(ctx, ac, vsockPath, c); err != nil { + if err := s.startRootfsContainer(ctx, ac, vsockPath, c); err != nil { return err } } @@ -906,16 +906,18 @@ func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentC // dumps the guest's view of the shared tree. // // Its spec binds every declared volume at its mount path. -func startRootfsContainer(ctx context.Context, ac *kata.AgentClient, vsockPath string, c actorContainer) error { +func (s *AteomService) startRootfsContainer(ctx context.Context, ac *kata.AgentClient, vsockPath string, c actorContainer) error { cCtx, cancel := context.WithTimeout(ctx, 30*time.Second) err := ac.StartRootfsContainer(cCtx, c.name, c.spec) cancel() if err != nil { - dump := kata.DebugConsoleDump(ctx, vsockPath, - "echo '== shared/containers =='; ls -la /run/kata-containers/shared/containers/ 2>&1 | head -40; "+ - "echo '== rootfs =='; ls /run/kata-containers/shared/containers/"+c.name+"/rootfs/ 2>&1 | head; "+ - "echo '== mounts =='; grep -E 'kata|virtiofs' /proc/mounts 2>&1") - slog.ErrorContext(ctx, "rootfs container failed; dump", slog.String("container", c.name), slog.String("dump", dump)) + if s.guestDebug { + dump := kata.DebugConsoleDump(ctx, vsockPath, + "echo '== shared/containers =='; ls -la /run/kata-containers/shared/containers/ 2>&1 | head -40; "+ + "echo '== rootfs =='; ls /run/kata-containers/shared/containers/"+c.name+"/rootfs/ 2>&1 | head; "+ + "echo '== mounts =='; grep -E 'kata|virtiofs' /proc/mounts 2>&1") + slog.ErrorContext(ctx, "rootfs container failed; dump", slog.String("container", c.name), slog.String("dump", dump)) + } return fmt.Errorf("while starting rootfs container %q: %w", c.name, err) } return nil diff --git a/cmd/ateom-microvm/run_test.go b/cmd/ateom-microvm/run_test.go index 9db11c3d6d..2d7bb92e7e 100644 --- a/cmd/ateom-microvm/run_test.go +++ b/cmd/ateom-microvm/run_test.go @@ -208,3 +208,19 @@ func TestWorkloadIDs(t *testing.T) { t.Errorf("workloadIDs() = %v, want %v", got, want) } } + +func TestGuestConfigDebugConsole(t *testing.T) { + _, _, prodParams := (&AteomService{guestDebug: false}).guestConfig() + for _, forbidden := range []string{"agent.debug_console", "agent.debug_console_vport", "1026"} { + if strings.Contains(prodParams, forbidden) { + t.Errorf("expected guestConfig() with guestDebug=false not to contain %q, but got %q", forbidden, prodParams) + } + } + + _, _, dbgParams := (&AteomService{guestDebug: true}).guestConfig() + for _, want := range []string{"agent.log=debug", "agent.debug_console", "agent.debug_console_vport=1026"} { + if !strings.Contains(dbgParams, want) { + t.Errorf("expected guestConfig() with guestDebug=true to contain %q, but got %q", want, dbgParams) + } + } +}