Add confidential WCOW debug mode and boot files overrides

cmd/containerd-shim-runhcs-v1/pod.go

@@ -49,8 +49,8 @@ func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs
 			return fmt.Errorf("security policy (confidential mode) only works with block CIM based layers")
 		}
 		// we use measured EFI & rootfs for confidential UVMs, use those instead of the ones passed in layers/rootfs
-		wopts.BootFiles.BlockCIMFiles.EFIVHDPath = uvm.GetDefaultConfidentialEFIPath()
-		wopts.BootFiles.BlockCIMFiles.BootCIMVHDPath = uvm.GetDefaultConfidentialBootCIMPath()
+		wopts.BootFiles.BlockCIMFiles.EFIVHDPath = uvm.ConfidentialEFIPath(wopts.BootFilesRootPath)
+		wopts.BootFiles.BlockCIMFiles.BootCIMVHDPath = uvm.ConfidentialBootCIMPath(wopts.BootFilesRootPath)
 
 		// make a copy of the vmgs file as the same vmgs can not be used by multiple pods in parallel
 		// TODO(ambarve): for C-LCOW we make a copy in the bundle directory, is it better

internal/oci/uvm.go

@@ -251,9 +251,8 @@ func handleWCOWSecurityPolicy(ctx context.Context, a map[string]string, wopts *u
 
 	wopts.SecurityPolicyEnforcer = ParseAnnotationsString(a, annotations.WCOWSecurityPolicyEnforcer, wopts.SecurityPolicyEnforcer)
 	wopts.DisableSecureBoot = ParseAnnotationsBool(ctx, a, annotations.WCOWDisableSecureBoot, false)
-	wopts.GuestStateFilePath = ParseAnnotationsString(a, annotations.WCOWGuestStateFile, uvm.GetDefaultConfidentialVMGSPath())
-	wopts.UVMReferenceInfoFile = ParseAnnotationsString(a, annotations.WCOWReferenceInfoFile, uvm.GetDefaultReferenceInfoFilePath())
-	wopts.UVMHashEnvelopeReferenceInfoFile = ParseAnnotationsString(a, annotations.UVMHashEnvelopeReferenceInfoFile, uvm.GetDefaultHashEnvelopeReferenceInfoFilePath())
+
+	// Resolve the isolation type first so the default VMGS file can be selected accordingly.
 	wopts.IsolationType = "SecureNestedPaging"
 	if noSecurityHardware := ParseAnnotationsBool(ctx, a, annotations.NoSecurityHardware, false); noSecurityHardware {
 		wopts.IsolationType = "GuestStateOnly"
@@ -262,6 +261,10 @@ func handleWCOWSecurityPolicy(ctx context.Context, a map[string]string, wopts *u
 		return err
 	}
 
+	wopts.GuestStateFilePath = ParseAnnotationsString(a, annotations.WCOWGuestStateFile, uvm.ConfidentialVMGSPath(wopts.BootFilesRootPath, wopts.IsolationType))
+	wopts.UVMReferenceInfoFile = ParseAnnotationsString(a, annotations.WCOWReferenceInfoFile, uvm.ConfidentialReferenceInfoFilePath(wopts.BootFilesRootPath))
+	wopts.UVMHashEnvelopeReferenceInfoFile = ParseAnnotationsString(a, annotations.UVMHashEnvelopeReferenceInfoFile, uvm.ConfidentialHashEnvelopeReferenceInfoFilePath(wopts.BootFilesRootPath))
+
 	return nil
 }
 
@@ -420,10 +423,27 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) (
 		// Writable EFI is valid for both confidential and regular Hyper-V isolated WCOW.
 		wopts.WritableEFI = ParseAnnotationsBool(ctx, s.Annotations, annotations.WCOWWritableEFI, wopts.WritableEFI)
 
+		// Debug mode (confidential WCOW only) saves the boot/EFI and scratch VHDs on teardown.
+		wopts.DebugMode = ParseAnnotationsBool(ctx, s.Annotations, annotations.WCOWDebugMode, wopts.DebugMode)
+		wopts.DebugDataPath = ParseAnnotationsString(s.Annotations, annotations.WCOWDebugDataPath, wopts.DebugDataPath)
+
+		// Optional override for the directory containing the confidential WCOW boot files.
+		wopts.BootFilesRootPath = ParseAnnotationsString(s.Annotations, annotations.BootFilesRootPath, wopts.BootFilesRootPath)
+
 		// Handle WCOW security policy settings
 		if err := handleWCOWSecurityPolicy(ctx, s.Annotations, wopts); err != nil {
 			return nil, err
 		}
+
+		// Debug mode is only supported for confidential WCOW and requires a non-empty data path.
+		if wopts.DebugMode {
+			if !wopts.SecurityPolicyEnabled {
+				return nil, fmt.Errorf("%q is only supported for confidential WCOW", annotations.WCOWDebugMode)
+			}
+			if wopts.DebugDataPath == "" {
+				return nil, fmt.Errorf("%q must be set to a non-empty path when %q is enabled", annotations.WCOWDebugDataPath, annotations.WCOWDebugMode)
+			}
+		}
 		// If security policy is enable, wopts.LogForwardingEnabled default value should be false (CWCOW should not allow log forwarding by default)
 		if wopts.SecurityPolicyEnabled {
 			wopts.LogForwardingEnabled = false

internal/oci/uvm_test.go

@@ -263,6 +263,109 @@ func Test_SpecToUVMCreateOptions_WCOW_Confidential_Overrides(t *testing.T) {
 	}
 }
 
+func Test_SpecToUVMCreateOptions_WCOW_Confidential_BootFilesRootPath(t *testing.T) {
+	root := "C:\\custom\\bootfiles"
+	s := &specs.Spec{
+		Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}},
+		Annotations: map[string]string{
+			annotations.WCOWSecurityPolicy: "test-policy",
+			annotations.BootFilesRootPath:  root,
+		},
+	}
+
+	opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "")
+	if err != nil {
+		t.Fatalf("unexpected error generating UVM opts: %v", err)
+	}
+
+	wopts := (opts).(*uvm.OptionsWCOW)
+	if wopts.BootFilesRootPath != root {
+		t.Fatalf("expected BootFilesRootPath=%q, got %q", root, wopts.BootFilesRootPath)
+	}
+	// Defaults derived from the boot files root (SNP isolation by default).
+	if want := uvm.ConfidentialVMGSPath(root, "SecureNestedPaging"); wopts.GuestStateFilePath != want {
+		t.Fatalf("expected GuestStateFilePath=%q, got %q", want, wopts.GuestStateFilePath)
+	}
+	if want := uvm.ConfidentialReferenceInfoFilePath(root); wopts.UVMReferenceInfoFile != want {
+		t.Fatalf("expected UVMReferenceInfoFile=%q, got %q", want, wopts.UVMReferenceInfoFile)
+	}
+}
+
+func Test_SpecToUVMCreateOptions_WCOW_Confidential_VMGSDefaultByIsolation(t *testing.T) {
+	s := &specs.Spec{
+		Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}},
+		Annotations: map[string]string{
+			annotations.WCOWSecurityPolicy: "test-policy",
+			annotations.WCOWIsolationType:  "VirtualizationBasedSecurity",
+		},
+	}
+
+	opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "")
+	if err != nil {
+		t.Fatalf("unexpected error generating UVM opts: %v", err)
+	}
+
+	wopts := (opts).(*uvm.OptionsWCOW)
+	// With no explicit VMGS override, the default VMGS must follow the isolation type.
+	if want := uvm.ConfidentialVMGSPath("", "VirtualizationBasedSecurity"); wopts.GuestStateFilePath != want {
+		t.Fatalf("expected GuestStateFilePath=%q for VBS isolation, got %q", want, wopts.GuestStateFilePath)
+	}
+}
+
+func Test_SpecToUVMCreateOptions_WCOW_Confidential_DebugMode_Enabled(t *testing.T) {
+	dataPath := "C:\\debug\\data"
+	s := &specs.Spec{
+		Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}},
+		Annotations: map[string]string{
+			annotations.WCOWSecurityPolicy: "test-policy",
+			annotations.WCOWDebugMode:      "true",
+			annotations.WCOWDebugDataPath:  dataPath,
+		},
+	}
+
+	opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "")
+	if err != nil {
+		t.Fatalf("unexpected error generating UVM opts: %v", err)
+	}
+
+	wopts := (opts).(*uvm.OptionsWCOW)
+	if !wopts.DebugMode {
+		t.Fatal("DebugMode should be true when WCOWDebugMode is set")
+	}
+	if wopts.DebugDataPath != dataPath {
+		t.Fatalf("expected DebugDataPath=%q, got %q", dataPath, wopts.DebugDataPath)
+	}
+}
+
+func Test_SpecToUVMCreateOptions_WCOW_Confidential_DebugMode_ErrorOnMissingDataPath(t *testing.T) {
+	s := &specs.Spec{
+		Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}},
+		Annotations: map[string]string{
+			annotations.WCOWSecurityPolicy: "test-policy",
+			annotations.WCOWDebugMode:      "true",
+		},
+	}
+
+	if _, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), ""); err == nil {
+		t.Fatal("expected error when debug mode is enabled without a data path, got nil")
+	}
+}
+
+func Test_SpecToUVMCreateOptions_WCOW_NonConfidential_DebugMode_Error(t *testing.T) {
+	s := &specs.Spec{
+		Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}},
+		Annotations: map[string]string{
+			// No WCOWSecurityPolicy means non-confidential path.
+			annotations.WCOWDebugMode:     "true",
+			annotations.WCOWDebugDataPath: "C:\\debug\\data",
+		},
+	}
+
+	if _, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), ""); err == nil {
+		t.Fatal("expected error when debug mode is enabled for non-confidential WCOW, got nil")
+	}
+}
+
 func Test_SpecToUVMCreateOptions_Common(t *testing.T) {
 	cpugroupid := "1"
 	lowmmiogap := 1024

internal/tools/uvmboot/conf_wcow.go

@@ -23,6 +23,8 @@ const (
 	disableSBArgName     = "disable-secure-boot"
 	isolationTypeArgName = "isolation-type"
 	writableEFIArgName   = "writable-efi"
+	debugModeArgName     = "debug-mode"
+	debugDataPathArgName = "debug-data-path"
 
 	// default policy (that allows all operations) used when no policy is provided
 	allowAllPolicy = "cGFja2FnZSBwb2xpY3kKCmFwaV92ZXJzaW9uIDo9ICIwLjExLjAiCmZyYW1ld29ya192ZXJzaW9uIDo9ICIwLjQuMCIKCm1vdW50X2NpbXMgOj0geyJhbGxvd2VkIjogdHJ1ZX0KbW91bnRfZGV2aWNlIDo9IHsiYWxsb3dlZCI6IHRydWV9Cm1vdW50X292ZXJsYXkgOj0geyJhbGxvd2VkIjogdHJ1ZX0KY3JlYXRlX2NvbnRhaW5lciA6PSB7ImFsbG93ZWQiOiB0cnVlLCAiZW52X2xpc3QiOiBudWxsLCAiYWxsb3dfc3RkaW9fYWNjZXNzIjogdHJ1ZX0KdW5tb3VudF9kZXZpY2UgOj0geyJhbGxvd2VkIjogdHJ1ZX0KdW5tb3VudF9vdmVybGF5IDo9IHsiYWxsb3dlZCI6IHRydWV9CmV4ZWNfaW5fY29udGFpbmVyIDo9IHsiYWxsb3dlZCI6IHRydWUsICJlbnZfbGlzdCI6IG51bGx9CmV4ZWNfZXh0ZXJuYWwgOj0geyJhbGxvd2VkIjogdHJ1ZSwgImVudl9saXN0IjogbnVsbCwgImFsbG93X3N0ZGlvX2FjY2VzcyI6IHRydWV9CnNodXRkb3duX2NvbnRhaW5lciA6PSB7ImFsbG93ZWQiOiB0cnVlfQpzaWduYWxfY29udGFpbmVyX3Byb2Nlc3MgOj0geyJhbGxvd2VkIjogdHJ1ZX0KcGxhbjlfbW91bnQgOj0geyJhbGxvd2VkIjogdHJ1ZX0KcGxhbjlfdW5tb3VudCA6PSB7ImFsbG93ZWQiOiB0cnVlfQpnZXRfcHJvcGVydGllcyA6PSB7ImFsbG93ZWQiOiB0cnVlfQpkdW1wX3N0YWNrcyA6PSB7ImFsbG93ZWQiOiB0cnVlfQpydW50aW1lX2xvZ2dpbmcgOj0geyJhbGxvd2VkIjogdHJ1ZX0KbG9hZF9mcmFnbWVudCA6PSB7ImFsbG93ZWQiOiB0cnVlfQpzY3JhdGNoX21vdW50IDo9IHsiYWxsb3dlZCI6IHRydWV9CnNjcmF0Y2hfdW5tb3VudCA6PSB7ImFsbG93ZWQiOiB0cnVlfQo="
@@ -33,10 +35,13 @@ var (
 	cwcowEFIVHD            string
 	cwcowScratchVHD        string
 	cwcowVMGSPath          string
+	cwcowBootFilesPath     string
 	cwcowDisableSecureBoot bool
 	cwcowIsolationMode     string
 	cwcowSecurityPolicy    string
 	cwcowWritableEFI       bool
+	cwcowDebugMode         bool
+	cwcowDebugDataPath     string
 )
 
 var cwcowCommand = cli.Command{
@@ -55,15 +60,13 @@ var cwcowCommand = cli.Command{
 		},
 		cli.StringFlag{
 			Name:        "efi-vhd",
-			Usage:       "VHD at the provided path MUST have the EFI boot partition and be properly formatted for UEFI boot.",
+			Usage:       "VHD at the provided path MUST have the EFI boot partition and be properly formatted for UEFI boot. Ignored when --" + bootFilesPathArgName + " is provided.",
 			Destination: &cwcowEFIVHD,
-			Required:    true,
 		},
 		cli.StringFlag{
 			Name:        "boot-cim-vhd",
-			Usage:       "A VHD containing the block CIM that contains the OS files.",
+			Usage:       "A VHD containing the block CIM that contains the OS files. Ignored when --" + bootFilesPathArgName + " is provided.",
 			Destination: &cwcowBootVHD,
-			Required:    true,
 		},
 		cli.StringFlag{
 			Name:        "scratch-vhd",
@@ -73,9 +76,13 @@ var cwcowCommand = cli.Command{
 		},
 		cli.StringFlag{
 			Name:        vmgsFilePathArgName,
-			Usage:       "VMGS file path (only applies when confidential mode is enabled). This option is only applicable in confidential mode.",
+			Usage:       "VMGS file path (only applies when confidential mode is enabled). Ignored when --" + bootFilesPathArgName + " is provided.",
 			Destination: &cwcowVMGSPath,
-			Required:    true,
+		},
+		cli.StringFlag{
+			Name:        bootFilesPathArgName,
+			Usage:       "Directory containing the confidential boot files. When provided, the EFI VHD (boot.vhd), boot CIM VHD (rootfs.vhd) and VMGS file (selected from --" + isolationTypeArgName + ") are derived from it, and --efi-vhd/--boot-cim-vhd/--" + vmgsFilePathArgName + " must not be set. Provide either this or all three individual files.",
+			Destination: &cwcowBootFilesPath,
 		},
 		cli.BoolFlag{
 			Name:        disableSBArgName,
@@ -99,9 +106,50 @@ var cwcowCommand = cli.Command{
 			Usage:       "Attaches the EFI VHD as read-write instead of read-only. This allows the UVM to modify the contents of the VHD, be careful when using this option!",
 			Destination: &cwcowWritableEFI,
 		},
+		cli.BoolFlag{
+			Name:        debugModeArgName,
+			Usage:       "Enables debug mode: the boot/EFI VHD and scratch VHD are saved to the directory given by --" + debugDataPathArgName + " when the UVM is torn down. Combine with --" + writableEFIArgName + " to capture the bootstat trace.",
+			Destination: &cwcowDebugMode,
+		},
+		cli.StringFlag{
+			Name:        debugDataPathArgName,
+			Usage:       "Directory to save the boot/EFI VHD and scratch VHD to when --" + debugModeArgName + " is enabled. Required when debug mode is enabled.",
+			Destination: &cwcowDebugDataPath,
+		},
 	},
 	Action: func(c *cli.Context) error {
 		runMany(c, func(id string) error {
+			// Resolve the boot files: either derive them from a single directory or use the
+			// individually-provided VHD/VMGS paths. The scratch VHD is always provided separately.
+			efiVHD, bootVHD, vmgsPath := cwcowEFIVHD, cwcowBootVHD, cwcowVMGSPath
+			if cwcowBootFilesPath != "" {
+				if cwcowEFIVHD != "" || cwcowBootVHD != "" || cwcowVMGSPath != "" {
+					return fmt.Errorf("--%s cannot be combined with --efi-vhd, --boot-cim-vhd or --%s", bootFilesPathArgName, vmgsFilePathArgName)
+				}
+				root, err := filepath.Abs(cwcowBootFilesPath)
+				if err != nil {
+					return err
+				}
+				efiVHD = uvm.ConfidentialEFIPath(root)
+				bootVHD = uvm.ConfidentialBootCIMPath(root)
+				vmgsPath = uvm.ConfidentialVMGSPath(root, cwcowIsolationMode)
+			} else if cwcowEFIVHD == "" || cwcowBootVHD == "" || cwcowVMGSPath == "" {
+				return fmt.Errorf("either --%s or all of --efi-vhd, --boot-cim-vhd and --%s must be provided", bootFilesPathArgName, vmgsFilePathArgName)
+			}
+
+			bootVHD, err := filepath.Abs(bootVHD)
+			if err != nil {
+				return err
+			}
+			efiVHD, err = filepath.Abs(efiVHD)
+			if err != nil {
+				return err
+			}
+			scratchVHD, err := filepath.Abs(cwcowScratchVHD)
+			if err != nil {
+				return err
+			}
+
 			options := uvm.NewDefaultOptionsWCOW(id, "")
 			options.ProcessorCount = 2
 			options.MemorySizeInMB = 2048
@@ -112,36 +160,26 @@ var cwcowCommand = cli.Command{
 			options.SecurityPolicyEnabled = true
 			options.SecurityPolicy = cwcowSecurityPolicy
 			options.DisableSecureBoot = cwcowDisableSecureBoot
-			options.GuestStateFilePath = cwcowVMGSPath
+			options.GuestStateFilePath = vmgsPath
 			options.IsolationType = cwcowIsolationMode
 
 			// graphics console helps with testing/debugging however, it
 			// doesn't work in SNP isolation mode.
 			options.EnableGraphicsConsole = cwcowIsolationMode != "SecureNestedPaging"
 			options.WritableEFI = cwcowWritableEFI
 
-			var err error
-			cwcowBootVHD, err = filepath.Abs(cwcowBootVHD)
-			if err != nil {
-				return err
-			}
-
-			cwcowEFIVHD, err = filepath.Abs(cwcowEFIVHD)
-			if err != nil {
-				return err
-			}
-
-			cwcowScratchVHD, err = filepath.Abs(cwcowScratchVHD)
-			if err != nil {
-				return err
+			options.DebugMode = cwcowDebugMode
+			options.DebugDataPath = cwcowDebugDataPath
+			if cwcowDebugMode && cwcowDebugDataPath == "" {
+				return fmt.Errorf("--%s must be set to a non-empty path when --%s is enabled", debugDataPathArgName, debugModeArgName)
 			}
 
 			options.BootFiles = &uvm.WCOWBootFiles{
 				BootType: uvm.BlockCIMBoot,
 				BlockCIMFiles: &uvm.BlockCIMBootFiles{
-					BootCIMVHDPath: cwcowBootVHD,
-					EFIVHDPath:     cwcowEFIVHD,
-					ScratchVHDPath: cwcowScratchVHD,
+					BootCIMVHDPath: bootVHD,
+					EFIVHDPath:     efiVHD,
+					ScratchVHDPath: scratchVHD,
 				},
 			}
 			setGlobalOptions(c, options.Options)

internal/uvm/create.go

@@ -15,6 +15,7 @@ import (
 	"go.opencensus.io/trace"
 	"golang.org/x/sys/windows"
 
+	"github.com/Microsoft/hcsshim/internal/copyfile"
 	"github.com/Microsoft/hcsshim/internal/cow"
 	hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
 	hcs "github.com/Microsoft/hcsshim/internal/hcs/v2"
@@ -349,13 +350,53 @@ func (uvm *UtilityVM) CloseCtx(ctx context.Context) (err error) {
 		}
 	}
 
+	// If debug mode is enabled, save the per-UVM boot/EFI VHD and scratch VHD to the debug data
+	// directory before they are cleaned up so they can be inspected (e.g. for boot failures).
+	if wopts, ok := uvm.createOpts.(*OptionsWCOW); ok &&
+		uvm.HasConfidentialPolicy() &&
+		wopts.DebugMode &&
+		wopts.DebugDataPath != "" &&
+		wopts.BootFiles != nil &&
+		wopts.BootFiles.BlockCIMFiles != nil {
+		uvm.preserveWCOWScratch(ctx, wopts.DebugDataPath, wopts.BootFiles.BlockCIMFiles)
+	}
+
 	if uvm.hcsSystem != nil {
 		return uvm.hcsSystem.CloseCtx(ctx)
 	}
 
 	return nil
 }
 
+// preserveWCOWScratch copies the per-UVM scratch VHD and boot/EFI VHD to destDir so they can be
+// inspected for troubleshooting after the UVM is torn down. The boot/EFI VHD contains the bootstat
+// trace when the writable EFI option is enabled. It is best-effort: any error is logged but does not
+// fail UVM close. Copied files are prefixed with the UVM id to avoid collisions when multiple UVMs
+// share the same directory.
+func (uvm *UtilityVM) preserveWCOWScratch(ctx context.Context, destDir string, blockFiles *BlockCIMBootFiles) {
+	e := log.G(ctx).WithFields(logrus.Fields{
+		logfields.UVMID: uvm.id,
+		"destination":   destDir,
+	})
+
+	if err := os.MkdirAll(destDir, 0755); err != nil {
+		e.WithError(err).Error("failed to create boot files preserve directory")
+		return
+	}
+
+	for _, src := range []string{blockFiles.ScratchVHDPath, blockFiles.EFIVHDPath} {
+		if src == "" {
+			continue
+		}
+		dst := filepath.Join(destDir, uvm.id+"_"+filepath.Base(src))
+		if err := copyfile.CopyFile(ctx, src, dst, true); err != nil {
+			e.WithField("source", src).WithError(err).Error("failed to preserve boot file")
+			continue
+		}
+		e.WithFields(logrus.Fields{"source": src, "copy": dst}).Debug("preserved boot file for troubleshooting")
+	}
+}
+
 // CreateContainer creates a container in the utility VM.
 func (uvm *UtilityVM) CreateContainer(ctx context.Context, id string, settings interface{}) (cow.Container, error) {
 	if uvm.gc != nil {