diff --git a/cmd/containerd-shim-runhcs-v1/pod.go b/cmd/containerd-shim-runhcs-v1/pod.go index c0f6b72597..3f4d5ee8cb 100644 --- a/cmd/containerd-shim-runhcs-v1/pod.go +++ b/cmd/containerd-shim-runhcs-v1/pod.go @@ -49,8 +49,8 @@ func initializeWCOWBootFiles(ctx context.Context, wopts *uvm.OptionsWCOW, rootfs return fmt.Errorf("security policy (confidential mode) only works with block CIM based layers") } // we use measured EFI & rootfs for confidential UVMs, use those instead of the ones passed in layers/rootfs - wopts.BootFiles.BlockCIMFiles.EFIVHDPath = uvm.GetDefaultConfidentialEFIPath() - wopts.BootFiles.BlockCIMFiles.BootCIMVHDPath = uvm.GetDefaultConfidentialBootCIMPath() + wopts.BootFiles.BlockCIMFiles.EFIVHDPath = uvm.ConfidentialEFIPath(wopts.BootFilesRootPath) + wopts.BootFiles.BlockCIMFiles.BootCIMVHDPath = uvm.ConfidentialBootCIMPath(wopts.BootFilesRootPath) // make a copy of the vmgs file as the same vmgs can not be used by multiple pods in parallel // TODO(ambarve): for C-LCOW we make a copy in the bundle directory, is it better diff --git a/internal/oci/uvm.go b/internal/oci/uvm.go index 3acf09ac42..9a1b906846 100644 --- a/internal/oci/uvm.go +++ b/internal/oci/uvm.go @@ -251,9 +251,8 @@ func handleWCOWSecurityPolicy(ctx context.Context, a map[string]string, wopts *u wopts.SecurityPolicyEnforcer = ParseAnnotationsString(a, annotations.WCOWSecurityPolicyEnforcer, wopts.SecurityPolicyEnforcer) wopts.DisableSecureBoot = ParseAnnotationsBool(ctx, a, annotations.WCOWDisableSecureBoot, false) - wopts.GuestStateFilePath = ParseAnnotationsString(a, annotations.WCOWGuestStateFile, uvm.GetDefaultConfidentialVMGSPath()) - wopts.UVMReferenceInfoFile = ParseAnnotationsString(a, annotations.WCOWReferenceInfoFile, uvm.GetDefaultReferenceInfoFilePath()) - wopts.UVMHashEnvelopeReferenceInfoFile = ParseAnnotationsString(a, annotations.UVMHashEnvelopeReferenceInfoFile, uvm.GetDefaultHashEnvelopeReferenceInfoFilePath()) + + // Resolve the isolation type first so the default VMGS file can be selected accordingly. wopts.IsolationType = "SecureNestedPaging" if noSecurityHardware := ParseAnnotationsBool(ctx, a, annotations.NoSecurityHardware, false); noSecurityHardware { wopts.IsolationType = "GuestStateOnly" @@ -262,6 +261,10 @@ func handleWCOWSecurityPolicy(ctx context.Context, a map[string]string, wopts *u return err } + wopts.GuestStateFilePath = ParseAnnotationsString(a, annotations.WCOWGuestStateFile, uvm.ConfidentialVMGSPath(wopts.BootFilesRootPath, wopts.IsolationType)) + wopts.UVMReferenceInfoFile = ParseAnnotationsString(a, annotations.WCOWReferenceInfoFile, uvm.ConfidentialReferenceInfoFilePath(wopts.BootFilesRootPath)) + wopts.UVMHashEnvelopeReferenceInfoFile = ParseAnnotationsString(a, annotations.UVMHashEnvelopeReferenceInfoFile, uvm.ConfidentialHashEnvelopeReferenceInfoFilePath(wopts.BootFilesRootPath)) + return nil } @@ -420,10 +423,27 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) ( // Writable EFI is valid for both confidential and regular Hyper-V isolated WCOW. wopts.WritableEFI = ParseAnnotationsBool(ctx, s.Annotations, annotations.WCOWWritableEFI, wopts.WritableEFI) + // Debug mode (confidential WCOW only) saves the boot/EFI and scratch VHDs on teardown. + wopts.DebugMode = ParseAnnotationsBool(ctx, s.Annotations, annotations.WCOWDebugMode, wopts.DebugMode) + wopts.DebugDataPath = ParseAnnotationsString(s.Annotations, annotations.WCOWDebugDataPath, wopts.DebugDataPath) + + // Optional override for the directory containing the confidential WCOW boot files. + wopts.BootFilesRootPath = ParseAnnotationsString(s.Annotations, annotations.BootFilesRootPath, wopts.BootFilesRootPath) + // Handle WCOW security policy settings if err := handleWCOWSecurityPolicy(ctx, s.Annotations, wopts); err != nil { return nil, err } + + // Debug mode is only supported for confidential WCOW and requires a non-empty data path. + if wopts.DebugMode { + if !wopts.SecurityPolicyEnabled { + return nil, fmt.Errorf("%q is only supported for confidential WCOW", annotations.WCOWDebugMode) + } + if wopts.DebugDataPath == "" { + return nil, fmt.Errorf("%q must be set to a non-empty path when %q is enabled", annotations.WCOWDebugDataPath, annotations.WCOWDebugMode) + } + } // If security policy is enable, wopts.LogForwardingEnabled default value should be false (CWCOW should not allow log forwarding by default) if wopts.SecurityPolicyEnabled { wopts.LogForwardingEnabled = false diff --git a/internal/oci/uvm_test.go b/internal/oci/uvm_test.go index 9d13271568..bc9798ab1c 100644 --- a/internal/oci/uvm_test.go +++ b/internal/oci/uvm_test.go @@ -263,6 +263,109 @@ func Test_SpecToUVMCreateOptions_WCOW_Confidential_Overrides(t *testing.T) { } } +func Test_SpecToUVMCreateOptions_WCOW_Confidential_BootFilesRootPath(t *testing.T) { + root := "C:\\custom\\bootfiles" + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + annotations.BootFilesRootPath: root, + }, + } + + opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "") + if err != nil { + t.Fatalf("unexpected error generating UVM opts: %v", err) + } + + wopts := (opts).(*uvm.OptionsWCOW) + if wopts.BootFilesRootPath != root { + t.Fatalf("expected BootFilesRootPath=%q, got %q", root, wopts.BootFilesRootPath) + } + // Defaults derived from the boot files root (SNP isolation by default). + if want := uvm.ConfidentialVMGSPath(root, "SecureNestedPaging"); wopts.GuestStateFilePath != want { + t.Fatalf("expected GuestStateFilePath=%q, got %q", want, wopts.GuestStateFilePath) + } + if want := uvm.ConfidentialReferenceInfoFilePath(root); wopts.UVMReferenceInfoFile != want { + t.Fatalf("expected UVMReferenceInfoFile=%q, got %q", want, wopts.UVMReferenceInfoFile) + } +} + +func Test_SpecToUVMCreateOptions_WCOW_Confidential_VMGSDefaultByIsolation(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + annotations.WCOWIsolationType: "VirtualizationBasedSecurity", + }, + } + + opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "") + if err != nil { + t.Fatalf("unexpected error generating UVM opts: %v", err) + } + + wopts := (opts).(*uvm.OptionsWCOW) + // With no explicit VMGS override, the default VMGS must follow the isolation type. + if want := uvm.ConfidentialVMGSPath("", "VirtualizationBasedSecurity"); wopts.GuestStateFilePath != want { + t.Fatalf("expected GuestStateFilePath=%q for VBS isolation, got %q", want, wopts.GuestStateFilePath) + } +} + +func Test_SpecToUVMCreateOptions_WCOW_Confidential_DebugMode_Enabled(t *testing.T) { + dataPath := "C:\\debug\\data" + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + annotations.WCOWDebugMode: "true", + annotations.WCOWDebugDataPath: dataPath, + }, + } + + opts, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), "") + if err != nil { + t.Fatalf("unexpected error generating UVM opts: %v", err) + } + + wopts := (opts).(*uvm.OptionsWCOW) + if !wopts.DebugMode { + t.Fatal("DebugMode should be true when WCOWDebugMode is set") + } + if wopts.DebugDataPath != dataPath { + t.Fatalf("expected DebugDataPath=%q, got %q", dataPath, wopts.DebugDataPath) + } +} + +func Test_SpecToUVMCreateOptions_WCOW_Confidential_DebugMode_ErrorOnMissingDataPath(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + annotations.WCOWSecurityPolicy: "test-policy", + annotations.WCOWDebugMode: "true", + }, + } + + if _, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), ""); err == nil { + t.Fatal("expected error when debug mode is enabled without a data path, got nil") + } +} + +func Test_SpecToUVMCreateOptions_WCOW_NonConfidential_DebugMode_Error(t *testing.T) { + s := &specs.Spec{ + Windows: &specs.Windows{HyperV: &specs.WindowsHyperV{}}, + Annotations: map[string]string{ + // No WCOWSecurityPolicy means non-confidential path. + annotations.WCOWDebugMode: "true", + annotations.WCOWDebugDataPath: "C:\\debug\\data", + }, + } + + if _, err := SpecToUVMCreateOpts(context.Background(), s, t.Name(), ""); err == nil { + t.Fatal("expected error when debug mode is enabled for non-confidential WCOW, got nil") + } +} + func Test_SpecToUVMCreateOptions_Common(t *testing.T) { cpugroupid := "1" lowmmiogap := 1024 diff --git a/internal/tools/uvmboot/conf_wcow.go b/internal/tools/uvmboot/conf_wcow.go index a66043b91b..2617ebb967 100644 --- a/internal/tools/uvmboot/conf_wcow.go +++ b/internal/tools/uvmboot/conf_wcow.go @@ -23,6 +23,8 @@ const ( disableSBArgName = "disable-secure-boot" isolationTypeArgName = "isolation-type" writableEFIArgName = "writable-efi" + debugModeArgName = "debug-mode" + debugDataPathArgName = "debug-data-path" // default policy (that allows all operations) used when no policy is provided allowAllPolicy = "cGFja2FnZSBwb2xpY3kKCmFwaV92ZXJzaW9uIDo9ICIwLjExLjAiCmZyYW1ld29ya192ZXJzaW9uIDo9ICIwLjQuMCIKCm1vdW50X2NpbXMgOj0geyJhbGxvd2VkIjogdHJ1ZX0KbW91bnRfZGV2aWNlIDo9IHsiYWxsb3dlZCI6IHRydWV9Cm1vdW50X292ZXJsYXkgOj0geyJhbGxvd2VkIjogdHJ1ZX0KY3JlYXRlX2NvbnRhaW5lciA6PSB7ImFsbG93ZWQiOiB0cnVlLCAiZW52X2xpc3QiOiBudWxsLCAiYWxsb3dfc3RkaW9fYWNjZXNzIjogdHJ1ZX0KdW5tb3VudF9kZXZpY2UgOj0geyJhbGxvd2VkIjogdHJ1ZX0KdW5tb3VudF9vdmVybGF5IDo9IHsiYWxsb3dlZCI6IHRydWV9CmV4ZWNfaW5fY29udGFpbmVyIDo9IHsiYWxsb3dlZCI6IHRydWUsICJlbnZfbGlzdCI6IG51bGx9CmV4ZWNfZXh0ZXJuYWwgOj0geyJhbGxvd2VkIjogdHJ1ZSwgImVudl9saXN0IjogbnVsbCwgImFsbG93X3N0ZGlvX2FjY2VzcyI6IHRydWV9CnNodXRkb3duX2NvbnRhaW5lciA6PSB7ImFsbG93ZWQiOiB0cnVlfQpzaWduYWxfY29udGFpbmVyX3Byb2Nlc3MgOj0geyJhbGxvd2VkIjogdHJ1ZX0KcGxhbjlfbW91bnQgOj0geyJhbGxvd2VkIjogdHJ1ZX0KcGxhbjlfdW5tb3VudCA6PSB7ImFsbG93ZWQiOiB0cnVlfQpnZXRfcHJvcGVydGllcyA6PSB7ImFsbG93ZWQiOiB0cnVlfQpkdW1wX3N0YWNrcyA6PSB7ImFsbG93ZWQiOiB0cnVlfQpydW50aW1lX2xvZ2dpbmcgOj0geyJhbGxvd2VkIjogdHJ1ZX0KbG9hZF9mcmFnbWVudCA6PSB7ImFsbG93ZWQiOiB0cnVlfQpzY3JhdGNoX21vdW50IDo9IHsiYWxsb3dlZCI6IHRydWV9CnNjcmF0Y2hfdW5tb3VudCA6PSB7ImFsbG93ZWQiOiB0cnVlfQo=" @@ -33,10 +35,13 @@ var ( cwcowEFIVHD string cwcowScratchVHD string cwcowVMGSPath string + cwcowBootFilesPath string cwcowDisableSecureBoot bool cwcowIsolationMode string cwcowSecurityPolicy string cwcowWritableEFI bool + cwcowDebugMode bool + cwcowDebugDataPath string ) var cwcowCommand = cli.Command{ @@ -55,15 +60,13 @@ var cwcowCommand = cli.Command{ }, cli.StringFlag{ Name: "efi-vhd", - Usage: "VHD at the provided path MUST have the EFI boot partition and be properly formatted for UEFI boot.", + Usage: "VHD at the provided path MUST have the EFI boot partition and be properly formatted for UEFI boot. Ignored when --" + bootFilesPathArgName + " is provided.", Destination: &cwcowEFIVHD, - Required: true, }, cli.StringFlag{ Name: "boot-cim-vhd", - Usage: "A VHD containing the block CIM that contains the OS files.", + Usage: "A VHD containing the block CIM that contains the OS files. Ignored when --" + bootFilesPathArgName + " is provided.", Destination: &cwcowBootVHD, - Required: true, }, cli.StringFlag{ Name: "scratch-vhd", @@ -73,9 +76,13 @@ var cwcowCommand = cli.Command{ }, cli.StringFlag{ Name: vmgsFilePathArgName, - Usage: "VMGS file path (only applies when confidential mode is enabled). This option is only applicable in confidential mode.", + Usage: "VMGS file path (only applies when confidential mode is enabled). Ignored when --" + bootFilesPathArgName + " is provided.", Destination: &cwcowVMGSPath, - Required: true, + }, + cli.StringFlag{ + Name: bootFilesPathArgName, + Usage: "Directory containing the confidential boot files. When provided, the EFI VHD (boot.vhd), boot CIM VHD (rootfs.vhd) and VMGS file (selected from --" + isolationTypeArgName + ") are derived from it, and --efi-vhd/--boot-cim-vhd/--" + vmgsFilePathArgName + " must not be set. Provide either this or all three individual files.", + Destination: &cwcowBootFilesPath, }, cli.BoolFlag{ Name: disableSBArgName, @@ -99,9 +106,50 @@ var cwcowCommand = cli.Command{ Usage: "Attaches the EFI VHD as read-write instead of read-only. This allows the UVM to modify the contents of the VHD, be careful when using this option!", Destination: &cwcowWritableEFI, }, + cli.BoolFlag{ + Name: debugModeArgName, + Usage: "Enables debug mode: the boot/EFI VHD and scratch VHD are saved to the directory given by --" + debugDataPathArgName + " when the UVM is torn down. Combine with --" + writableEFIArgName + " to capture the bootstat trace.", + Destination: &cwcowDebugMode, + }, + cli.StringFlag{ + Name: debugDataPathArgName, + Usage: "Directory to save the boot/EFI VHD and scratch VHD to when --" + debugModeArgName + " is enabled. Required when debug mode is enabled.", + Destination: &cwcowDebugDataPath, + }, }, Action: func(c *cli.Context) error { runMany(c, func(id string) error { + // Resolve the boot files: either derive them from a single directory or use the + // individually-provided VHD/VMGS paths. The scratch VHD is always provided separately. + efiVHD, bootVHD, vmgsPath := cwcowEFIVHD, cwcowBootVHD, cwcowVMGSPath + if cwcowBootFilesPath != "" { + if cwcowEFIVHD != "" || cwcowBootVHD != "" || cwcowVMGSPath != "" { + return fmt.Errorf("--%s cannot be combined with --efi-vhd, --boot-cim-vhd or --%s", bootFilesPathArgName, vmgsFilePathArgName) + } + root, err := filepath.Abs(cwcowBootFilesPath) + if err != nil { + return err + } + efiVHD = uvm.ConfidentialEFIPath(root) + bootVHD = uvm.ConfidentialBootCIMPath(root) + vmgsPath = uvm.ConfidentialVMGSPath(root, cwcowIsolationMode) + } else if cwcowEFIVHD == "" || cwcowBootVHD == "" || cwcowVMGSPath == "" { + return fmt.Errorf("either --%s or all of --efi-vhd, --boot-cim-vhd and --%s must be provided", bootFilesPathArgName, vmgsFilePathArgName) + } + + bootVHD, err := filepath.Abs(bootVHD) + if err != nil { + return err + } + efiVHD, err = filepath.Abs(efiVHD) + if err != nil { + return err + } + scratchVHD, err := filepath.Abs(cwcowScratchVHD) + if err != nil { + return err + } + options := uvm.NewDefaultOptionsWCOW(id, "") options.ProcessorCount = 2 options.MemorySizeInMB = 2048 @@ -112,7 +160,7 @@ var cwcowCommand = cli.Command{ options.SecurityPolicyEnabled = true options.SecurityPolicy = cwcowSecurityPolicy options.DisableSecureBoot = cwcowDisableSecureBoot - options.GuestStateFilePath = cwcowVMGSPath + options.GuestStateFilePath = vmgsPath options.IsolationType = cwcowIsolationMode // graphics console helps with testing/debugging however, it @@ -120,28 +168,18 @@ var cwcowCommand = cli.Command{ options.EnableGraphicsConsole = cwcowIsolationMode != "SecureNestedPaging" options.WritableEFI = cwcowWritableEFI - var err error - cwcowBootVHD, err = filepath.Abs(cwcowBootVHD) - if err != nil { - return err - } - - cwcowEFIVHD, err = filepath.Abs(cwcowEFIVHD) - if err != nil { - return err - } - - cwcowScratchVHD, err = filepath.Abs(cwcowScratchVHD) - if err != nil { - return err + options.DebugMode = cwcowDebugMode + options.DebugDataPath = cwcowDebugDataPath + if cwcowDebugMode && cwcowDebugDataPath == "" { + return fmt.Errorf("--%s must be set to a non-empty path when --%s is enabled", debugDataPathArgName, debugModeArgName) } options.BootFiles = &uvm.WCOWBootFiles{ BootType: uvm.BlockCIMBoot, BlockCIMFiles: &uvm.BlockCIMBootFiles{ - BootCIMVHDPath: cwcowBootVHD, - EFIVHDPath: cwcowEFIVHD, - ScratchVHDPath: cwcowScratchVHD, + BootCIMVHDPath: bootVHD, + EFIVHDPath: efiVHD, + ScratchVHDPath: scratchVHD, }, } setGlobalOptions(c, options.Options) diff --git a/internal/uvm/create.go b/internal/uvm/create.go index ceae6ea18c..d53f356052 100644 --- a/internal/uvm/create.go +++ b/internal/uvm/create.go @@ -15,6 +15,7 @@ import ( "go.opencensus.io/trace" "golang.org/x/sys/windows" + "github.com/Microsoft/hcsshim/internal/copyfile" "github.com/Microsoft/hcsshim/internal/cow" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" hcs "github.com/Microsoft/hcsshim/internal/hcs/v2" @@ -349,6 +350,17 @@ func (uvm *UtilityVM) CloseCtx(ctx context.Context) (err error) { } } + // If debug mode is enabled, save the per-UVM boot/EFI VHD and scratch VHD to the debug data + // directory before they are cleaned up so they can be inspected (e.g. for boot failures). + if wopts, ok := uvm.createOpts.(*OptionsWCOW); ok && + uvm.HasConfidentialPolicy() && + wopts.DebugMode && + wopts.DebugDataPath != "" && + wopts.BootFiles != nil && + wopts.BootFiles.BlockCIMFiles != nil { + uvm.preserveWCOWScratch(ctx, wopts.DebugDataPath, wopts.BootFiles.BlockCIMFiles) + } + if uvm.hcsSystem != nil { return uvm.hcsSystem.CloseCtx(ctx) } @@ -356,6 +368,35 @@ func (uvm *UtilityVM) CloseCtx(ctx context.Context) (err error) { return nil } +// preserveWCOWScratch copies the per-UVM scratch VHD and boot/EFI VHD to destDir so they can be +// inspected for troubleshooting after the UVM is torn down. The boot/EFI VHD contains the bootstat +// trace when the writable EFI option is enabled. It is best-effort: any error is logged but does not +// fail UVM close. Copied files are prefixed with the UVM id to avoid collisions when multiple UVMs +// share the same directory. +func (uvm *UtilityVM) preserveWCOWScratch(ctx context.Context, destDir string, blockFiles *BlockCIMBootFiles) { + e := log.G(ctx).WithFields(logrus.Fields{ + logfields.UVMID: uvm.id, + "destination": destDir, + }) + + if err := os.MkdirAll(destDir, 0755); err != nil { + e.WithError(err).Error("failed to create boot files preserve directory") + return + } + + for _, src := range []string{blockFiles.ScratchVHDPath, blockFiles.EFIVHDPath} { + if src == "" { + continue + } + dst := filepath.Join(destDir, uvm.id+"_"+filepath.Base(src)) + if err := copyfile.CopyFile(ctx, src, dst, true); err != nil { + e.WithField("source", src).WithError(err).Error("failed to preserve boot file") + continue + } + e.WithFields(logrus.Fields{"source": src, "copy": dst}).Debug("preserved boot file for troubleshooting") + } +} + // CreateContainer creates a container in the utility VM. func (uvm *UtilityVM) CreateContainer(ctx context.Context, id string, settings interface{}) (cow.Container, error) { if uvm.gc != nil { diff --git a/internal/uvm/create_test.go b/internal/uvm/create_test.go index b79f37b7e7..2d60047b51 100644 --- a/internal/uvm/create_test.go +++ b/internal/uvm/create_test.go @@ -19,3 +19,20 @@ func TestCreateBadBootFilesPath(t *testing.T) { t.Fatal(err) } } + +func TestConfidentialVMGSFileName(t *testing.T) { + for _, tc := range []struct { + isolationType string + want string + }{ + {"SecureNestedPaging", "cwcow.snp.vmgs"}, + {"VirtualizationBasedSecurity", "cwcow.vbs.vmgs"}, + {"GuestStateOnly", "cwcow.gso.vmgs"}, + {"", "cwcow.snp.vmgs"}, // unknown defaults to SNP + {"SomethingElse", "cwcow.snp.vmgs"}, // unknown defaults to SNP + } { + if got := ConfidentialVMGSFileName(tc.isolationType); got != tc.want { + t.Errorf("ConfidentialVMGSFileName(%q) = %q, want %q", tc.isolationType, got, tc.want) + } + } +} diff --git a/internal/uvm/create_wcow.go b/internal/uvm/create_wcow.go index 188201ec56..1567ad8417 100644 --- a/internal/uvm/create_wcow.go +++ b/internal/uvm/create_wcow.go @@ -52,6 +52,13 @@ type ConfidentialWCOWOptions struct { DisableSecureBoot bool FirmwareParameters string WritableEFI bool + // DebugMode, when enabled, saves the per-UVM boot/EFI VHD and scratch VHD to DebugDataPath when the + // UVM is torn down, so they can be inspected for troubleshooting (e.g. boot failures). The boot/EFI VHD + // contains the bootstat trace when WritableEFI is also enabled. Only valid for confidential WCOW. + DebugMode bool + // DebugDataPath is the directory to which the boot/EFI VHD and scratch VHD are saved when DebugMode is + // enabled. It must be non-empty when DebugMode is set. + DebugDataPath string } // OptionsWCOW are the set of options passed to CreateWCOW() to create a utility vm. @@ -61,6 +68,11 @@ type OptionsWCOW struct { BootFiles *WCOWBootFiles + // BootFilesRootPath optionally overrides the directory used to locate the confidential WCOW boot + // files (boot.vhd, rootfs.vhd, VMGS, reference info). If empty, the default location next to the + // shim executable is used. + BootFilesRootPath string + // NoDirectMap specifies that no direct mapping should be used for any VSMBs added to the UVM NoDirectMap bool @@ -76,28 +88,77 @@ type OptionsWCOW struct { DefaultLogSourcesEnabled bool // Whether to enable using default log sources } -func defaultConfidentialWCOWOSBootFilesPath() string { +// confidentialWCOWOSBootFilesPath returns the directory that holds the confidential WCOW boot +// files. If root is non-empty it is used as-is (allowing the location to be overridden via the +// BootFilesRootPath annotation), otherwise the default location next to the shim executable is used. +func confidentialWCOWOSBootFilesPath(root string) string { + if root != "" { + return root + } return filepath.Join(filepath.Dir(os.Args[0]), "WindowsBootFiles", "confidential") } +// ConfidentialVMGSFileName returns the VMGS file name for the given UVM isolation type. +func ConfidentialVMGSFileName(isolationType string) string { + switch isolationType { + case "VirtualizationBasedSecurity": + return "cwcow.vbs.vmgs" + case "GuestStateOnly": + return "cwcow.gso.vmgs" + default: // SecureNestedPaging (and any unknown value) defaults to SNP. + return "cwcow.snp.vmgs" + } +} + +// ConfidentialVMGSPath returns the VMGS file path under the given boot files root for the given +// isolation type. An empty root resolves to the default confidential boot files location. +func ConfidentialVMGSPath(root, isolationType string) string { + return filepath.Join(confidentialWCOWOSBootFilesPath(root), ConfidentialVMGSFileName(isolationType)) +} + +// ConfidentialBootCIMPath returns the BootCIM (rootfs.vhd) path under the given boot files root. +// An empty root resolves to the default confidential boot files location. +func ConfidentialBootCIMPath(root string) string { + return filepath.Join(confidentialWCOWOSBootFilesPath(root), "rootfs.vhd") +} + +// ConfidentialEFIPath returns the EFI/boot (boot.vhd) path under the given boot files root. +// An empty root resolves to the default confidential boot files location. +func ConfidentialEFIPath(root string) string { + return filepath.Join(confidentialWCOWOSBootFilesPath(root), "boot.vhd") +} + +// ConfidentialReferenceInfoFilePath returns the UVM reference info file path under the given boot +// files root. An empty root resolves to the default confidential boot files location. +func ConfidentialReferenceInfoFilePath(root string) string { + return filepath.Join(confidentialWCOWOSBootFilesPath(root), vmutils.DefaultUVMReferenceInfoFile) +} + +// ConfidentialHashEnvelopeReferenceInfoFilePath returns the hash envelope UVM reference info file +// path under the given boot files root. An empty root resolves to the default confidential boot +// files location. +func ConfidentialHashEnvelopeReferenceInfoFilePath(root string) string { + return filepath.Join(confidentialWCOWOSBootFilesPath(root), vmutils.DefaultUVMHashEnvelopeReferenceInfoFile) +} + func GetDefaultConfidentialVMGSPath() string { - return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "cwcow.snp.vmgs") + return ConfidentialVMGSPath("", "SecureNestedPaging") } func GetDefaultConfidentialBootCIMPath() string { - return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "rootfs.vhd") + return ConfidentialBootCIMPath("") } func GetDefaultConfidentialEFIPath() string { - return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), "boot.vhd") + return ConfidentialEFIPath("") } func GetDefaultReferenceInfoFilePath() string { - return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), vmutils.DefaultUVMReferenceInfoFile) + return ConfidentialReferenceInfoFilePath("") } func GetDefaultHashEnvelopeReferenceInfoFilePath() string { - return filepath.Join(defaultConfidentialWCOWOSBootFilesPath(), vmutils.DefaultUVMHashEnvelopeReferenceInfoFile) + return ConfidentialHashEnvelopeReferenceInfoFilePath("") } // NewDefaultOptionsWCOW creates the default options for a bootable version of diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 79289eed4d..054754a6bd 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -280,6 +280,18 @@ const ( // Attaches the EFI/boot VHD in the writable mode (instead of the default read-only mode). This is usually required // when debugging boot to capture bootstat traces. WCOWWritableEFI = "io.microsoft.virtualmachine.wcow.writable_efi" + + // WCOWDebugMode enables debug mode for confidential WCOW. When enabled, the per-UVM boot/EFI VHD and + // scratch VHD are saved to WCOWDebugDataPath when the UVM is torn down, so they can be inspected for + // troubleshooting (e.g. boot failures). To also capture the bootstat trace in the saved boot/EFI VHD, + // enable WCOWWritableEFI alongside this option. This option is only valid for confidential WCOW and + // requires WCOWDebugDataPath to be set to a non-empty path. + WCOWDebugMode = "io.microsoft.wcow.debug" + + // WCOWDebugDataPath specifies the directory to which the boot/EFI VHD and scratch VHD are saved when + // WCOWDebugMode is enabled. The directory is created if it does not exist and the saved files are + // prefixed with the UVM id to avoid collisions when multiple UVMs share the directory. + WCOWDebugDataPath = "io.microsoft.wcow.debug_data_path" ) // WCOW host process container annotations. @@ -297,6 +309,11 @@ const ( // uVM annotations. const ( + // BootFilesRootPath indicates the path to find the boot files to use when creating the UVM. It applies + // to LCOW and confidential WCOW (where it locates boot.vhd, rootfs.vhd, VMGS, and reference info). If + // unset, a platform-specific default location is used. + BootFilesRootPath = "io.microsoft.virtualmachine.bootfilesrootpath" + // DumpDirectoryPath provides a path to the directory in which dumps for a UVM will be collected in // case the UVM crashes. DumpDirectoryPath = "io.microsoft.virtualmachine.dump-directory-path" @@ -485,9 +502,6 @@ const ( // LCOW uVM annotations. const ( - // BootFilesRootPath indicates the path to find the LCOW boot files to use when creating the UVM. - BootFilesRootPath = "io.microsoft.virtualmachine.lcow.bootfilesrootpath" - // DisableLCOWTimeSyncService is used to disable the chronyd time // synchronization service inside the LCOW UVM. DisableLCOWTimeSyncService = "io.microsoft.virtualmachine.lcow.timesync.disable"