From 68e2037cd3740f56e02c9e9778afd7611e9eefcf Mon Sep 17 00:00:00 2001 From: Harsh Rawat Date: Thu, 2 Jul 2026 10:35:48 +0530 Subject: [PATCH] Add LCOW live migration source and destination pathways Introduce an end-to-end live-migration workflow for LCOW sandboxes, letting a running sandbox be handed off from a source shim to a destination shim. - Add a migration Controller that sequences a single session through a source or destination lifecycle: the source prepares and exports an opaque sandbox snapshot (VM + per-pod state), while the destination imports it, patches each container onto its new IDs, builds the destination VM, transfers memory over a shared socket, and finalizes with a resume or stop. - Expose the migration gRPC/ttrpc surface on the LCOW shim service (PrepareAndExportSandbox, ImportSandbox, PrepareSandbox, TransferSandbox, FinalizeSandbox, Cancel, Cleanup, CreateDuplicateSocket, Notifications) and register it alongside the task and sandbox services. - Wire migration into the task lifecycle: route destination-side CreateTask for rehydrated containers into a patch path, and reject task mutations (state, delete, kill) while a session is in progress. - Add duplicated transport-socket adoption, a subscriber-based progress notification stream, and proto/HCS conversion helpers for migration options and events. Signed-off-by: Harsh Rawat --- .../service/mocks/mock_service.go | 170 +++++- .../service/service.go | 16 + .../service/service_migration.go | 196 +++++++ .../service/service_migration_internal.go | 238 ++++++++ .../service/service_sandbox_internal_test.go | 2 + .../service/service_task_internal.go | 31 +- .../service/service_task_internal_test.go | 26 +- cmd/containerd-shim-lcow-v2/service/types.go | 36 ++ internal/controller/migration/controller.go | 417 ++++++++++++++ .../migration/controller_destination.go | 263 +++++++++ .../migration/controller_destination_test.go | 414 ++++++++++++++ .../controller/migration/controller_source.go | 126 +++++ .../migration/controller_source_test.go | 279 +++++++++ .../controller/migration/controller_test.go | 534 ++++++++++++++++++ internal/controller/migration/doc.go | 100 ++++ .../migration/mocks/mock_lcow_migration.go | 337 +++++++++++ .../controller/migration/notifications.go | 214 +++++++ .../migration/notifications_test.go | 329 +++++++++++ internal/controller/migration/socket.go | 114 ++++ internal/controller/migration/socket_test.go | 68 +++ internal/controller/migration/state.go | 116 ++++ internal/controller/migration/types.go | 90 +++ pkg/migration/parse.go | 102 ++++ pkg/migration/parse_test.go | 310 ++++++++++ 24 files changed, 4496 insertions(+), 32 deletions(-) create mode 100644 cmd/containerd-shim-lcow-v2/service/service_migration.go create mode 100644 cmd/containerd-shim-lcow-v2/service/service_migration_internal.go create mode 100644 internal/controller/migration/controller.go create mode 100644 internal/controller/migration/controller_destination.go create mode 100644 internal/controller/migration/controller_destination_test.go create mode 100644 internal/controller/migration/controller_source.go create mode 100644 internal/controller/migration/controller_source_test.go create mode 100644 internal/controller/migration/controller_test.go create mode 100644 internal/controller/migration/doc.go create mode 100644 internal/controller/migration/mocks/mock_lcow_migration.go create mode 100644 internal/controller/migration/notifications.go create mode 100644 internal/controller/migration/notifications_test.go create mode 100644 internal/controller/migration/socket.go create mode 100644 internal/controller/migration/socket_test.go create mode 100644 internal/controller/migration/state.go create mode 100644 internal/controller/migration/types.go create mode 100644 pkg/migration/parse_test.go diff --git a/cmd/containerd-shim-lcow-v2/service/mocks/mock_service.go b/cmd/containerd-shim-lcow-v2/service/mocks/mock_service.go index 72388413da..f3b20c371f 100644 --- a/cmd/containerd-shim-lcow-v2/service/mocks/mock_service.go +++ b/cmd/containerd-shim-lcow-v2/service/mocks/mock_service.go @@ -26,13 +26,16 @@ import ( process "github.com/Microsoft/hcsshim/internal/controller/process" vm "github.com/Microsoft/hcsshim/internal/controller/vm" schema2 "github.com/Microsoft/hcsshim/internal/hcs/schema2" + v2 "github.com/Microsoft/hcsshim/internal/hcs/v2" guestresource "github.com/Microsoft/hcsshim/internal/protocol/guestresource" shimdiag "github.com/Microsoft/hcsshim/internal/shimdiag" guestmanager "github.com/Microsoft/hcsshim/internal/vm/guestmanager" + vmmanager "github.com/Microsoft/hcsshim/internal/vm/vmmanager" v3 "github.com/containerd/containerd/api/runtime/task/v3" task "github.com/containerd/containerd/api/types/task" specs_go "github.com/opencontainers/runtime-spec/specs-go" gomock "go.uber.org/mock/gomock" + anypb "google.golang.org/protobuf/types/known/anypb" ) // MockvmController is a mock of vmController interface. @@ -118,6 +121,20 @@ func (mr *MockvmControllerMockRecorder) ExitStatus() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExitStatus", reflect.TypeOf((*MockvmController)(nil).ExitStatus)) } +// FinalizeLiveMigration mocks base method. +func (m *MockvmController) FinalizeLiveMigration(ctx context.Context, options *schema2.MigrationFinalizedOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FinalizeLiveMigration", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// FinalizeLiveMigration indicates an expected call of FinalizeLiveMigration. +func (mr *MockvmControllerMockRecorder) FinalizeLiveMigration(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FinalizeLiveMigration", reflect.TypeOf((*MockvmController)(nil).FinalizeLiveMigration), ctx, options) +} + // Guest mocks base method. func (m *MockvmController) Guest() *guestmanager.Guest { m.ctrl.T.Helper() @@ -132,6 +149,49 @@ func (mr *MockvmControllerMockRecorder) Guest() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Guest", reflect.TypeOf((*MockvmController)(nil).Guest)) } +// Import mocks base method. +func (m *MockvmController) Import(ctx context.Context, env *anypb.Any) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Import", ctx, env) + ret0, _ := ret[0].(error) + return ret0 +} + +// Import indicates an expected call of Import. +func (mr *MockvmControllerMockRecorder) Import(ctx, env any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Import", reflect.TypeOf((*MockvmController)(nil).Import), ctx, env) +} + +// InitializeLiveMigrationOnSource mocks base method. +func (m *MockvmController) InitializeLiveMigrationOnSource(ctx context.Context, options *schema2.MigrationInitializeOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitializeLiveMigrationOnSource", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// InitializeLiveMigrationOnSource indicates an expected call of InitializeLiveMigrationOnSource. +func (mr *MockvmControllerMockRecorder) InitializeLiveMigrationOnSource(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitializeLiveMigrationOnSource", reflect.TypeOf((*MockvmController)(nil).InitializeLiveMigrationOnSource), ctx, options) +} + +// MigrationNotifications mocks base method. +func (m *MockvmController) MigrationNotifications() (<-chan schema2.OperationSystemMigrationNotificationInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MigrationNotifications") + ret0, _ := ret[0].(<-chan schema2.OperationSystemMigrationNotificationInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MigrationNotifications indicates an expected call of MigrationNotifications. +func (mr *MockvmControllerMockRecorder) MigrationNotifications() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MigrationNotifications", reflect.TypeOf((*MockvmController)(nil).MigrationNotifications)) +} + // NetworkController mocks base method. func (m *MockvmController) NetworkController(networkNamespaceID string) *network.Controller { m.ctrl.T.Helper() @@ -146,6 +206,20 @@ func (mr *MockvmControllerMockRecorder) NetworkController(networkNamespaceID any return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetworkController", reflect.TypeOf((*MockvmController)(nil).NetworkController), networkNamespaceID) } +// Patch mocks base method. +func (m *MockvmController) Patch(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Patch", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Patch indicates an expected call of Patch. +func (mr *MockvmControllerMockRecorder) Patch(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockvmController)(nil).Patch), ctx) +} + // Plan9Controller mocks base method. func (m *MockvmController) Plan9Controller() *plan9.Controller { m.ctrl.T.Helper() @@ -160,6 +234,20 @@ func (mr *MockvmControllerMockRecorder) Plan9Controller() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Plan9Controller", reflect.TypeOf((*MockvmController)(nil).Plan9Controller)) } +// Resume mocks base method. +func (m *MockvmController) Resume(ctx context.Context, rebuildBridge bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resume", ctx, rebuildBridge) + ret0, _ := ret[0].(error) + return ret0 +} + +// Resume indicates an expected call of Resume. +func (mr *MockvmControllerMockRecorder) Resume(ctx, rebuildBridge any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resume", reflect.TypeOf((*MockvmController)(nil).Resume), ctx, rebuildBridge) +} + // RuntimeID mocks base method. func (m *MockvmController) RuntimeID() string { m.ctrl.T.Helper() @@ -175,17 +263,18 @@ func (mr *MockvmControllerMockRecorder) RuntimeID() *gomock.Call { } // SCSIController mocks base method. -func (m *MockvmController) SCSIController() *scsi.Controller { +func (m *MockvmController) SCSIController(ctx context.Context) (*scsi.Controller, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SCSIController") + ret := m.ctrl.Call(m, "SCSIController", ctx) ret0, _ := ret[0].(*scsi.Controller) - return ret0 + ret1, _ := ret[1].(error) + return ret0, ret1 } // SCSIController indicates an expected call of SCSIController. -func (mr *MockvmControllerMockRecorder) SCSIController() *gomock.Call { +func (mr *MockvmControllerMockRecorder) SCSIController(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SCSIController", reflect.TypeOf((*MockvmController)(nil).SCSIController)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SCSIController", reflect.TypeOf((*MockvmController)(nil).SCSIController), ctx) } // SandboxOptions mocks base method. @@ -202,6 +291,49 @@ func (mr *MockvmControllerMockRecorder) SandboxOptions() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SandboxOptions", reflect.TypeOf((*MockvmController)(nil).SandboxOptions)) } +// Save mocks base method. +func (m *MockvmController) Save(ctx context.Context) (*anypb.Any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Save", ctx) + ret0, _ := ret[0].(*anypb.Any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Save indicates an expected call of Save. +func (mr *MockvmControllerMockRecorder) Save(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockvmController)(nil).Save), ctx) +} + +// StartLiveMigrationOnSource mocks base method. +func (m *MockvmController) StartLiveMigrationOnSource(ctx context.Context, config *v2.MigrationConfig) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartLiveMigrationOnSource", ctx, config) + ret0, _ := ret[0].(error) + return ret0 +} + +// StartLiveMigrationOnSource indicates an expected call of StartLiveMigrationOnSource. +func (mr *MockvmControllerMockRecorder) StartLiveMigrationOnSource(ctx, config any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartLiveMigrationOnSource", reflect.TypeOf((*MockvmController)(nil).StartLiveMigrationOnSource), ctx, config) +} + +// StartLiveMigrationTransfer mocks base method. +func (m *MockvmController) StartLiveMigrationTransfer(ctx context.Context, options *schema2.MigrationTransferOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartLiveMigrationTransfer", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// StartLiveMigrationTransfer indicates an expected call of StartLiveMigrationTransfer. +func (mr *MockvmControllerMockRecorder) StartLiveMigrationTransfer(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartLiveMigrationTransfer", reflect.TypeOf((*MockvmController)(nil).StartLiveMigrationTransfer), ctx, options) +} + // StartTime mocks base method. func (m *MockvmController) StartTime() time.Time { m.ctrl.T.Helper() @@ -230,6 +362,20 @@ func (mr *MockvmControllerMockRecorder) StartVM(ctx, opts any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartVM", reflect.TypeOf((*MockvmController)(nil).StartVM), ctx, opts) } +// StartWithMigrationOptions mocks base method. +func (m *MockvmController) StartWithMigrationOptions(ctx context.Context, config *v2.MigrationConfig) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartWithMigrationOptions", ctx, config) + ret0, _ := ret[0].(error) + return ret0 +} + +// StartWithMigrationOptions indicates an expected call of StartWithMigrationOptions. +func (mr *MockvmControllerMockRecorder) StartWithMigrationOptions(ctx, config any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartWithMigrationOptions", reflect.TypeOf((*MockvmController)(nil).StartWithMigrationOptions), ctx, config) +} + // State mocks base method. func (m *MockvmController) State() vm.State { m.ctrl.T.Helper() @@ -329,6 +475,20 @@ func (mr *MockvmControllerMockRecorder) UpdatePolicyFragment(ctx, fragment any) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdatePolicyFragment", reflect.TypeOf((*MockvmController)(nil).UpdatePolicyFragment), ctx, fragment) } +// VM mocks base method. +func (m *MockvmController) VM() *vmmanager.UtilityVM { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VM") + ret0, _ := ret[0].(*vmmanager.UtilityVM) + return ret0 +} + +// VM indicates an expected call of VM. +func (mr *MockvmControllerMockRecorder) VM() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VM", reflect.TypeOf((*MockvmController)(nil).VM)) +} + // VPCIController mocks base method. func (m *MockvmController) VPCIController() *vpci.Controller { m.ctrl.T.Helper() diff --git a/cmd/containerd-shim-lcow-v2/service/service.go b/cmd/containerd-shim-lcow-v2/service/service.go index 536295d04b..8135dc1e2d 100644 --- a/cmd/containerd-shim-lcow-v2/service/service.go +++ b/cmd/containerd-shim-lcow-v2/service/service.go @@ -7,11 +7,13 @@ import ( "fmt" "sync" + "github.com/Microsoft/hcsshim/internal/controller/migration" "github.com/Microsoft/hcsshim/internal/controller/pod" "github.com/Microsoft/hcsshim/internal/controller/vm" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/shim" "github.com/Microsoft/hcsshim/internal/shimdiag" + migrationsvc "github.com/Microsoft/hcsshim/pkg/migration" sandboxsvc "github.com/containerd/containerd/api/runtime/sandbox/v1" tasksvc "github.com/containerd/containerd/api/runtime/task/v3" @@ -54,6 +56,10 @@ type Service struct { // from podControllers. containerPodMapping map[string]string + // migrationController orchestrates the live-migration session for the + // sandbox. There is at most one active session per shim. + migrationController *migration.Controller + // shutdown manages graceful shutdown operations and allows registration of cleanup callbacks. shutdown shutdown.Service } @@ -68,6 +74,7 @@ func NewService(ctx context.Context, eventsPublisher shim.Publisher, sd shutdown vmController: vm.New(), podControllers: make(map[string]*pod.Controller), containerPodMapping: make(map[string]string), + migrationController: migration.New(), shutdown: sd, } @@ -95,6 +102,7 @@ func (s *Service) RegisterTTRPC(server *ttrpc.Server) error { tasksvc.RegisterTTRPCTaskService(server, s) sandboxsvc.RegisterTTRPCSandboxService(server, s) shimdiag.RegisterShimDiagService(server, s) + migrationsvc.RegisterMigrationService(server, s) return nil } @@ -106,6 +114,14 @@ func (s *Service) ensureVMRunning() error { return nil } +// ensureMigrationIdle returns an error if a migration session is in progress. +func (s *Service) ensureMigrationIdle() error { + if state := s.migrationController.State(); state != migration.StateIdle { + return fmt.Errorf("migration session is in progress (state: %s): %w", state, errdefs.ErrFailedPrecondition) + } + return nil +} + // SandboxID returns the unique identifier for the sandbox managed by this Service. func (s *Service) SandboxID() string { return s.sandboxID diff --git a/cmd/containerd-shim-lcow-v2/service/service_migration.go b/cmd/containerd-shim-lcow-v2/service/service_migration.go new file mode 100644 index 0000000000..dfc06e04e5 --- /dev/null +++ b/cmd/containerd-shim-lcow-v2/service/service_migration.go @@ -0,0 +1,196 @@ +//go:build windows && lcow + +package service + +import ( + "context" + + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/logfields" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/pkg/migration" + + "github.com/containerd/errdefs/pkg/errgrpc" + "github.com/sirupsen/logrus" + "go.opencensus.io/trace" +) + +// Ensure Service implements the MigrationService interface at compile time. +var _ migration.MigrationService = &Service{} + +// PrepareAndExportSandbox prepares the source sandbox for live migration and +// exports an opaque config that the destination shim can use to import it. +// This method is part of the instrumentation layer and business logic is included in prepareAndExportSandboxInternal. +func (s *Service) PrepareAndExportSandbox(ctx context.Context, request *migration.PrepareAndExportSandboxRequest) (resp *migration.PrepareAndExportSandboxResponse, err error) { + ctx, span := oc.StartSpan(ctx, "PrepareAndExportSandbox") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.prepareAndExportSandboxInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// ImportSandbox imports a sandbox on the destination shim from the opaque +// config produced by PrepareAndExportSandbox on the source. +// This method is part of the instrumentation layer and business logic is included in importSandboxInternal. +func (s *Service) ImportSandbox(ctx context.Context, request *migration.ImportSandboxRequest) (resp *migration.ImportSandboxResponse, err error) { + ctx, span := oc.StartSpan(ctx, "ImportSandbox") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.importSandboxInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// PrepareSandbox prepares the destination-side compute system to receive the +// migrated sandbox state. +// This method is part of the instrumentation layer and business logic is included in prepareSandboxInternal. +func (s *Service) PrepareSandbox(ctx context.Context, request *migration.PrepareSandboxRequest) (resp *migration.PrepareSandboxResponse, err error) { + ctx, span := oc.StartSpan(ctx, "PrepareSandbox") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.prepareSandboxInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// TransferSandbox transfers sandbox state between source and destination +// over the previously established migration transport. +// This method is part of the instrumentation layer and business logic is included in transferSandboxInternal. +func (s *Service) TransferSandbox(ctx context.Context, request *migration.TransferSandboxRequest) (resp *migration.TransferSandboxResponse, err error) { + ctx, span := oc.StartSpan(ctx, "TransferSandbox") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + if request.Timeout != nil { + span.AddAttributes(trace.Int64Attribute(logfields.Timeout, int64(request.Timeout.AsDuration()))) + } + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.transferSandboxInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// FinalizeSandbox finalizes the migration on either side: stop on the +// source, resume on the destination (per the requested action). +// This method is part of the instrumentation layer and business logic is included in finalizeSandboxInternal. +func (s *Service) FinalizeSandbox(ctx context.Context, request *migration.FinalizeSandboxRequest) (resp *migration.FinalizeSandboxResponse, err error) { + ctx, span := oc.StartSpan(ctx, "FinalizeSandbox") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID), + trace.StringAttribute(logfields.Action, request.Action.String())) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.finalizeSandboxInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// Notifications streams migration progress notifications to the caller for +// the lifetime of the migration session. +// This method is part of the instrumentation layer and business logic is included in notificationsInternal. +func (s *Service) Notifications(ctx context.Context, request *migration.NotificationsRequest, server migration.Migration_NotificationsServer) (err error) { + ctx, span := oc.StartSpan(ctx, "Notifications") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + return errgrpc.ToGRPC(s.notificationsInternal(ctx, request, server)) +} + +// CreateDuplicateSocket duplicates a socket handle from the caller into the +// shim process for use as the migration transport. +// This method is part of the instrumentation layer and business logic is included in createDuplicateSocketInternal. +func (s *Service) CreateDuplicateSocket(ctx context.Context, request *migration.CreateDuplicateSocketRequest) (resp *migration.CreateDuplicateSocketResponse, err error) { + ctx, span := oc.StartSpan(ctx, "CreateDuplicateSocket") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.createDuplicateSocketInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// Cancel aborts an in-flight migration. It is mandatory on both source and +// destination to abort a migration that is already underway; it stops the +// in-flight transfer and closes the migration socket but leaves the shim in +// the migrating state until Cleanup is called. +// This method is part of the instrumentation layer and business logic is included in cancelInternal. +func (s *Service) Cancel(ctx context.Context, request *migration.CancelRequest) (resp *migration.CancelResponse, err error) { + ctx, span := oc.StartSpan(ctx, "Cancel") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.cancelInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} + +// Cleanup is the terminal call of a migration on both sides, issued regardless +// of whether the migration succeeded, failed, or was cancelled. It reverts the +// migration controller's state machine back to the normal state. +// This method is part of the instrumentation layer and business logic is included in cleanupInternal. +func (s *Service) Cleanup(ctx context.Context, request *migration.CleanupRequest) (resp *migration.CleanupResponse, err error) { + ctx, span := oc.StartSpan(ctx, "Cleanup") + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + + span.AddAttributes( + trace.StringAttribute(logfields.SandboxID, s.sandboxID), + trace.StringAttribute(logfields.SessionID, request.SessionID)) + + // Set the session id in the logger context for all subsequent logs in this request. + ctx, _ = log.WithContext(ctx, logrus.WithField(logfields.SessionID, request.SessionID)) + + r, e := s.cleanupInternal(ctx, request) + return r, errgrpc.ToGRPC(e) +} diff --git a/cmd/containerd-shim-lcow-v2/service/service_migration_internal.go b/cmd/containerd-shim-lcow-v2/service/service_migration_internal.go new file mode 100644 index 0000000000..f27ff69da7 --- /dev/null +++ b/cmd/containerd-shim-lcow-v2/service/service_migration_internal.go @@ -0,0 +1,238 @@ +//go:build windows && lcow + +package service + +import ( + "context" + "fmt" + "time" + + migrationcontroller "github.com/Microsoft/hcsshim/internal/controller/migration" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/logfields" + "github.com/Microsoft/hcsshim/pkg/migration" + + "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/opencontainers/runtime-spec/specs-go" +) + +// prepareAndExportSandboxInternal is the implementation for PrepareAndExportSandbox. +// +// It initializes the source sandbox for migration and exports the +// opaque snapshot consumed by the destination shim's ImportSandbox API. +func (s *Service) prepareAndExportSandboxInternal(ctx context.Context, request *migration.PrepareAndExportSandboxRequest) (*migration.PrepareAndExportSandboxResponse, error) { + // Convert the protobuf migration options to the HCS representation. + migrationOpts, err := migration.InitializeOptionsFromProto(request.InitOptions) + if err != nil { + return nil, fmt.Errorf("convert migration initialize options: %w", err) + } + + // Arm the source for migration. + if err = s.migrationController.PrepareSource(ctx, + &migrationcontroller.PrepareSourceOptions{ + InitOptions: migrationcontroller.InitOptions{ + SessionID: request.SessionID, + Origin: hcsschema.MigrationOriginSource, + VMController: s.vmController, + PodControllers: s.podControllers, + }, + MigrationOpts: migrationOpts, + }); err != nil { + return nil, fmt.Errorf("source: prepare migration: %w", err) + } + + // Produce the opaque sandbox snapshot (VM plus per-pod state) that the + // destination shim consumes through ImportSandbox. + cfg, err := s.migrationController.ExportState(ctx, request.SessionID) + if err != nil { + return nil, fmt.Errorf("source: export migration state: %w", err) + } + + return &migration.PrepareAndExportSandboxResponse{Config: cfg}, nil +} + +// importSandboxInternal is the implementation for ImportSandbox. +// +// It rehydrates the destination shim from the source's opaque snapshot, +// mutating the Service-owned vm controller and pod controllers in place. +func (s *Service) importSandboxInternal(ctx context.Context, request *migration.ImportSandboxRequest) (*migration.ImportSandboxResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + + // Rehydrate the destination from the source snapshot, recreating the VM + // and pod state and indexing each container to its owning pod. + if err := s.migrationController.ImportState(ctx, + &migrationcontroller.ImportStateOptions{ + InitOptions: migrationcontroller.InitOptions{ + SessionID: request.SessionID, + Origin: hcsschema.MigrationOriginDestination, + VMController: s.vmController, + PodControllers: s.podControllers, + }, + SandboxID: request.SandboxID, + SavedState: request.Config, + ContainerPodMapping: s.containerPodMapping, + }); err != nil { + return nil, fmt.Errorf("destination: import migration state: %w", err) + } + + s.sandboxID = request.SandboxID + return &migration.ImportSandboxResponse{}, nil +} + +// patchMigratedContainerInternal completes CreateTask for a container already +// rehydrated by migration: it rebinds the container's host-side resources to +// this shim and returns its existing init process PID without starting anything. +func (s *Service) patchMigratedContainerInternal( + ctx context.Context, + request *task.CreateTaskRequest, + spec specs.Spec, +) (*task.CreateTaskResponse, error) { + // Rebind the migrated container's host-side resources to this side's IDs. + err := s.migrationController.PatchResourcePaths(ctx, request, spec) + if err != nil { + return nil, fmt.Errorf("patch migrated container %q: %w", request.ID, err) + } + + ctrCtrl, err := getContainerController(s, request.ID) + if err != nil { + return nil, fmt.Errorf("lookup migrated container %q: %w", request.ID, err) + } + + initProc, err := ctrCtrl.GetProcess("") + if err != nil { + return nil, fmt.Errorf("get init process for migrated container %q: %w", request.ID, err) + } + + return &task.CreateTaskResponse{Pid: uint32(initProc.Pid())}, nil +} + +// prepareSandboxInternal is the implementation for PrepareSandbox. +// +// It creates the destination VM's HCS compute system and re-ACLs its disks so +// it is ready to start; all migrated containers must already be patched. +func (s *Service) prepareSandboxInternal(ctx context.Context, request *migration.PrepareSandboxRequest) (*migration.PrepareSandboxResponse, error) { + migrationOpts, err := migration.InitializeOptionsFromProto(request.InitOptions) + if err != nil { + return nil, fmt.Errorf("convert migration initialize options: %w", err) + } + + // Build the destination VM and prep its disks; every container must be patched first. + if err := s.migrationController.PrepareDestination( + ctx, + request.SessionID, + migrationOpts, + ); err != nil { + return nil, fmt.Errorf("destination: prepare migration: %w", err) + } + + return &migration.PrepareSandboxResponse{}, nil +} + +// transferSandboxInternal is the implementation for TransferSandbox. +// +// It drives the memory transfer between source and destination over the +// duplicated socket. +func (s *Service) transferSandboxInternal(ctx context.Context, request *migration.TransferSandboxRequest) (*migration.TransferSandboxResponse, error) { + var timeout time.Duration + if request.Timeout != nil { + timeout = request.Timeout.AsDuration() + } + + // Wait for the transport socket, then drive the memory transfer within the timeout. + if err := s.migrationController.Transfer( + ctx, + request.SessionID, + timeout, + ); err != nil { + return nil, fmt.Errorf("transfer migration state: %w", err) + } + return &migration.TransferSandboxResponse{}, nil +} + +// finalizeSandboxInternal is the implementation for FinalizeSandbox. +// +// It commits the migration's final outcome on each side per the requested +// action — resuming the sandbox back to running or stopping it — and ends the session. +func (s *Service) finalizeSandboxInternal(ctx context.Context, request *migration.FinalizeSandboxRequest) (*migration.FinalizeSandboxResponse, error) { + // Apply the requested resume/stop outcome and end the session. + if err := s.migrationController.Finalize(ctx, request.SessionID, request.Action, s.events); err != nil { + return nil, fmt.Errorf("finalize migration session: %w", err) + } + + return &migration.FinalizeSandboxResponse{}, nil +} + +// notificationsInternal is the implementation for Notifications. +// +// It forwards migration notifications to the calling stream until the session +// terminates or the client disconnects. +func (s *Service) notificationsInternal(ctx context.Context, request *migration.NotificationsRequest, server migration.Migration_NotificationsServer) error { + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + + // Attach to this session's migration progress notifications. + ch, err := s.migrationController.Subscribe(subCtx, request.SessionID) + if err != nil { + return fmt.Errorf("subscribe to migration notifications: %w", err) + } + + logger := log.G(ctx).WithField(logfields.SessionID, request.SessionID) + for { + select { + case <-ctx.Done(): + return ctx.Err() + case resp, ok := <-ch: + if !ok { + // Session terminated; close the stream cleanly. + return nil + } + + // Forward each notification to the client stream. + if err := server.Send(resp); err != nil { + logger.WithError(err).Warn("send migration notification failed") + return fmt.Errorf("send migration notification: %w", err) + } + } + } +} + +// createDuplicateSocketInternal is the implementation for CreateDuplicateSocket. +// +// It reconstructs the connected migration transport socket from the caller's +// duplicated protocol info and adopts it for the session, unblocking the transfer. +func (s *Service) createDuplicateSocketInternal(ctx context.Context, request *migration.CreateDuplicateSocketRequest) (*migration.CreateDuplicateSocketResponse, error) { + // Adopt the caller's duplicated socket as the transport and unblock the transfer. + if err := s.migrationController.RegisterDuplicateSocket(ctx, request.SessionID, request.ProtocolInfo); err != nil { + return nil, fmt.Errorf("register duplicate migration socket: %w", err) + } + + return &migration.CreateDuplicateSocketResponse{}, nil +} + +// cancelInternal is the implementation for Cancel. +// +// It aborts the in-flight transfer and closes the migration socket without +// reverting the controller state machine; Cleanup performs the final revert. +func (s *Service) cancelInternal(ctx context.Context, request *migration.CancelRequest) (*migration.CancelResponse, error) { + // Abort the in-flight transfer now; the session stays until cleanup. + if err := s.migrationController.Cancel(ctx, request.SessionID); err != nil { + return nil, fmt.Errorf("cancel migration session: %w", err) + } + + return &migration.CancelResponse{}, nil +} + +// cleanupInternal is the implementation for Cleanup. +// +// It reverts the migration controller state machine back to idle, resuming or +// aborting the underlying controllers based on the current migration state. +func (s *Service) cleanupInternal(ctx context.Context, request *migration.CleanupRequest) (*migration.CleanupResponse, error) { + // Tear down the session and revert this side back to idle. + if err := s.migrationController.Cleanup(ctx, request.SessionID, s.events); err != nil { + return nil, fmt.Errorf("cleanup migration session: %w", err) + } + + return &migration.CleanupResponse{}, nil +} diff --git a/cmd/containerd-shim-lcow-v2/service/service_sandbox_internal_test.go b/cmd/containerd-shim-lcow-v2/service/service_sandbox_internal_test.go index 0036831d12..29aecbf06b 100644 --- a/cmd/containerd-shim-lcow-v2/service/service_sandbox_internal_test.go +++ b/cmd/containerd-shim-lcow-v2/service/service_sandbox_internal_test.go @@ -21,6 +21,7 @@ import ( "github.com/Microsoft/hcsshim/cmd/containerd-shim-lcow-v2/service/mocks" "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" "github.com/Microsoft/hcsshim/internal/builder/vm/lcow" + "github.com/Microsoft/hcsshim/internal/controller/migration" "github.com/Microsoft/hcsshim/internal/controller/pod" "github.com/Microsoft/hcsshim/internal/controller/vm" ) @@ -48,6 +49,7 @@ func newTestService(t *testing.T) (*Service, *mocks.MockvmController) { events: make(chan interface{}, 128), podControllers: make(map[string]*pod.Controller), containerPodMapping: make(map[string]string), + migrationController: migration.New(), shutdown: sd, }, mockCtrl } diff --git a/cmd/containerd-shim-lcow-v2/service/service_task_internal.go b/cmd/containerd-shim-lcow-v2/service/service_task_internal.go index bfa9765516..0fc246a619 100644 --- a/cmd/containerd-shim-lcow-v2/service/service_task_internal.go +++ b/cmd/containerd-shim-lcow-v2/service/service_task_internal.go @@ -75,6 +75,11 @@ func (s *Service) getPodController(podID string) (*pod.Controller, bool) { // stateInternal returns the current status of a process within a container. func (s *Service) stateInternal(_ context.Context, request *task.StateRequest) (*task.StateResponse, error) { + // Task queries are only valid while no migration session is in progress. + if err := s.ensureMigrationIdle(); err != nil { + return nil, err + } + // Look up the container controller for the requested container. ctrCtrl, err := getContainerController(s, request.ID) if err != nil { @@ -93,8 +98,8 @@ func (s *Service) stateInternal(_ context.Context, request *task.StateRequest) ( // createInternal creates a new pod sandbox or workload container based on the OCI spec. func (s *Service) createInternal(ctx context.Context, request *task.CreateTaskRequest) (*task.CreateTaskResponse, error) { - if err := s.ensureVMRunning(); err != nil { - return nil, err + if state := s.vmController.State(); state != vm.StateRunning && state != vm.StateMigratingImported { + return nil, fmt.Errorf("vm is not running or migrating (state: %s): %w", state, errdefs.ErrFailedPrecondition) } // Parse the OCI spec from the bundle. @@ -116,6 +121,14 @@ func (s *Service) createInternal(ctx context.Context, request *task.CreateTaskRe return nil, err } + // Live-migration destination path: the container is already rehydrated + // from the imported snapshot. Delegate to the migration helper, which + // patches host-side resource paths, fixes bookkeeping, and publishes the + // TaskCreate event without driving the creation/start lifecycle. + if _, ok := spec.Annotations[annotations.LiveMigrationSourceContainerID]; ok { + return s.patchMigratedContainerInternal(ctx, request, spec) + } + s.mu.Lock() defer s.mu.Unlock() @@ -279,6 +292,11 @@ func (s *Service) startInternal(ctx context.Context, request *task.StartRequest) // deleteInternal deletes a process, container, or pod sandbox depending on the request. func (s *Service) deleteInternal(ctx context.Context, request *task.DeleteRequest) (*task.DeleteResponse, error) { + // Deletion is only valid while no migration session is in progress. + if err := s.ensureMigrationIdle(); err != nil { + return nil, err + } + // Look up the container controller for the target ID. ctrCtrl, err := getContainerController(s, request.ID) if err != nil { @@ -401,7 +419,8 @@ func (s *Service) checkpointInternal(_ context.Context, _ *task.CheckpointTaskRe // killInternal sends a signal to a process or, when All is set, to every process in the pod. func (s *Service) killInternal(ctx context.Context, request *task.KillRequest) (*emptypb.Empty, error) { - if err := s.ensureVMRunning(); err != nil { + // Kill is only valid while no migration session is in progress. + if err := s.ensureMigrationIdle(); err != nil { return nil, err } @@ -607,10 +626,8 @@ func (s *Service) updateVMResources(ctx context.Context, resources interface{}, // waitInternal blocks until the specified process exits and returns its exit status. func (s *Service) waitInternal(ctx context.Context, request *task.WaitRequest) (*task.WaitResponse, error) { - if err := s.ensureVMRunning(); err != nil { - return nil, err - } - + // No ensureVMRunning: Wait/Status only read controller-cached state so + // they're safe even after the VM has been terminated. ctrCtrl, err := getContainerController(s, request.ID) if err != nil { return nil, fmt.Errorf("failed to find container for wait request: %w", err) diff --git a/cmd/containerd-shim-lcow-v2/service/service_task_internal_test.go b/cmd/containerd-shim-lcow-v2/service/service_task_internal_test.go index 92fbc132f7..a68f3985b1 100644 --- a/cmd/containerd-shim-lcow-v2/service/service_task_internal_test.go +++ b/cmd/containerd-shim-lcow-v2/service/service_task_internal_test.go @@ -41,10 +41,11 @@ var ( // one representative not-running state (NotCreated); a regression in the guard // would let containerd issue task RPCs against a VM that has not booted. // -// state and delete are intentionally excluded: containerd issues them during -// task teardown, so they operate on container bookkeeping and surface NotFound -// when the container is absent (see TestTaskMethods_RejectUnknownContainer) -// rather than a precondition error. +// state, delete, kill, and wait are intentionally excluded: they omit the +// VM-running guard so they can run during task teardown / migration aborts, so +// they operate on container bookkeeping and surface NotFound when the container +// is absent (see TestTaskMethods_RejectUnknownContainer) rather than a +// precondition error. func TestTaskMethods_RejectVMNotRunning(t *testing.T) { tests := []struct { name string @@ -71,13 +72,6 @@ func TestTaskMethods_RejectVMNotRunning(t *testing.T) { return err }, }, - { - name: "killInternal", - call: func(svc *Service) error { - _, err := svc.killInternal(context.Background(), &task.KillRequest{ID: "ctr"}) - return err - }, - }, { name: "execInternal", call: func(svc *Service) error { @@ -106,13 +100,6 @@ func TestTaskMethods_RejectVMNotRunning(t *testing.T) { return err }, }, - { - name: "waitInternal", - call: func(svc *Service) error { - _, err := svc.waitInternal(context.Background(), &task.WaitRequest{ID: "ctr"}) - return err - }, - }, { name: "statsInternal", call: func(svc *Service) error { @@ -655,8 +642,7 @@ func TestDeleteProcess_Success(t *testing.T) { // TestKill_Success verifies the single-container kill path: killInternal // forwards the exec ID, signal, and all=false to the container controller. func TestKill_Success(t *testing.T) { - svc, mockVM := newTestService(t) - mockVM.EXPECT().State().Return(vm.StateRunning) + svc, _ := newTestService(t) mockCtr := mocks.NewMockcontainerController(gomock.NewController(t)) swapGetContainerController(t, mockCtr) diff --git a/cmd/containerd-shim-lcow-v2/service/types.go b/cmd/containerd-shim-lcow-v2/service/types.go index af6382a555..c3c9f57a88 100644 --- a/cmd/containerd-shim-lcow-v2/service/types.go +++ b/cmd/containerd-shim-lcow-v2/service/types.go @@ -16,12 +16,14 @@ import ( "github.com/Microsoft/hcsshim/internal/controller/process" "github.com/Microsoft/hcsshim/internal/controller/vm" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + hcs "github.com/Microsoft/hcsshim/internal/hcs/v2" "github.com/Microsoft/hcsshim/internal/protocol/guestresource" "github.com/Microsoft/hcsshim/internal/shimdiag" "github.com/Microsoft/hcsshim/internal/vm/guestmanager" "github.com/containerd/containerd/api/runtime/task/v3" containerdtypes "github.com/containerd/containerd/api/types/task" "github.com/opencontainers/runtime-spec/specs-go" + "google.golang.org/protobuf/types/known/anypb" ) // vmController is the subset of the VM controller that [Service] depends on. @@ -94,6 +96,40 @@ type vmController interface { // NetworkController returns the network controller for the VM. NetworkController(networkNamespaceID string) *network.Controller + + // The methods below are required by the migration controller, which the + // Service hands this vmController to. Declaring them here lets the same + // field satisfy the migration controller's vmController interface. + + // InitializeLiveMigrationOnSource arms the running source VM for migration. + InitializeLiveMigrationOnSource(ctx context.Context, options *hcsschema.MigrationInitializeOptions) error + + // Save captures the migrating VM's state into a portable snapshot. + Save(ctx context.Context) (*anypb.Any, error) + + // Import rehydrates a destination VM controller from a saved snapshot. + Import(ctx context.Context, env *anypb.Any) error + + // Patch re-ACLs the destination VM's disks after creation. + Patch(ctx context.Context) error + + // StartLiveMigrationOnSource starts the outgoing migration on the source VM. + StartLiveMigrationOnSource(ctx context.Context, config *hcs.MigrationConfig) error + + // StartWithMigrationOptions starts the destination VM against the transport. + StartWithMigrationOptions(ctx context.Context, config *hcs.MigrationConfig) error + + // StartLiveMigrationTransfer begins the memory transfer. + StartLiveMigrationTransfer(ctx context.Context, options *hcsschema.MigrationTransferOptions) error + + // FinalizeLiveMigration applies the migration's final operation to the VM. + FinalizeLiveMigration(ctx context.Context, options *hcsschema.MigrationFinalizedOptions) error + + // Resume returns a migrating VM to the running state. + Resume(ctx context.Context, rebuildBridge bool) error + + // MigrationNotifications returns the VM's migration event stream. + MigrationNotifications() (<-chan hcsschema.OperationSystemMigrationNotificationInfo, error) } // containerController is the subset of the container controller that [Service] diff --git a/internal/controller/migration/controller.go b/internal/controller/migration/controller.go new file mode 100644 index 0000000000..fdabe730de --- /dev/null +++ b/internal/controller/migration/controller.go @@ -0,0 +1,417 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + "sync" + "syscall" + "time" + + "github.com/Microsoft/hcsshim/internal/controller/pod" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + hcs "github.com/Microsoft/hcsshim/internal/hcs/v2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/logfields" + + "github.com/Microsoft/go-winio/pkg/guid" + "github.com/Microsoft/hcsshim/pkg/migration" + "github.com/containerd/errdefs" + "golang.org/x/sys/windows" +) + +// defaultSocketReadyTimeout caps the Transfer wait for the duplicate socket. +const defaultSocketReadyTimeout = 10 * time.Minute + +// Controller sequences a single live-migration session for an LCOW sandbox, +// driving the source or destination side through its lifecycle states. +type Controller struct { + // mu guards all mutable fields below. + mu sync.RWMutex + + // state is the session's current lifecycle state. + state State + + // sessionID identifies the active migration session. + sessionID string + + // sandboxID is the sandbox this session migrates; set on the destination. + sandboxID string + + // origin is this host's role in the session: source or destination. + origin hcsschema.MigrationOrigin + + // vmController drives the VM being migrated; borrowed from the service. + vmController vmController + + // podControllers are the sandbox's pods keyed by pod ID; borrowed from the service. + podControllers map[string]*pod.Controller + + // containerPodMapping aliases the service-owned containerID -> podID + // index so ImportState and PatchResourcePaths can keep it in sync with + // pod/container renames. Guarded by mu. + containerPodMapping map[string]string + + // pendingPatches is the set of source container IDs imported by + // ImportState that still need a PatchResourcePaths call; a container + // drops out once it has been patched. Guarded by mu. + pendingPatches map[string]struct{} + + // dupSocket is the duplicated transport socket used for the memory transfer. + dupSocket windows.Handle + + // socketReady is closed by RegisterDuplicateSocket on transition to StateSocketReady. + socketReady chan struct{} + + // notifier is created lazily on the first Subscribe. Guarded by mu. + notifier *notifications +} + +// New returns an idle controller ready to host a migration session. +func New() *Controller { + return &Controller{ + state: StateIdle, + socketReady: make(chan struct{}), + } +} + +// State returns the current session state. +func (c *Controller) State() State { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.state +} + +// Transfer starts the session's memory transfer and returns immediately. The +// wait for the duplicate socket and the transfer itself run in the background; +// failures reach notification subscribers rather than the caller. Duplicate +// calls are no-ops. +func (c *Controller) Transfer(ctx context.Context, sessionID string, timeout time.Duration) error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.sessionID != sessionID { + return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) + } + + // Only one background goroutine should drive the transfer, so duplicate + // Transfer calls must be no-ops. If the duplicate socket has not arrived + // yet, the first call claims StateSocketWaiting and later calls return + // below. If the socket has already arrived, the state is left unchanged here + // and the goroutines dedup instead: only one goroutine moves + // StateSocketReady to StateTransferring and the rest bail. + switch c.state { + case StateSocketReady: + // Socket already arrived: fall through and start the transfer goroutine. + case StateSourceExported, StateDestinationPrepared: + // Socket not here yet: claim the wait so later calls no-op below. + c.state = StateSocketWaiting + case StateSocketWaiting, StateTransferring, StateTransferCompleted: + // A transfer is already waiting, running, or done: nothing to do. + return nil + default: + // No other state can begin a transfer. + return fmt.Errorf("transfer not valid in state %s: %w", c.state, errdefs.ErrFailedPrecondition) + } + + // Fall back to the default wait when the caller gives no timeout. + if timeout <= 0 { + timeout = defaultSocketReadyTimeout + } + socketReady := c.socketReady + + // Drive the transfer in the background so the caller returns immediately; + // its outcome is surfaced to notification subscribers, not returned here. + go func() { + // Detached ctx so the transfer outlives the gRPC call. + ctx = context.WithoutCancel(ctx) + + // Wait for the duplicate socket to arrive, or give up after the timeout. + var socketTimeoutErr error + select { + case <-socketReady: + case <-time.After(timeout): + socketTimeoutErr = fmt.Errorf("timed out waiting for socket ready after %s: %w", timeout, errdefs.ErrFailedPrecondition) + } + + c.mu.Lock() + defer c.mu.Unlock() + + // If the socket connection was not ready within timeout, return an error. + if socketTimeoutErr != nil { + c.failTransfer(ctx, socketTimeoutErr) + return + } + + // Gate: only the first goroutine finds StateSocketReady and claims the + // transfer by moving to StateTransferring; any later goroutine (or a + // torn-down session) bails. + if c.state != StateSocketReady { + return + } + + // Move the state to transferring so that other concurrent callers + // will return early from above and there is a single driver for transfer. + c.state = StateTransferring + + // Run the transfer; a failure marks the session failed for subscribers. + if err := c.runTransfer(ctx); err != nil { + c.failTransfer(ctx, err) + return + } + + c.state = StateTransferCompleted + log.G(ctx).Info("migration transfer completed") + }() + + return nil +} + +// failTransfer marks the session failed and surfaces err to notification +// subscribers as a PHASE_FAILED event. +func (c *Controller) failTransfer(ctx context.Context, err error) { + c.state = StateFailed + + log.G(ctx).WithError(err).Error("migration transfer failed") + + // If there are no subscribers, then we should return early. + if c.notifier == nil { + return + } + + result := hcsschema.MigrationResultDestinationMigrationFailed + if c.origin == hcsschema.MigrationOriginSource { + result = hcsschema.MigrationResultSourceMigrationFailed + } + + // Broadcast the failure to subscribers as a migration-failed event. + c.notifier.broadcast(hcsschema.OperationSystemMigrationNotificationInfo{ + Origin: c.origin, + Event: hcsschema.MigrationEventMigrationFailed, + Result: result, + }) +} + +// sessionIDToUint32 derives a stable uint32 from a session ID. SHA-256 is +// deterministic and uniformly distributed, so the same session ID always maps +// to the same value on both hosts. +func sessionIDToUint32(sessionID string) uint32 { + // Session IDs that parse as a GUID are normalized first so + // formatting/case differences map to the same value. + if g, err := guid.FromString(sessionID); err == nil { + sessionID = g.String() + } + + sum := sha256.Sum256([]byte(sessionID)) + return binary.BigEndian.Uint32(sum[:4]) +} + +// runTransfer issues the per-origin HCS calls for the memory transfer. +func (c *Controller) runTransfer(ctx context.Context) error { + config := &hcs.MigrationConfig{ + Socket: syscall.Handle(c.dupSocket), + SessionID: sessionIDToUint32(c.sessionID), + } + + // Start the live migration on the VM according to this host's role. + switch c.origin { + case hcsschema.MigrationOriginSource: + if err := c.vmController.StartLiveMigrationOnSource(ctx, config); err != nil { + return fmt.Errorf("start live migration on source: %w", err) + } + case hcsschema.MigrationOriginDestination: + if err := c.vmController.StartWithMigrationOptions(ctx, config); err != nil { + return fmt.Errorf("start with migration options: %w", err) + } + default: + return fmt.Errorf("unsupported migration origin %q: %w", c.origin, errdefs.ErrInvalidArgument) + } + + // Begin streaming the memory transfer over the socket. + transferOpts := &hcsschema.MigrationTransferOptions{Origin: c.origin} + if err := c.vmController.StartLiveMigrationTransfer(ctx, transferOpts); err != nil { + return fmt.Errorf("start live migration transfer: %w", err) + } + + return nil +} + +// Finalize applies the session's final outcome (resume or stop) on this host +// and, on success, leaves the session finalized so Cleanup can run. A repeat +// call for an already-finalized session is a no-op. +func (c *Controller) Finalize(ctx context.Context, sessionID string, action migration.FinalizeAction, events chan interface{}) error { + if action == migration.FinalizeAction_FINALIZE_ACTION_UNSPECIFIED { + return fmt.Errorf("finalize action must be specified: %w", errdefs.ErrInvalidArgument) + } + + c.mu.Lock() + defer c.mu.Unlock() + + // Idempotent: an already-finalized session succeeds without redoing work. + if c.state == StateFinalized { + return nil + } + + // Finalize is valid only after a completed transfer or a cancellation. + if c.state != StateTransferCompleted && c.state != StateCancelled { + return fmt.Errorf("finalize not valid in state %s: %w", c.state, errdefs.ErrFailedPrecondition) + } + + // Map the gRPC finalize action to the HCS final operation. + var finalOp hcsschema.MigrationFinalOperation + switch action { + case migration.FinalizeAction_FINALIZE_ACTION_RESUME: + finalOp = hcsschema.MigrationFinalOperationResume + case migration.FinalizeAction_FINALIZE_ACTION_STOP: + finalOp = hcsschema.MigrationFinalOperationStop + default: + return fmt.Errorf("unsupported finalize action %s: %w", action, errdefs.ErrInvalidArgument) + } + + // Apply the final operation (resume or stop) to the underlying VM. + if err := c.vmController.FinalizeLiveMigration(ctx, + &hcsschema.MigrationFinalizedOptions{ + Origin: c.origin, + FinalizedOperation: finalOp, + }); err != nil { + return fmt.Errorf("finalize live migration (origin=%s, action=%s): %w", c.origin, action, err) + } + + // On RESUME, rebuild the GCS bridge. Destination also walks pods to + // reattach gcs.Container/gcs.Process; source has nothing else to restore. + if finalOp == hcsschema.MigrationFinalOperationResume { + switch c.origin { + case hcsschema.MigrationOriginDestination: + // Resume the destination VM. + if err := c.vmController.Resume(ctx, false /* rebuildBridge */); err != nil { + return fmt.Errorf("resume destination vm from migration: %w", err) + } + + // Reattach each migrated pod's containers and processes on this host. + for podID, podCtrl := range c.podControllers { + if err := podCtrl.Resume(ctx, c.vmController, events, true /* isDestination */); err != nil { + return fmt.Errorf("resume pod %q from migration: %w", podID, err) + } + } + case hcsschema.MigrationOriginSource: + // Resume the source VM, rebuilding its GCS bridge. + if err := c.vmController.Resume(ctx, true /* rebuildBridge */); err != nil { + return fmt.Errorf("resume source vm from migration: %w", err) + } + + // Resume pods to lift the source freeze, skipping destination-only + // steps such as network reset. + for podID, podCtrl := range c.podControllers { + if err := podCtrl.Resume(ctx, c.vmController, events, false /* isDestination */); err != nil { + return fmt.Errorf("resume pod %q from migration: %w", podID, err) + } + } + } + } + + if finalOp == hcsschema.MigrationFinalOperationStop { + // For both source and destination, there is nothing to do for VM. + // vm.FinalizeLiveMigration + STOP will stop the VM itself. + // We only need to take care of tasks. + switch c.origin { + case hcsschema.MigrationOriginDestination: + // On the destination side, we would not create the exit handlers until resume + // which happens in Finalize + Resume. Therefore, containers would not have + // any way to report the exit event. Hence, we explicitly abort the + // in-migration tasks. + for _, podCtrl := range c.podControllers { + podCtrl.AbortMigrated(ctx, events) + } + case hcsschema.MigrationOriginSource: + // On the source side, all the exit handlers for containers and processes + // are already running. Therefore, during vm.FinalizeLiveMigration, if the + // operation was stop, we would collapse the bridge which would lead to all + // the exit handlers observing the VM exiting and report the exit event. + // Therefore, nothing to do here. + } + } + + // Mark the session finalized so Cleanup can run and repeat calls no-op. + c.state = StateFinalized + + log.G(ctx).WithField(logfields.Action, action.String()).Info("migration session finalized") + return nil +} + +// Cancel aborts an in-flight migration session, marking it cancelled so a +// subsequent Finalize and Cleanup can wind it down. A repeat call is a no-op. +func (c *Controller) Cancel(ctx context.Context, sessionID string) error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.sessionID != sessionID { + return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrFailedPrecondition) + } + + // Idempotent: already canceled. + if c.state == StateCancelled { + return nil + } + + // TODO: call into HCS to cancel the in-flight migration once the cancel API is available. + + // Mark the session canceled; Cleanup later returns it to idle. + c.state = StateCancelled + + log.G(ctx).Info("migration session cancelled") + return nil +} + +// Cleanup is the terminal call of a migration session on either side. It +// reverts the controller back to [StateIdle], releasing the transport socket +// and tearing down notification subscribers, regardless of whether the +// migration completed, failed, or was cancelled. A call against an already-idle +// controller is a no-op. +func (c *Controller) Cleanup(ctx context.Context, sessionID string, events chan interface{}) error { + c.mu.Lock() + defer c.mu.Unlock() + + // Already cleaned up / no active session. + if c.state == StateIdle { + return nil + } + + if c.sessionID != sessionID { + return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrFailedPrecondition) + } + + // Cleanup is only valid once the session has been finalized. + if c.state != StateFinalized { + return fmt.Errorf("cleanup not valid in state %s: %w", c.state, errdefs.ErrFailedPrecondition) + } + + // The transport socket is unused past this point and the notifier is + // scoped to the session, so release both before resetting state. + if c.dupSocket != 0 { + if err := windows.Closesocket(c.dupSocket); err != nil { + log.G(ctx).WithError(err).Warn("close duplicate migration socket") + } + c.dupSocket = 0 + } + + if c.notifier != nil { + c.notifier.close() + c.notifier = nil + } + + // Reset all session-scoped state so the controller can host a new session. + c.sessionID, c.sandboxID, c.origin = "", "", "" + c.vmController = nil + c.podControllers = nil + c.containerPodMapping = nil + c.pendingPatches = nil + c.socketReady = make(chan struct{}) + c.state = StateIdle + + log.G(ctx).Info("migration session cleaned up") + return nil +} diff --git a/internal/controller/migration/controller_destination.go b/internal/controller/migration/controller_destination.go new file mode 100644 index 0000000000..1309eb6cd0 --- /dev/null +++ b/internal/controller/migration/controller_destination.go @@ -0,0 +1,263 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "fmt" + "maps" + "slices" + "strings" + + save "github.com/Microsoft/hcsshim/internal/controller/migration/save" + "github.com/Microsoft/hcsshim/internal/controller/pod" + "github.com/Microsoft/hcsshim/internal/controller/vm" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/logfields" + "github.com/Microsoft/hcsshim/internal/oci" + hcsannotations "github.com/Microsoft/hcsshim/pkg/annotations" + + "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/containerd/errdefs" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" +) + +// ImportState rehydrates a source-side migration snapshot onto this controller. +// A repeat call for the same session is a no-op. +func (c *Controller) ImportState(ctx context.Context, opts *ImportStateOptions) error { + // Reject malformed input up front so the controller is never mutated + // on the basis of a half-specified request. + switch { + case opts == nil: + return fmt.Errorf("options are required: %w", errdefs.ErrInvalidArgument) + case opts.SessionID == "": + return fmt.Errorf("session id is required: %w", errdefs.ErrInvalidArgument) + case opts.VMController == nil: + return fmt.Errorf("vm controller is required: %w", errdefs.ErrInvalidArgument) + case opts.SandboxID == "": + return fmt.Errorf("sandbox id is required: %w", errdefs.ErrInvalidArgument) + case opts.PodControllers == nil: + return fmt.Errorf("pod controllers map is required: %w", errdefs.ErrInvalidArgument) + case opts.ContainerPodMapping == nil: + return fmt.Errorf("container-pod mapping is required: %w", errdefs.ErrInvalidArgument) + case opts.SavedState == nil: + return fmt.Errorf("sandbox saved state is required: %w", errdefs.ErrInvalidArgument) + case opts.SavedState.TypeUrl != save.TypeURL: + return fmt.Errorf("unsupported sandbox saved-state type %q: %w", opts.SavedState.TypeUrl, errdefs.ErrInvalidArgument) + case opts.VMController.State() != vm.StateNotCreated: + return fmt.Errorf("vm controller is in invalid state %s: %w", opts.VMController.State(), errdefs.ErrFailedPrecondition) + default: + } + + decoded := &save.Payload{} + if err := proto.Unmarshal(opts.SavedState.Value, decoded); err != nil { + return fmt.Errorf("unmarshal sandbox saved state: %w", err) + } + if decoded.GetSchemaVersion() != save.SchemaVersion { + return fmt.Errorf("sandbox saved-state schema version %d not supported (want %d): %w", + decoded.GetSchemaVersion(), save.SchemaVersion, errdefs.ErrInvalidArgument) + } + + c.mu.Lock() + defer c.mu.Unlock() + + // Idempotent retry for the same session; any other non-idle state is a conflict. + if c.state == StateDestinationImported && c.sessionID == opts.SessionID { + return nil + } + if c.state != StateIdle { + return fmt.Errorf("controller is in state %s for session %q: %w", c.state, c.sessionID, errdefs.ErrAlreadyExists) + } + + // Rehydrate the VM controller from the saved VM payload. + if err := opts.VMController.Import(ctx, decoded.GetVm()); err != nil { + return fmt.Errorf("import vm controller: %w", err) + } + + // Rehydrate each pod and index its containers so PatchResourcePaths + // can look up the owning pod. + pending := make(map[string]struct{}) + for _, podAny := range decoded.GetPods() { + // Rebuild the pod controller from its saved payload. + importedPod, err := pod.Import(ctx, podAny) + if err != nil { + return fmt.Errorf("import pod: %w", err) + } + + opts.PodControllers[importedPod.PodID()] = importedPod + + // Source container IDs must be unique across pods so the later patch + // lookup is unambiguous. + for containerID := range importedPod.ListContainers() { + if _, dup := pending[containerID]; dup { + return fmt.Errorf("duplicate source container id %q across imported pods: %w", containerID, errdefs.ErrInvalidArgument) + } + + // Track the container as awaiting a patch and map it to its pod. + pending[containerID] = struct{}{} + opts.ContainerPodMapping[containerID] = importedPod.PodID() + } + } + + c.sessionID = opts.SessionID + c.sandboxID = opts.SandboxID + c.origin = opts.Origin + c.vmController = opts.VMController + c.podControllers = opts.PodControllers + c.containerPodMapping = opts.ContainerPodMapping + c.pendingPatches = pending + c.state = StateDestinationImported + + log.G(ctx).Info("migration destination state imported") + return nil +} + +// PatchResourcePaths rewrites the imported source container's identifiers to +// the destination IDs carried by request and spec. A container may be patched +// only once; a repeat call for an already-patched container fails. +func (c *Controller) PatchResourcePaths( + ctx context.Context, + request *task.CreateTaskRequest, + spec specs.Spec, +) error { + switch { + case request == nil: + return fmt.Errorf("request is required: %w", errdefs.ErrInvalidArgument) + case request.ID == "": + return fmt.Errorf("destination container id is required: %w", errdefs.ErrInvalidArgument) + case spec.Annotations == nil: + return fmt.Errorf("annotations are required: %w", errdefs.ErrInvalidArgument) + } + + // The source container being rebound is identified by the annotation. + sourceContainerID := spec.Annotations[hcsannotations.LiveMigrationSourceContainerID] + if sourceContainerID == "" { + return fmt.Errorf("annotation %q is required: %w", hcsannotations.LiveMigrationSourceContainerID, errdefs.ErrInvalidArgument) + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.state != StateDestinationImported { + return fmt.Errorf("patch not valid in state %s: %w", c.state, errdefs.ErrFailedPrecondition) + } + + // A container may be patched only once; once patched, it drops out of + // pendingPatches. Hence, a non-pending ID has already been patched. + if _, pending := c.pendingPatches[sourceContainerID]; !pending { + return fmt.Errorf("source container %q is not pending a patch: %w", sourceContainerID, errdefs.ErrAlreadyExists) + } + + // containerPodMapping is the source-of-truth index built in ImportState + // and rewritten in place here, so a direct lookup beats scanning pods. + sourcePodID, ok := c.containerPodMapping[sourceContainerID] + if !ok { + return fmt.Errorf("source container %q not found: %w", sourceContainerID, errdefs.ErrNotFound) + } + podCtrl := c.podControllers[sourcePodID] + + // Sandbox is detected structurally as container ID == pod ID. + isSandbox := sourceContainerID == sourcePodID + + // The K8s container-type annotation, if present, must agree with the + // structural detection; a mismatch would desync the in-pod and outer-map renames. + if v := spec.Annotations[hcsannotations.KubernetesContainerType]; v != "" { + annotationSaysSandbox := v == string(oci.KubernetesContainerTypeSandbox) + if annotationSaysSandbox != isSandbox { + return fmt.Errorf("annotation %q=%q disagrees with structural sandbox detection (sourceContainerID=%q, sourcePodID=%q): %w", + hcsannotations.KubernetesContainerType, v, sourceContainerID, sourcePodID, errdefs.ErrInvalidArgument) + } + } + + // Fetch the SCSI controller so that we can patch the VHD paths on destination. + scsiCtrl, err := c.vmController.SCSIController(ctx) + if err != nil { + return fmt.Errorf("get scsi controller for patch: %w", err) + } + + // Rebind the container's resources within its pod to the destination IDs. + if err := podCtrl.Patch(ctx, sourceContainerID, isSandbox, scsiCtrl, request, spec); err != nil { + return fmt.Errorf("patch source container %q in pod %q: %w", sourceContainerID, sourcePodID, err) + } + + // Patching the sandbox renames the pod: re-key its podControllers entry and + // repoint every container still mapped to the old pod ID at the new one. + if isSandbox { + // Move the pod controller from the old pod ID to the destination ID. + delete(c.podControllers, sourcePodID) + c.podControllers[request.ID] = podCtrl + + // Repoint any container (this sandbox plus worker containers patched earlier) + // that still references the old pod ID at the new pod ID. + for cid, pid := range c.containerPodMapping { + if pid == sourcePodID { + c.containerPodMapping[cid] = request.ID + } + } + } + + // Re-key this container's mapping entry from its source to destination ID. + delete(c.containerPodMapping, sourceContainerID) + c.containerPodMapping[request.ID] = podCtrl.PodID() + + // The container is now patched, so drop it from the pending set. + delete(c.pendingPatches, sourceContainerID) + + log.G(ctx).WithFields(logrus.Fields{ + logfields.SourceContainerID: sourceContainerID, + logfields.DestinationContainerID: request.ID, + }).Info("migration container resource paths patched") + + return nil +} + +// PrepareDestination materialises the destination HCS compute system once every +// imported container has been patched to its destination identifiers. +func (c *Controller) PrepareDestination(ctx context.Context, sessionID string, migrationOpts *hcsschema.MigrationInitializeOptions) error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.sessionID != sessionID { + return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) + } + + if c.state != StateDestinationImported { + return fmt.Errorf("prepare destination not valid in state %s: %w", c.state, errdefs.ErrFailedPrecondition) + } + + // Every imported container must be patched before the destination VM is built. + if len(c.pendingPatches) > 0 { + pending := slices.Sorted(maps.Keys(c.pendingPatches)) + return fmt.Errorf("%d container patches still pending [%s]: %w", + len(pending), strings.Join(pending, ", "), errdefs.ErrFailedPrecondition) + } + + // Default options and stamp the destination origin so HCS sees a + // complete config regardless of caller input. + if migrationOpts == nil { + migrationOpts = &hcsschema.MigrationInitializeOptions{} + } + migrationOpts.Origin = c.origin + + // Build the destination VM's HCS compute system from the imported config. + if err := c.vmController.CreateVM(ctx, + &vm.CreateOptions{ + ID: fmt.Sprintf("%s@vm", c.sandboxID), + MigrationOptions: migrationOpts, + }); err != nil { + return fmt.Errorf("create destination vm: %w", err) + } + + // Re-ACL the patched (destination-host) VHDs against the freshly + // created VM's SID so it can open them once it starts. + if err := c.vmController.Patch(ctx); err != nil { + return fmt.Errorf("patch destination vm: %w", err) + } + + c.state = StateDestinationPrepared + log.G(ctx).Info("migration destination prepared") + return nil +} diff --git a/internal/controller/migration/controller_destination_test.go b/internal/controller/migration/controller_destination_test.go new file mode 100644 index 0000000000..53f79eb7e1 --- /dev/null +++ b/internal/controller/migration/controller_destination_test.go @@ -0,0 +1,414 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/containerd/errdefs" + "github.com/opencontainers/runtime-spec/specs-go" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/Microsoft/hcsshim/internal/controller/migration/mocks" + save "github.com/Microsoft/hcsshim/internal/controller/migration/save" + "github.com/Microsoft/hcsshim/internal/controller/pod" + "github.com/Microsoft/hcsshim/internal/controller/vm" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/oci" + hcsannotations "github.com/Microsoft/hcsshim/pkg/annotations" +) + +// validImportPayload returns a decodable, current-schema sandbox payload with a +// VM blob and no pods, so import never has to construct a real pod controller. +func validImportPayload() *save.Payload { + return &save.Payload{ + SchemaVersion: save.SchemaVersion, + Vm: &anypb.Any{TypeUrl: "type.example/vm", Value: []byte("vm")}, + } +} + +// importEnvelope marshals a payload and wraps it in an envelope with the +// well-known type URL, matching what the source's ExportState emits. +func importEnvelope(t *testing.T, p *save.Payload) *anypb.Any { + t.Helper() + b, err := proto.Marshal(p) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + return &anypb.Any{TypeUrl: save.TypeURL, Value: b} +} + +// importOptions returns a valid ImportStateOptions bound to the given VM mock +// and saved-state envelope. +func importOptions(vmc vmController, saved *anypb.Any) *ImportStateOptions { + return &ImportStateOptions{ + InitOptions: InitOptions{ + SessionID: "sess-1", + Origin: hcsschema.MigrationOriginDestination, + VMController: vmc, + PodControllers: map[string]*pod.Controller{}, + }, + SandboxID: "sandbox-1", + SavedState: saved, + ContainerPodMapping: map[string]string{}, + } +} + +// importVM returns a VM mock reporting the not-created state that ImportState +// requires before rehydrating a snapshot. +func importVM(ctrl *gomock.Controller) *mocks.MockvmController { + vmc := mocks.NewMockvmController(ctrl) + vmc.EXPECT().State().Return(vm.StateNotCreated).AnyTimes() + return vmc +} + +// patchSpec builds a spec carrying the source-container annotation plus any extras. +func patchSpec(sourceContainerID string, extra map[string]string) specs.Spec { + ann := map[string]string{hcsannotations.LiveMigrationSourceContainerID: sourceContainerID} + for k, v := range extra { + ann[k] = v + } + return specs.Spec{Annotations: ann} +} + +// ───────────────────────────────────────────────────────────────────────────── +// ImportState +// ───────────────────────────────────────────────────────────────────────────── + +// TestImportState_RejectsInvalidArgs verifies a half-specified request is +// refused before the controller is mutated. +func TestImportState_RejectsInvalidArgs(t *testing.T) { + valid := func() *ImportStateOptions { + return importOptions(&mocks.MockvmController{}, importEnvelope(t, validImportPayload())) + } + cases := map[string]*ImportStateOptions{ + "NilOptions": nil, + "EmptySessionID": func() *ImportStateOptions { o := valid(); o.SessionID = ""; return o }(), + "NilVMController": func() *ImportStateOptions { o := valid(); o.VMController = nil; return o }(), + "EmptySandboxID": func() *ImportStateOptions { o := valid(); o.SandboxID = ""; return o }(), + "NilPodControllers": func() *ImportStateOptions { o := valid(); o.PodControllers = nil; return o }(), + "NilContainerPodMapping": func() *ImportStateOptions { o := valid(); o.ContainerPodMapping = nil; return o }(), + "NilSavedState": func() *ImportStateOptions { o := valid(); o.SavedState = nil; return o }(), + "WrongTypeURL": func() *ImportStateOptions { o := valid(); o.SavedState = &anypb.Any{TypeUrl: "type.bogus"}; return o }(), + } + for name, opts := range cases { + t.Run(name, func(t *testing.T) { + c := New() + + err := c.ImportState(t.Context(), opts) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } + if c.state != StateIdle { + t.Errorf("expected state Idle after rejected args, got %s", c.state) + } + }) + } +} + +// TestImportState_RejectsUndecodableState verifies a payload this build cannot +// decode is rejected before any controller state is rehydrated. +func TestImportState_RejectsUndecodableState(t *testing.T) { + cases := map[string]*anypb.Any{ + "CorruptPayload": {TypeUrl: save.TypeURL, Value: []byte{0x08}}, + "SchemaMismatch": importEnvelope(t, &save.Payload{SchemaVersion: save.SchemaVersion + 1}), + } + for name, saved := range cases { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + c := New() + + if err := c.ImportState(t.Context(), importOptions(importVM(ctrl), saved)); err == nil { + t.Fatal("expected error, got nil") + } + if c.state != StateIdle { + t.Errorf("expected state Idle, got %s", c.state) + } + }) + } +} + +// TestImportState_IdempotentSameSession verifies a repeat import for the active +// session is a no-op that does not re-import the VM. +func TestImportState_IdempotentSameSession(t *testing.T) { + ctrl := gomock.NewController(t) + c := New() + c.state = StateDestinationImported + c.sessionID = "sess-1" + + if err := c.ImportState(t.Context(), importOptions(importVM(ctrl), importEnvelope(t, validImportPayload()))); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// TestImportState_RejectsConflictingState verifies a snapshot cannot be imported +// while the controller is busy with another (non-idle) session. +func TestImportState_RejectsConflictingState(t *testing.T) { + ctrl := gomock.NewController(t) + c := New() + c.state = StateDestinationPrepared + c.sessionID = "other" + + err := c.ImportState(t.Context(), importOptions(importVM(ctrl), importEnvelope(t, validImportPayload()))) + if !errors.Is(err, errdefs.ErrAlreadyExists) { + t.Fatalf("expected ErrAlreadyExists, got %v", err) + } +} + +// TestImportState_RejectsWrongVMState verifies a snapshot cannot be imported +// unless the VM controller has not yet been created. +func TestImportState_RejectsWrongVMState(t *testing.T) { + ctrl := gomock.NewController(t) + vmc := mocks.NewMockvmController(ctrl) + vmc.EXPECT().State().Return(vm.StateRunning).AnyTimes() + + c := New() + err := c.ImportState(t.Context(), importOptions(vmc, importEnvelope(t, validImportPayload()))) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestImportState_VMImportError verifies a failure rehydrating the VM aborts the +// import without advancing the state. +func TestImportState_VMImportError(t *testing.T) { + ctrl := gomock.NewController(t) + vmc := importVM(ctrl) + vmc.EXPECT().Import(gomock.Any(), gomock.Any()).Return(errors.New("boom")) + + c := New() + if err := c.ImportState(t.Context(), importOptions(vmc, importEnvelope(t, validImportPayload()))); err == nil { + t.Fatal("expected error, got nil") + } + if c.state != StateIdle { + t.Errorf("expected state Idle after failure, got %s", c.state) + } +} + +// TestImportState_Success verifies a valid snapshot with no pods rehydrates the +// VM, binds the session, and advances the state. +func TestImportState_Success(t *testing.T) { + ctrl := gomock.NewController(t) + vmc := importVM(ctrl) + vmc.EXPECT().Import(gomock.Any(), gomock.Any()).Return(nil) + + c := New() + opts := importOptions(vmc, importEnvelope(t, validImportPayload())) + if err := c.ImportState(t.Context(), opts); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if c.state != StateDestinationImported { + t.Errorf("expected state DestinationImported, got %s", c.state) + } + if c.sessionID != "sess-1" || c.sandboxID != "sandbox-1" || c.vmController != vmc { + t.Errorf("session state not bound: %+v", c) + } + if c.pendingPatches == nil || len(c.pendingPatches) != 0 { + t.Errorf("expected empty pending set, got %+v", c.pendingPatches) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// PatchResourcePaths +// ───────────────────────────────────────────────────────────────────────────── + +// TestPatchResourcePaths_RejectsInvalidArgs verifies a malformed request or spec +// is refused before the controller is touched. +func TestPatchResourcePaths_RejectsInvalidArgs(t *testing.T) { + cases := map[string]struct { + request *task.CreateTaskRequest + spec specs.Spec + }{ + "NilRequest": {nil, patchSpec("ctr-1", nil)}, + "EmptyRequestID": {&task.CreateTaskRequest{ID: ""}, patchSpec("ctr-1", nil)}, + "NilAnnotations": {&task.CreateTaskRequest{ID: "dst-1"}, specs.Spec{}}, + "MissingSourceAnnotation": {&task.CreateTaskRequest{ID: "dst-1"}, specs.Spec{Annotations: map[string]string{}}}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + c := New() + + err := c.PatchResourcePaths(t.Context(), tc.request, tc.spec) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } + }) + } +} + +// TestPatchResourcePaths_RejectsWrongState verifies patching is only valid once +// the destination has imported a snapshot. +func TestPatchResourcePaths_RejectsWrongState(t *testing.T) { + c := New() // StateIdle + + err := c.PatchResourcePaths(t.Context(), &task.CreateTaskRequest{ID: "dst-1"}, patchSpec("ctr-1", nil)) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestPatchResourcePaths_RejectsAlreadyPatched verifies a container that is no +// longer pending cannot be patched again. +func TestPatchResourcePaths_RejectsAlreadyPatched(t *testing.T) { + c := New() + c.state = StateDestinationImported + c.pendingPatches = map[string]struct{}{} + c.containerPodMapping = map[string]string{} + + err := c.PatchResourcePaths(t.Context(), &task.CreateTaskRequest{ID: "dst-1"}, patchSpec("ctr-1", nil)) + if !errors.Is(err, errdefs.ErrAlreadyExists) { + t.Fatalf("expected ErrAlreadyExists, got %v", err) + } +} + +// TestPatchResourcePaths_RejectsUnknownContainer verifies a pending container +// missing from the index is reported as not found. +func TestPatchResourcePaths_RejectsUnknownContainer(t *testing.T) { + c := New() + c.state = StateDestinationImported + c.pendingPatches = map[string]struct{}{"ctr-1": {}} + c.containerPodMapping = map[string]string{} + + err := c.PatchResourcePaths(t.Context(), &task.CreateTaskRequest{ID: "dst-1"}, patchSpec("ctr-1", nil)) + if !errors.Is(err, errdefs.ErrNotFound) { + t.Fatalf("expected ErrNotFound, got %v", err) + } +} + +// TestPatchResourcePaths_RejectsAnnotationMismatch verifies a container-type +// annotation that contradicts the structural sandbox detection is rejected. +func TestPatchResourcePaths_RejectsAnnotationMismatch(t *testing.T) { + c := New() + c.state = StateDestinationImported + c.pendingPatches = map[string]struct{}{"ctr-1": {}} + c.containerPodMapping = map[string]string{"ctr-1": "pod-1"} // ctr-1 != pod-1, so not a sandbox + c.podControllers = map[string]*pod.Controller{} + + // Annotation claims sandbox, contradicting the structural detection. + spec := patchSpec("ctr-1", map[string]string{ + hcsannotations.KubernetesContainerType: string(oci.KubernetesContainerTypeSandbox), + }) + + err := c.PatchResourcePaths(t.Context(), &task.CreateTaskRequest{ID: "dst-1"}, spec) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestPatchResourcePaths_SCSIControllerError verifies a failure obtaining the +// SCSI controller surfaces before the pod is patched. +func TestPatchResourcePaths_SCSIControllerError(t *testing.T) { + ctrl := gomock.NewController(t) + vmc := mocks.NewMockvmController(ctrl) + vmc.EXPECT().SCSIController(gomock.Any()).Return(nil, errors.New("boom")) + + c := New() + c.state = StateDestinationImported + c.pendingPatches = map[string]struct{}{"ctr-1": {}} + c.containerPodMapping = map[string]string{"ctr-1": "pod-1"} + c.podControllers = map[string]*pod.Controller{} + c.vmController = vmc + + if err := c.PatchResourcePaths(t.Context(), &task.CreateTaskRequest{ID: "dst-1"}, patchSpec("ctr-1", nil)); err == nil { + t.Fatal("expected error, got nil") + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// PrepareDestination +// ───────────────────────────────────────────────────────────────────────────── + +// TestPrepareDestination_RejectsWrongState verifies the destination VM is only +// built from an imported controller. +func TestPrepareDestination_RejectsWrongState(t *testing.T) { + c := New() // StateIdle + c.sessionID = "sess-1" + + if err := c.PrepareDestination(t.Context(), "sess-1", nil); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestPrepareDestination_RejectsPendingPatches verifies the VM is not built while +// any imported container is still awaiting a patch. +func TestPrepareDestination_RejectsPendingPatches(t *testing.T) { + c := New() + c.state = StateDestinationImported + c.sessionID = "sess-1" + c.pendingPatches = map[string]struct{}{"ctr-1": {}, "ctr-2": {}} + + err := c.PrepareDestination(t.Context(), "sess-1", nil) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } + if !strings.Contains(err.Error(), "ctr-1") || !strings.Contains(err.Error(), "ctr-2") { + t.Errorf("expected error to list pending containers, got: %v", err) + } +} + +// TestPrepareDestination_CreateVMError verifies a failure creating the VM aborts +// the preparation without advancing the state. +func TestPrepareDestination_CreateVMError(t *testing.T) { + ctrl := gomock.NewController(t) + vmc := mocks.NewMockvmController(ctrl) + vmc.EXPECT().CreateVM(gomock.Any(), gomock.Any()).Return(errors.New("boom")) + + c := New() + c.state = StateDestinationImported + c.sandboxID = "sandbox-1" + c.sessionID = "sess-1" + c.pendingPatches = map[string]struct{}{} + c.vmController = vmc + + if err := c.PrepareDestination(t.Context(), "sess-1", nil); err == nil { + t.Fatal("expected error, got nil") + } + if c.state != StateDestinationImported { + t.Errorf("expected state DestinationImported after failure, got %s", c.state) + } +} + +// TestPrepareDestination_Success verifies a successful preparation stamps the +// origin onto the (defaulted) options, builds the VM, and advances the state. +func TestPrepareDestination_Success(t *testing.T) { + ctrl := gomock.NewController(t) + vmc := mocks.NewMockvmController(ctrl) + + var gotOpts *vm.CreateOptions + vmc.EXPECT().CreateVM(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, o *vm.CreateOptions) error { + gotOpts = o + return nil + }) + vmc.EXPECT().Patch(gomock.Any()).Return(nil) + + c := New() + c.state = StateDestinationImported + c.sandboxID = "sandbox-1" + c.sessionID = "sess-1" + c.origin = hcsschema.MigrationOriginDestination + c.pendingPatches = map[string]struct{}{} + c.vmController = vmc + + if err := c.PrepareDestination(t.Context(), "sess-1", nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if c.state != StateDestinationPrepared { + t.Errorf("expected state DestinationPrepared, got %s", c.state) + } + if gotOpts == nil || gotOpts.ID != "sandbox-1@vm" { + t.Fatalf("unexpected create options: %+v", gotOpts) + } + // MigrationOptions is defaulted when nil and stamped with the origin. + if gotOpts.MigrationOptions == nil || gotOpts.MigrationOptions.Origin != hcsschema.MigrationOriginDestination { + t.Errorf("expected migration options stamped with destination origin, got %+v", gotOpts.MigrationOptions) + } +} diff --git a/internal/controller/migration/controller_source.go b/internal/controller/migration/controller_source.go new file mode 100644 index 0000000000..60f4a25ece --- /dev/null +++ b/internal/controller/migration/controller_source.go @@ -0,0 +1,126 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "fmt" + + save "github.com/Microsoft/hcsshim/internal/controller/migration/save" + "github.com/Microsoft/hcsshim/internal/controller/vm" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + + "github.com/containerd/errdefs" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +// PrepareSource readies the source side of a migration session so a subsequent +// [Controller.ExportState] can capture its state. A repeat call for the same +// session is a no-op. +func (c *Controller) PrepareSource(ctx context.Context, opts *PrepareSourceOptions) error { + switch { + case opts == nil: + return fmt.Errorf("options are required: %w", errdefs.ErrInvalidArgument) + case opts.SessionID == "": + return fmt.Errorf("session id is required: %w", errdefs.ErrInvalidArgument) + case opts.VMController == nil: + return fmt.Errorf("vm controller is required: %w", errdefs.ErrInvalidArgument) + case opts.PodControllers == nil: + return fmt.Errorf("pod controllers map is required: %w", errdefs.ErrInvalidArgument) + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.state != StateIdle { + // If we already called this API, then this is a no-op. + if c.state == StateSourcePrepared && c.sessionID == opts.SessionID { + return nil + } + return fmt.Errorf("controller is in state %s for session %q: %w", c.state, c.sessionID, errdefs.ErrFailedPrecondition) + } + + // VM must be running in order to initiate the migration. + if opts.VMController.State() != vm.StateRunning { + return fmt.Errorf("vm controller is in invalid state %s: %w", opts.VMController.State(), errdefs.ErrFailedPrecondition) + } + + // The sandbox must have been created with live migration enabled. + // Reject otherwise so callers learn the session cannot proceed before + // any source-side state is mutated. + sandboxOpts := opts.VMController.SandboxOptions() + if sandboxOpts == nil || !sandboxOpts.LiveMigrationSupportEnabled { + return fmt.Errorf("sandbox is not configured to allow live migration: %w", errdefs.ErrFailedPrecondition) + } + + if opts.MigrationOpts == nil { + opts.MigrationOpts = &hcsschema.MigrationInitializeOptions{} + } + opts.MigrationOpts.Origin = opts.Origin + + // Prepare the running source VM; afterward it accepts only live-migration compatible calls. + if err := opts.VMController.InitializeLiveMigrationOnSource(ctx, opts.MigrationOpts); err != nil { + return fmt.Errorf("initialize live migration on source vm: %w", err) + } + + c.sessionID = opts.SessionID + c.origin = opts.Origin + c.vmController = opts.VMController + c.podControllers = opts.PodControllers + c.state = StateSourcePrepared + + log.G(ctx).Info("migration source prepared") + return nil +} + +// ExportState captures the prepared source sandbox into an opaque, versioned +// saved state that the destination consumes via [Controller.ImportState]. The VM +// and per-pod payloads it carries are themselves opaque, owned by their +// respective controllers. A repeat call returns a fresh snapshot. +func (c *Controller) ExportState(ctx context.Context, sessionID string) (*anypb.Any, error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.sessionID != sessionID { + return nil, fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) + } + + // Allow re-export after a prior success so duplicate calls are idempotent. + if c.state != StateSourcePrepared && c.state != StateSourceExported { + return nil, fmt.Errorf("export requires state %s or %s (current: %s): %w", StateSourcePrepared, StateSourceExported, c.state, errdefs.ErrFailedPrecondition) + } + + // Save the VM state. + vmAny, err := c.vmController.Save(ctx) + if err != nil { + return nil, fmt.Errorf("save vm controller: %w", err) + } + + // Save all the pod controller states as opaque envelopes. + pods := make([]*anypb.Any, 0, len(c.podControllers)) + for podID, podCtrl := range c.podControllers { + ps, err := podCtrl.Save(ctx) + if err != nil { + return nil, fmt.Errorf("save pod %s: %w", podID, err) + } + + pods = append(pods, ps) + } + + // Wrap the VM and pod payloads in the versioned sandbox-level envelope. + payload, err := proto.Marshal(&save.Payload{ + SchemaVersion: save.SchemaVersion, + Vm: vmAny, + Pods: pods, + }) + if err != nil { + return nil, fmt.Errorf("marshal sandbox saved state: %w", err) + } + + c.state = StateSourceExported + + log.G(ctx).Info("migration source state exported") + return &anypb.Any{TypeUrl: save.TypeURL, Value: payload}, nil +} diff --git a/internal/controller/migration/controller_source_test.go b/internal/controller/migration/controller_source_test.go new file mode 100644 index 0000000000..5ae7d24514 --- /dev/null +++ b/internal/controller/migration/controller_source_test.go @@ -0,0 +1,279 @@ +//go:build windows && lcow + +package migration + +import ( + "errors" + "testing" + + "github.com/containerd/errdefs" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/Microsoft/hcsshim/internal/builder/vm/lcow" + "github.com/Microsoft/hcsshim/internal/controller/migration/mocks" + save "github.com/Microsoft/hcsshim/internal/controller/migration/save" + "github.com/Microsoft/hcsshim/internal/controller/pod" + vmpkg "github.com/Microsoft/hcsshim/internal/controller/vm" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" +) + +// migrationEnabledOptions returns sandbox options that permit live migration. +func migrationEnabledOptions() *lcow.SandboxOptions { + return &lcow.SandboxOptions{LiveMigrationSupportEnabled: true} +} + +// sourceOptions returns a valid PrepareSourceOptions bound to the given VM mock. +func sourceOptions(vm vmController) *PrepareSourceOptions { + return &PrepareSourceOptions{ + InitOptions: InitOptions{ + SessionID: "sess-1", + Origin: hcsschema.MigrationOriginSource, + VMController: vm, + PodControllers: map[string]*pod.Controller{}, + }, + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// PrepareSource +// ───────────────────────────────────────────────────────────────────────────── + +// TestPrepareSource_RejectsInvalidArgs verifies a half-specified request is +// refused before any source-side state is touched. +func TestPrepareSource_RejectsInvalidArgs(t *testing.T) { + cases := map[string]*PrepareSourceOptions{ + "NilOptions": nil, + "EmptySessionID": {InitOptions: InitOptions{VMController: &mocks.MockvmController{}, PodControllers: map[string]*pod.Controller{}}}, + "NilVMController": {InitOptions: InitOptions{SessionID: "sess-1", PodControllers: map[string]*pod.Controller{}}}, + "NilPodControllers": {InitOptions: InitOptions{SessionID: "sess-1", VMController: &mocks.MockvmController{}}}, + } + for name, opts := range cases { + t.Run(name, func(t *testing.T) { + c := New() + + err := c.PrepareSource(t.Context(), opts) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } + if c.state != StateIdle { + t.Errorf("expected state Idle after rejected args, got %s", c.state) + } + }) + } +} + +// TestPrepareSource_RejectsWrongState verifies a session can only be armed from +// idle; a controller busy with another session is rejected. +func TestPrepareSource_RejectsWrongState(t *testing.T) { + ctrl := gomock.NewController(t) + c := New() + c.state = StateSourceExported + c.sessionID = "other" + + err := c.PrepareSource(t.Context(), sourceOptions(mocks.NewMockvmController(ctrl))) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestPrepareSource_IdempotentSameSession verifies re-arming the same session is +// a no-op that does not touch the VM. +func TestPrepareSource_IdempotentSameSession(t *testing.T) { + ctrl := gomock.NewController(t) + c := New() + c.state = StateSourcePrepared + c.sessionID = "sess-1" + + // No VM calls are expected on a duplicate arm. + if err := c.PrepareSource(t.Context(), sourceOptions(mocks.NewMockvmController(ctrl))); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.state != StateSourcePrepared { + t.Errorf("expected state SourcePrepared, got %s", c.state) + } +} + +// TestPrepareSource_RejectsMigrationDisabled verifies a sandbox that was not +// created with live migration enabled cannot be armed. +func TestPrepareSource_RejectsMigrationDisabled(t *testing.T) { + cases := map[string]*lcow.SandboxOptions{ + "NilOptions": nil, + "FeatureDisabled": {LiveMigrationSupportEnabled: false}, + } + for name, sandboxOpts := range cases { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateRunning) + vm.EXPECT().SandboxOptions().Return(sandboxOpts) + + c := New() + err := c.PrepareSource(t.Context(), sourceOptions(vm)) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } + if c.state != StateIdle { + t.Errorf("expected state Idle, got %s", c.state) + } + }) + } +} + +// TestPrepareSource_RejectsWrongVMState verifies the source cannot be armed +// unless the VM is running. +func TestPrepareSource_RejectsWrongVMState(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateCreated).AnyTimes() + + c := New() + if err := c.PrepareSource(t.Context(), sourceOptions(vm)); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } + if c.state != StateIdle { + t.Errorf("expected state Idle, got %s", c.state) + } +} + +// TestPrepareSource_InitializeError verifies a failure arming the VM leaves the +// controller idle so the session can be retried. +func TestPrepareSource_InitializeError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateRunning) + vm.EXPECT().SandboxOptions().Return(migrationEnabledOptions()) + vm.EXPECT().InitializeLiveMigrationOnSource(gomock.Any(), gomock.Any()).Return(errors.New("boom")) + + c := New() + if err := c.PrepareSource(t.Context(), sourceOptions(vm)); err == nil { + t.Fatal("expected error, got nil") + } + if c.state != StateIdle { + t.Errorf("expected state Idle after failure, got %s", c.state) + } +} + +// TestPrepareSource_Success verifies a successful arm records the session, stamps +// the origin onto the (defaulted) migration options, and advances the state. +func TestPrepareSource_Success(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().State().Return(vmpkg.StateRunning) + vm.EXPECT().SandboxOptions().Return(migrationEnabledOptions()) + vm.EXPECT().InitializeLiveMigrationOnSource(gomock.Any(), gomock.Any()).Return(nil) + + c := New() + opts := sourceOptions(vm) + if err := c.PrepareSource(t.Context(), opts); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if c.state != StateSourcePrepared { + t.Errorf("expected state SourcePrepared, got %s", c.state) + } + if c.sessionID != "sess-1" || c.origin != hcsschema.MigrationOriginSource || c.vmController != vm { + t.Errorf("session state not bound: %+v", c) + } + // MigrationOpts is defaulted when nil and stamped with the origin. + if opts.MigrationOpts == nil || opts.MigrationOpts.Origin != hcsschema.MigrationOriginSource { + t.Errorf("expected migration options stamped with source origin, got %+v", opts.MigrationOpts) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// ExportState +// ───────────────────────────────────────────────────────────────────────────── + +// TestExportState_RejectsSessionMismatch verifies a snapshot is only produced +// for the active session. +func TestExportState_RejectsSessionMismatch(t *testing.T) { + c := New() + c.state = StateSourcePrepared + c.sessionID = "sess-1" + + env, err := c.ExportState(t.Context(), "other") + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } + if env != nil { + t.Errorf("expected nil envelope on failure, got %+v", env) + } +} + +// TestExportState_RejectsWrongState verifies a snapshot is only produced from a +// prepared (or already-exported) source. +func TestExportState_RejectsWrongState(t *testing.T) { + c := New() + c.sessionID = "sess-1" + + env, err := c.ExportState(t.Context(), "sess-1") + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } + if env != nil { + t.Errorf("expected nil envelope on failure, got %+v", env) + } +} + +// TestExportState_VMSaveError verifies a failure saving the VM aborts the export +// without advancing the state. +func TestExportState_VMSaveError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().Save(gomock.Any()).Return(nil, errors.New("boom")) + + c := New() + c.state = StateSourcePrepared + c.sessionID = "sess-1" + c.vmController = vm + c.podControllers = map[string]*pod.Controller{} + + if _, err := c.ExportState(t.Context(), "sess-1"); err == nil { + t.Fatal("expected error, got nil") + } + if c.state != StateSourcePrepared { + t.Errorf("expected state SourcePrepared after failure, got %s", c.state) + } +} + +// TestExportState_Success verifies the produced envelope is self-describing and +// carries the VM payload, and that the source advances to exported. +func TestExportState_Success(t *testing.T) { + ctrl := gomock.NewController(t) + vmAny := &anypb.Any{TypeUrl: "type.example/vm", Value: []byte("vm-state")} + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().Save(gomock.Any()).Return(vmAny, nil) + + c := New() + c.state = StateSourcePrepared + c.sessionID = "sess-1" + c.vmController = vm + c.podControllers = map[string]*pod.Controller{} + + env, err := c.ExportState(t.Context(), "sess-1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if env.GetTypeUrl() != save.TypeURL { + t.Errorf("expected type URL %q, got %q", save.TypeURL, env.GetTypeUrl()) + } + + got := &save.Payload{} + if err := proto.Unmarshal(env.GetValue(), got); err != nil { + t.Fatalf("unmarshal saved payload: %v", err) + } + if got.GetSchemaVersion() != save.SchemaVersion { + t.Errorf("expected schema version %d, got %d", save.SchemaVersion, got.GetSchemaVersion()) + } + if got.GetVm().GetTypeUrl() != vmAny.TypeUrl || string(got.GetVm().GetValue()) != string(vmAny.Value) { + t.Errorf("vm payload not preserved: %+v", got.GetVm()) + } + if len(got.GetPods()) != 0 { + t.Errorf("expected no pod payloads, got %d", len(got.GetPods())) + } + if c.state != StateSourceExported { + t.Errorf("expected state SourceExported after export, got %s", c.state) + } +} diff --git a/internal/controller/migration/controller_test.go b/internal/controller/migration/controller_test.go new file mode 100644 index 0000000000..94448acb34 --- /dev/null +++ b/internal/controller/migration/controller_test.go @@ -0,0 +1,534 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/containerd/errdefs" + "go.uber.org/mock/gomock" + "golang.org/x/sys/windows" + + "github.com/Microsoft/hcsshim/internal/controller/migration/mocks" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/pkg/migration" +) + +// waitForState polls until the controller reaches want or the deadline elapses, +// so background transfer goroutines can be observed deterministically. +func waitForState(t *testing.T, c *Controller, want State) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if c.State() == want { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("state = %s; want %s", c.State(), want) +} + +// ───────────────────────────────────────────────────────────────────────────── +// New / State +// ───────────────────────────────────────────────────────────────────────────── + +// TestNew verifies a fresh controller starts idle and ready to host a session. +func TestNew(t *testing.T) { + c := New() + if c.State() != StateIdle { + t.Errorf("expected state Idle, got %s", c.State()) + } + if c.socketReady == nil { + t.Error("expected socketReady channel to be initialized") + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// sessionIDToUint32 +// ───────────────────────────────────────────────────────────────────────────── + +// TestSessionIDToUint32 verifies the mapping is deterministic and that a GUID +// maps to the same value regardless of letter case. +func TestSessionIDToUint32(t *testing.T) { + const lower = "5f0b1190-63be-4e0c-b974-bd0f55675a42" + const upper = "5F0B1190-63BE-4E0C-B974-BD0F55675A42" + + if got, want := sessionIDToUint32(lower), sessionIDToUint32(lower); got != want { + t.Errorf("not deterministic: %d != %d", got, want) + } + if got, want := sessionIDToUint32(upper), sessionIDToUint32(lower); got != want { + t.Errorf("GUID case affected result: %d != %d", got, want) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Transfer +// ───────────────────────────────────────────────────────────────────────────── + +// TestTransfer_RejectsSessionMismatch verifies a call for a different session is +// rejected. +func TestTransfer_RejectsSessionMismatch(t *testing.T) { + c := New() + c.sessionID = "sess-1" + + if err := c.Transfer(t.Context(), "other", time.Minute); !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestTransfer_NoopWhenInProgress verifies a transfer that is already waiting, +// running, or done is a no-op that leaves the state unchanged. +func TestTransfer_NoopWhenInProgress(t *testing.T) { + for _, st := range []State{StateSocketWaiting, StateTransferring, StateTransferCompleted} { + t.Run(st.String(), func(t *testing.T) { + c := New() + c.sessionID = "sess-1" + c.state = st + + if err := c.Transfer(t.Context(), "sess-1", time.Minute); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != st { + t.Errorf("state = %s; want unchanged %s", c.State(), st) + } + }) + } +} + +// TestTransfer_RejectsInvalidState verifies a transfer cannot start from a state +// that has no path to it. +func TestTransfer_RejectsInvalidState(t *testing.T) { + c := New() // StateIdle + + if err := c.Transfer(t.Context(), "", time.Minute); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestTransfer_ClaimsSocketWaiting verifies that starting a transfer before the +// socket arrives claims the wait so later calls no-op. +func TestTransfer_ClaimsSocketWaiting(t *testing.T) { + c := New() + c.sessionID = "sess-1" + c.state = StateSourceExported + + // Long timeout so the background goroutine simply parks on the socket. + if err := c.Transfer(t.Context(), "sess-1", time.Hour); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateSocketWaiting { + t.Errorf("state = %s; want SocketWaiting", c.State()) + } + + // Unpark the goroutine so it exits without driving a transfer. + close(c.socketReady) +} + +// TestTransfer_Success verifies that with the socket ready the background +// transfer runs to completion. +func TestTransfer_Success(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().StartLiveMigrationOnSource(gomock.Any(), gomock.Any()).Return(nil) + vm.EXPECT().StartLiveMigrationTransfer(gomock.Any(), gomock.Any()).Return(nil) + + c := New() + c.sessionID = "sess-1" + c.origin = hcsschema.MigrationOriginSource + c.vmController = vm + c.state = StateSocketReady + close(c.socketReady) // socket already arrived + + if err := c.Transfer(t.Context(), "sess-1", time.Minute); err != nil { + t.Fatalf("unexpected error: %v", err) + } + waitForState(t, c, StateTransferCompleted) +} + +// TestTransfer_SocketTimeout verifies the session fails if the socket never +// arrives within the timeout. +func TestTransfer_SocketTimeout(t *testing.T) { + c := New() + c.sessionID = "sess-1" + c.state = StateSocketReady // socketReady stays open so the wait times out + + if err := c.Transfer(t.Context(), "sess-1", time.Millisecond); err != nil { + t.Fatalf("unexpected error: %v", err) + } + waitForState(t, c, StateFailed) +} + +// ───────────────────────────────────────────────────────────────────────────── +// Finalize +// ───────────────────────────────────────────────────────────────────────────── + +// TestFinalize_RejectsUnspecifiedAction verifies an unspecified action is refused. +func TestFinalize_RejectsUnspecifiedAction(t *testing.T) { + c := New() + + err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_UNSPECIFIED, nil) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestFinalize_IdempotentFinalized verifies finalizing an already-finalized +// session is a no-op. +func TestFinalize_IdempotentFinalized(t *testing.T) { + c := New() + c.state = StateFinalized + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_RESUME, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// TestFinalize_RejectsWrongState verifies finalize is rejected before a transfer +// has completed or been cancelled. +func TestFinalize_RejectsWrongState(t *testing.T) { + c := New() // StateIdle + + err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_RESUME, nil) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestFinalize_RejectsUnsupportedAction verifies an unknown action is rejected. +func TestFinalize_RejectsUnsupportedAction(t *testing.T) { + c := New() + c.state = StateTransferCompleted + + err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction(99), nil) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestFinalize_FinalizeError verifies a failure finalizing the VM aborts without +// advancing the state. +func TestFinalize_FinalizeError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().FinalizeLiveMigration(gomock.Any(), gomock.Any()).Return(errors.New("boom")) + + c := New() + c.state = StateTransferCompleted + c.origin = hcsschema.MigrationOriginSource + c.vmController = vm + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_STOP, nil); err == nil { + t.Fatal("expected error, got nil") + } + if c.State() != StateTransferCompleted { + t.Errorf("state = %s; want unchanged TransferCompleted", c.State()) + } +} + +// TestFinalize_ResumeSucceeds verifies a resume finalizes the VM, resumes it, +// and advances to finalized on both origins. +func TestFinalize_ResumeSucceeds(t *testing.T) { + cases := map[string]struct { + origin hcsschema.MigrationOrigin + rebuildBridge bool + }{ + "destination": {hcsschema.MigrationOriginDestination, false}, + "source": {hcsschema.MigrationOriginSource, true}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().FinalizeLiveMigration(gomock.Any(), gomock.Any()).Return(nil) + vm.EXPECT().Resume(gomock.Any(), tc.rebuildBridge).Return(nil) + + c := New() + c.state = StateTransferCompleted + c.origin = tc.origin + c.vmController = vm + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_RESUME, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateFinalized { + t.Errorf("state = %s; want Finalized", c.State()) + } + }) + } +} + +// TestFinalize_StopSucceeds verifies a stop finalizes the VM and advances to +// finalized without resuming it. +func TestFinalize_StopSucceeds(t *testing.T) { + for _, origin := range []hcsschema.MigrationOrigin{hcsschema.MigrationOriginSource, hcsschema.MigrationOriginDestination} { + t.Run(string(origin), func(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().FinalizeLiveMigration(gomock.Any(), gomock.Any()).Return(nil) + + c := New() + c.state = StateTransferCompleted + c.origin = origin + c.vmController = vm + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_STOP, nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateFinalized { + t.Errorf("state = %s; want Finalized", c.State()) + } + }) + } +} + +// TestFinalize_ResumeError verifies a failure resuming the VM aborts without +// advancing the state. +func TestFinalize_ResumeError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().FinalizeLiveMigration(gomock.Any(), gomock.Any()).Return(nil) + vm.EXPECT().Resume(gomock.Any(), false).Return(errors.New("boom")) + + c := New() + c.state = StateTransferCompleted + c.origin = hcsschema.MigrationOriginDestination + c.vmController = vm + + if err := c.Finalize(t.Context(), "sess-1", migration.FinalizeAction_FINALIZE_ACTION_RESUME, nil); err == nil { + t.Fatal("expected error, got nil") + } + if c.State() != StateTransferCompleted { + t.Errorf("state = %s; want unchanged TransferCompleted", c.State()) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Cancel +// ───────────────────────────────────────────────────────────────────────────── + +// TestCancel_RejectsSessionMismatch verifies a cancel for a different session is +// rejected. +func TestCancel_RejectsSessionMismatch(t *testing.T) { + c := New() + c.sessionID = "sess-1" + + if err := c.Cancel(t.Context(), "other"); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestCancel_Idempotent verifies cancelling an already-cancelled session is a +// no-op. +func TestCancel_Idempotent(t *testing.T) { + c := New() + c.sessionID = "sess-1" + c.state = StateCancelled + + if err := c.Cancel(t.Context(), "sess-1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateCancelled { + t.Errorf("state = %s; want Cancelled", c.State()) + } +} + +// TestCancel_Success verifies cancelling an in-flight session moves it to +// cancelled. +func TestCancel_Success(t *testing.T) { + c := New() + c.sessionID = "sess-1" + c.state = StateTransferCompleted + + if err := c.Cancel(t.Context(), "sess-1"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateCancelled { + t.Errorf("state = %s; want Cancelled", c.State()) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Cleanup +// ───────────────────────────────────────────────────────────────────────────── + +// TestCleanup_NoopWhenIdle verifies cleaning up an idle controller is a no-op. +func TestCleanup_NoopWhenIdle(t *testing.T) { + c := New() + + if err := c.Cleanup(t.Context(), "sess-1", nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// TestCleanup_RejectsSessionMismatch verifies cleanup for a different session is +// rejected. +func TestCleanup_RejectsSessionMismatch(t *testing.T) { + c := New() + c.state = StateFinalized + c.sessionID = "sess-1" + + if err := c.Cleanup(t.Context(), "other", nil); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestCleanup_RejectsNotFinalized verifies cleanup is only valid once the +// session has been finalized. +func TestCleanup_RejectsNotFinalized(t *testing.T) { + c := New() + c.state = StateTransferCompleted + c.sessionID = "sess-1" + + if err := c.Cleanup(t.Context(), "sess-1", nil); !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("expected ErrFailedPrecondition, got %v", err) + } +} + +// TestCleanup_Success verifies cleanup returns a finalized session to idle and +// clears its session-scoped state. +func TestCleanup_Success(t *testing.T) { + c := New() + c.state = StateFinalized + c.sessionID = "sess-1" + c.origin = hcsschema.MigrationOriginSource + + if err := c.Cleanup(t.Context(), "sess-1", nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c.State() != StateIdle { + t.Errorf("state = %s; want Idle", c.State()) + } + if c.sessionID != "" || c.vmController != nil || c.podControllers != nil { + t.Errorf("session state not cleared: %+v", c) + } +} + +// TestCleanup_ClosesSocketAndNotifier verifies cleanup releases the transport +// socket and tears down the notifier before returning to idle. +func TestCleanup_ClosesSocketAndNotifier(t *testing.T) { + if err := ensureWinsock(); err != nil { + t.Skipf("winsock unavailable: %v", err) + } + sock, err := windows.Socket(windows.AF_INET, windows.SOCK_STREAM, windows.IPPROTO_TCP) + if err != nil { + t.Fatalf("create socket: %v", err) + } + + c := New() + c.state = StateFinalized + c.sessionID = "s" + c.dupSocket = sock + c.notifier = newTestNotifications(hcsschema.MigrationOriginSource) + + if err := c.Cleanup(context.Background(), "s", nil); err != nil { + t.Fatalf("cleanup: %v", err) + } + if c.State() != StateIdle { + t.Errorf("state = %s; want Idle", c.State()) + } + if c.dupSocket != 0 { + t.Error("expected dupSocket to be cleared") + } + if c.notifier != nil { + t.Error("expected notifier to be cleared") + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// runTransfer +// ───────────────────────────────────────────────────────────────────────────── + +// TestRunTransfer_Destination verifies the destination issues the destination +// start call followed by the transfer. +func TestRunTransfer_Destination(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().StartWithMigrationOptions(gomock.Any(), gomock.Any()).Return(nil) + vm.EXPECT().StartLiveMigrationTransfer(gomock.Any(), gomock.Any()).Return(nil) + + c := New() + c.sessionID = "s" + c.origin = hcsschema.MigrationOriginDestination + c.vmController = vm + + if err := c.runTransfer(t.Context()); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// TestRunTransfer_UnsupportedOrigin verifies an unknown origin is rejected. +func TestRunTransfer_UnsupportedOrigin(t *testing.T) { + c := New() + c.origin = hcsschema.MigrationOrigin("bogus") + + if err := c.runTransfer(t.Context()); !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestRunTransfer_StartError verifies a failure starting the migration surfaces. +func TestRunTransfer_StartError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().StartLiveMigrationOnSource(gomock.Any(), gomock.Any()).Return(errors.New("boom")) + + c := New() + c.origin = hcsschema.MigrationOriginSource + c.vmController = vm + + if err := c.runTransfer(t.Context()); err == nil { + t.Fatal("expected error, got nil") + } +} + +// TestRunTransfer_TransferError verifies a failure starting the transfer surfaces. +func TestRunTransfer_TransferError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().StartLiveMigrationOnSource(gomock.Any(), gomock.Any()).Return(nil) + vm.EXPECT().StartLiveMigrationTransfer(gomock.Any(), gomock.Any()).Return(errors.New("boom")) + + c := New() + c.origin = hcsschema.MigrationOriginSource + c.vmController = vm + + if err := c.runTransfer(t.Context()); err == nil { + t.Fatal("expected error, got nil") + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// failTransfer +// ───────────────────────────────────────────────────────────────────────────── + +// TestFailTransfer_BroadcastsToSubscribers verifies a failed transfer marks the +// session failed and delivers a failure event to subscribers on either origin. +func TestFailTransfer_BroadcastsToSubscribers(t *testing.T) { + for _, origin := range []hcsschema.MigrationOrigin{hcsschema.MigrationOriginSource, hcsschema.MigrationOriginDestination} { + t.Run(string(origin), func(t *testing.T) { + n := newTestNotifications(origin) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sub, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + c := New() + c.origin = origin + c.notifier = n + + c.failTransfer(context.Background(), errors.New("boom")) + + if c.State() != StateFailed { + t.Errorf("state = %s; want Failed", c.State()) + } + if got := recvWithin(t, sub, time.Second); got == nil { + t.Fatal("expected a failure notification") + } + }) + } +} diff --git a/internal/controller/migration/doc.go b/internal/controller/migration/doc.go new file mode 100644 index 0000000000..75db1c582a --- /dev/null +++ b/internal/controller/migration/doc.go @@ -0,0 +1,100 @@ +//go:build windows && lcow + +// Package migration provides a controller for sequencing a single live-migration +// session of an LCOW sandbox between two shims: a source that hands off a running +// sandbox and a destination that receives it. +// +// The [Controller] drives one session at a time. The source captures its sandbox +// as an opaque snapshot ([Controller.PrepareSource], [Controller.ExportState]); +// the destination rehydrates that snapshot ([Controller.ImportState]), rebinds +// each migrated container onto its own IDs ([Controller.PatchResourcePaths]), and +// materializes the VM ([Controller.PrepareDestination]). Once both sides share the +// duplicated transport socket ([Controller.RegisterDuplicateSocket]), the memory +// transfer runs ([Controller.Transfer]); the session is then committed with a +// resume or stop ([Controller.Finalize]) and torn down ([Controller.Cleanup]). +// The VM and pod controllers it drives are owned by the service and only borrowed +// for the session. +// +// # Lifecycle +// +// A session created via [New] starts at [StateIdle] and follows a source or a +// destination path; the two converge once the transport socket is ready: +// +// source destination +// ┌─────────────────────┐ ┌───────────────────────────┐ +// │ StateIdle │ │ StateIdle │ +// └──────────┬──────────┘ └─────────────┬─────────────┘ +// │ PrepareSource │ ImportState +// ▼ ▼ +// ┌─────────────────────┐ ┌───────────────────────────┐ +// │ StateSourcePrepared │ │ StateDestinationImported │ +// └──────────┬──────────┘ └─────────────┬─────────────┘ +// │ ExportState │ PatchResourcePaths (per container) +// ▼ │ then PrepareDestination +// ┌─────────────────────┐ ▼ +// │ StateSourceExported │ ┌───────────────────────────┐ +// └──────────┬──────────┘ │ StateDestinationPrepared │ +// │ └─────────────┬─────────────┘ +// │ RegisterDuplicateSocket │ RegisterDuplicateSocket +// └───────────────┬───────────────────────┘ +// ▼ +// ┌────────────────────┐ +// │ StateSocketReady │ +// └─────────┬──────────┘ +// │ Transfer +// ▼ +// ┌────────────────────┐ +// │ StateTransferring │ +// └─────────┬──────────┘ +// │ transfer ok +// ▼ +// ┌────────────────────────┐ +// │ StateTransferCompleted │ +// └───────────┬────────────┘ +// │ Finalize +// ▼ +// ┌──────────────────┐ +// │ StateFinalized │ +// └────────┬─────────┘ +// │ Cleanup +// ▼ +// ┌──────────────┐ +// │ StateIdle │ +// └──────────────┘ +// +// State descriptions: +// +// - [StateIdle]: no session is active; the initial state and where +// [Controller.Cleanup] returns the controller. +// - [StateSourcePrepared]: [Controller.PrepareSource] has armed the source; the +// next call is [Controller.ExportState]. +// - [StateSourceExported]: [Controller.ExportState] has produced the opaque +// sandbox snapshot; the next call is [Controller.RegisterDuplicateSocket]. +// - [StateDestinationImported]: [Controller.ImportState] has rehydrated the +// snapshot; the next calls are [Controller.PatchResourcePaths] (one per +// container) followed by [Controller.PrepareDestination]. +// - [StateDestinationPrepared]: [Controller.PrepareDestination] has materialized +// the destination HCS compute system; the next call is +// [Controller.RegisterDuplicateSocket]. +// - [StateSocketReady]: the duplicated transport socket has been adopted via +// [Controller.RegisterDuplicateSocket]; the next call is [Controller.Transfer]. +// - [StateSocketWaiting]: [Controller.Transfer] ran before the socket arrived and +// is waiting for it in the background; it advances to [StateTransferring] once +// the socket is registered. +// - [StateTransferring]: [Controller.Transfer] has started the memory transfer; +// it auto-advances to [StateTransferCompleted] or [StateFailed]. +// - [StateTransferCompleted]: the memory transfer finished; the next call is +// [Controller.Finalize]. +// - [StateFinalized]: [Controller.Finalize] has applied the resume or stop; the +// next call is [Controller.Cleanup]. +// - [StateCancelled]: [Controller.Cancel] aborted the session; the next call is +// [Controller.Finalize] followed by [Controller.Cleanup]. +// - [StateFailed]: the memory transfer failed; the next call is +// [Controller.Cancel]. +// +// # Notifications +// +// Callers observe session progress by streaming events with +// [Controller.Subscribe]; a failed transfer is reported to subscribers rather +// than returned from [Controller.Transfer]. +package migration diff --git a/internal/controller/migration/mocks/mock_lcow_migration.go b/internal/controller/migration/mocks/mock_lcow_migration.go new file mode 100644 index 0000000000..d97956308b --- /dev/null +++ b/internal/controller/migration/mocks/mock_lcow_migration.go @@ -0,0 +1,337 @@ +//go:build windows && lcow + +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/Microsoft/hcsshim/internal/controller/migration (interfaces: vmController) +// +// Generated by this command: +// +// mockgen -build_flags=-tags=windows,lcow -build_constraint=windows && lcow -package mocks -destination internal/controller/migration/mocks/mock_lcow_migration.go github.com/Microsoft/hcsshim/internal/controller/migration vmController +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + lcow "github.com/Microsoft/hcsshim/internal/builder/vm/lcow" + plan9 "github.com/Microsoft/hcsshim/internal/controller/device/plan9" + scsi "github.com/Microsoft/hcsshim/internal/controller/device/scsi" + vpci "github.com/Microsoft/hcsshim/internal/controller/device/vpci" + network "github.com/Microsoft/hcsshim/internal/controller/network" + vm "github.com/Microsoft/hcsshim/internal/controller/vm" + schema2 "github.com/Microsoft/hcsshim/internal/hcs/schema2" + v2 "github.com/Microsoft/hcsshim/internal/hcs/v2" + guestmanager "github.com/Microsoft/hcsshim/internal/vm/guestmanager" + vmmanager "github.com/Microsoft/hcsshim/internal/vm/vmmanager" + gomock "go.uber.org/mock/gomock" + anypb "google.golang.org/protobuf/types/known/anypb" +) + +// MockvmController is a mock of vmController interface. +type MockvmController struct { + ctrl *gomock.Controller + recorder *MockvmControllerMockRecorder + isgomock struct{} +} + +// MockvmControllerMockRecorder is the mock recorder for MockvmController. +type MockvmControllerMockRecorder struct { + mock *MockvmController +} + +// NewMockvmController creates a new mock instance. +func NewMockvmController(ctrl *gomock.Controller) *MockvmController { + mock := &MockvmController{ctrl: ctrl} + mock.recorder = &MockvmControllerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockvmController) EXPECT() *MockvmControllerMockRecorder { + return m.recorder +} + +// CreateVM mocks base method. +func (m *MockvmController) CreateVM(ctx context.Context, opts *vm.CreateOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateVM", ctx, opts) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateVM indicates an expected call of CreateVM. +func (mr *MockvmControllerMockRecorder) CreateVM(ctx, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateVM", reflect.TypeOf((*MockvmController)(nil).CreateVM), ctx, opts) +} + +// FinalizeLiveMigration mocks base method. +func (m *MockvmController) FinalizeLiveMigration(ctx context.Context, options *schema2.MigrationFinalizedOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FinalizeLiveMigration", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// FinalizeLiveMigration indicates an expected call of FinalizeLiveMigration. +func (mr *MockvmControllerMockRecorder) FinalizeLiveMigration(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FinalizeLiveMigration", reflect.TypeOf((*MockvmController)(nil).FinalizeLiveMigration), ctx, options) +} + +// Guest mocks base method. +func (m *MockvmController) Guest() *guestmanager.Guest { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Guest") + ret0, _ := ret[0].(*guestmanager.Guest) + return ret0 +} + +// Guest indicates an expected call of Guest. +func (mr *MockvmControllerMockRecorder) Guest() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Guest", reflect.TypeOf((*MockvmController)(nil).Guest)) +} + +// Import mocks base method. +func (m *MockvmController) Import(ctx context.Context, env *anypb.Any) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Import", ctx, env) + ret0, _ := ret[0].(error) + return ret0 +} + +// Import indicates an expected call of Import. +func (mr *MockvmControllerMockRecorder) Import(ctx, env any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Import", reflect.TypeOf((*MockvmController)(nil).Import), ctx, env) +} + +// InitializeLiveMigrationOnSource mocks base method. +func (m *MockvmController) InitializeLiveMigrationOnSource(ctx context.Context, options *schema2.MigrationInitializeOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitializeLiveMigrationOnSource", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// InitializeLiveMigrationOnSource indicates an expected call of InitializeLiveMigrationOnSource. +func (mr *MockvmControllerMockRecorder) InitializeLiveMigrationOnSource(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitializeLiveMigrationOnSource", reflect.TypeOf((*MockvmController)(nil).InitializeLiveMigrationOnSource), ctx, options) +} + +// MigrationNotifications mocks base method. +func (m *MockvmController) MigrationNotifications() (<-chan schema2.OperationSystemMigrationNotificationInfo, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MigrationNotifications") + ret0, _ := ret[0].(<-chan schema2.OperationSystemMigrationNotificationInfo) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MigrationNotifications indicates an expected call of MigrationNotifications. +func (mr *MockvmControllerMockRecorder) MigrationNotifications() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MigrationNotifications", reflect.TypeOf((*MockvmController)(nil).MigrationNotifications)) +} + +// NetworkController mocks base method. +func (m *MockvmController) NetworkController(networkNamespaceID string) *network.Controller { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NetworkController", networkNamespaceID) + ret0, _ := ret[0].(*network.Controller) + return ret0 +} + +// NetworkController indicates an expected call of NetworkController. +func (mr *MockvmControllerMockRecorder) NetworkController(networkNamespaceID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetworkController", reflect.TypeOf((*MockvmController)(nil).NetworkController), networkNamespaceID) +} + +// Patch mocks base method. +func (m *MockvmController) Patch(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Patch", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// Patch indicates an expected call of Patch. +func (mr *MockvmControllerMockRecorder) Patch(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Patch", reflect.TypeOf((*MockvmController)(nil).Patch), ctx) +} + +// Plan9Controller mocks base method. +func (m *MockvmController) Plan9Controller() *plan9.Controller { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Plan9Controller") + ret0, _ := ret[0].(*plan9.Controller) + return ret0 +} + +// Plan9Controller indicates an expected call of Plan9Controller. +func (mr *MockvmControllerMockRecorder) Plan9Controller() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Plan9Controller", reflect.TypeOf((*MockvmController)(nil).Plan9Controller)) +} + +// Resume mocks base method. +func (m *MockvmController) Resume(ctx context.Context, rebuildBridge bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Resume", ctx, rebuildBridge) + ret0, _ := ret[0].(error) + return ret0 +} + +// Resume indicates an expected call of Resume. +func (mr *MockvmControllerMockRecorder) Resume(ctx, rebuildBridge any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resume", reflect.TypeOf((*MockvmController)(nil).Resume), ctx, rebuildBridge) +} + +// RuntimeID mocks base method. +func (m *MockvmController) RuntimeID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RuntimeID") + ret0, _ := ret[0].(string) + return ret0 +} + +// RuntimeID indicates an expected call of RuntimeID. +func (mr *MockvmControllerMockRecorder) RuntimeID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RuntimeID", reflect.TypeOf((*MockvmController)(nil).RuntimeID)) +} + +// SCSIController mocks base method. +func (m *MockvmController) SCSIController(ctx context.Context) (*scsi.Controller, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SCSIController", ctx) + ret0, _ := ret[0].(*scsi.Controller) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// SCSIController indicates an expected call of SCSIController. +func (mr *MockvmControllerMockRecorder) SCSIController(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SCSIController", reflect.TypeOf((*MockvmController)(nil).SCSIController), ctx) +} + +// SandboxOptions mocks base method. +func (m *MockvmController) SandboxOptions() *lcow.SandboxOptions { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SandboxOptions") + ret0, _ := ret[0].(*lcow.SandboxOptions) + return ret0 +} + +// SandboxOptions indicates an expected call of SandboxOptions. +func (mr *MockvmControllerMockRecorder) SandboxOptions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SandboxOptions", reflect.TypeOf((*MockvmController)(nil).SandboxOptions)) +} + +// Save mocks base method. +func (m *MockvmController) Save(ctx context.Context) (*anypb.Any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Save", ctx) + ret0, _ := ret[0].(*anypb.Any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Save indicates an expected call of Save. +func (mr *MockvmControllerMockRecorder) Save(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockvmController)(nil).Save), ctx) +} + +// StartLiveMigrationOnSource mocks base method. +func (m *MockvmController) StartLiveMigrationOnSource(ctx context.Context, config *v2.MigrationConfig) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartLiveMigrationOnSource", ctx, config) + ret0, _ := ret[0].(error) + return ret0 +} + +// StartLiveMigrationOnSource indicates an expected call of StartLiveMigrationOnSource. +func (mr *MockvmControllerMockRecorder) StartLiveMigrationOnSource(ctx, config any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartLiveMigrationOnSource", reflect.TypeOf((*MockvmController)(nil).StartLiveMigrationOnSource), ctx, config) +} + +// StartLiveMigrationTransfer mocks base method. +func (m *MockvmController) StartLiveMigrationTransfer(ctx context.Context, options *schema2.MigrationTransferOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartLiveMigrationTransfer", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// StartLiveMigrationTransfer indicates an expected call of StartLiveMigrationTransfer. +func (mr *MockvmControllerMockRecorder) StartLiveMigrationTransfer(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartLiveMigrationTransfer", reflect.TypeOf((*MockvmController)(nil).StartLiveMigrationTransfer), ctx, options) +} + +// StartWithMigrationOptions mocks base method. +func (m *MockvmController) StartWithMigrationOptions(ctx context.Context, config *v2.MigrationConfig) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StartWithMigrationOptions", ctx, config) + ret0, _ := ret[0].(error) + return ret0 +} + +// StartWithMigrationOptions indicates an expected call of StartWithMigrationOptions. +func (mr *MockvmControllerMockRecorder) StartWithMigrationOptions(ctx, config any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartWithMigrationOptions", reflect.TypeOf((*MockvmController)(nil).StartWithMigrationOptions), ctx, config) +} + +// State mocks base method. +func (m *MockvmController) State() vm.State { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "State") + ret0, _ := ret[0].(vm.State) + return ret0 +} + +// State indicates an expected call of State. +func (mr *MockvmControllerMockRecorder) State() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "State", reflect.TypeOf((*MockvmController)(nil).State)) +} + +// VM mocks base method. +func (m *MockvmController) VM() *vmmanager.UtilityVM { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VM") + ret0, _ := ret[0].(*vmmanager.UtilityVM) + return ret0 +} + +// VM indicates an expected call of VM. +func (mr *MockvmControllerMockRecorder) VM() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VM", reflect.TypeOf((*MockvmController)(nil).VM)) +} + +// VPCIController mocks base method. +func (m *MockvmController) VPCIController() *vpci.Controller { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VPCIController") + ret0, _ := ret[0].(*vpci.Controller) + return ret0 +} + +// VPCIController indicates an expected call of VPCIController. +func (mr *MockvmControllerMockRecorder) VPCIController() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VPCIController", reflect.TypeOf((*MockvmController)(nil).VPCIController)) +} diff --git a/internal/controller/migration/notifications.go b/internal/controller/migration/notifications.go new file mode 100644 index 0000000000..c69847f8a1 --- /dev/null +++ b/internal/controller/migration/notifications.go @@ -0,0 +1,214 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "fmt" + "sync" + "time" + + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/pkg/migration" + + "github.com/containerd/errdefs" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// subscriberBuffer caps how many notifications a subscriber may queue before +// slow readers start dropping updates. +const subscriberBuffer = 64 + +// notifications fans migration events out to every subscriber and replays +// the latest event to late subscribers, all sharing one messageID sequence. +type notifications struct { + // mu guards the mutable fields below. + mu sync.RWMutex + + // subscribers is the set of active streams each event is delivered to. + subscribers map[chan *migration.NotificationsResponse]struct{} + + // lastResponse is the most recent event, replayed to new subscribers. + lastResponse *migration.NotificationsResponse + + // messageID is the monotonically increasing sequence number on each event. + messageID uint32 + + // startTime is when this notifier was created, reported as StartTime on every event. + startTime time.Time + + // done is closed by close to end the session and all subscriber streams. + done chan struct{} + + // origin identifies this host's role (source or destination) in the migration. + origin hcsschema.MigrationOrigin +} + +// newNotifications begins forwarding the VM's migration events to subscribers. +func newNotifications(vmController vmController, origin hcsschema.MigrationOrigin) (*notifications, error) { + // Subscribe to the VM's migration events before any subscriber attaches. + src, err := vmController.MigrationNotifications() + if err != nil { + return nil, fmt.Errorf("get migration notifications channel: %w", err) + } + + notif := ¬ifications{ + subscribers: map[chan *migration.NotificationsResponse]struct{}{}, + startTime: time.Now(), + done: make(chan struct{}), + origin: origin, + } + + // Forward each VM migration event to subscribers until the session is torn down or + // the source stops producing. + go func() { + for { + select { + // Session torn down: stop forwarding. + case <-notif.done: + return + + // Next VM migration event, or the source channel was closed. + case info, ok := <-src: + // Source closed: there is nothing left to forward. + if !ok { + return + } + + // broadcast returns false once the notifier is closed. + if !notif.broadcast(info) { + return + } + } + } + }() + + return notif, nil +} + +// Subscribe returns a stream of migration notifications for the session, +// beginning with the most recent one. The stream ends when ctx is canceled or +// the session is torn down. +func (c *Controller) Subscribe(ctx context.Context, sessionID string) (<-chan *migration.NotificationsResponse, error) { + c.mu.Lock() + defer c.mu.Unlock() + + // Reject callers without an active session or whose sessionID does not + // match the active one. + if c.sessionID != sessionID { + return nil, fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) + } + + // Create the notifier on first use; it begins forwarding VM events as + // soon as it exists, independent of whether any subscriber attaches. + if c.notifier == nil { + notifier, err := newNotifications(c.vmController, c.origin) + if err != nil { + return nil, err + } + + c.notifier = notifier + } + + log.G(ctx).Debug("migration notification subscriber attached") + return c.notifier.subscribe(ctx) +} + +// subscribe returns a channel that first replays the latest notification, then +// delivers every later one until ctx is canceled or the notifier closes. +func (n *notifications) subscribe(ctx context.Context) (<-chan *migration.NotificationsResponse, error) { + subscriber := make(chan *migration.NotificationsResponse, subscriberBuffer) + + n.mu.Lock() + defer n.mu.Unlock() + + // Session already terminated: reject the subscription. + select { + case <-n.done: + return nil, fmt.Errorf("migration session already terminated: %w", errdefs.ErrFailedPrecondition) + default: + } + + // Replay the latest event so a late subscriber has immediate context; + // the buffered channel keeps this send non-blocking. + if n.lastResponse != nil { + subscriber <- n.lastResponse + } + n.subscribers[subscriber] = struct{}{} + + // Drop the subscriber once its context ends or the notifier closes. + go func() { + select { + case <-ctx.Done(): + case <-n.done: + return + } + + n.mu.Lock() + defer n.mu.Unlock() + + // Skip if broadcast or close already removed this subscriber, to + // avoid a double close. + if _, ok := n.subscribers[subscriber]; ok { + delete(n.subscribers, subscriber) + close(subscriber) + } + }() + + return subscriber, nil +} + +// broadcast delivers info to every subscriber and caches it for replay, +// returning false once the notifier has been closed. +func (n *notifications) broadcast(info hcsschema.OperationSystemMigrationNotificationInfo) bool { + n.mu.Lock() + defer n.mu.Unlock() + + // Notifier closed: drop the event and signal the forwarder to stop. + select { + case <-n.done: + return false + default: + } + + // Stamp the next sequence number and cache the event for replay. + n.messageID++ + n.lastResponse = &migration.NotificationsResponse{ + MessageID: n.messageID, + Notification: migration.ToNotification(info, n.origin), + StartTime: timestamppb.New(n.startTime), + UpdateTime: timestamppb.Now(), + } + + // Non-blocking send so one slow subscriber cannot stall the others. + for subscriber := range n.subscribers { + select { + case subscriber <- n.lastResponse: + default: + } + } + + return true +} + +// close stops forwarding events and closes every subscriber channel. Safe to +// call more than once. +func (n *notifications) close() { + n.mu.Lock() + defer n.mu.Unlock() + + // Already closed: nothing to do. + select { + case <-n.done: + return + default: + } + + // Signal the forwarder to stop, then close every subscriber stream. + close(n.done) + for subscriber := range n.subscribers { + close(subscriber) + delete(n.subscribers, subscriber) + } +} diff --git a/internal/controller/migration/notifications_test.go b/internal/controller/migration/notifications_test.go new file mode 100644 index 0000000000..dcb6da717b --- /dev/null +++ b/internal/controller/migration/notifications_test.go @@ -0,0 +1,329 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "errors" + "testing" + "time" + + "go.uber.org/mock/gomock" + + "github.com/Microsoft/hcsshim/internal/controller/migration/mocks" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/pkg/migration" +) + +// newTestNotifications builds a notifier without the source-forwarding +// goroutine so the fan-out logic can be driven directly via broadcast. +func newTestNotifications(origin hcsschema.MigrationOrigin) *notifications { + return ¬ifications{ + subscribers: map[chan *migration.NotificationsResponse]struct{}{}, + startTime: time.Now(), + done: make(chan struct{}), + origin: origin, + } +} + +func setupDoneInfo() hcsschema.OperationSystemMigrationNotificationInfo { + return hcsschema.OperationSystemMigrationNotificationInfo{Event: hcsschema.MigrationEventSetupDone} +} + +// recvWithin returns the next notification or fails if none arrives in time. +func recvWithin(t *testing.T, ch <-chan *migration.NotificationsResponse, d time.Duration) *migration.NotificationsResponse { + t.Helper() + select { + case r := <-ch: + return r + case <-time.After(d): + t.Fatal("timed out waiting for notification") + return nil + } +} + +// waitChannelClosed drains ch until it is closed or fails on timeout. +func waitChannelClosed(t *testing.T, ch <-chan *migration.NotificationsResponse, d time.Duration) { + t.Helper() + deadline := time.After(d) + for { + select { + case _, ok := <-ch: + if !ok { + return + } + case <-deadline: + t.Fatal("channel not closed within timeout") + } + } +} + +// TestNotificationsBroadcastDeliversToSubscriber verifies a broadcast reaches +// an attached subscriber with a populated response. +func TestNotificationsBroadcastDeliversToSubscriber(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sub, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + if ok := n.broadcast(setupDoneInfo()); !ok { + t.Fatal("broadcast returned false on an open notifier") + } + + got := recvWithin(t, sub, time.Second) + if got.MessageID != 1 { + t.Fatalf("messageID: got %d want 1", got.MessageID) + } + if got.Notification == nil || got.Notification.Phase != migration.Phase_PHASE_SETUP_DONE { + t.Fatalf("unexpected notification: %+v", got.Notification) + } + if got.Notification.Origin != migration.Origin_ORIGIN_SOURCE { + t.Fatalf("origin: got %s want %s", got.Notification.Origin, migration.Origin_ORIGIN_SOURCE) + } + if got.StartTime == nil || got.UpdateTime == nil { + t.Fatal("expected StartTime and UpdateTime to be set") + } +} + +// TestNotificationsBroadcastFansOutToAllSubscribers verifies every subscriber +// receives the same event. +func TestNotificationsBroadcastFansOutToAllSubscribers(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sub1, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe sub1: %v", err) + } + sub2, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe sub2: %v", err) + } + + n.broadcast(setupDoneInfo()) + + if r := recvWithin(t, sub1, time.Second); r.MessageID != 1 { + t.Fatalf("sub1 messageID: got %d want 1", r.MessageID) + } + if r := recvWithin(t, sub2, time.Second); r.MessageID != 1 { + t.Fatalf("sub2 messageID: got %d want 1", r.MessageID) + } +} + +// TestNotificationsBroadcastIncrementsMessageID verifies the per-stream +// counter increases monotonically across events. +func TestNotificationsBroadcastIncrementsMessageID(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sub, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + n.broadcast(setupDoneInfo()) + n.broadcast(setupDoneInfo()) + + if r := recvWithin(t, sub, time.Second); r.MessageID != 1 { + t.Fatalf("first messageID: got %d want 1", r.MessageID) + } + if r := recvWithin(t, sub, time.Second); r.MessageID != 2 { + t.Fatalf("second messageID: got %d want 2", r.MessageID) + } +} + +// TestNotificationsSubscribeReplaysLatest verifies a late subscriber +// immediately receives the most recent event. +func TestNotificationsSubscribeReplaysLatest(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Broadcast before anyone subscribes. + n.broadcast(setupDoneInfo()) + + sub, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + if r := recvWithin(t, sub, time.Second); r.MessageID != 1 { + t.Fatalf("replayed messageID: got %d want 1", r.MessageID) + } +} + +// TestNotificationsSubscribeAfterCloseFails verifies subscribing to a +// terminated notifier is rejected. +func TestNotificationsSubscribeAfterCloseFails(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + n.close() + + if _, err := n.subscribe(context.Background()); err == nil { + t.Fatal("expected error subscribing to a closed notifier") + } +} + +// TestNotificationsBroadcastAfterCloseReturnsFalse verifies broadcast signals +// the forwarder to stop once the notifier is closed. +func TestNotificationsBroadcastAfterCloseReturnsFalse(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + n.close() + + if n.broadcast(setupDoneInfo()) { + t.Fatal("broadcast should return false after close") + } +} + +// TestNotificationsCloseClosesSubscribers verifies close ends every active +// subscriber stream. +func TestNotificationsCloseClosesSubscribers(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sub, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + n.close() + waitChannelClosed(t, sub, time.Second) +} + +// TestNotificationsCloseIsIdempotent verifies close can be called repeatedly. +func TestNotificationsCloseIsIdempotent(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + n.close() + n.close() +} + +// TestNotificationsSubscribeContextCancelDropsSubscriber verifies a canceled +// context removes and closes the subscriber, leaving other delivery intact. +func TestNotificationsSubscribeContextCancelDropsSubscriber(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + + sub, err := n.subscribe(ctx) + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + cancel() + waitChannelClosed(t, sub, time.Second) + + // The notifier is still open and broadcasting must not panic on the + // dropped subscriber. + if ok := n.broadcast(setupDoneInfo()); !ok { + t.Fatal("broadcast on an open notifier returned false") + } +} + +// TestNotificationsBroadcastDoesNotBlockOnSlowSubscriber verifies a subscriber +// that never reads cannot stall the broadcaster. +func TestNotificationsBroadcastDoesNotBlockOnSlowSubscriber(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Subscribe but never drain the channel. + if _, err := n.subscribe(ctx); err != nil { + t.Fatalf("subscribe: %v", err) + } + + done := make(chan struct{}) + go func() { + // Broadcasting well past the buffer must not block; extras are dropped. + for i := 0; i < subscriberBuffer*2; i++ { + n.broadcast(setupDoneInfo()) + } + close(done) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("broadcast blocked on a slow subscriber") + } +} + +// TestControllerSubscribeNoActiveSession verifies Subscribe is rejected when +// no session is active. +func TestControllerSubscribeNoActiveSession(t *testing.T) { + c := &Controller{} + if _, err := c.Subscribe(context.Background(), "any"); err == nil { + t.Fatal("expected error when no session is active") + } +} + +// TestControllerSubscribeSessionMismatch verifies Subscribe rejects a +// sessionID that does not match the active one. +func TestControllerSubscribeSessionMismatch(t *testing.T) { + c := &Controller{sessionID: "active"} + if _, err := c.Subscribe(context.Background(), "other"); err == nil { + t.Fatal("expected error on session mismatch") + } +} + +// TestControllerSubscribeReusesExistingNotifier verifies Subscribe attaches to +// an already-created notifier and delivers its events. +func TestControllerSubscribeReusesExistingNotifier(t *testing.T) { + n := newTestNotifications(hcsschema.MigrationOriginSource) + c := &Controller{sessionID: "s", notifier: n} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sub, err := c.Subscribe(ctx, "s") + if err != nil { + t.Fatalf("subscribe: %v", err) + } + + n.broadcast(setupDoneInfo()) + if r := recvWithin(t, sub, time.Second); r.MessageID != 1 { + t.Fatalf("messageID: got %d want 1", r.MessageID) + } +} + +// TestControllerSubscribeCreatesNotifier verifies the first Subscribe builds the +// notifier from the VM's event stream and forwards its events to the subscriber. +func TestControllerSubscribeCreatesNotifier(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + + src := make(chan hcsschema.OperationSystemMigrationNotificationInfo, 1) + var recv <-chan hcsschema.OperationSystemMigrationNotificationInfo = src + vm.EXPECT().MigrationNotifications().Return(recv, nil) + + c := &Controller{sessionID: "s", origin: hcsschema.MigrationOriginSource, vmController: vm} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sub, err := c.Subscribe(ctx, "s") + if err != nil { + t.Fatalf("subscribe: %v", err) + } + defer c.notifier.close() + + // An event on the VM stream is forwarded to the subscriber. + src <- setupDoneInfo() + if got := recvWithin(t, sub, time.Second); got == nil { + t.Fatal("expected forwarded notification") + } +} + +// TestControllerSubscribeNotifierError verifies Subscribe surfaces a failure to +// obtain the VM's notification stream. +func TestControllerSubscribeNotifierError(t *testing.T) { + ctrl := gomock.NewController(t) + vm := mocks.NewMockvmController(ctrl) + vm.EXPECT().MigrationNotifications().Return(nil, errors.New("boom")) + + c := &Controller{sessionID: "s", vmController: vm} + if _, err := c.Subscribe(context.Background(), "s"); err == nil { + t.Fatal("expected error, got nil") + } +} diff --git a/internal/controller/migration/socket.go b/internal/controller/migration/socket.go new file mode 100644 index 0000000000..a97645f14e --- /dev/null +++ b/internal/controller/migration/socket.go @@ -0,0 +1,114 @@ +//go:build windows && lcow + +package migration + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "sync" + "unsafe" + + "github.com/Microsoft/hcsshim/internal/log" + "github.com/containerd/errdefs" + "golang.org/x/sys/windows" +) + +// wsaVersion is the Winsock version requested by [ensureWinsock]. +const wsaVersion uint32 = 0x0202 + +// ensureWinsock initializes Winsock once for the process so a socket can be +// recreated without depending on other code having initialized it first. +// WSACleanup is deliberately never called, so Winsock stays up for the +// process lifetime. +var ensureWinsock = sync.OnceValue(func() error { + var data windows.WSAData + if err := windows.WSAStartup(wsaVersion, &data); err != nil { + return fmt.Errorf("WSAStartup: %w", err) + } + + return nil +}) + +// soConnectTime is the SO_CONNECT_TIME socket option. Querying it with +// getsockopt yields how many seconds the socket has been connected, or +// 0xFFFFFFFF if it has never connected. It is not exported by +// [golang.org/x/sys/windows]. +const soConnectTime int32 = 0x700C + +// connectTimeNotConnected is the SO_CONNECT_TIME sentinel returned for an +// unconnected socket. +const connectTimeNotConnected uint32 = 0xFFFFFFFF + +// RegisterDuplicateSocket adopts a duplicated migration transport socket, +// described by protocolInfo, into this process and makes it available to the +// pending transfer for the given session. A repeat call for an already-adopted +// session is a no-op. +func (c *Controller) RegisterDuplicateSocket(ctx context.Context, sessionID string, protocolInfo []byte) error { + // Reject input too small to hold a serialized socket descriptor before + // attempting to decode it. + wantSize := int(unsafe.Sizeof(windows.WSAProtocolInfo{})) + if len(protocolInfo) < wantSize { + return fmt.Errorf("protocol info is %d bytes, want at least %d: %w", len(protocolInfo), wantSize, errdefs.ErrInvalidArgument) + } + + // Decode the opaque caller-supplied bytes into the socket descriptor + // used to recreate the duplicated socket. + var info windows.WSAProtocolInfo + if err := binary.Read(bytes.NewReader(protocolInfo), binary.LittleEndian, &info); err != nil { + return fmt.Errorf("decode WSAProtocolInfo: %w", err) + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.sessionID != sessionID { + return fmt.Errorf("session id %q does not match active session %q: %w", sessionID, c.sessionID, errdefs.ErrInvalidArgument) + } + + // Idempotent: a repeat call for the same session is a no-op once the + // socket has been adopted. + if c.dupSocket != 0 { + return nil + } + + // Transfer may have already claimed the session (StateSocketWaiting) and + // be waiting on the socket; allow registration in that case too. + if c.state != StateSourceExported && c.state != StateDestinationPrepared && c.state != StateSocketWaiting { + return fmt.Errorf("register duplicate socket requires state %s or %s (current: %s): %w", StateSourceExported, StateDestinationPrepared, c.state, errdefs.ErrFailedPrecondition) + } + + // Make sure Winsock is up for this process before recreating the socket. + if err := ensureWinsock(); err != nil { + return err + } + + // Recreate the duplicated socket in this process from the descriptor so + // the transfer can use it as its transport. + sock, err := windows.WSASocket(info.AddressFamily, info.SocketType, info.Protocol, &info, 0, 0) + if err != nil { + return fmt.Errorf("WSASocket: %w", err) + } + + // Verify the duplicated handle actually represents a connected socket; + // HCS will fail the migration if we hand it an unconnected endpoint. + var connectTime uint32 + optLen := int32(unsafe.Sizeof(connectTime)) + if err := windows.Getsockopt(sock, windows.SOL_SOCKET, soConnectTime, (*byte)(unsafe.Pointer(&connectTime)), &optLen); err != nil { + _ = windows.Closesocket(sock) + return fmt.Errorf("getsockopt SO_CONNECT_TIME: %w", err) + } + + if connectTime == connectTimeNotConnected { + _ = windows.Closesocket(sock) + return fmt.Errorf("duplicated socket is not connected: %w", errdefs.ErrFailedPrecondition) + } + + c.dupSocket = sock + c.state = StateSocketReady + close(c.socketReady) + + log.G(ctx).Info("duplicate migration socket registered") + return nil +} diff --git a/internal/controller/migration/socket_test.go b/internal/controller/migration/socket_test.go new file mode 100644 index 0000000000..71d2be6759 --- /dev/null +++ b/internal/controller/migration/socket_test.go @@ -0,0 +1,68 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + "errors" + "testing" + "unsafe" + + "github.com/containerd/errdefs" + "golang.org/x/sys/windows" +) + +// validProtocolInfo returns a correctly-sized (all-zero) serialized descriptor +// that passes the size check and decode, so tests can exercise the later guards. +func validProtocolInfo() []byte { + return make([]byte, int(unsafe.Sizeof(windows.WSAProtocolInfo{}))) +} + +// TestRegisterDuplicateSocket_TooSmall verifies a buffer too short to hold a +// socket descriptor is rejected as an invalid argument before any decode. +func TestRegisterDuplicateSocket_TooSmall(t *testing.T) { + c := New() + + err := c.RegisterDuplicateSocket(context.Background(), "", []byte{0x00}) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("got %v, want ErrInvalidArgument", err) + } +} + +// TestRegisterDuplicateSocket_SessionMismatch verifies a call for a session +// other than the active one is rejected as an invalid argument. +func TestRegisterDuplicateSocket_SessionMismatch(t *testing.T) { + c := New() + c.sessionID = "active" + + err := c.RegisterDuplicateSocket(context.Background(), "other", validProtocolInfo()) + if !errors.Is(err, errdefs.ErrInvalidArgument) { + t.Fatalf("got %v, want ErrInvalidArgument", err) + } +} + +// TestRegisterDuplicateSocket_Idempotent verifies a repeat call once a socket +// has been adopted is a no-op, regardless of state. +func TestRegisterDuplicateSocket_Idempotent(t *testing.T) { + c := New() + c.sessionID = "s" + c.dupSocket = windows.Handle(1) + c.state = StateSourceExported + + if err := c.RegisterDuplicateSocket(context.Background(), "s", validProtocolInfo()); err != nil { + t.Fatalf("got %v, want nil", err) + } +} + +// TestRegisterDuplicateSocket_InvalidState verifies registration is rejected as +// a failed precondition when the session is not awaiting a socket. +func TestRegisterDuplicateSocket_InvalidState(t *testing.T) { + c := New() + c.sessionID = "s" + c.state = StateIdle + + err := c.RegisterDuplicateSocket(context.Background(), "s", validProtocolInfo()) + if !errors.Is(err, errdefs.ErrFailedPrecondition) { + t.Fatalf("got %v, want ErrFailedPrecondition", err) + } +} diff --git a/internal/controller/migration/state.go b/internal/controller/migration/state.go new file mode 100644 index 0000000000..dbd2532e71 --- /dev/null +++ b/internal/controller/migration/state.go @@ -0,0 +1,116 @@ +//go:build windows && lcow + +package migration + +// State is the current state of a live migration session, advanced by the +// [Controller] APIs. The source and destination follow distinct paths that +// converge once the transport socket is ready, and both return to [StateIdle]. +// +// Source progression: +// +// StateIdle → StateSourcePrepared → StateSourceExported → StateSocketReady +// → StateTransferring → StateTransferCompleted → StateFinalized → StateIdle +// +// Destination progression: +// +// StateIdle → StateDestinationImported → StateDestinationPrepared → StateSocketReady +// → StateTransferring → StateTransferCompleted → StateFinalized → StateIdle +// +// [StateSocketWaiting] is entered instead of [StateSocketReady] when +// [Controller.Transfer] runs before the socket arrives. A failed transfer +// enters [StateFailed], from which [Controller.Cancel] moves to +// [StateCancelled]; [Controller.Finalize] then [Controller.Cleanup] wind a +// canceled or completed session down to [StateIdle]. +type State int32 + +const ( + // StateIdle indicates no migration session is active. This is the + // initial state and the state the controller returns to after + // [Controller.Cleanup] completes. + StateIdle State = iota + + // StateSourcePrepared indicates [Controller.PrepareSource] has armed + // the source-side migration. The next valid call is + // [Controller.ExportState]. + StateSourcePrepared + + // StateSourceExported indicates the source has produced its opaque saved-state + // envelope via [Controller.ExportState]. The next valid call is + // [Controller.RegisterDuplicateSocket]. + StateSourceExported + + // StateDestinationImported indicates the destination has rehydrated the source + // snapshot via [Controller.ImportState]. The next valid calls are + // [Controller.PatchResourcePaths] (one per container) followed by + // [Controller.PrepareDestination]. + StateDestinationImported + + // StateDestinationPrepared indicates the destination has materialized the + // HCS compute system via [Controller.PrepareDestination]. The next + // valid call is [Controller.RegisterDuplicateSocket]. + StateDestinationPrepared + + // StateSocketWaiting indicates [Controller.Transfer] has claimed the + // session and a single goroutine is driving it, waiting for the duplicate + // socket if it is not yet ready. + StateSocketWaiting + + // StateSocketReady indicates the duplicated migration transport socket + // has been adopted via [Controller.RegisterDuplicateSocket]. The next + // valid call is [Controller.Transfer]. + StateSocketReady + + // StateTransferring indicates [Controller.Transfer] has kicked off the + // memory transfer. The controller automatically transitions to + // [StateTransferCompleted] or [StateFailed] when the transfer ends. + StateTransferring + + // StateTransferCompleted indicates the memory transfer finished + // successfully. The next valid call is [Controller.Finalize]. + StateTransferCompleted + + // StateFinalized indicates [Controller.Finalize] has applied the final + // operation. The next valid call is [Controller.Cleanup]. + StateFinalized + + // StateCancelled indicates the session was aborted via [Controller.Cancel]. + // The next valid call is [Controller.Finalize], followed by + // [Controller.Cleanup] to return the controller to [StateIdle]. + StateCancelled + + // StateFailed indicates the memory transfer failed. The next valid call is + // [Controller.Cancel] to abort the session. + StateFailed +) + +// String returns a human-readable representation of the migration State. +func (s State) String() string { + switch s { + case StateIdle: + return "Idle" + case StateSourcePrepared: + return "SourcePrepared" + case StateSourceExported: + return "SourceExported" + case StateDestinationImported: + return "DestinationImported" + case StateDestinationPrepared: + return "DestinationPrepared" + case StateSocketWaiting: + return "SocketWaiting" + case StateSocketReady: + return "SocketReady" + case StateTransferring: + return "Transferring" + case StateTransferCompleted: + return "TransferCompleted" + case StateFinalized: + return "Finalized" + case StateCancelled: + return "Cancelled" + case StateFailed: + return "Failed" + default: + return "Unknown" + } +} diff --git a/internal/controller/migration/types.go b/internal/controller/migration/types.go new file mode 100644 index 0000000000..800be0ecf4 --- /dev/null +++ b/internal/controller/migration/types.go @@ -0,0 +1,90 @@ +//go:build windows && lcow + +package migration + +import ( + "context" + + "github.com/Microsoft/hcsshim/internal/builder/vm/lcow" + "github.com/Microsoft/hcsshim/internal/controller/device/plan9" + "github.com/Microsoft/hcsshim/internal/controller/device/scsi" + "github.com/Microsoft/hcsshim/internal/controller/device/vpci" + "github.com/Microsoft/hcsshim/internal/controller/network" + "github.com/Microsoft/hcsshim/internal/controller/pod" + "github.com/Microsoft/hcsshim/internal/controller/vm" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + hcs "github.com/Microsoft/hcsshim/internal/hcs/v2" + "github.com/Microsoft/hcsshim/internal/vm/guestmanager" + "github.com/Microsoft/hcsshim/internal/vm/vmmanager" + + "google.golang.org/protobuf/types/known/anypb" +) + +// vmController is the subset of the VM controller that the migration controller +// drives. It is declared as an interface so tests can substitute a mock; it is +// implemented by [vm.Controller]. +type vmController interface { + State() vm.State + SandboxOptions() *lcow.SandboxOptions + InitializeLiveMigrationOnSource(ctx context.Context, options *hcsschema.MigrationInitializeOptions) error + Save(ctx context.Context) (*anypb.Any, error) + Import(ctx context.Context, env *anypb.Any) error + CreateVM(ctx context.Context, opts *vm.CreateOptions) error + Patch(ctx context.Context) error + StartLiveMigrationOnSource(ctx context.Context, config *hcs.MigrationConfig) error + StartWithMigrationOptions(ctx context.Context, config *hcs.MigrationConfig) error + StartLiveMigrationTransfer(ctx context.Context, options *hcsschema.MigrationTransferOptions) error + FinalizeLiveMigration(ctx context.Context, options *hcsschema.MigrationFinalizedOptions) error + Resume(ctx context.Context, rebuildBridge bool) error + MigrationNotifications() (<-chan hcsschema.OperationSystemMigrationNotificationInfo, error) + + RuntimeID() string + VM() *vmmanager.UtilityVM + Guest() *guestmanager.Guest + SCSIController(ctx context.Context) (*scsi.Controller, error) + VPCIController() *vpci.Controller + Plan9Controller() *plan9.Controller + NetworkController(networkNamespaceID string) *network.Controller +} + +// InitOptions carries the fields common to every migration entry point that +// binds a controller to a session. +type InitOptions struct { + // SessionID correlates all calls belonging to a single migration session. + SessionID string + + // Origin selects whether this side acts as the migration source or destination. + Origin hcsschema.MigrationOrigin + + // VMController is the VM driven by the session: saved on the source, rehydrated + // on the destination. Owned by the service; the controller only borrows it. + VMController vmController + + // PodControllers are the sandbox's pods, keyed by pod ID, migrated alongside the + // VM. Owned by the service; the controller only borrows the map. + PodControllers map[string]*pod.Controller +} + +// PrepareSourceOptions configures the source side of a migration session. +type PrepareSourceOptions struct { + InitOptions + + // MigrationOpts tunes the HCS migration workflow; optional, defaulted when nil. + MigrationOpts *hcsschema.MigrationInitializeOptions +} + +// ImportStateOptions configures rehydrating a source snapshot on the destination side. +type ImportStateOptions struct { + InitOptions + + // SandboxID is the destination sandbox ID the snapshot is imported into. + SandboxID string + + // SavedState is the opaque snapshot produced by the source's ExportState. + SavedState *anypb.Any + + // ContainerPodMapping is the service-owned containerID -> podID index. + // ImportState populates it and PatchResourcePaths renames its entries + // in place; the service continues to own its lifetime. + ContainerPodMapping map[string]string +} diff --git a/pkg/migration/parse.go b/pkg/migration/parse.go index e6c66c4e31..fe2bb15505 100644 --- a/pkg/migration/parse.go +++ b/pkg/migration/parse.go @@ -3,9 +3,11 @@ package migration import ( + "encoding/json" "fmt" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "google.golang.org/protobuf/types/known/timestamppb" ) // InitializeOptionsFromProto converts a protobuf [InitializeOptions] to the @@ -71,3 +73,103 @@ func compressionSettingsFromProto(p *CompressionSettings) *hcsschema.MigrationCo ThrottleWorkerCount: p.ThrottleWorkerCount, } } + +// ToOrigin maps an HCS migration origin to its wire form, falling back to the +// controller-known origin when HCS leaves the field empty. +func ToOrigin(origin, fallback hcsschema.MigrationOrigin) Origin { + if origin == "" { + origin = fallback + } + + switch origin { + case hcsschema.MigrationOriginSource: + return Origin_ORIGIN_SOURCE + case hcsschema.MigrationOriginDestination: + return Origin_ORIGIN_DESTINATION + } + + return Origin_ORIGIN_UNSPECIFIED +} + +// ToPhase maps an HCS migration event to its wire-form phase, returning +// PHASE_UNSPECIFIED for an unrecognized event. +func ToPhase(event hcsschema.MigrationEvent) Phase { + switch event { + case hcsschema.MigrationEventSetupDone: + return Phase_PHASE_SETUP_DONE + case hcsschema.MigrationEventTransferInProgress: + return Phase_PHASE_TRANSFER_IN_PROGRESS + case hcsschema.MigrationEventBlackoutStarted: + return Phase_PHASE_BLACKOUT_STARTED + case hcsschema.MigrationEventOfflineDone: + return Phase_PHASE_OFFLINE_DONE + case hcsschema.MigrationEventBlackoutExited: + return Phase_PHASE_BLACKOUT_EXITED + case hcsschema.MigrationEventMigrationDone: + return Phase_PHASE_DONE + case hcsschema.MigrationEventMigrationRecoveryDone: + return Phase_PHASE_RECOVERY_DONE + case hcsschema.MigrationEventMigrationFailed: + return Phase_PHASE_FAILED + } + + return Phase_PHASE_UNSPECIFIED +} + +// ToPhaseState maps an HCS migration result to its wire-form state. +func ToPhaseState(result hcsschema.MigrationResult, phase Phase) PhaseState { + switch result { + case hcsschema.MigrationResultSuccess: + return PhaseState_PHASE_STATE_SUCCESS + case hcsschema.MigrationResultMigrationCancelled: + return PhaseState_PHASE_STATE_CANCELLED + case hcsschema.MigrationResultGuestInitiatedCancellation: + return PhaseState_PHASE_STATE_GUEST_INITIATED_CANCELLATION + case hcsschema.MigrationResultSourceMigrationFailed: + return PhaseState_PHASE_STATE_SOURCE_FAILED + case hcsschema.MigrationResultDestinationMigrationFailed: + return PhaseState_PHASE_STATE_DESTINATION_FAILED + case hcsschema.MigrationResultMigrationRecoveryFailed: + return PhaseState_PHASE_STATE_RECOVERY_FAILED + } + + // No HCS result: progress phases imply forward progress (failures arrive + // as PHASE_FAILED), so default to SUCCESS; terminal phases stay UNSPECIFIED + // so callers can tell "HCS did not say" from a real outcome. + switch phase { + case Phase_PHASE_SETUP_DONE, + Phase_PHASE_TRANSFER_IN_PROGRESS, + Phase_PHASE_BLACKOUT_STARTED, + Phase_PHASE_OFFLINE_DONE, + Phase_PHASE_BLACKOUT_EXITED: + return PhaseState_PHASE_STATE_SUCCESS + } + + return PhaseState_PHASE_STATE_UNSPECIFIED +} + +// ToNotification converts an HCS migration event into its wire-form notification. +func ToNotification(info hcsschema.OperationSystemMigrationNotificationInfo, fallbackOrigin hcsschema.MigrationOrigin) *Notification { + phase := ToPhase(info.Event) + notification := &Notification{ + Origin: ToOrigin(info.Origin, fallbackOrigin), + Phase: phase, + State: ToPhaseState(info.Result, phase), + } + + if info.Event == hcsschema.MigrationEventBlackoutExited && len(info.AdditionalDetails) > 0 { + // On unmarshal failure we drop PhaseDetails rather than the whole + // notification; the core phase/state info is still useful. + var details hcsschema.BlackoutExitedEventDetails + if err := json.Unmarshal(info.AdditionalDetails, &details); err == nil { + notification.PhaseDetails = &Notification_BlackoutExited{ + BlackoutExited: &BlackoutExitedEventDetails{ + BlackoutDurationMilliseconds: details.BlackoutDurationMilliseconds, + BlackoutStopTimestamp: timestamppb.New(details.BlackoutStopTimestamp), + }, + } + } + } + + return notification +} diff --git a/pkg/migration/parse_test.go b/pkg/migration/parse_test.go new file mode 100644 index 0000000000..2ac7eff2ea --- /dev/null +++ b/pkg/migration/parse_test.go @@ -0,0 +1,310 @@ +//go:build windows + +package migration + +import ( + "encoding/json" + "reflect" + "testing" + "time" + + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func ptr[T any](v T) *T { return &v } + +// TestInitializeOptionsFromProto verifies the wire init options are converted to +// the HCS options: nil and unsupported transports are handled, nested params are +// copied (or left nil), and an out-of-range throttle percentage wraps to uint8. +func TestInitializeOptionsFromProto(t *testing.T) { + tests := []struct { + name string + in *InitializeOptions + want *hcsschema.MigrationInitializeOptions + wantErr bool + }{ + { + name: "nil input returns nil", + }, + { + name: "unsupported transport returns error", + in: &InitializeOptions{MemoryTransport: MemoryTransport_MEMORY_TRANSPORT_UNSPECIFIED}, + wantErr: true, + }, + { + name: "nil nested params are preserved as nil", + in: &InitializeOptions{ + MemoryTransport: MemoryTransport_MEMORY_TRANSPORT_TCP, + ChecksumVerification: true, + }, + want: &hcsschema.MigrationInitializeOptions{ + MemoryTransport: hcsschema.MigrationMemoryTransportTCP, + ChecksumVerification: true, + }, + }, + { + name: "full conversion with nested params", + in: &InitializeOptions{ + MemoryTransport: MemoryTransport_MEMORY_TRANSPORT_TCP, + MemoryTransferThrottleParams: &MemoryTransferThrottleParams{ + SkipThrottling: ptr(true), + ThrottlingScale: ptr(42.5), + MinimumThrottlePercentage: ptr(uint32(50)), + TargetNumberOfBrownoutTransferPasses: ptr(uint32(3)), + StartingBrownoutPassNumberForThrottling: ptr(uint32(1)), + MaximumNumberOfBrownoutTransferPasses: ptr(uint32(7)), + TargetBlackoutTransferTime: ptr(uint32(100)), + BlackoutTimeThresholdForCancellingMigration: ptr(uint32(200)), + }, + CompressionSettings: &CompressionSettings{ThrottleWorkerCount: ptr(uint32(4))}, + ChecksumVerification: true, + PerfTracingEnabled: true, + CancelIfBlackoutThresholdExceeds: true, + PrepareMemoryTransferMode: true, + }, + want: &hcsschema.MigrationInitializeOptions{ + MemoryTransport: hcsschema.MigrationMemoryTransportTCP, + MemoryTransferThrottleParams: &hcsschema.MemoryMigrationTransferThrottleParams{ + SkipThrottling: ptr(true), + ThrottlingScale: ptr(42.5), + MinimumThrottlePercentage: ptr(uint8(50)), + TargetNumberOfBrownoutTransferPasses: ptr(uint32(3)), + StartingBrownoutPassNumberForThrottling: ptr(uint32(1)), + MaximumNumberOfBrownoutTransferPasses: ptr(uint32(7)), + TargetBlackoutTransferTime: ptr(uint32(100)), + BlackoutTimeThresholdForCancellingMigration: ptr(uint32(200)), + }, + CompressionSettings: &hcsschema.MigrationCompressionSettings{ThrottleWorkerCount: ptr(uint32(4))}, + ChecksumVerification: true, + PerfTracingEnabled: true, + CancelIfBlackoutThresholdExceeds: true, + PrepareMemoryTransferMode: true, + }, + }, + { + // A throttle percentage above 255 wraps when narrowed to uint8. + name: "minimum throttle percentage truncates to uint8", + in: &InitializeOptions{ + MemoryTransport: MemoryTransport_MEMORY_TRANSPORT_TCP, + MemoryTransferThrottleParams: &MemoryTransferThrottleParams{MinimumThrottlePercentage: ptr(uint32(300))}, + }, + want: &hcsschema.MigrationInitializeOptions{ + MemoryTransport: hcsschema.MigrationMemoryTransportTCP, + MemoryTransferThrottleParams: &hcsschema.MemoryMigrationTransferThrottleParams{MinimumThrottlePercentage: ptr(uint8(44))}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := InitializeOptionsFromProto(tt.in) + if (err != nil) != tt.wantErr { + t.Fatalf("error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("got %+v, want %+v", got, tt.want) + } + }) + } +} + +// TestToOrigin verifies a caller sees the migration side mapped to the wire +// origin, with the controller-known fallback used only when HCS omits the origin +// and unknown origins reported as unspecified. +func TestToOrigin(t *testing.T) { + tests := []struct { + name string + origin hcsschema.MigrationOrigin + fallback hcsschema.MigrationOrigin + want Origin + }{ + {name: "empty falls back to source", fallback: hcsschema.MigrationOriginSource, want: Origin_ORIGIN_SOURCE}, + {name: "empty falls back to destination", fallback: hcsschema.MigrationOriginDestination, want: Origin_ORIGIN_DESTINATION}, + {name: "empty with empty fallback is unspecified", want: Origin_ORIGIN_UNSPECIFIED}, + {name: "source ignores fallback", origin: hcsschema.MigrationOriginSource, fallback: hcsschema.MigrationOriginDestination, want: Origin_ORIGIN_SOURCE}, + {name: "destination", origin: hcsschema.MigrationOriginDestination, want: Origin_ORIGIN_DESTINATION}, + {name: "unknown non-empty origin ignores fallback", origin: hcsschema.MigrationOrigin("Bogus"), fallback: hcsschema.MigrationOriginSource, want: Origin_ORIGIN_UNSPECIFIED}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToOrigin(tt.origin, tt.fallback); got != tt.want { + t.Fatalf("got %v, want %v", got, tt.want) + } + }) + } +} + +// TestToPhase verifies every HCS migration event maps to its wire phase and that +// unknown or empty events fall back to the unspecified phase. +func TestToPhase(t *testing.T) { + tests := []struct { + event hcsschema.MigrationEvent + want Phase + }{ + {hcsschema.MigrationEventSetupDone, Phase_PHASE_SETUP_DONE}, + {hcsschema.MigrationEventTransferInProgress, Phase_PHASE_TRANSFER_IN_PROGRESS}, + {hcsschema.MigrationEventBlackoutStarted, Phase_PHASE_BLACKOUT_STARTED}, + {hcsschema.MigrationEventOfflineDone, Phase_PHASE_OFFLINE_DONE}, + {hcsschema.MigrationEventBlackoutExited, Phase_PHASE_BLACKOUT_EXITED}, + {hcsschema.MigrationEventMigrationDone, Phase_PHASE_DONE}, + {hcsschema.MigrationEventMigrationRecoveryDone, Phase_PHASE_RECOVERY_DONE}, + {hcsschema.MigrationEventMigrationFailed, Phase_PHASE_FAILED}, + {hcsschema.MigrationEventUnknown, Phase_PHASE_UNSPECIFIED}, + {hcsschema.MigrationEvent(""), Phase_PHASE_UNSPECIFIED}, + } + + for _, tt := range tests { + t.Run(string(tt.event), func(t *testing.T) { + if got := ToPhase(tt.event); got != tt.want { + t.Fatalf("got %v, want %v", got, tt.want) + } + }) + } +} + +// TestToPhaseState verifies an HCS result takes precedence when mapping to the +// wire state, and that with no result the phase decides the outcome: progress +// phases default to success while terminal/unspecified phases stay unspecified. +func TestToPhaseState(t *testing.T) { + tests := []struct { + name string + result hcsschema.MigrationResult + phase Phase + want PhaseState + }{ + {name: "success", result: hcsschema.MigrationResultSuccess, want: PhaseState_PHASE_STATE_SUCCESS}, + {name: "cancelled", result: hcsschema.MigrationResultMigrationCancelled, want: PhaseState_PHASE_STATE_CANCELLED}, + {name: "guest cancellation", result: hcsschema.MigrationResultGuestInitiatedCancellation, want: PhaseState_PHASE_STATE_GUEST_INITIATED_CANCELLATION}, + {name: "source failed", result: hcsschema.MigrationResultSourceMigrationFailed, want: PhaseState_PHASE_STATE_SOURCE_FAILED}, + {name: "destination failed", result: hcsschema.MigrationResultDestinationMigrationFailed, want: PhaseState_PHASE_STATE_DESTINATION_FAILED}, + {name: "recovery failed", result: hcsschema.MigrationResultMigrationRecoveryFailed, want: PhaseState_PHASE_STATE_RECOVERY_FAILED}, + {name: "result wins over phase", result: hcsschema.MigrationResultSuccess, phase: Phase_PHASE_FAILED, want: PhaseState_PHASE_STATE_SUCCESS}, + {name: "no result progress phase defaults to success", phase: Phase_PHASE_TRANSFER_IN_PROGRESS, want: PhaseState_PHASE_STATE_SUCCESS}, + {name: "no result terminal phase is unspecified", phase: Phase_PHASE_DONE, want: PhaseState_PHASE_STATE_UNSPECIFIED}, + {name: "no result unspecified phase is unspecified", phase: Phase_PHASE_UNSPECIFIED, want: PhaseState_PHASE_STATE_UNSPECIFIED}, + {name: "invalid result falls through to progress phase", result: hcsschema.MigrationResultInvalid, phase: Phase_PHASE_SETUP_DONE, want: PhaseState_PHASE_STATE_SUCCESS}, + {name: "invalid result falls through to terminal phase", result: hcsschema.MigrationResultInvalid, phase: Phase_PHASE_FAILED, want: PhaseState_PHASE_STATE_UNSPECIFIED}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ToPhaseState(tt.result, tt.phase); got != tt.want { + t.Fatalf("got %v, want %v", got, tt.want) + } + }) + } +} + +// TestToNotification verifies an HCS notification is converted to its wire form: +// origin/phase/state are mapped (with fallback origin), and blackout-exited +// details are attached only on a valid payload and dropped otherwise. +func TestToNotification(t *testing.T) { + stopTime := time.Unix(1700000000, 0).UTC() + blackoutDetails, err := json.Marshal(hcsschema.BlackoutExitedEventDetails{ + BlackoutDurationMilliseconds: 1234, + BlackoutStopTimestamp: stopTime, + }) + if err != nil { + t.Fatalf("marshal details: %v", err) + } + + tests := []struct { + name string + info hcsschema.OperationSystemMigrationNotificationInfo + fallback hcsschema.MigrationOrigin + want *Notification + }{ + { + name: "maps origin phase and state", + info: hcsschema.OperationSystemMigrationNotificationInfo{ + Origin: hcsschema.MigrationOriginSource, + Event: hcsschema.MigrationEventMigrationFailed, + Result: hcsschema.MigrationResultSourceMigrationFailed, + }, + want: &Notification{ + Origin: Origin_ORIGIN_SOURCE, + Phase: Phase_PHASE_FAILED, + State: PhaseState_PHASE_STATE_SOURCE_FAILED, + }, + }, + { + name: "empty origin uses fallback", + info: hcsschema.OperationSystemMigrationNotificationInfo{Event: hcsschema.MigrationEventSetupDone}, + fallback: hcsschema.MigrationOriginDestination, + want: &Notification{ + Origin: Origin_ORIGIN_DESTINATION, + Phase: Phase_PHASE_SETUP_DONE, + State: PhaseState_PHASE_STATE_SUCCESS, + }, + }, + { + name: "blackout exited with valid details", + info: hcsschema.OperationSystemMigrationNotificationInfo{ + Origin: hcsschema.MigrationOriginSource, + Event: hcsschema.MigrationEventBlackoutExited, + AdditionalDetails: blackoutDetails, + }, + want: &Notification{ + Origin: Origin_ORIGIN_SOURCE, + Phase: Phase_PHASE_BLACKOUT_EXITED, + State: PhaseState_PHASE_STATE_SUCCESS, + PhaseDetails: &Notification_BlackoutExited{ + BlackoutExited: &BlackoutExitedEventDetails{ + BlackoutDurationMilliseconds: 1234, + BlackoutStopTimestamp: timestamppb.New(stopTime), + }, + }, + }, + }, + { + name: "blackout exited with invalid details drops phase details", + info: hcsschema.OperationSystemMigrationNotificationInfo{ + Origin: hcsschema.MigrationOriginSource, + Event: hcsschema.MigrationEventBlackoutExited, + AdditionalDetails: json.RawMessage("{invalid"), + }, + want: &Notification{ + Origin: Origin_ORIGIN_SOURCE, + Phase: Phase_PHASE_BLACKOUT_EXITED, + State: PhaseState_PHASE_STATE_SUCCESS, + }, + }, + { + name: "blackout exited without details has no phase details", + info: hcsschema.OperationSystemMigrationNotificationInfo{ + Event: hcsschema.MigrationEventBlackoutExited, + }, + want: &Notification{ + Phase: Phase_PHASE_BLACKOUT_EXITED, + State: PhaseState_PHASE_STATE_SUCCESS, + }, + }, + { + name: "details ignored for non-blackout event", + info: hcsschema.OperationSystemMigrationNotificationInfo{ + Event: hcsschema.MigrationEventSetupDone, + AdditionalDetails: blackoutDetails, + }, + want: &Notification{ + Phase: Phase_PHASE_SETUP_DONE, + State: PhaseState_PHASE_STATE_SUCCESS, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ToNotification(tt.info, tt.fallback) + if !proto.Equal(got, tt.want) { + t.Fatalf("got %+v, want %+v", got, tt.want) + } + }) + } +}