Trwały fsync (rclone mount2 + syncfs w punchfs) i czyste zatrzymanie VM (vm-hold)

launchd/com.renacode.cloudmachine.vm-start.plist.template

@@ -5,20 +5,21 @@
   <key>Label</key>
   <string>com.renacode.cloudmachine.vm-start</string>
   <!--
-    Starts the machine at login. `limactl start` on a running machine only
-    says so and exits, which is why RunAtLoad without KeepAlive is enough.
-    limactl comes from Homebrew, found through PATH below.
+    Starts the machine at login and stays alive (`vm-hold`): launchd's SIGTERM
+    at logout or shutdown becomes a clean `limactl stop` of the machine, which
+    otherwise dies hard with the Mac. ExitTimeOut is how long launchd waits
+    before SIGKILL; it must exceed VMHold.stopTimeout (a test keeps them
+    together). limactl comes from Homebrew, found through PATH below.
   -->
   <key>ProgramArguments</key>
   <array>
-    <string>/usr/bin/env</string>
-    <string>limactl</string>
-    <string>start</string>
-    <string>--tty=false</string>
-    <string>cmsmb</string>
+    <string>__CM_AGENT_BIN__</string>
+    <string>vm-hold</string>
   </array>
   <key>RunAtLoad</key>
   <true/>
+  <key>ExitTimeOut</key>
+  <integer>120</integer>
   <key>EnvironmentVariables</key>
   <dict>
     <key>PATH</key>

mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift

@@ -21,6 +21,7 @@ struct CloudMachineAgent: AsyncParsableCommand {
       VMStatusCommand.self,
       VMUpdateCommand.self,
       VMGuardCommand.self,
+      VMHoldCommand.self,
       VMEncrypt.self,
       BackupHealthCommand.self,
       PrepareShutdown.self,

mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift

@@ -0,0 +1,20 @@
+import ArgumentParser
+import CloudMachineCore
+import Foundation
+
+struct VMHoldCommand: AsyncParsableCommand {
+  static let configuration = CommandConfiguration(
+    commandName: "vm-hold",
+    // Not translated: only the launchd job runs this, nobody reads its help.
+    abstract:
+      "Starts the machine and stays running; when it is stopped (logout, shutdown) it stops the machine cleanly."
+  )
+
+  func run() async throws {
+    // The handlers must be in place before the (minutes long) start: a SIGTERM
+    // that arrives during it is remembered and cancels the start.
+    let signals = TerminationSignals()
+    let code = await VMHold.run(steps: VMHold.live, termination: signals.latch)
+    if code != 0 { throw ExitCode(code) }
+  }
+}

mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift

@@ -50,8 +50,11 @@ public enum LaunchdInstaller {
     // app bundle.
     CMTooling.linkCommandIntoPath()
 
-    // Reloading touches nothing that holds data: `vm-start` on a running
-    // machine only says so, the others are one-shot checks.
+    // Reloading is `unload` (SIGTERM) and `load`. The one-shot agents are
+    // checks and lose nothing. `vm-start` is `vm-hold`, which stops the
+    // machine on SIGTERM: a running one is NOT reloaded (see
+    // `installAgents(isRunning:)`), its new plist is picked up at the next
+    // login or boot.
     guard let templatesDir = CMPaths.launchdTemplatesDir else {
       return CMActionResult(
         succeeded: false, message: L10n.tr("Could not find the launchd/ directory with templates."))
@@ -131,6 +134,21 @@ public enum LaunchdInstaller {
     var failed: [Failure] = []
   }
 
+  /// Agents whose process IS the machine's lifetime: never reloaded while running.
+  static let holdingLabels: Set<String> = ["com.renacode.cloudmachine.vm-start"]
+
+  /// `launchctl list <label>` prints `"PID" = n;` only while the job runs.
+  static func jobIsRunning(listOutput: String) -> Bool {
+    listOutput.contains("\"PID\" =")
+  }
+
+  static func launchctlJobIsRunning(label: String) async -> Bool {
+    guard let result = try? await ProcessRunner.run("/bin/launchctl", ["list", label]),
+      result.succeeded
+    else { return false }
+    return jobIsRunning(listOutput: result.stdout)
+  }
+
   /// Generates `.plist` files from the templates and reloads the agents,
   /// COLLECTING failures.
   ///
@@ -151,6 +169,7 @@ public enum LaunchdInstaller {
       try $0.write(to: $1, atomically: true, encoding: .utf8)
     },
     reload: @Sendable (URL) async -> Bool = { await launchctlReload($0) },
+    isRunning: @Sendable (String) async -> Bool = { await launchctlJobIsRunning(label: $0) },
     log: @Sendable (String) -> Void = { CMLogger.log($0) }
   ) async -> InstallOutcome {
     var outcome = InstallOutcome()
@@ -193,6 +212,20 @@ public enum LaunchdInstaller {
       }
       log("Generated \(destURL.path)")
 
+      // `vm-start` holds the machine: unloading it stops the machine (in the
+      // middle of a backup, if one runs) and the new job would race the old
+      // one's `limactl stop`. Leave a running one alone; it reads the plist
+      // written above at its next start. A job that is not running (the old
+      // `limactl start` job has exited, or a first installation) is loaded
+      // as usual - that is what starts `vm-hold`.
+      if holdingLabels.contains(label), await isRunning(label) {
+        outcome.installed.append(label)
+        log(
+          "Left \(label) running: reloading would stop the machine; the new plist takes effect at the next login or boot"
+        )
+        continue
+      }
+
       if await reload(destURL) {
         outcome.installed.append(label)
         log("Loaded \(label) via launchctl")

mac-app/Sources/CloudMachineCore/ProcessRunner.swift

@@ -186,9 +186,71 @@ final class TimeoutTargets: @unchecked Sendable {
 /// only in the GUI as `Shell.run`; moved here so that the CLI watchdogs have
 /// exactly the same, already proven timeout semantics.
 public enum ProcessRunner {
+  /// After `timeout` the process gets SIGTERM, after `timeout + 5` SIGKILL, and
+  /// after `timeout + giveUpSlack` the call returns `timedOut` regardless.
+  /// A caller with a deadline (`VMHold`) adds this to every limit it sets.
+  public static let giveUpSlack: TimeInterval = 10
+
+  /// A cancelled call sends SIGTERM and, if the process is still there after
+  /// this many seconds, SIGKILL (a process that ignores SIGTERM must not hold
+  /// the caller past a launchd deadline). `VMHold` counts it in its budget.
+  public static let cancelKillDelay: TimeInterval = 5
+
+  /// Holds the process of one call, so that cancelling the calling task can
+  /// reach it (it is created before the process exists).
+  private final class CancelBox: @unchecked Sendable {
+    private let lock = NSLock()
+    private var process: Process?
+    private var cancelled = false
+    /// `true` when the call was cancelled before the process was set.
+    func set(_ process: Process) -> Bool {
+      lock.lock()
+      defer { lock.unlock() }
+      self.process = process
+      return cancelled
+    }
+    /// Cancelled before the process existed: end it as soon as it does.
+    func terminateNow(_ process: Process) {
+      process.terminate()
+      let pid = process.processIdentifier
+      DispatchQueue.global().asyncAfter(deadline: .now() + ProcessRunner.cancelKillDelay) {
+        if process.isRunning { kill(pid, SIGKILL) }
+      }
+    }
+    func cancel() {
+      lock.lock()
+      cancelled = true
+      let target = process
+      lock.unlock()
+      guard let target, target.isRunning else { return }
+      target.terminate()
+      let pid = target.processIdentifier
+      DispatchQueue.global().asyncAfter(deadline: .now() + ProcessRunner.cancelKillDelay) {
+        // The pid cannot have been reused within seconds unless the process
+        // exited, and then `target` says so.
+        if target.isRunning { kill(pid, SIGKILL) }
+      }
+    }
+  }
+
+  /// Cancelling the calling task sends the process SIGTERM (`limactl start`
+  /// can run for 50 minutes and must end when the job is told to stop); the
+  /// call then returns when the process has exited.
   public static func run(
     _ executable: String, _ args: [String], env: [String: String] = [:],
     timeout: TimeInterval? = nil
+  ) async throws -> ProcessResult {
+    let box = CancelBox()
+    return try await withTaskCancellationHandler {
+      try await start(executable, args, env: env, timeout: timeout, box: box)
+    } onCancel: {
+      box.cancel()
+    }
+  }
+
+  private static func start(
+    _ executable: String, _ args: [String], env: [String: String], timeout: TimeInterval?,
+    box: CancelBox
   ) async throws -> ProcessResult {
     try await withCheckedThrowingContinuation { continuation in
       let process = Process()
@@ -260,6 +322,7 @@ public enum ProcessRunner {
         }
         return
       }
+      if box.set(process), process.isRunning { box.terminateNow(process) }
 
       if let timeout {
         // The closures below capture `targets` only - never `process` or a
@@ -288,7 +351,7 @@ public enum ProcessRunner {
         // the background (harmlessly - it will finish by itself some day, when
         // the kernel finally gets an answer, and `resumeGuard` prevents a
         // double resume if `terminationHandler` fires later).
-        DispatchQueue.global().asyncAfter(deadline: .now() + timeout + 10) {
+        DispatchQueue.global().asyncAfter(deadline: .now() + timeout + giveUpSlack) {
           if resumeGuard.claim() {
             // IMPORTANT: we do NOT stop reading - that leaves the pipe with NO
             // reader at all. If the process survived even SIGKILL (stuck in the

mac-app/Sources/CloudMachineCore/TerminationSignals.swift

@@ -0,0 +1,127 @@
+import Foundation
+
+/// "A termination signal has arrived", as a state that can be waited for.
+///
+/// A state, not an event: SIGTERM can arrive during the (minutes long) start,
+/// before anybody waits, and must not be lost. `wait()` ends when `fire()` was
+/// called, now or earlier, or when the waiting task is cancelled (so that a
+/// waiter that lost a race does not hang its task group).
+public final class TerminationLatch: @unchecked Sendable {
+  private let lock = NSLock()
+  private var arrived = false
+  private var waiters: [UUID: CheckedContinuation<Void, Never>] = [:]
+
+  public init() {}
+
+  public func fire() {
+    lock.lock()
+    arrived = true
+    let released = Array(waiters.values)
+    waiters.removeAll()
+    lock.unlock()
+    for waiter in released { waiter.resume() }
+  }
+
+  public var hasArrived: Bool {
+    lock.lock()
+    defer { lock.unlock() }
+    return arrived
+  }
+
+  public func wait() async {
+    let id = UUID()
+    await withTaskCancellationHandler {
+      await withCheckedContinuation { (continuation: CheckedContinuation<Void, Never>) in
+        lock.lock()
+        if arrived || Task.isCancelled {
+          lock.unlock()
+          continuation.resume()
+        } else {
+          waiters[id] = continuation
+          lock.unlock()
+        }
+      }
+    } onCancel: {
+      lock.lock()
+      let waiter = waiters.removeValue(forKey: id)
+      lock.unlock()
+      waiter?.resume()
+    }
+  }
+}
+
+/// SIGTERM is what launchd sends; SIGINT and SIGHUP cover a terminal.
+///
+/// A real handler (`sigaction`) that writes one byte to a pipe, not `SIG_IGN`
+/// plus a dispatch source. An ignored signal is inherited through exec, so
+/// `limactl` and the Lima host agent would have started with SIGTERM ignored
+/// and `Process.terminate()` (or launchd's group kill) would not reach them.
+/// A caught signal is reset to the default by exec.
+///
+/// One instance at a time: the handler reaches the pipe through a global.
+public final class TerminationSignals: @unchecked Sendable {
+  public let latch = TerminationLatch()
+  private let signals: [Int32]
+  private var previous: [Int32: sigaction] = [:]
+  private var pipeFDs: [Int32] = [-1, -1]
+  private var source: DispatchSourceRead?
+
+  public init(signals: [Int32] = [SIGTERM, SIGINT, SIGHUP]) {
+    self.signals = signals
+    var fds: [Int32] = [-1, -1]
+    guard pipe(&fds) == 0 else { return }
+    pipeFDs = fds
+    for fd in fds {
+      _ = fcntl(fd, F_SETFD, FD_CLOEXEC)
+      _ = fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK)
+    }
+    terminationPipeWriteFD = fds[1]
+
+    let readFD = fds[0]
+    let latch = self.latch
+    let source = DispatchSource.makeReadSource(fileDescriptor: readFD, queue: .global())
+    source.setEventHandler {
+      var buffer = [UInt8](repeating: 0, count: 16)
+      _ = read(readFD, &buffer, buffer.count)
+      latch.fire()
+    }
+    // The descriptor is closed only after the source has really let go of it.
+    source.setCancelHandler { close(readFD) }
+    source.resume()
+    self.source = source
+
+    for sig in signals {
+      var action = sigaction()
+      action.__sigaction_u.__sa_handler = { signalNumber in
+        // Only async-signal-safe calls here: one byte into the pipe, and errno
+        // put back for the code the signal interrupted.
+        let saved = errno
+        var byte = UInt8(truncatingIfNeeded: signalNumber)
+        _ = write(terminationPipeWriteFD, &byte, 1)
+        errno = saved
+      }
+      sigemptyset(&action.sa_mask)
+      action.sa_flags = SA_RESTART
+      var old = sigaction()
+      sigaction(sig, &action, &old)
+      previous[sig] = old
+    }
+  }
+
+  /// Puts the previous dispositions back (tests; the job simply exits).
+  public func uninstall() {
+    for (sig, old) in previous {
+      var restore = old
+      sigaction(sig, &restore, nil)
+    }
+    previous.removeAll()
+    source?.cancel()
+    source = nil
+    terminationPipeWriteFD = -1
+    if pipeFDs[1] >= 0 { close(pipeFDs[1]) }  // the read end closes in the cancel handler
+    pipeFDs = [-1, -1]
+  }
+}
+
+/// Read by the signal handler, which cannot capture anything.
+nonisolated(unsafe) private var terminationPipeWriteFD: Int32 = -1