From f44137404c42fc3e3b6cfbe2fbefc607d623a268 Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Thu, 12 Nov 2026 10:38:06 +0100 Subject: [PATCH 1/6] =?UTF-8?q?Trwa=C5=82y=20fsync=20w=20=C5=82a=C5=84cuch?= =?UTF-8?q?u=20TM=20->=20SMB=20->=20punchfs=20->=20rclone=20i=20czyste=20z?= =?UTF-8?q?atrzymanie=20VM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Przyczyna uszkodzenia obrazu Time Machine po wyłączeniu Maca: rclone mount ignoruje fsync, a metadane cache (vfsMeta/Rs) są zapisywane bez fsync, więc twarde zabicie VM gubiło zapisy, które APFS uznał za trwałe. - rclone-tm.service: rclone mount -> mount2 (Fsync dociera do Item.Sync) - punchfs: opcja -o syncfs=; lo_fsync/lo_fsyncdir wołają syncfs() na katalogu cache rclone, błąd -> errno - vm-hold: job launchd vm-start trzyma się przy życiu i na SIGTERM robi limactl stop (ExitTimeOut 120 s) - testy: VMDurabilityTests, VMHoldTests, rozszerzony test-punchfs.sh Co-Authored-By: Claude Sonnet 5.5 --- ...acode.cloudmachine.vm-start.plist.template | 17 ++-- .../CloudMachineAgent/CloudMachineAgent.swift | 1 + .../CloudMachineAgent/VMHoldCommand.swift | 61 ++++++++++++ mac-app/Sources/CloudMachineCore/VMHold.swift | 86 +++++++++++++++++ .../VMDurabilityTests.swift | 84 ++++++++++++++++ .../CloudMachineAppTests/VMHoldTests.swift | 95 +++++++++++++++++++ vm/README.md | 2 +- vm/punchfs/README.md | 13 ++- vm/punchfs/punchfs.c | 43 ++++++++- vm/punchfs/test-punchfs.sh | 50 +++++++++- vm/systemd/punchfs.service | 4 +- vm/systemd/rclone-tm.service | 6 +- 12 files changed, 444 insertions(+), 18 deletions(-) create mode 100644 mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift create mode 100644 mac-app/Sources/CloudMachineCore/VMHold.swift create mode 100644 mac-app/Tests/CloudMachineAppTests/VMDurabilityTests.swift create mode 100644 mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift diff --git a/launchd/com.renacode.cloudmachine.vm-start.plist.template b/launchd/com.renacode.cloudmachine.vm-start.plist.template index e990144..21969c8 100644 --- a/launchd/com.renacode.cloudmachine.vm-start.plist.template +++ b/launchd/com.renacode.cloudmachine.vm-start.plist.template @@ -5,20 +5,21 @@ Label com.renacode.cloudmachine.vm-start ProgramArguments - /usr/bin/env - limactl - start - --tty=false - cmsmb + __CM_AGENT_BIN__ + vm-hold RunAtLoad + ExitTimeOut + 120 EnvironmentVariables PATH diff --git a/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift b/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift index 65c5daa..b755bf0 100644 --- a/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift +++ b/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift @@ -21,6 +21,7 @@ struct CloudMachineAgent: AsyncParsableCommand { VMStatusCommand.self, VMUpdateCommand.self, VMGuardCommand.self, + VMHoldCommand.self, VMEncrypt.self, BackupHealthCommand.self, PrepareShutdown.self, diff --git a/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift b/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift new file mode 100644 index 0000000..b1b9604 --- /dev/null +++ b/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift @@ -0,0 +1,61 @@ +import ArgumentParser +import CloudMachineCore +import Foundation + +struct VMHoldCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "vm-hold", + // Not translated: only the launchd job runs this, nobody reads its help. + abstract: + "Starts the machine and stays running; when it is stopped (logout, shutdown) it stops the machine cleanly." + ) + + func run() async throws { + // The handlers must be in place before the (minutes long) start: a SIGTERM + // that arrives during it is remembered and ends the wait at once. + let signals = TerminationSignals() + let code = await VMHold.run( + steps: VMHold.live, awaitTermination: { await signals.wait() }) + if code != 0 { throw ExitCode(code) } + } +} + +/// SIGTERM is what launchd sends; SIGINT and SIGHUP cover a terminal. +private final class TerminationSignals: @unchecked Sendable { + private let lock = NSLock() + private var arrived = false + private var waiter: CheckedContinuation? + private var sources: [DispatchSourceSignal] = [] + + init() { + for sig in [SIGTERM, SIGINT, SIGHUP] { + signal(sig, SIG_IGN) + let source = DispatchSource.makeSignalSource(signal: sig, queue: .global()) + source.setEventHandler { [weak self] in self?.fire() } + source.resume() + sources.append(source) + } + } + + private func fire() { + lock.lock() + arrived = true + let w = waiter + waiter = nil + lock.unlock() + w?.resume() + } + + func wait() async { + await withCheckedContinuation { (continuation: CheckedContinuation) in + lock.lock() + if arrived { + lock.unlock() + continuation.resume() + } else { + waiter = continuation + lock.unlock() + } + } + } +} diff --git a/mac-app/Sources/CloudMachineCore/VMHold.swift b/mac-app/Sources/CloudMachineCore/VMHold.swift new file mode 100644 index 0000000..d10ac1b --- /dev/null +++ b/mac-app/Sources/CloudMachineCore/VMHold.swift @@ -0,0 +1,86 @@ +import Foundation + +/// `vm-hold`: the launchd job behind the machine. +/// +/// Before, the job was `limactl start`, which exits once the machine is up. +/// Nothing then stopped the machine when the Mac logged out or shut down: the +/// Lima host agent died with the session, the guest lost power, and ext4 (and +/// the Time Machine image on top of it) had to recover from a hard stop. +/// +/// Now the job starts the machine and stays alive. launchd sends SIGTERM when +/// the job is stopped (logout, shutdown, `launchctl bootout`) and waits +/// `ExitTimeOut` seconds before SIGKILL; on SIGTERM this runs `limactl stop`, +/// which asks the guest to power off (Lima's vz driver: RequestStop, up to 30 s, +/// never a forced stop) so systemd stops smbd, punchfs and rclone in order. +public enum VMHold { + /// Limit of the `limactl stop` itself. + public static let stopTimeout: TimeInterval = 100 + /// `ExitTimeOut` in the launchd template: the stop plus a margin. A test + /// keeps the template and this value together. + public static let exitTimeOut: TimeInterval = 120 + + /// The effects, injected so the order of events is testable. + public struct Steps: Sendable { + public var start: @Sendable () async -> Bool + public var status: @Sendable () async throws -> String? + public var stop: @Sendable () async -> Bool + public var log: @Sendable (String) -> Void + + public init( + start: @escaping @Sendable () async -> Bool, + status: @escaping @Sendable () async throws -> String?, + stop: @escaping @Sendable () async -> Bool, + log: @escaping @Sendable (String) -> Void + ) { + self.start = start + self.status = status + self.stop = stop + self.log = log + } + } + + /// The exit code of the job: 0 when the machine was started and then + /// stopped (or was already stopped), 1 when it did not start or did not stop. + public static func run( + steps: Steps, awaitTermination: @Sendable () async -> Void + ) async -> Int32 { + guard await steps.start() else { + steps.log("vm-hold: the machine did not start") + return 1 + } + steps.log("vm-hold: the machine is up, waiting for the job to be stopped") + await awaitTermination() + + // `nil` = no such instance. When Lima does not answer we cannot tell, and + // a machine left running is worse than a failed `limactl stop`, so we stop. + do { + let status = try await steps.status() + if status != "Running" { + steps.log("vm-hold: the machine is \(status ?? "missing"), nothing to stop") + return 0 + } + } catch { + steps.log("vm-hold: Lima did not answer about the status (\(error)), stopping anyway") + } + steps.log("vm-hold: stopping the machine") + guard await steps.stop() else { + steps.log("vm-hold: limactl stop failed or ran out of time") + return 1 + } + steps.log("vm-hold: the machine is stopped") + return 0 + } + + /// The real effects. + public static var live: Steps { + Steps( + start: { await VMLayout.start().succeeded }, + status: { try await VMLayout.instanceStatus() }, + stop: { + let result = try? await VMLayout.limactl( + ["stop", VMLayout.instanceName], timeout: stopTimeout) + return result?.succeeded == true + }, + log: { CMLogger.log($0) }) + } +} diff --git a/mac-app/Tests/CloudMachineAppTests/VMDurabilityTests.swift b/mac-app/Tests/CloudMachineAppTests/VMDurabilityTests.swift new file mode 100644 index 0000000..0cfbf5c --- /dev/null +++ b/mac-app/Tests/CloudMachineAppTests/VMDurabilityTests.swift @@ -0,0 +1,84 @@ +import XCTest + +/// Durability of a Time Machine write: APFS inside the image relies on fsync +/// (F_FULLFSYNC -> SMB flush -> Samba fsync -> punchfs -> rclone) really +/// reaching the disk, because the machine can die hard with the Mac. These +/// tests pin the pieces of that chain that live in the repo's templates. +final class VMDurabilityTests: XCTestCase { + private static let repoRoot = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent() + .deletingLastPathComponent() + + private func repoFile(_ path: String) throws -> String { + try String( + contentsOf: Self.repoRoot.appendingPathComponent(path), encoding: .utf8) + } + + private func execStart(_ unit: String) throws -> String { + let text = try repoFile(unit) + // ExecStart may continue over several lines with a trailing backslash. + var lines: [String] = [] + var inside = false + for line in text.components(separatedBy: "\n") { + if line.hasPrefix("ExecStart=") { inside = true } + guard inside else { continue } + let continued = line.hasSuffix("\\") + lines.append(continued ? String(line.dropLast()) : line) + if !continued { break } + } + return lines.joined(separator: " ") + } + + /// `rclone mount` (cgofuse-less libfuse path) answers fsync with OK and + /// does nothing; `mount2` (go-fuse) passes it on to the VFS cache item. + func testRcloneUsesMount2() throws { + let exec = try execStart("vm/systemd/rclone-tm.service") + XCTAssertTrue(exec.contains("/usr/local/bin/rclone mount2 "), exec) + XCTAssertFalse(exec.contains("/usr/local/bin/rclone mount "), exec) + } + + /// punchfs must sync the filesystem that holds rclone's cache metadata + /// (written without fsync), and it must be the cache dir rclone really uses. + func testPunchfsSyncsRcloneCacheDirectory() throws { + let punch = try execStart("vm/systemd/punchfs.service") + let rclone = try execStart("vm/systemd/rclone-tm.service") + let option = try XCTUnwrap( + punch.range(of: "syncfs=[^,\\s]+", options: .regularExpression).map { String(punch[$0]) }, + punch) + let dir = String(option.dropFirst("syncfs=".count)) + XCTAssertEqual(dir, "/var/cache/rclone") + XCTAssertTrue(rclone.contains("--cache-dir \(dir)"), "syncfs dir must be rclone's cache dir") + XCTAssertTrue(punch.contains("source=/srv/tm,"), "other options keep working: \(punch)") + XCTAssertTrue(punch.contains("allow_other"), punch) + } + + /// The C side: the option exists, the directory is opened once, and both + /// fsync handlers call syncfs and report its errno. + func testPunchfsCodeSyncsOnFsync() throws { + let source = try repoFile("vm/punchfs/punchfs.c") + XCTAssertTrue(source.contains("\"syncfs=%s\""), "option syncfs= is parsed") + func body(of function: String, returning type: String = "void") throws -> String { + let start = try XCTUnwrap(source.range(of: "static \(type) \(function)("), function) + let rest = source[start.lowerBound...] + let end = rest.range(of: "\n}\n")?.upperBound ?? rest.endIndex + return String(rest[..syncfs_fd)"), helper) + XCTAssertTrue(helper.contains("errno"), helper) + XCTAssertTrue(source.contains("O_DIRECTORY"), "the directory is opened at start") + } + + /// The machine test exercises the chain for real and the template ships it. + func testMachineTestChecksFsyncDurability() throws { + let script = try repoFile("vm/punchfs/test-punchfs.sh") + XCTAssertTrue(script.contains("mount2"), "the test mounts like production") + XCTAssertTrue(script.contains("syncfs=")) + XCTAssertTrue(script.contains("Dirty"), "fsync must leave ~0 dirty pages") + XCTAssertTrue(script.contains("vfsMeta"), "metadata file must cover the written range") + } +} diff --git a/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift b/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift new file mode 100644 index 0000000..9c2a8b2 --- /dev/null +++ b/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift @@ -0,0 +1,95 @@ +import XCTest + +@testable import CloudMachineCore + +/// `vm-hold`: starts the machine at login and stays alive, so that launchd's +/// SIGTERM at logout/shutdown turns into a clean `limactl stop` (the machine +/// otherwise dies together with the Mac and the guest's ext4 needs recovery). +final class VMHoldTests: XCTestCase { + + /// Records the order of events. + private final class Recorder: @unchecked Sendable { + private let lock = NSLock() + private var items: [String] = [] + func add(_ item: String) { lock.lock(); items.append(item); lock.unlock() } + var events: [String] { lock.lock(); defer { lock.unlock() }; return items } + } + + private func steps( + _ rec: Recorder, start: Bool = true, status: String? = "Running", stop: Bool = true + ) -> VMHold.Steps { + VMHold.Steps( + start: { rec.add("start"); return start }, + status: { rec.add("status"); return status }, + stop: { rec.add("stop"); return stop }, + log: { rec.add("log: \($0)") }) + } + + func testStartsThenWaitsThenStopsOnTermination() async { + let rec = Recorder() + let code = await VMHold.run( + steps: steps(rec), awaitTermination: { rec.add("wait") }) + XCTAssertEqual(code, 0) + let order = rec.events.filter { !$0.hasPrefix("log") } + XCTAssertEqual(order, ["start", "wait", "status", "stop"]) + } + + /// A failed start is a failure of the job, and there is nothing to stop. + func testFailedStartExitsNonZeroWithoutWaitingOrStopping() async { + let rec = Recorder() + let code = await VMHold.run( + steps: steps(rec, start: false), awaitTermination: { rec.add("wait") }) + XCTAssertEqual(code, 1) + XCTAssertEqual(rec.events.filter { !$0.hasPrefix("log") }, ["start"]) + } + + /// The machine was already stopped (the app's Stop, prepare-shutdown): + /// `limactl stop` on it would only fail. + func testDoesNotStopAMachineThatIsNotRunning() async { + for status in ["Stopped", nil] as [String?] { + let rec = Recorder() + let code = await VMHold.run( + steps: steps(rec, status: status), awaitTermination: {}) + XCTAssertEqual(code, 0) + XCTAssertFalse(rec.events.contains("stop"), "status \(String(describing: status))") + } + } + + /// A stop that fails must be visible (exit code and log), not swallowed. + func testFailedStopIsReported() async { + let rec = Recorder() + let code = await VMHold.run(steps: steps(rec, stop: false), awaitTermination: {}) + XCTAssertEqual(code, 1) + XCTAssertTrue(rec.events.contains { $0.hasPrefix("log") && $0.contains("stop") }) + } + + /// When Lima does not answer about the status we still try to stop: a + /// machine left running is worse than a failed `limactl stop`. + func testUnknownStatusStillStops() async { + let rec = Recorder() + let steps = VMHold.Steps( + start: { true }, status: { throw ProcessRunnerError.launchFailed("x") }, + stop: { rec.add("stop"); return true }, log: { _ in }) + let code = await VMHold.run(steps: steps, awaitTermination: {}) + XCTAssertEqual(code, 0) + XCTAssertTrue(rec.events.contains("stop")) + } + + /// launchd sends SIGKILL ExitTimeOut seconds after SIGTERM: the stop's own + /// limit must fit inside it, and the plist must ask for that much. + func testStopLimitFitsInsideLaunchdExitTimeout() throws { + XCTAssertGreaterThanOrEqual(VMHold.exitTimeOut, VMHold.stopTimeout + 10) + let root = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent() + .deletingLastPathComponent() + let plist = try String( + contentsOf: root.appendingPathComponent( + "launchd/com.renacode.cloudmachine.vm-start.plist.template"), encoding: .utf8) + XCTAssertTrue(plist.contains("vm-hold"), "the job runs vm-hold") + XCTAssertTrue(plist.contains("__CM_AGENT_BIN__")) + XCTAssertFalse(plist.contains("limactl"), "no bare limactl start") + XCTAssertTrue( + plist.contains("ExitTimeOut\n \(Int(VMHold.exitTimeOut))"), + "ExitTimeOut must equal VMHold.exitTimeOut") + } +} diff --git a/vm/README.md b/vm/README.md index 761f094..452bc78 100644 --- a/vm/README.md +++ b/vm/README.md @@ -50,7 +50,7 @@ Each layer is there for a reason: cloudmachine-agent vm-update # after an app upgrade: move the machine onto its template `install-launchd` (also run by `--install-agents`) installs four agents: -`app`, `backup-health`, `vm-start` (boots the machine at login) and `vm-guard` +`app`, `backup-health`, `vm-start` (runs `vm-hold`: boots the machine at login and stays alive, so launchd's SIGTERM at logout/shutdown becomes a clean `limactl stop`) and `vm-guard` (pauses Time Machine when more than 110 GiB waits to be uploaded or the machine has less than 40 GiB free, resumes below 40 GiB - only what it paused). diff --git a/vm/punchfs/README.md b/vm/punchfs/README.md index 6d71e9d..6d6bd4c 100644 --- a/vm/punchfs/README.md +++ b/vm/punchfs/README.md @@ -30,7 +30,18 @@ The machine's provisioning builds it (`vm/cmsmb.yaml`): gcc -Wall -O2 -D_FILE_OFFSET_BITS=64 -DHAVE_FALLOCATE punchfs.c \ $(pkg-config fuse3 --cflags --libs) -o /usr/local/bin/punchfs -`test-punchfs.sh` checks both changes on a throwaway rclone mount of a local +### Durable fsync (`-o syncfs=`) + +`rclone mount` answers fsync with OK and does nothing, and rclone writes its +cache metadata (`vfsMeta`, with the list of cached ranges `Rs`) without fsync, +so APFS barriers inside the Time Machine image were not durable and a hard stop +of the machine lost writes APFS considered safe. Production therefore uses +`rclone mount2` (passes fsync to the cache item) and `-o syncfs=/var/cache/rclone`: +after a successful fsync/fsyncdir punchfs also calls `syncfs()` on that +directory's filesystem and replies with its errno on failure. Without the +option nothing changes. + +`test-punchfs.sh` checks all of this: both changes above and the fsync on a throwaway rclone mount of a local directory, without touching Google Drive: limactl shell cmsmb sudo /opt/cloudmachine/punchfs/test-punchfs.sh diff --git a/vm/punchfs/punchfs.c b/vm/punchfs/punchfs.c index 78ec009..8ef731c 100644 --- a/vm/punchfs/punchfs.c +++ b/vm/punchfs/punchfs.c @@ -98,6 +98,10 @@ struct lo_data { int flock; int xattr; const char *source; + /* -o syncfs=: after fsync, also syncfs() the filesystem of + * (rclone's cache dir, whose metadata files are written without fsync) */ + const char *syncfs; + int syncfs_fd; double timeout; int cache; int timeout_set; @@ -111,6 +115,8 @@ static const struct fuse_opt lo_opts[] = { offsetof(struct lo_data, writeback), 0 }, { "source=%s", offsetof(struct lo_data, source), 0 }, + { "syncfs=%s", + offsetof(struct lo_data, syncfs), 0 }, { "flock", offsetof(struct lo_data, flock), 1 }, { "no_flock", @@ -136,6 +142,7 @@ static const struct fuse_opt lo_opts[] = { static void passthrough_ll_help(void) { printf( +" -o syncfs= After fsync also syncfs() the filesystem of \n" " -o writeback Enable writeback\n" " -o no_writeback Disable write back\n" " -o source=/home/dir Source directory to be mounted\n" @@ -790,6 +797,16 @@ static void lo_create(fuse_req_t req, fuse_ino_t parent, const char *name, fuse_reply_create(req, &e, fi); } +/* 0 or an errno. Without -o syncfs this does nothing. */ +static int lo_syncfs(fuse_req_t req) +{ + struct lo_data *lo = lo_data(req); + + if (lo->syncfs_fd < 0) + return 0; + return syncfs(lo->syncfs_fd) == -1 ? errno : 0; +} + static void lo_fsyncdir(fuse_req_t req, fuse_ino_t ino, int datasync, struct fuse_file_info *fi) { @@ -800,7 +817,11 @@ static void lo_fsyncdir(fuse_req_t req, fuse_ino_t ino, int datasync, res = fdatasync(fd); else res = fsync(fd); - fuse_reply_err(req, res == -1 ? errno : 0); + if (res == -1) { + fuse_reply_err(req, errno); + return; + } + fuse_reply_err(req, lo_syncfs(req)); } static void lo_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) @@ -867,7 +888,11 @@ static void lo_fsync(fuse_req_t req, fuse_ino_t ino, int datasync, res = fdatasync(fi->fh); else res = fsync(fi->fh); - fuse_reply_err(req, res == -1 ? errno : 0); + if (res == -1) { + fuse_reply_err(req, errno); + return; + } + fuse_reply_err(req, lo_syncfs(req)); } static void lo_read(fuse_req_t req, fuse_ino_t ino, size_t size, @@ -1167,7 +1192,8 @@ int main(int argc, char *argv[]) struct fuse_cmdline_opts opts; struct fuse_loop_config config; struct lo_data lo = { .debug = 0, - .writeback = 0 }; + .writeback = 0, + .syncfs_fd = -1 }; int ret = -1; /* Don't mask creation mode, kernel already did that */ @@ -1251,6 +1277,15 @@ int main(int argc, char *argv[]) exit(1); } + if (lo.syncfs) { + lo.syncfs_fd = open(lo.syncfs, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (lo.syncfs_fd == -1) { + fuse_log(FUSE_LOG_ERR, "open(\"%s\") for syncfs: %m\n", + lo.syncfs); + exit(1); + } + } + se = fuse_session_new(&args, &lo_oper, sizeof(lo_oper), &lo); if (se == NULL) goto err_out1; @@ -1283,6 +1318,8 @@ int main(int argc, char *argv[]) if (lo.root.fd >= 0) close(lo.root.fd); + if (lo.syncfs_fd >= 0) + close(lo.syncfs_fd); return ret ? 1 : 0; } diff --git a/vm/punchfs/test-punchfs.sh b/vm/punchfs/test-punchfs.sh index 3ff1206..6fdbed1 100755 --- a/vm/punchfs/test-punchfs.sh +++ b/vm/punchfs/test-punchfs.sh @@ -6,7 +6,12 @@ # with no fallocate and no extended attributes - the same gaps as the Google # Drive mount), puts punchfs on top, and checks the two things macOS needs: # 1. hole punch / zero range succeed, read back as zeros, keep the size; -# 2. extended attributes can be set, read, listed and removed. +# 2. extended attributes can be set, read, listed and removed; +# 3. fsync through punchfs is durable: it leaves ~0 Dirty pages (rclone's +# cache metadata included, via -o syncfs=) and rclone's metadata file +# (vfsMeta) already lists the written range in "Rs". +# The mount is `rclone mount2` like production (`mount` ignores fsync); set +# RCLONE_MOUNT_CMD=mount and PUNCHFS_SYNCFS=0 to see the old behaviour fail. # Nothing touches /srv/tm or Google Drive. Attributes of the test file land in # /srv/tm-xattr/ and are removed at the end. set -euo pipefail @@ -34,10 +39,19 @@ wait_mounted() { echo "not mounted: $1" >&2; return 1 } -rclone mount "$work/src" "$work/rc" --config /dev/null --vfs-cache-mode full \ +# An existing file the cache has never seen, so that the partial write below +# leaves a sparse "Rs" (only the written range is in the cache). +head -c $((64 * 1048576)) /dev/zero | tr "\0" B > "$work/src/durable" + +RCLONE_MOUNT_CMD=${RCLONE_MOUNT_CMD:-mount2} +PUNCHFS_SYNCFS=${PUNCHFS_SYNCFS:-1} +punch_opts="source=$work/rc" +[ "$PUNCHFS_SYNCFS" = 1 ] && punch_opts="$punch_opts,syncfs=$work/cache" + +rclone "$RCLONE_MOUNT_CMD" "$work/src" "$work/rc" --config /dev/null --vfs-cache-mode full \ --cache-dir "$work/cache" --log-level ERROR & wait_mounted "$work/rc" -"$PUNCHFS" -f -o source="$work/rc" "$work/punch" & +"$PUNCHFS" -f -o "$punch_opts" "$work/punch" & wait_mounted "$work/punch" f="$work/punch/band" @@ -71,6 +85,36 @@ check "XATTR_CREATE refuses an existing name" \ check "remove attributes" bash -c "setfattr -x user.cmtest '$f' && setfattr -x user.cmtest.bin '$f'" check "removed attribute is gone" bash -c "! getfattr -n user.cmtest '$f' 2>/dev/null" +# --- durability of fsync --------------------------------------------------- +dirty_kb() { awk '/^Dirty:/ {print $2}' /proc/meminfo; } + +d="$work/punch/durable" +sync # start from a quiet system; the next write is a PARTIAL one into an uncached file +# 8 MiB of new data at 8 MiB of a 64 MiB file, then fsync (dd conv=fsync = one fsync at the end) +head -c $((8 * MiB)) /dev/zero | tr "\0" C | dd of="$d" bs=$MiB seek=8 conv=notrunc,fsync status=none +dirty=$(dirty_kb) +echo "Dirty after fsync: ${dirty} kB" +check "fsync leaves almost nothing dirty (< 2 MiB)" test "$dirty" -lt 2048 + +# rclone's metadata file for the cache item: "Rs" = ranges present in the cache. +meta=$(find "$work/cache/vfsMeta" -type f -name durable | head -1) +check "vfsMeta file exists" test -n "$meta" +check "vfsMeta Rs covers the range written before fsync" \ + python3 - "$meta" $((8 * MiB)) $((16 * MiB)) <<'PY' +import json, sys +meta = json.load(open(sys.argv[1])) +start, end = int(sys.argv[2]), int(sys.argv[3]) +rs = sorted((r["Pos"], r["Pos"] + r["Size"]) for r in meta.get("Rs", [])) +merged = [] +for a, b in rs: + if merged and a <= merged[-1][1]: + merged[-1] = (merged[-1][0], max(merged[-1][1], b)) + else: + merged.append((a, b)) +print("Rs:", merged) +sys.exit(0 if any(a <= start and end <= b for a, b in merged) else 1) +PY + # The side directory is keyed by the inode punchfs sees, i.e. rclone's. ino=$(printf '%x' "$(stat -c %i "$work/rc/band")") rmdir --ignore-fail-on-non-empty "/srv/tm-xattr/$ino" 2>/dev/null || true diff --git a/vm/systemd/punchfs.service b/vm/systemd/punchfs.service index afb168e..c466e97 100644 --- a/vm/systemd/punchfs.service +++ b/vm/systemd/punchfs.service @@ -12,7 +12,9 @@ Type=simple # passthrough_ll keeps one descriptor open for every inode the kernel has seen; # a full backup has tens of thousands of bands, and the default 1024 ran out (EMFILE). LimitNOFILE=524288 -ExecStart=/usr/local/bin/punchfs -f -o source=/srv/tm,allow_other /srv/tm-punch +# syncfs=: after fsync(fd) punchfs also syncfs() the filesystem of rclone's +# --cache-dir, so the cache metadata (vfsMeta, written without fsync) is durable too. +ExecStart=/usr/local/bin/punchfs -f -o source=/srv/tm,syncfs=/var/cache/rclone,allow_other /srv/tm-punch ExecStop=/bin/fusermount3 -uz /srv/tm-punch Restart=always RestartSec=5 diff --git a/vm/systemd/rclone-tm.service b/vm/systemd/rclone-tm.service index a04b64c..060663f 100644 --- a/vm/systemd/rclone-tm.service +++ b/vm/systemd/rclone-tm.service @@ -10,13 +10,17 @@ ConditionPathExists=/etc/rclone/rclone.conf ConditionPathExists=/etc/cloudmachine/vm.env [Service] +# mount2 (go-fuse), not mount: `rclone mount` answers fsync with OK and does +# nothing, so APFS barriers inside the Time Machine image were never durable. +# mount2 passes fsync to the VFS cache item (data + metadata with an up-to-date +# range list); punchfs adds syncfs() for the metadata file (see punchfs.service). Type=notify # rclone scans the whole VFS cache (up to 120G) before it mounts; after an # interrupted upload that takes minutes, and the default 90 s kills it in a loop. TimeoutStartSec=45min EnvironmentFile=/etc/cloudmachine/vm.env EnvironmentFile=/etc/cloudmachine/tm-ids.env -ExecStart=/usr/local/bin/rclone mount ${CM_REMOTE} /srv/tm \ +ExecStart=/usr/local/bin/rclone mount2 ${CM_REMOTE} /srv/tm \ --config /etc/rclone/rclone.conf --allow-other --uid ${TM_UID} --gid ${TM_GID} --umask 002 \ --vfs-cache-mode full --vfs-cache-max-size 120G --vfs-cache-max-age 9999h --vfs-write-back 600s \ --cache-dir /var/cache/rclone --dir-cache-time 9999h --poll-interval 0 --attr-timeout 5m \ From 2fc2a18696816497017d83d082ff7c34f0409960 Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Thu, 12 Nov 2026 10:40:07 +0100 Subject: [PATCH 2/6] =?UTF-8?q?vm-hold:=20wolny=20start=20pod=20obci=C4=85?= =?UTF-8?q?=C5=BCeniem=20nie=20jest=20pora=C5=BCk=C4=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Po nieudanym limactl start (12.11: "did not receive an event with the running status" po 10 min, a VM wstała) vm-hold dalej pyta Limę o stan, do 60 min, bez zatrzymywania ani restartu maszyny. Błąd kończy dopiero stan Stopped/Broken/brak instancji albo upływ limitu. SIGTERM przerywa czekanie. Co-Authored-By: Claude Sonnet 5.5 --- .../CloudMachineAgent/VMHoldCommand.swift | 8 +- mac-app/Sources/CloudMachineCore/VMHold.swift | 59 +++++++++++++-- .../CloudMachineAppTests/VMHoldTests.swift | 74 ++++++++++++++++++- 3 files changed, 133 insertions(+), 8 deletions(-) diff --git a/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift b/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift index b1b9604..45f286e 100644 --- a/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift +++ b/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift @@ -15,7 +15,8 @@ struct VMHoldCommand: AsyncParsableCommand { // that arrives during it is remembered and ends the wait at once. let signals = TerminationSignals() let code = await VMHold.run( - steps: VMHold.live, awaitTermination: { await signals.wait() }) + steps: VMHold.live, awaitTermination: { await signals.wait() }, + terminated: { signals.hasArrived }) if code != 0 { throw ExitCode(code) } } } @@ -46,6 +47,11 @@ private final class TerminationSignals: @unchecked Sendable { w?.resume() } + var hasArrived: Bool { + lock.lock(); defer { lock.unlock() } + return arrived + } + func wait() async { await withCheckedContinuation { (continuation: CheckedContinuation) in lock.lock() diff --git a/mac-app/Sources/CloudMachineCore/VMHold.swift b/mac-app/Sources/CloudMachineCore/VMHold.swift index d10ac1b..6475718 100644 --- a/mac-app/Sources/CloudMachineCore/VMHold.swift +++ b/mac-app/Sources/CloudMachineCore/VMHold.swift @@ -25,31 +25,80 @@ public enum VMHold { public var status: @Sendable () async throws -> String? public var stop: @Sendable () async -> Bool public var log: @Sendable (String) -> Void + public var sleep: @Sendable (TimeInterval) async -> Void public init( start: @escaping @Sendable () async -> Bool, status: @escaping @Sendable () async throws -> String?, stop: @escaping @Sendable () async -> Bool, - log: @escaping @Sendable (String) -> Void + log: @escaping @Sendable (String) -> Void, + sleep: @escaping @Sendable (TimeInterval) async -> Void = { + try? await Task.sleep(nanoseconds: UInt64($0 * 1_000_000_000)) + } ) { self.start = start self.status = status self.stop = stop self.log = log + self.sleep = sleep + } + } + + /// How long a machine that `limactl start` gave up on may still take to + /// come up. The Mac often runs under a load of ~100 (renders); on 12.11 + /// `limactl start` failed with "did not receive an event with the `running` + /// status" after 10 min and the machine was up a little later. + public static let startGrace: TimeInterval = 60 * 60 + static let checkInterval: TimeInterval = 30 + + enum StartOutcome { case up, failed, terminated } + + /// After a failed `limactl start`: is the machine coming up? Waits while + /// Lima says anything but gone/stopped/broken, or does not answer. Never + /// touches the machine. + static func waitForStart( + steps: Steps, terminated: @Sendable () -> Bool + ) async -> StartOutcome { + var waited: TimeInterval = 0 + var seen = "unknown" + while true { + do { + let status = try await steps.status() + if status == "Running" { return .up } + guard let status, !["Stopped", "Broken"].contains(status) else { return .failed } + seen = status + } catch { + seen = "unreachable" // Lima busy or slow under load: ask again + } + if terminated() { return .terminated } + if waited >= startGrace { return .failed } + steps.log("vm-hold: start did not finish, the machine is \(seen); waiting") + await steps.sleep(checkInterval) + waited += checkInterval } } /// The exit code of the job: 0 when the machine was started and then /// stopped (or was already stopped), 1 when it did not start or did not stop. public static func run( - steps: Steps, awaitTermination: @Sendable () async -> Void + steps: Steps, awaitTermination: @Sendable () async -> Void, + terminated: @Sendable () -> Bool = { false } ) async -> Int32 { - guard await steps.start() else { + var outcome = StartOutcome.up + if await !steps.start() { + steps.log("vm-hold: limactl start failed; checking whether the machine comes up anyway") + outcome = await waitForStart(steps: steps, terminated: terminated) + } + switch outcome { + case .failed: steps.log("vm-hold: the machine did not start") return 1 + case .up: + steps.log("vm-hold: the machine is up, waiting for the job to be stopped") + await awaitTermination() + case .terminated: + break } - steps.log("vm-hold: the machine is up, waiting for the job to be stopped") - await awaitTermination() // `nil` = no such instance. When Lima does not answer we cannot tell, and // a machine left running is worse than a failed `limactl stop`, so we stop. diff --git a/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift b/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift index 9c2a8b2..b4297d4 100644 --- a/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift @@ -38,9 +38,10 @@ final class VMHoldTests: XCTestCase { func testFailedStartExitsNonZeroWithoutWaitingOrStopping() async { let rec = Recorder() let code = await VMHold.run( - steps: steps(rec, start: false), awaitTermination: { rec.add("wait") }) + steps: steps(rec, start: false, status: "Stopped"), awaitTermination: { rec.add("wait") }) XCTAssertEqual(code, 1) - XCTAssertEqual(rec.events.filter { !$0.hasPrefix("log") }, ["start"]) + // Start failed, Lima says the machine is not there: not slow, failed. + XCTAssertEqual(rec.events.filter { !$0.hasPrefix("log") }, ["start", "status"]) } /// The machine was already stopped (the app's Stop, prepare-shutdown): @@ -75,6 +76,75 @@ final class VMHoldTests: XCTestCase { XCTAssertTrue(rec.events.contains("stop")) } + // MARK: - Slow start (Mac under load) + + /// Statuses answered one per call; the last one repeats. Records sleeps. + private final class Script: @unchecked Sendable { + private let lock = NSLock() + private var statuses: [String?] + private(set) var slept: TimeInterval = 0 + private(set) var stopCalls = 0 + init(_ statuses: [String?]) { self.statuses = statuses } + func next() -> String? { + lock.lock(); defer { lock.unlock() } + return statuses.count > 1 ? statuses.removeFirst() : statuses[0] + } + func sleep(_ t: TimeInterval) { lock.lock(); slept += t; lock.unlock() } + func stopped() { lock.lock(); stopCalls += 1; lock.unlock() } + } + + private func slowSteps(_ script: Script, startSucceeds: Bool) -> VMHold.Steps { + VMHold.Steps( + start: { startSucceeds }, + status: { script.next() }, + stop: { script.stopped(); return true }, + log: { _ in }, + sleep: { script.sleep($0) }) + } + + /// 12.11: `limactl start` gave up after 10 min ("did not receive an event + /// with the `running` status") although the machine came up. A failed start + /// is not a failed machine: keep asking Lima, do not stop or restart it. + func testFailedStartIsToleratedWhileTheMachineComesUp() async { + let script = Script(["Starting", "Starting", "Starting", "Running"]) + let code = await VMHold.run( + steps: slowSteps(script, startSucceeds: false), awaitTermination: {}) + XCTAssertEqual(code, 0) + XCTAssertGreaterThan(script.slept, 0, "it waited between checks") + // 3 "Starting" + 1 "Running" while waiting, then the status at shutdown. + XCTAssertEqual(script.stopCalls, 1, "stopped once, at termination - never during the wait") + } + + func testWaitsLongerThanTenMinutes() async { + let script = Script(Array(repeating: "Starting", count: 100) + ["Running"]) + let code = await VMHold.run( + steps: slowSteps(script, startSucceeds: false), awaitTermination: {}) + XCTAssertEqual(code, 0) + XCTAssertGreaterThan(script.slept, 20 * 60) + } + + /// A machine that never comes up within the grace period is a failure of + /// the job - but even then the machine is left alone (no stop, no kill). + func testNeverRunningFailsAfterGraceWithoutStopping() async { + let script = Script(["Starting"]) + let code = await VMHold.run( + steps: slowSteps(script, startSucceeds: false), awaitTermination: {}) + XCTAssertEqual(code, 1) + XCTAssertEqual(script.stopCalls, 0) + XCTAssertGreaterThanOrEqual(script.slept, VMHold.startGrace) + } + + /// A SIGTERM during the wait ends it at once. + func testTerminationDuringStartEndsTheWait() async { + let script = Script(["Starting"]) + let code = await VMHold.run( + steps: slowSteps(script, startSucceeds: false), awaitTermination: {}, + terminated: { true }) + XCTAssertEqual(script.slept, 0) + XCTAssertEqual(code, 0) + XCTAssertEqual(script.stopCalls, 0, "not Running: nothing to stop") + } + /// launchd sends SIGKILL ExitTimeOut seconds after SIGTERM: the stop's own /// limit must fit inside it, and the plist must ask for that much. func testStopLimitFitsInsideLaunchdExitTimeout() throws { From 896d6c7ee2820a12b4235582b323162dba571a1b Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Thu, 12 Nov 2026 10:45:11 +0100 Subject: [PATCH 3/6] =?UTF-8?q?swift-format:=20usu=C5=84=20=C5=9Bredniki?= =?UTF-8?q?=20w=20VMHoldCommand=20i=20VMHoldTests=20(lint=20CI)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- .../CloudMachineAgent/VMHoldCommand.swift | 3 +- .../CloudMachineAppTests/VMHoldTests.swift | 52 +++++++++++++++---- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift b/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift index 45f286e..bdba266 100644 --- a/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift +++ b/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift @@ -48,7 +48,8 @@ private final class TerminationSignals: @unchecked Sendable { } var hasArrived: Bool { - lock.lock(); defer { lock.unlock() } + lock.lock() + defer { lock.unlock() } return arrived } diff --git a/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift b/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift index b4297d4..f44ff4c 100644 --- a/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift @@ -11,17 +11,34 @@ final class VMHoldTests: XCTestCase { private final class Recorder: @unchecked Sendable { private let lock = NSLock() private var items: [String] = [] - func add(_ item: String) { lock.lock(); items.append(item); lock.unlock() } - var events: [String] { lock.lock(); defer { lock.unlock() }; return items } + func add(_ item: String) { + lock.lock() + items.append(item) + lock.unlock() + } + var events: [String] { + lock.lock() + defer { lock.unlock() } + return items + } } private func steps( _ rec: Recorder, start: Bool = true, status: String? = "Running", stop: Bool = true ) -> VMHold.Steps { VMHold.Steps( - start: { rec.add("start"); return start }, - status: { rec.add("status"); return status }, - stop: { rec.add("stop"); return stop }, + start: { + rec.add("start") + return start + }, + status: { + rec.add("status") + return status + }, + stop: { + rec.add("stop") + return stop + }, log: { rec.add("log: \($0)") }) } @@ -70,7 +87,10 @@ final class VMHoldTests: XCTestCase { let rec = Recorder() let steps = VMHold.Steps( start: { true }, status: { throw ProcessRunnerError.launchFailed("x") }, - stop: { rec.add("stop"); return true }, log: { _ in }) + stop: { + rec.add("stop") + return true + }, log: { _ in }) let code = await VMHold.run(steps: steps, awaitTermination: {}) XCTAssertEqual(code, 0) XCTAssertTrue(rec.events.contains("stop")) @@ -86,18 +106,30 @@ final class VMHoldTests: XCTestCase { private(set) var stopCalls = 0 init(_ statuses: [String?]) { self.statuses = statuses } func next() -> String? { - lock.lock(); defer { lock.unlock() } + lock.lock() + defer { lock.unlock() } return statuses.count > 1 ? statuses.removeFirst() : statuses[0] } - func sleep(_ t: TimeInterval) { lock.lock(); slept += t; lock.unlock() } - func stopped() { lock.lock(); stopCalls += 1; lock.unlock() } + func sleep(_ t: TimeInterval) { + lock.lock() + slept += t + lock.unlock() + } + func stopped() { + lock.lock() + stopCalls += 1 + lock.unlock() + } } private func slowSteps(_ script: Script, startSucceeds: Bool) -> VMHold.Steps { VMHold.Steps( start: { startSucceeds }, status: { script.next() }, - stop: { script.stopped(); return true }, + stop: { + script.stopped() + return true + }, log: { _ in }, sleep: { script.sleep($0) }) } From 071f1b5e7acb7e7134ae88436fd48e4ee3ddb4c3 Mon Sep 17 00:00:00 2001 From: Marcin Beczynski Date: Thu, 12 Nov 2026 11:22:42 +0100 Subject: [PATCH 4/6] Poprawki po ocenie PR #58: xattr po sciezce, vm-hold, instalator, testy, limit Limy - punchfs: xattr kluczowane sciezka wzgledem source (mount2 daje inode z licznika procesu); unlink/rmdir kasuja, rename przenosi, stary uklad usuwany przy starcie - punchfs.service: stara binarka bez -o syncfs konczy sie raz kodem 78, bez petli restartow - test-punchfs.sh: wykrywa brak syncfs (ftrace ext4_sync_fs, brudne strony pliku danych), xattr po restarcie rclone/rename/unlink, bez zaleznosci od globalnego Dirty - vm-hold: nieudany start nie konczy joba, SIGTERM anuluje start, budzet czasu w ExitTimeOut, przerywalne czekanie, prawdziwe statusy Limy, quiesce smbd+sync w gosciu, log o poprzednim zamknieciu, prawdziwy handler sygnalu zamiast SIG_IGN, eskalacja do SIGKILL przy anulowaniu - install-launchd nie przeladowuje dzialajacego vm-start - limactl list --json pyta o jedna instancje; zrodla punchfs w jednym archiwum (limit 64 KiB JSON Limy przy dzialajacej maszynie) Co-Authored-By: Claude Sonnet 5.5 --- .../CloudMachineAgent/VMHoldCommand.swift | 52 +-- .../CloudMachineCore/LaunchdInstaller.swift | 37 +- .../CloudMachineCore/ProcessRunner.swift | 65 ++- .../CloudMachineCore/TerminationSignals.swift | 127 ++++++ mac-app/Sources/CloudMachineCore/VMHold.swift | 234 +++++++---- .../Sources/CloudMachineCore/VMLayout.swift | 53 ++- .../LaunchdInstallerTests.swift | 72 ++++ .../TerminationSignalsTests.swift | 147 +++++++ .../VMDurabilityTests.swift | 109 ++++- .../CloudMachineAppTests/VMHoldTests.swift | 391 ++++++++++++------ .../CloudMachineAppTests/VMLayoutTests.swift | 19 + vm/README.md | 42 +- vm/cmsmb.yaml | 17 +- vm/punchfs/README.md | 37 +- vm/punchfs/pack.sh | 14 + vm/punchfs/punchfs.c | 230 ++++++++++- vm/punchfs/src.tar.gz.b64 | 1 + vm/punchfs/test-punchfs.sh | 206 +++++++-- vm/systemd/punchfs.service | 11 +- 19 files changed, 1557 insertions(+), 307 deletions(-) create mode 100644 mac-app/Sources/CloudMachineCore/TerminationSignals.swift create mode 100644 mac-app/Tests/CloudMachineAppTests/TerminationSignalsTests.swift create mode 100755 vm/punchfs/pack.sh create mode 100644 vm/punchfs/src.tar.gz.b64 diff --git a/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift b/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift index bdba266..6e87b37 100644 --- a/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift +++ b/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift @@ -12,57 +12,9 @@ struct VMHoldCommand: AsyncParsableCommand { func run() async throws { // The handlers must be in place before the (minutes long) start: a SIGTERM - // that arrives during it is remembered and ends the wait at once. + // that arrives during it is remembered and cancels the start. let signals = TerminationSignals() - let code = await VMHold.run( - steps: VMHold.live, awaitTermination: { await signals.wait() }, - terminated: { signals.hasArrived }) + let code = await VMHold.run(steps: VMHold.live, termination: signals.latch) if code != 0 { throw ExitCode(code) } } } - -/// SIGTERM is what launchd sends; SIGINT and SIGHUP cover a terminal. -private final class TerminationSignals: @unchecked Sendable { - private let lock = NSLock() - private var arrived = false - private var waiter: CheckedContinuation? - private var sources: [DispatchSourceSignal] = [] - - init() { - for sig in [SIGTERM, SIGINT, SIGHUP] { - signal(sig, SIG_IGN) - let source = DispatchSource.makeSignalSource(signal: sig, queue: .global()) - source.setEventHandler { [weak self] in self?.fire() } - source.resume() - sources.append(source) - } - } - - private func fire() { - lock.lock() - arrived = true - let w = waiter - waiter = nil - lock.unlock() - w?.resume() - } - - var hasArrived: Bool { - lock.lock() - defer { lock.unlock() } - return arrived - } - - func wait() async { - await withCheckedContinuation { (continuation: CheckedContinuation) in - lock.lock() - if arrived { - lock.unlock() - continuation.resume() - } else { - waiter = continuation - lock.unlock() - } - } - } -} diff --git a/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift b/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift index 6f4f342..4367d7c 100644 --- a/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift +++ b/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift @@ -50,8 +50,11 @@ public enum LaunchdInstaller { // app bundle. CMTooling.linkCommandIntoPath() - // Reloading touches nothing that holds data: `vm-start` on a running - // machine only says so, the others are one-shot checks. + // Reloading is `unload` (SIGTERM) and `load`. The one-shot agents are + // checks and lose nothing. `vm-start` is `vm-hold`, which stops the + // machine on SIGTERM: a running one is NOT reloaded (see + // `installAgents(isRunning:)`), its new plist is picked up at the next + // login or boot. guard let templatesDir = CMPaths.launchdTemplatesDir else { return CMActionResult( succeeded: false, message: L10n.tr("Could not find the launchd/ directory with templates.")) @@ -131,6 +134,21 @@ public enum LaunchdInstaller { var failed: [Failure] = [] } + /// Agents whose process IS the machine's lifetime: never reloaded while running. + static let holdingLabels: Set = ["com.renacode.cloudmachine.vm-start"] + + /// `launchctl list