diff --git a/launchd/com.renacode.cloudmachine.vm-start.plist.template b/launchd/com.renacode.cloudmachine.vm-start.plist.template
index e990144..21969c8 100644
--- a/launchd/com.renacode.cloudmachine.vm-start.plist.template
+++ b/launchd/com.renacode.cloudmachine.vm-start.plist.template
@@ -5,20 +5,21 @@
Label
com.renacode.cloudmachine.vm-start
ProgramArguments
- /usr/bin/env
- limactl
- start
- --tty=false
- cmsmb
+ __CM_AGENT_BIN__
+ vm-hold
RunAtLoad
ExitTimeOut
+ 120
EnvironmentVariables
PATH
diff --git a/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift b/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift
index 65c5daa..b755bf0 100644
--- a/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift
+++ b/mac-app/Sources/CloudMachineAgent/CloudMachineAgent.swift
@@ -21,6 +21,7 @@ struct CloudMachineAgent: AsyncParsableCommand {
VMStatusCommand.self,
VMUpdateCommand.self,
VMGuardCommand.self,
+ VMHoldCommand.self,
VMEncrypt.self,
BackupHealthCommand.self,
PrepareShutdown.self,
diff --git a/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift b/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift
new file mode 100644
index 0000000..6e87b37
--- /dev/null
+++ b/mac-app/Sources/CloudMachineAgent/VMHoldCommand.swift
@@ -0,0 +1,20 @@
+import ArgumentParser
+import CloudMachineCore
+import Foundation
+
+struct VMHoldCommand: AsyncParsableCommand {
+ static let configuration = CommandConfiguration(
+ commandName: "vm-hold",
+ // Not translated: only the launchd job runs this, nobody reads its help.
+ abstract:
+ "Starts the machine and stays running; when it is stopped (logout, shutdown) it stops the machine cleanly."
+ )
+
+ func run() async throws {
+ // The handlers must be in place before the (minutes long) start: a SIGTERM
+ // that arrives during it is remembered and cancels the start.
+ let signals = TerminationSignals()
+ let code = await VMHold.run(steps: VMHold.live, termination: signals.latch)
+ if code != 0 { throw ExitCode(code) }
+ }
+}
diff --git a/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift b/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift
index 6f4f342..4367d7c 100644
--- a/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift
+++ b/mac-app/Sources/CloudMachineCore/LaunchdInstaller.swift
@@ -50,8 +50,11 @@ public enum LaunchdInstaller {
// app bundle.
CMTooling.linkCommandIntoPath()
- // Reloading touches nothing that holds data: `vm-start` on a running
- // machine only says so, the others are one-shot checks.
+ // Reloading is `unload` (SIGTERM) and `load`. The one-shot agents are
+ // checks and lose nothing. `vm-start` is `vm-hold`, which stops the
+ // machine on SIGTERM: a running one is NOT reloaded (see
+ // `installAgents(isRunning:)`), its new plist is picked up at the next
+ // login or boot.
guard let templatesDir = CMPaths.launchdTemplatesDir else {
return CMActionResult(
succeeded: false, message: L10n.tr("Could not find the launchd/ directory with templates."))
@@ -131,6 +134,21 @@ public enum LaunchdInstaller {
var failed: [Failure] = []
}
+ /// Agents whose process IS the machine's lifetime: never reloaded while running.
+ static let holdingLabels: Set = ["com.renacode.cloudmachine.vm-start"]
+
+ /// `launchctl list ` prints `"PID" = n;` only while the job runs.
+ static func jobIsRunning(listOutput: String) -> Bool {
+ listOutput.contains("\"PID\" =")
+ }
+
+ static func launchctlJobIsRunning(label: String) async -> Bool {
+ guard let result = try? await ProcessRunner.run("/bin/launchctl", ["list", label]),
+ result.succeeded
+ else { return false }
+ return jobIsRunning(listOutput: result.stdout)
+ }
+
/// Generates `.plist` files from the templates and reloads the agents,
/// COLLECTING failures.
///
@@ -151,6 +169,7 @@ public enum LaunchdInstaller {
try $0.write(to: $1, atomically: true, encoding: .utf8)
},
reload: @Sendable (URL) async -> Bool = { await launchctlReload($0) },
+ isRunning: @Sendable (String) async -> Bool = { await launchctlJobIsRunning(label: $0) },
log: @Sendable (String) -> Void = { CMLogger.log($0) }
) async -> InstallOutcome {
var outcome = InstallOutcome()
@@ -193,6 +212,20 @@ public enum LaunchdInstaller {
}
log("Generated \(destURL.path)")
+ // `vm-start` holds the machine: unloading it stops the machine (in the
+ // middle of a backup, if one runs) and the new job would race the old
+ // one's `limactl stop`. Leave a running one alone; it reads the plist
+ // written above at its next start. A job that is not running (the old
+ // `limactl start` job has exited, or a first installation) is loaded
+ // as usual - that is what starts `vm-hold`.
+ if holdingLabels.contains(label), await isRunning(label) {
+ outcome.installed.append(label)
+ log(
+ "Left \(label) running: reloading would stop the machine; the new plist takes effect at the next login or boot"
+ )
+ continue
+ }
+
if await reload(destURL) {
outcome.installed.append(label)
log("Loaded \(label) via launchctl")
diff --git a/mac-app/Sources/CloudMachineCore/ProcessRunner.swift b/mac-app/Sources/CloudMachineCore/ProcessRunner.swift
index 269ac28..7069430 100644
--- a/mac-app/Sources/CloudMachineCore/ProcessRunner.swift
+++ b/mac-app/Sources/CloudMachineCore/ProcessRunner.swift
@@ -186,9 +186,71 @@ final class TimeoutTargets: @unchecked Sendable {
/// only in the GUI as `Shell.run`; moved here so that the CLI watchdogs have
/// exactly the same, already proven timeout semantics.
public enum ProcessRunner {
+ /// After `timeout` the process gets SIGTERM, after `timeout + 5` SIGKILL, and
+ /// after `timeout + giveUpSlack` the call returns `timedOut` regardless.
+ /// A caller with a deadline (`VMHold`) adds this to every limit it sets.
+ public static let giveUpSlack: TimeInterval = 10
+
+ /// A cancelled call sends SIGTERM and, if the process is still there after
+ /// this many seconds, SIGKILL (a process that ignores SIGTERM must not hold
+ /// the caller past a launchd deadline). `VMHold` counts it in its budget.
+ public static let cancelKillDelay: TimeInterval = 5
+
+ /// Holds the process of one call, so that cancelling the calling task can
+ /// reach it (it is created before the process exists).
+ private final class CancelBox: @unchecked Sendable {
+ private let lock = NSLock()
+ private var process: Process?
+ private var cancelled = false
+ /// `true` when the call was cancelled before the process was set.
+ func set(_ process: Process) -> Bool {
+ lock.lock()
+ defer { lock.unlock() }
+ self.process = process
+ return cancelled
+ }
+ /// Cancelled before the process existed: end it as soon as it does.
+ func terminateNow(_ process: Process) {
+ process.terminate()
+ let pid = process.processIdentifier
+ DispatchQueue.global().asyncAfter(deadline: .now() + ProcessRunner.cancelKillDelay) {
+ if process.isRunning { kill(pid, SIGKILL) }
+ }
+ }
+ func cancel() {
+ lock.lock()
+ cancelled = true
+ let target = process
+ lock.unlock()
+ guard let target, target.isRunning else { return }
+ target.terminate()
+ let pid = target.processIdentifier
+ DispatchQueue.global().asyncAfter(deadline: .now() + ProcessRunner.cancelKillDelay) {
+ // The pid cannot have been reused within seconds unless the process
+ // exited, and then `target` says so.
+ if target.isRunning { kill(pid, SIGKILL) }
+ }
+ }
+ }
+
+ /// Cancelling the calling task sends the process SIGTERM (`limactl start`
+ /// can run for 50 minutes and must end when the job is told to stop); the
+ /// call then returns when the process has exited.
public static func run(
_ executable: String, _ args: [String], env: [String: String] = [:],
timeout: TimeInterval? = nil
+ ) async throws -> ProcessResult {
+ let box = CancelBox()
+ return try await withTaskCancellationHandler {
+ try await start(executable, args, env: env, timeout: timeout, box: box)
+ } onCancel: {
+ box.cancel()
+ }
+ }
+
+ private static func start(
+ _ executable: String, _ args: [String], env: [String: String], timeout: TimeInterval?,
+ box: CancelBox
) async throws -> ProcessResult {
try await withCheckedThrowingContinuation { continuation in
let process = Process()
@@ -260,6 +322,7 @@ public enum ProcessRunner {
}
return
}
+ if box.set(process), process.isRunning { box.terminateNow(process) }
if let timeout {
// The closures below capture `targets` only - never `process` or a
@@ -288,7 +351,7 @@ public enum ProcessRunner {
// the background (harmlessly - it will finish by itself some day, when
// the kernel finally gets an answer, and `resumeGuard` prevents a
// double resume if `terminationHandler` fires later).
- DispatchQueue.global().asyncAfter(deadline: .now() + timeout + 10) {
+ DispatchQueue.global().asyncAfter(deadline: .now() + timeout + giveUpSlack) {
if resumeGuard.claim() {
// IMPORTANT: we do NOT stop reading - that leaves the pipe with NO
// reader at all. If the process survived even SIGKILL (stuck in the
diff --git a/mac-app/Sources/CloudMachineCore/TerminationSignals.swift b/mac-app/Sources/CloudMachineCore/TerminationSignals.swift
new file mode 100644
index 0000000..36afaea
--- /dev/null
+++ b/mac-app/Sources/CloudMachineCore/TerminationSignals.swift
@@ -0,0 +1,127 @@
+import Foundation
+
+/// "A termination signal has arrived", as a state that can be waited for.
+///
+/// A state, not an event: SIGTERM can arrive during the (minutes long) start,
+/// before anybody waits, and must not be lost. `wait()` ends when `fire()` was
+/// called, now or earlier, or when the waiting task is cancelled (so that a
+/// waiter that lost a race does not hang its task group).
+public final class TerminationLatch: @unchecked Sendable {
+ private let lock = NSLock()
+ private var arrived = false
+ private var waiters: [UUID: CheckedContinuation] = [:]
+
+ public init() {}
+
+ public func fire() {
+ lock.lock()
+ arrived = true
+ let released = Array(waiters.values)
+ waiters.removeAll()
+ lock.unlock()
+ for waiter in released { waiter.resume() }
+ }
+
+ public var hasArrived: Bool {
+ lock.lock()
+ defer { lock.unlock() }
+ return arrived
+ }
+
+ public func wait() async {
+ let id = UUID()
+ await withTaskCancellationHandler {
+ await withCheckedContinuation { (continuation: CheckedContinuation) in
+ lock.lock()
+ if arrived || Task.isCancelled {
+ lock.unlock()
+ continuation.resume()
+ } else {
+ waiters[id] = continuation
+ lock.unlock()
+ }
+ }
+ } onCancel: {
+ lock.lock()
+ let waiter = waiters.removeValue(forKey: id)
+ lock.unlock()
+ waiter?.resume()
+ }
+ }
+}
+
+/// SIGTERM is what launchd sends; SIGINT and SIGHUP cover a terminal.
+///
+/// A real handler (`sigaction`) that writes one byte to a pipe, not `SIG_IGN`
+/// plus a dispatch source. An ignored signal is inherited through exec, so
+/// `limactl` and the Lima host agent would have started with SIGTERM ignored
+/// and `Process.terminate()` (or launchd's group kill) would not reach them.
+/// A caught signal is reset to the default by exec.
+///
+/// One instance at a time: the handler reaches the pipe through a global.
+public final class TerminationSignals: @unchecked Sendable {
+ public let latch = TerminationLatch()
+ private let signals: [Int32]
+ private var previous: [Int32: sigaction] = [:]
+ private var pipeFDs: [Int32] = [-1, -1]
+ private var source: DispatchSourceRead?
+
+ public init(signals: [Int32] = [SIGTERM, SIGINT, SIGHUP]) {
+ self.signals = signals
+ var fds: [Int32] = [-1, -1]
+ guard pipe(&fds) == 0 else { return }
+ pipeFDs = fds
+ for fd in fds {
+ _ = fcntl(fd, F_SETFD, FD_CLOEXEC)
+ _ = fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK)
+ }
+ terminationPipeWriteFD = fds[1]
+
+ let readFD = fds[0]
+ let latch = self.latch
+ let source = DispatchSource.makeReadSource(fileDescriptor: readFD, queue: .global())
+ source.setEventHandler {
+ var buffer = [UInt8](repeating: 0, count: 16)
+ _ = read(readFD, &buffer, buffer.count)
+ latch.fire()
+ }
+ // The descriptor is closed only after the source has really let go of it.
+ source.setCancelHandler { close(readFD) }
+ source.resume()
+ self.source = source
+
+ for sig in signals {
+ var action = sigaction()
+ action.__sigaction_u.__sa_handler = { signalNumber in
+ // Only async-signal-safe calls here: one byte into the pipe, and errno
+ // put back for the code the signal interrupted.
+ let saved = errno
+ var byte = UInt8(truncatingIfNeeded: signalNumber)
+ _ = write(terminationPipeWriteFD, &byte, 1)
+ errno = saved
+ }
+ sigemptyset(&action.sa_mask)
+ action.sa_flags = SA_RESTART
+ var old = sigaction()
+ sigaction(sig, &action, &old)
+ previous[sig] = old
+ }
+ }
+
+ /// Puts the previous dispositions back (tests; the job simply exits).
+ public func uninstall() {
+ for (sig, old) in previous {
+ var restore = old
+ sigaction(sig, &restore, nil)
+ }
+ previous.removeAll()
+ source?.cancel()
+ source = nil
+ terminationPipeWriteFD = -1
+ if pipeFDs[1] >= 0 { close(pipeFDs[1]) } // the read end closes in the cancel handler
+ pipeFDs = [-1, -1]
+ }
+}
+
+/// Read by the signal handler, which cannot capture anything.
+nonisolated(unsafe) private var terminationPipeWriteFD: Int32 = -1
diff --git a/mac-app/Sources/CloudMachineCore/VMHold.swift b/mac-app/Sources/CloudMachineCore/VMHold.swift
new file mode 100644
index 0000000..91c6ae0
--- /dev/null
+++ b/mac-app/Sources/CloudMachineCore/VMHold.swift
@@ -0,0 +1,229 @@
+import Foundation
+
+/// `vm-hold`: the launchd job behind the machine.
+///
+/// Before, the job was `limactl start`, which exits once the machine is up.
+/// Nothing then stopped the machine when the Mac logged out or shut down: the
+/// Lima host agent died with the session, the guest lost power, and ext4 (and
+/// the Time Machine image on top of it) had to recover from a hard stop.
+///
+/// Now the job starts the machine and stays alive, whatever happens to the
+/// start. launchd sends SIGTERM when the job is stopped (logout, shutdown,
+/// `launchctl bootout`) and waits `ExitTimeOut` seconds before SIGKILL; on
+/// SIGTERM this stops EVERYTHING that is not already Stopped: it asks the
+/// guest to stop smbd and sync (Lima's vz driver waits for the guest only 30 s
+/// in `limactl stop`, which is too little for a Samba with open files and a
+/// dirty cache disk), then `limactl stop` (RequestStop, never a forced stop)
+/// so systemd stops punchfs and rclone in order.
+public enum VMHold {
+ // The stop sequence after SIGTERM is status, quiesce, stop, one after
+ // another. Each limit is that of its `ProcessRunner` call, which can give up
+ // `ProcessRunner.giveUpSlack` seconds later; `stopSequenceWorstCase` adds it
+ // up and a test keeps it inside `exitTimeOut`.
+ public static let statusTimeout: TimeInterval = 10
+ public static let quiesceTimeout: TimeInterval = 15
+ public static let stopTimeout: TimeInterval = 55
+ /// `ExitTimeOut` in the launchd template. A test keeps the template and this
+ /// value together.
+ public static let exitTimeOut: TimeInterval = 120
+ public static var stopSequenceWorstCase: TimeInterval {
+ [statusTimeout, quiesceTimeout, stopTimeout].reduce(0) {
+ $0 + $1 + ProcessRunner.giveUpSlack
+ }
+ }
+
+ /// A start cancelled by SIGTERM may take `cancelKillDelay` seconds to end
+ /// (SIGTERM, then SIGKILL) before the stop sequence begins.
+ public static var shutdownWorstCase: TimeInterval {
+ ProcessRunner.cancelKillDelay + stopSequenceWorstCase
+ }
+
+ /// How long a machine that `limactl start` gave up on may still take to
+ /// come up. The Mac often runs under a load of ~100-200 (renders); on 12.11
+ /// `limactl start` failed with "did not receive an event with the `running`
+ /// status" after 10 min and the machine was up a little later.
+ public static let startGrace: TimeInterval = 60 * 60
+ static let checkInterval: TimeInterval = 30
+ /// The status check while waiting for a start is not on the SIGTERM clock.
+ static let waitStatusTimeout: TimeInterval = 60
+
+ /// The effects, injected so the order of events is testable.
+ public struct Steps: Sendable {
+ /// `limactl start`; ends (false) when its task is cancelled.
+ public var start: @Sendable () async -> Bool
+ /// Lima's status of the instance (Running, Stopped, Broken, Installing,
+ /// Uninitialized, "") or `nil` when there is none; throws when Lima does
+ /// not answer within the given time.
+ public var status: @Sendable (TimeInterval) async throws -> String?
+ /// In the guest: stop smbd and sync, so that nothing is open and nothing
+ /// is dirty when the power goes.
+ public var quiesce: @Sendable (TimeInterval) async -> Bool
+ public var stop: @Sendable (TimeInterval) async -> Bool
+ /// After a start: how did the previous boot end? (one line for the log)
+ public var lastShutdown: @Sendable () async -> String?
+ public var log: @Sendable (String) -> Void
+ public var sleep: @Sendable (TimeInterval) async -> Void
+
+ public init(
+ start: @escaping @Sendable () async -> Bool,
+ status: @escaping @Sendable (TimeInterval) async throws -> String?,
+ quiesce: @escaping @Sendable (TimeInterval) async -> Bool = { _ in true },
+ stop: @escaping @Sendable (TimeInterval) async -> Bool,
+ lastShutdown: @escaping @Sendable () async -> String? = { nil },
+ log: @escaping @Sendable (String) -> Void,
+ sleep: @escaping @Sendable (TimeInterval) async -> Void = {
+ try? await Task.sleep(nanoseconds: UInt64($0 * 1_000_000_000))
+ }
+ ) {
+ self.start = start
+ self.status = status
+ self.quiesce = quiesce
+ self.stop = stop
+ self.lastShutdown = lastShutdown
+ self.log = log
+ self.sleep = sleep
+ }
+ }
+
+ /// The result of `op`, or `nil` when termination came first (and `op` was
+ /// cancelled, which for a process kills it).
+ static func interruptible(
+ _ latch: TerminationLatch, _ op: @escaping @Sendable () async -> T
+ ) async -> T? {
+ await withTaskGroup(of: T?.self) { group in
+ group.addTask { await op() }
+ group.addTask {
+ await latch.wait()
+ return nil
+ }
+ let first = await group.next() ?? nil
+ group.cancelAll()
+ return first
+ }
+ }
+
+ enum StartOutcome { case up, neverCameUp, terminated }
+
+ private enum Answer: Sendable {
+ case status(String?)
+ case unreachable(String)
+ }
+
+ private static func ask(_ steps: Steps, timeout: TimeInterval) async -> Answer {
+ do { return .status(try await steps.status(timeout)) } catch { return .unreachable("\(error)") }
+ }
+
+ /// After a failed `limactl start`: is the machine coming up anyway? Waits
+ /// while Lima says anything but Stopped/missing, or does not answer. Never
+ /// touches the machine.
+ static func waitForStart(steps: Steps, termination: TerminationLatch) async -> StartOutcome {
+ var waited: TimeInterval = 0
+ while true {
+ guard
+ let answer = await interruptible(
+ termination, { await ask(steps, timeout: waitStatusTimeout) })
+ else { return .terminated }
+ var seen = "not answering"
+ switch answer {
+ case .status("Running"): return .up
+ case .status(nil), .status("Stopped"): return .neverCameUp // nothing is coming up
+ case .status(let status?): seen = status.isEmpty ? "in an unknown state" : status
+ case .unreachable: break // Lima busy or slow under load: ask again
+ }
+ if waited >= startGrace { return .neverCameUp }
+ steps.log("vm-hold: limactl start did not finish; the machine is \(seen); waiting")
+ guard await interruptible(termination, { await steps.sleep(checkInterval) }) != nil
+ else { return .terminated }
+ waited += checkInterval
+ }
+ }
+
+ /// The exit code of the job: 0 when the machine was started and then
+ /// stopped (or was already stopped), 1 when it never came up or did not stop.
+ public static func run(steps: Steps, termination: TerminationLatch) async -> Int32 {
+ var neverCameUp = false
+ if let started = await interruptible(termination, { await steps.start() }) {
+ var outcome = StartOutcome.up
+ if !started {
+ steps.log("vm-hold: limactl start failed; checking whether the machine comes up anyway")
+ outcome = await waitForStart(steps: steps, termination: termination)
+ }
+ switch outcome {
+ case .up:
+ if let line = await interruptible(termination, { await steps.lastShutdown() }), let line {
+ steps.log("vm-hold: \(line)")
+ }
+ steps.log("vm-hold: the machine is up, waiting for the job to be stopped")
+ await termination.wait()
+ case .neverCameUp:
+ // Do not exit: nobody would hold the machine if it came up later, and
+ // an exit would leave a half-started one to die hard with the Mac.
+ neverCameUp = true
+ steps.log(
+ "vm-hold: the machine did not come up; holding anyway, whatever runs is stopped at termination"
+ )
+ await termination.wait()
+ case .terminated:
+ break
+ }
+ } else {
+ steps.log("vm-hold: terminated during limactl start; stopping whatever started")
+ }
+
+ let stopped = await stopMachine(steps: steps)
+ return stopped && !neverCameUp ? 0 : 1
+ }
+
+ /// After SIGTERM. Everything that is not Stopped gets stopped, a machine in
+ /// a state we do not know included (Lima's statuses: Running, Stopped,
+ /// Broken, Installing, Uninitialized, ""): one that is left alone dies hard
+ /// with the Mac. Returns false only when the stop itself failed.
+ static func stopMachine(steps: Steps) async -> Bool {
+ var guestMayAnswer = true
+ switch await ask(steps, timeout: statusTimeout) {
+ case .status(nil):
+ steps.log("vm-hold: the machine is missing, nothing to stop")
+ return true
+ case .status("Stopped"):
+ steps.log("vm-hold: the machine is stopped, nothing to stop")
+ return true
+ case .status("Running"):
+ break
+ case .status(let other?):
+ steps.log("vm-hold: the machine is '\(other)'; stopping it anyway")
+ guestMayAnswer = false
+ case .unreachable(let why):
+ // A machine left running is worse than a failed `limactl stop`.
+ steps.log("vm-hold: Lima did not answer about the status (\(why)), stopping anyway")
+ }
+ if guestMayAnswer {
+ if await steps.quiesce(quiesceTimeout) {
+ steps.log("vm-hold: smbd stopped and the guest synced")
+ } else {
+ steps.log("vm-hold: could not quiesce the guest (smbd, sync); stopping it regardless")
+ }
+ }
+ steps.log("vm-hold: stopping the machine")
+ guard await steps.stop(stopTimeout) else {
+ steps.log("vm-hold: limactl stop failed or ran out of time")
+ return false
+ }
+ steps.log("vm-hold: the machine is stopped")
+ return true
+ }
+
+ /// The real effects.
+ public static var live: Steps {
+ Steps(
+ start: { await VMLayout.start().succeeded },
+ status: { try await VMLayout.instanceStatus(timeout: $0) },
+ quiesce: { await VMLayout.quiesceGuest(timeout: $0) },
+ stop: { timeout in
+ let result = try? await VMLayout.limactl(
+ ["stop", VMLayout.instanceName], timeout: timeout)
+ return result?.succeeded == true
+ },
+ lastShutdown: { await VMLayout.lastShutdownSummary() },
+ log: { CMLogger.log($0) })
+ }
+}
diff --git a/mac-app/Sources/CloudMachineCore/VMLayout.swift b/mac-app/Sources/CloudMachineCore/VMLayout.swift
index 0daf412..d84cbcf 100644
--- a/mac-app/Sources/CloudMachineCore/VMLayout.swift
+++ b/mac-app/Sources/CloudMachineCore/VMLayout.swift
@@ -356,13 +356,56 @@ public enum VMLayout {
/// `Running`, `Stopped`, ... or `nil` when there is no instance. Throws
/// when `limactl` itself does not answer.
- public static func instanceStatus() async throws -> String? {
- let result = try await limactl(["list", "--json"], timeout: 60)
- guard result.succeeded else {
+ public static func instanceStatus(timeout: TimeInterval = 60) async throws -> String? {
+ let result = try await limactl(["list", "--json", instanceName], timeout: timeout)
+ return try instanceStatus(
+ exitCode: result.exitCode, stdout: result.stdout, stderr: result.stderr)
+ }
+
+ /// The answer of `limactl list --json `: "unmatched instances" is the
+ /// instance not being there (`nil`); any other failure is no answer.
+ static func instanceStatus(exitCode: Int32, stdout: String, stderr: String) throws -> String? {
+ guard exitCode == 0 else {
+ if stderr.contains("unmatched instances") { return nil }
throw ProcessRunnerError.launchFailed(
L10n.tr(
- "limactl list failed: %@", result.stderr.trimmingCharacters(in: .whitespacesAndNewlines)))
+ "limactl list failed: %@", stderr.trimmingCharacters(in: .whitespacesAndNewlines)))
}
- return instanceStatus(listJSON: result.stdout)
+ return instanceStatus(listJSON: stdout)
+ }
+
+ /// Before `limactl stop`: stop smbd (no open files, no writes arriving) and
+ /// sync the guest. Lima's vz driver gives the guest only 30 s to power off;
+ /// this takes the Samba shutdown and the flush of the dirty cache disk out
+ /// of that budget. `;` and not `&&`: the sync matters most when smbd is
+ /// already gone, masked, or slow to stop.
+ static let quiesceScript = "systemctl stop smbd; sync"
+
+ public static func quiesceGuest(timeout: TimeInterval) async -> Bool {
+ let result = try? await rootShell(quiesceScript, timeout: timeout)
+ return result?.succeeded == true
+ }
+
+ /// One line for the log at the next start: did the guest's previous boot end
+ /// cleanly? The kernel's word at THIS boot is authoritative: ext4 says
+ /// "recovery required" when the journal of the root filesystem has to be
+ /// replayed (a hard stop, also one so early that the previous boot left no
+ /// journal at all). Otherwise the previous boot's journal must end with the
+ /// power-off sequence ("Journal stopped").
+ static let lastShutdownScript = """
+ if journalctl -k -b 0 --no-pager 2>/dev/null | grep -q "recovery required"; then \
+ echo "the previous boot did NOT end cleanly: ext4 needed journal recovery (hard stop)"; \
+ elif journalctl -b -1 --no-pager -o cat 2>/dev/null | tail -n 5 | grep -q "Journal stopped"; then \
+ echo "the previous boot ended cleanly"; \
+ elif journalctl -b -1 --no-pager -o cat 2>/dev/null | grep -q .; then \
+ echo "the previous boot's journal ends without a shutdown sequence (hard stop?)"; \
+ else echo "no journal of a previous boot"; fi
+ """
+
+ public static func lastShutdownSummary() async -> String? {
+ guard let result = try? await rootShell(lastShutdownScript, timeout: 60), result.succeeded
+ else { return nil }
+ let lines = result.stdout.split(separator: "\n").map(String.init)
+ return lines.isEmpty ? nil : "guest: " + lines.joined(separator: "; ")
}
}
diff --git a/mac-app/Tests/CloudMachineAppTests/LaunchdInstallerTests.swift b/mac-app/Tests/CloudMachineAppTests/LaunchdInstallerTests.swift
index 38a9fd2..a8f10a7 100644
--- a/mac-app/Tests/CloudMachineAppTests/LaunchdInstallerTests.swift
+++ b/mac-app/Tests/CloudMachineAppTests/LaunchdInstallerTests.swift
@@ -174,6 +174,78 @@ final class LaunchdInstallerTests: XCTestCase {
XCTAssertTrue(outcome.failed.isEmpty, "a README is not a failed agent")
}
+ // MARK: - vm-start holds the machine
+
+ private let vmStart = "com.renacode.cloudmachine.vm-start"
+
+ /// `vm-start` is `vm-hold`: while it runs, the machine runs under it. A
+ /// reload (`unload` = SIGTERM, then `load`) would stop the machine in the
+ /// middle of a backup and race the old and the new process. A running job
+ /// gets its new plist written, and takes it over at the next login or boot.
+ func testRunningVMHoldIsNotReloaded() async {
+ let reloaded = Captured()
+ let written = Captured()
+ let outcome = await LaunchdInstaller.installAgents(
+ templates: templates([vmStart, "com.renacode.cloudmachine.buffer-guard"]),
+ into: destination, agentBin: agentBin, logDir: logDir,
+ read: { _ in "x" }, write: { content, url in written.add(content, url) },
+ reload: { url in
+ reloaded.add("", url)
+ return true
+ },
+ isRunning: { $0 == "com.renacode.cloudmachine.vm-start" },
+ log: { _ in })
+ XCTAssertEqual(
+ reloaded.paths,
+ [destination.appendingPathComponent("com.renacode.cloudmachine.buffer-guard.plist").path],
+ "only the other agent is reloaded")
+ XCTAssertEqual(written.paths.count, 2, "the new plist of vm-start is still written")
+ XCTAssertTrue(outcome.failed.isEmpty)
+ XCTAssertTrue(outcome.installed.contains(vmStart), "left running counts as installed")
+ }
+
+ /// Not running (the old `limactl start` job that has exited, or a first
+ /// installation): reload as usual, that is what starts `vm-hold` now.
+ func testStoppedVMStartIsReloaded() async {
+ let reloaded = Captured()
+ _ = await LaunchdInstaller.installAgents(
+ templates: templates([vmStart]),
+ into: destination, agentBin: agentBin, logDir: logDir,
+ read: { _ in "x" }, write: { _, _ in },
+ reload: { url in
+ reloaded.add("", url)
+ return true
+ },
+ isRunning: { _ in false }, log: { _ in })
+ XCTAssertEqual(reloaded.paths.count, 1)
+ }
+
+ /// Other agents are one-shot checks: a running one is still reloaded.
+ func testOtherAgentsAreReloadedEvenWhenRunning() async {
+ let reloaded = Captured()
+ _ = await LaunchdInstaller.installAgents(
+ templates: templates(["com.renacode.cloudmachine.buffer-guard"]),
+ into: destination, agentBin: agentBin, logDir: logDir,
+ read: { _ in "x" }, write: { _, _ in },
+ reload: { url in
+ reloaded.add("", url)
+ return true
+ },
+ isRunning: { _ in true }, log: { _ in })
+ XCTAssertEqual(reloaded.paths.count, 1)
+ }
+
+ /// `launchctl list ` prints a `"PID" = n;` line only while it runs.
+ func testParsesWhetherAJobIsRunning() {
+ XCTAssertTrue(
+ LaunchdInstaller.jobIsRunning(
+ listOutput: "{\n\t\"Label\" = \"x\";\n\t\"PID\" = 4242;\n\t\"Program\" = \"y\";\n};"))
+ XCTAssertFalse(
+ LaunchdInstaller.jobIsRunning(
+ listOutput: "{\n\t\"Label\" = \"x\";\n\t\"LastExitStatus\" = 0;\n};"))
+ XCTAssertFalse(LaunchdInstaller.jobIsRunning(listOutput: ""))
+ }
+
// MARK: - Helpers
/// Classes, because the substituted closures are `@Sendable`.
diff --git a/mac-app/Tests/CloudMachineAppTests/TerminationSignalsTests.swift b/mac-app/Tests/CloudMachineAppTests/TerminationSignalsTests.swift
new file mode 100644
index 0000000..0f4cc21
--- /dev/null
+++ b/mac-app/Tests/CloudMachineAppTests/TerminationSignalsTests.swift
@@ -0,0 +1,147 @@
+import XCTest
+
+@testable import CloudMachineCore
+
+/// What turns launchd's SIGTERM into `TerminationLatch.fire()`, and what it
+/// must not do to the processes `vm-hold` starts.
+final class TerminationSignalsTests: XCTestCase {
+
+ // MARK: - TerminationLatch
+
+ func testWaitReturnsAfterFire() async {
+ let latch = TerminationLatch()
+ Task {
+ try? await Task.sleep(nanoseconds: 50_000_000)
+ latch.fire()
+ }
+ await latch.wait()
+ XCTAssertTrue(latch.hasArrived)
+ }
+
+ func testWaitReturnsAtOnceWhenAlreadyFired() async {
+ let latch = TerminationLatch()
+ latch.fire()
+ await latch.wait()
+ }
+
+ func testEveryWaiterIsReleased() async {
+ let latch = TerminationLatch()
+ async let a: Void = latch.wait()
+ async let b: Void = latch.wait()
+ try? await Task.sleep(nanoseconds: 50_000_000)
+ latch.fire()
+ _ = await (a, b)
+ }
+
+ /// Without this a race between "the start finished" and "SIGTERM" would hang
+ /// the job: the waiter that lost the race has to end when it is cancelled.
+ func testWaitEndsWhenTheWaiterIsCancelled() async {
+ let latch = TerminationLatch()
+ let task = Task { await latch.wait() }
+ try? await Task.sleep(nanoseconds: 50_000_000)
+ task.cancel()
+ await task.value
+ XCTAssertFalse(latch.hasArrived, "cancelling is not a termination")
+ }
+
+ // MARK: - Signals
+
+ /// A caught signal fires the latch.
+ func testSignalFiresTheLatch() async throws {
+ let signals = TerminationSignals(signals: [SIGUSR2])
+ defer { signals.uninstall() }
+ XCTAssertFalse(signals.latch.hasArrived)
+ kill(getpid(), SIGUSR2)
+ let latch = signals.latch
+ let waited = Task { await latch.wait() }
+ let timeout = Task {
+ try await Task.sleep(nanoseconds: 5_000_000_000)
+ waited.cancel()
+ }
+ await waited.value
+ timeout.cancel()
+ XCTAssertTrue(latch.hasArrived, "the signal did not arrive within 5 s")
+ }
+
+ /// Ignoring a signal (`SIG_IGN`) is inherited through fork AND exec: the
+ /// first version of `vm-hold` ignored SIGTERM, so `limactl` and its host
+ /// agent ignored it too and `Process.terminate()` did nothing to them. The
+ /// handler must be a real handler, which exec resets to the default.
+ func testChildProcessesDoNotInheritAnIgnoredSignal() async throws {
+ // Detector first: with SIG_IGN the child DOES inherit it and survives.
+ let previous = signal(SIGUSR2, SIG_IGN)
+ let ignoring = try childSurvivesSIGUSR2()
+ signal(SIGUSR2, previous)
+ XCTAssertTrue(ignoring, "the check cannot see an inherited SIG_IGN")
+
+ let signals = TerminationSignals(signals: [SIGUSR2])
+ defer { signals.uninstall() }
+ let survives = try childSurvivesSIGUSR2()
+ XCTAssertFalse(survives, "a child inherited an ignored SIGUSR2")
+ }
+
+ /// A fresh child sends itself SIGUSR2 (default action: terminate). It prints
+ /// "survived" only if the signal was ignored. Spawned with plain
+ /// `posix_spawn` and no signal attributes: Foundation's `Process` happens to
+ /// reset dispositions itself, which would hide the inheritance this test is
+ /// about (macOS `ps` has no ignored-mask column to look at instead).
+ private func childSurvivesSIGUSR2() throws -> Bool {
+ let out = FileManager.default.temporaryDirectory
+ .appendingPathComponent("cm-sig-\(UUID().uuidString)").path
+ defer { try? FileManager.default.removeItem(atPath: out) }
+ var actions: posix_spawn_file_actions_t?
+ posix_spawn_file_actions_init(&actions)
+ defer { posix_spawn_file_actions_destroy(&actions) }
+ posix_spawn_file_actions_addopen(&actions, 1, out, O_WRONLY | O_CREAT | O_TRUNC, 0o600)
+ // Only the signal MASK is cleared (a worker thread may have signals blocked,
+ // which a child would inherit and which would look like "ignored");
+ // dispositions are left as they are.
+ var attrs: posix_spawnattr_t?
+ posix_spawnattr_init(&attrs)
+ defer { posix_spawnattr_destroy(&attrs) }
+ var empty = sigset_t()
+ sigemptyset(&empty)
+ posix_spawnattr_setsigmask(&attrs, &empty)
+ posix_spawnattr_setflags(&attrs, Int16(POSIX_SPAWN_SETSIGMASK))
+ var pid: pid_t = 0
+ let argv: [String] = ["sh", "-c", "kill -USR2 $$; echo survived"]
+ var cArgs: [UnsafeMutablePointer?] = argv.map { strdup($0) } + [nil]
+ defer { for p in cArgs { free(p) } }
+ guard posix_spawn(&pid, "/bin/sh", &actions, &attrs, &cArgs, environ) == 0 else {
+ throw XCTSkip("posix_spawn failed")
+ }
+ var status: Int32 = 0
+ waitpid(pid, &status, 0)
+ return (try? String(contentsOfFile: out, encoding: .utf8))?.contains("survived") == true
+ }
+
+ // MARK: - Cancelling a running process
+
+ /// `limactl start` can run for up to 50 minutes; SIGTERM must end it. A
+ /// cancelled task terminates the child instead of waiting for it.
+ func testCancellingTheTaskTerminatesTheProcess() async {
+ let began = Date()
+ let task = Task { try await ProcessRunner.run("/bin/sleep", ["60"]) }
+ try? await Task.sleep(nanoseconds: 300_000_000)
+ task.cancel()
+ let result = try? await task.value
+ XCTAssertLessThan(Date().timeIntervalSince(began), 10)
+ XCTAssertNotEqual(result?.exitCode, 0, "the process was ended by the signal")
+ }
+
+ /// A process that ignores SIGTERM must not hold the job past launchd's
+ /// ExitTimeOut: after `cancelKillDelay` it gets SIGKILL.
+ func testCancellingEscalatesToKillForAProcessThatIgnoresSIGTERM() async {
+ let began = Date()
+ let task = Task {
+ try await ProcessRunner.run("/bin/sh", ["-c", "trap '' TERM; exec /bin/sleep 60"])
+ }
+ try? await Task.sleep(nanoseconds: 500_000_000)
+ task.cancel()
+ let result = try? await task.value
+ let took = Date().timeIntervalSince(began)
+ XCTAssertLessThan(took, ProcessRunner.cancelKillDelay + 10)
+ XCTAssertNotEqual(result?.exitCode, 0)
+ XCTAssertGreaterThanOrEqual(took, 0.5)
+ }
+}
diff --git a/mac-app/Tests/CloudMachineAppTests/VMDurabilityTests.swift b/mac-app/Tests/CloudMachineAppTests/VMDurabilityTests.swift
new file mode 100644
index 0000000..97ac110
--- /dev/null
+++ b/mac-app/Tests/CloudMachineAppTests/VMDurabilityTests.swift
@@ -0,0 +1,191 @@
+import XCTest
+
+/// Durability of a Time Machine write: APFS inside the image relies on fsync
+/// (F_FULLFSYNC -> SMB flush -> Samba fsync -> punchfs -> rclone) really
+/// reaching the disk, because the machine can die hard with the Mac. These
+/// tests pin the pieces of that chain that live in the repo's templates.
+final class VMDurabilityTests: XCTestCase {
+ private static let repoRoot = URL(fileURLWithPath: #filePath)
+ .deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
+ .deletingLastPathComponent()
+
+ private func repoFile(_ path: String) throws -> String {
+ try String(
+ contentsOf: Self.repoRoot.appendingPathComponent(path), encoding: .utf8)
+ }
+
+ private func execStart(_ unit: String) throws -> String {
+ let text = try repoFile(unit)
+ // ExecStart may continue over several lines with a trailing backslash.
+ var lines: [String] = []
+ var inside = false
+ for line in text.components(separatedBy: "\n") {
+ if line.hasPrefix("ExecStart=") { inside = true }
+ guard inside else { continue }
+ let continued = line.hasSuffix("\\")
+ lines.append(continued ? String(line.dropLast()) : line)
+ if !continued { break }
+ }
+ return lines.joined(separator: " ")
+ }
+
+ /// `rclone mount` (cgofuse-less libfuse path) answers fsync with OK and
+ /// does nothing; `mount2` (go-fuse) passes it on to the VFS cache item.
+ func testRcloneUsesMount2() throws {
+ let exec = try execStart("vm/systemd/rclone-tm.service")
+ XCTAssertTrue(exec.contains("/usr/local/bin/rclone mount2 "), exec)
+ XCTAssertFalse(exec.contains("/usr/local/bin/rclone mount "), exec)
+ }
+
+ /// punchfs must sync the filesystem that holds rclone's cache metadata
+ /// (written without fsync), and it must be the cache dir rclone really uses.
+ func testPunchfsSyncsRcloneCacheDirectory() throws {
+ let punch = try execStart("vm/systemd/punchfs.service")
+ let rclone = try execStart("vm/systemd/rclone-tm.service")
+ let option = try XCTUnwrap(
+ punch.range(of: "syncfs=/[^,\\s']+", options: .regularExpression).map { String(punch[$0]) },
+ punch)
+ let dir = String(option.dropFirst("syncfs=".count))
+ XCTAssertEqual(dir, "/var/cache/rclone")
+ XCTAssertTrue(rclone.contains("--cache-dir \(dir)"), "syncfs dir must be rclone's cache dir")
+ XCTAssertTrue(punch.contains("source=/srv/tm,"), "other options keep working: \(punch)")
+ XCTAssertTrue(punch.contains("allow_other"), punch)
+ }
+
+ /// The C side: the option exists, the directory is opened once, and both
+ /// fsync handlers call syncfs and report its errno.
+ func testPunchfsCodeSyncsOnFsync() throws {
+ let source = try repoFile("vm/punchfs/punchfs.c")
+ XCTAssertTrue(source.contains("\"syncfs=%s\""), "option syncfs= is parsed")
+ func body(of function: String, returning type: String = "void") throws -> String {
+ let start = try XCTUnwrap(source.range(of: "static \(type) \(function)("), function)
+ let rest = source[start.lowerBound...]
+ let end = rest.range(of: "\n}\n")?.upperBound ?? rest.endIndex
+ return String(rest[..syncfs_fd)"), helper)
+ XCTAssertTrue(helper.contains("errno"), helper)
+ XCTAssertTrue(source.contains("O_DIRECTORY"), "the directory is opened at start")
+ }
+
+ /// The machine test exercises the chain for real and the template ships it.
+ func testMachineTestChecksFsyncDurability() throws {
+ let script = try repoFile("vm/punchfs/test-punchfs.sh")
+ XCTAssertTrue(script.contains("mount2"), "the test mounts like production")
+ XCTAssertTrue(script.contains("syncfs="))
+ XCTAssertTrue(script.contains("Dirty"), "fsync must leave ~0 dirty pages")
+ XCTAssertTrue(script.contains("vfsMeta"), "metadata file must cover the written range")
+ // Detectors that can see a missing syncfs (the review found a test that
+ // read the page cache and could not): dirty pages of the cache data file
+ // per file, and the kernel's own record of syncfs calls.
+ XCTAssertTrue(script.contains("kpageflags"), "dirty pages are read per file")
+ XCTAssertTrue(script.contains("ext4_sync_fs"), "the kernel must have seen syncfs")
+ XCTAssertTrue(script.contains("restart of rclone"), "xattr must survive rclone's restart")
+ XCTAssertFalse(
+ script.contains("| head -1"), "a `find | head` under pipefail aborted without a FAIL")
+ XCTAssertFalse(
+ script.contains("/proc/meminfo | grep") || script.contains("test \"$dirty\" -lt"),
+ "the machine-wide Dirty figure is informational only: other writers fail it")
+ }
+
+ /// Partial deployment: the unit reaches the machine before the binary is
+ /// rebuilt. An old punchfs does not know `-o syncfs`; with Restart=always
+ /// that was a restart loop. It must end once, visibly (exit 78).
+ func testPunchfsUnitRefusesAnOldBinaryWithoutLooping() throws {
+ let unit = try repoFile("vm/systemd/punchfs.service")
+ let exec = try execStart("vm/systemd/punchfs.service")
+ XCTAssertTrue(exec.contains("--help"), "the binary is asked whether it has the option")
+ XCTAssertTrue(exec.contains("syncfs="), exec)
+ XCTAssertTrue(exec.contains("exit 78"), exec)
+ XCTAssertTrue(unit.contains("RestartPreventExitStatus=78"), "no restart loop")
+ XCTAssertTrue(exec.contains("exec /usr/local/bin/punchfs"), "the real process is punchfs")
+ }
+
+ /// rclone's mount2 inodes come from a counter of the process, so they
+ /// change when rclone restarts: the attributes (sparse flag, Apple streams)
+ /// must be keyed by the path below `source`, not by inode.
+ func testPunchfsKeysExtendedAttributesByPathNotInode() throws {
+ let source = try repoFile("vm/punchfs/punchfs.c")
+ func body(of function: String) throws -> String {
+ let start = try XCTUnwrap(source.range(of: "static int \(function)("), function)
+ let rest = source[start.lowerBound...]
+ return String(rest[..<(rest.range(of: "\n}\n")?.upperBound ?? rest.endIndex)])
+ }
+ let dir = try body(of: "xdir_of")
+ XCTAssertTrue(dir.contains("rel_path_of"), dir)
+ XCTAssertFalse(dir.contains("->ino"), "the inode must not be the key: \(dir)")
+ XCTAssertFalse(
+ source.contains("rclone's inode is a hash of the path"), "the false claim is gone")
+ XCTAssertTrue(source.contains("xattr_move("), "rename moves the attributes")
+ XCTAssertTrue(source.contains("xattr_forget("), "unlink removes them")
+ XCTAssertTrue(source.contains("xattr_drop_old_layout"), "the old inode-keyed layout goes")
+ }
+
+ // MARK: - Lima's 64 KiB limit
+
+ /// `limactl list --json` of a RUNNING machine fails with "bufio.Scanner:
+ /// token too long" once its JSON (which carries every embedded `file:` of the
+ /// template) passes 64 KiB. Measured 12.11: 61,980 bytes worked, a template
+ /// with 12.8 KB more did not - and with it `vm-status`, `vm-guard`,
+ /// `backup-health` and `vm-hold` lose sight of the machine. The punchfs
+ /// sources (the bulk) therefore ship as ONE compressed, base64 archive that
+ /// the machine unpacks; `vm/punchfs/pack.sh` makes it.
+ func testTemplateStaysFarBelowLimasLineLimit() throws {
+ let template = try repoFile("vm/cmsmb.yaml")
+ let files = template.components(separatedBy: "\n")
+ .compactMap { line -> String? in
+ guard line.hasPrefix(" file: ") else { return nil }
+ return String(line.dropFirst(" file: ".count))
+ }
+ XCTAssertFalse(files.isEmpty)
+ var total = 0
+ for file in files {
+ total += try Data(contentsOf: Self.repoRoot.appendingPathComponent("vm/" + file)).count
+ }
+ // The JSON limit is 65,536 bytes, reached at about 54 KB of embedded files
+ // (61,980 bytes of JSON for 51 KB): keep a wide margin below it.
+ XCTAssertLessThan(total, 42_000, "embedded files: \(total) bytes")
+ for big in ["punchfs/punchfs.c", "punchfs/test-punchfs.sh", "punchfs/passthrough_helpers.h"] {
+ XCTAssertFalse(files.contains(big), "\(big) must ship inside the archive")
+ }
+ XCTAssertTrue(files.contains("punchfs/src.tar.gz.b64"))
+ }
+
+ /// The archive is generated; it must hold exactly the files in the repo.
+ func testPunchfsArchiveMatchesTheSources() throws {
+ let work = FileManager.default.temporaryDirectory
+ .appendingPathComponent("cm-pack-\(UUID().uuidString)")
+ try FileManager.default.createDirectory(at: work, withIntermediateDirectories: true)
+ defer { try? FileManager.default.removeItem(at: work) }
+ let archive = Self.repoRoot.appendingPathComponent("vm/punchfs/src.tar.gz.b64").path
+ let unpack = Process()
+ unpack.executableURL = URL(fileURLWithPath: "/bin/sh")
+ unpack.arguments = ["-c", "base64 -D < '\(archive)' | tar -xz -C '\(work.path)'"]
+ try unpack.run()
+ unpack.waitUntilExit()
+ XCTAssertEqual(unpack.terminationStatus, 0, "the archive unpacks")
+ for name in ["punchfs.c", "passthrough_helpers.h", "test-punchfs.sh"] {
+ let packed = try Data(contentsOf: work.appendingPathComponent(name))
+ let plain = try Data(
+ contentsOf: Self.repoRoot.appendingPathComponent("vm/punchfs/" + name))
+ XCTAssertEqual(packed, plain, "\(name) in src.tar.gz.b64 is stale: run vm/punchfs/pack.sh")
+ }
+ let attributes = try FileManager.default.attributesOfItem(
+ atPath: work.appendingPathComponent("test-punchfs.sh").path)
+ XCTAssertEqual((attributes[.posixPermissions] as? Int ?? 0) & 0o111 != 0, true)
+ }
+
+ /// The machine unpacks it before it builds.
+ func testProvisioningUnpacksBeforeBuilding() throws {
+ let template = try repoFile("vm/cmsmb.yaml")
+ let unpack = try XCTUnwrap(template.range(of: "base64 -d"))
+ let build = try XCTUnwrap(template.range(of: "gcc -Wall"))
+ XCTAssertLessThan(unpack.lowerBound, build.lowerBound)
+ XCTAssertTrue(template.contains("/opt/cloudmachine/punchfs/src.tar.gz.b64"))
+ }
+}
diff --git a/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift b/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift
new file mode 100644
index 0000000..7f96199
--- /dev/null
+++ b/mac-app/Tests/CloudMachineAppTests/VMHoldTests.swift
@@ -0,0 +1,356 @@
+import XCTest
+
+@testable import CloudMachineCore
+
+/// `vm-hold`: starts the machine at login and stays alive, so that launchd's
+/// SIGTERM at logout/shutdown turns into a clean stop of the machine (which
+/// otherwise dies together with the Mac and the guest's ext4 needs recovery).
+///
+/// Lima's real statuses are Running, Stopped, Broken, Installing,
+/// Uninitialized and "" (no `Starting`); the tests use those.
+final class VMHoldTests: XCTestCase {
+
+ /// Records the order of events and the timeouts the steps were given.
+ private final class Recorder: @unchecked Sendable {
+ private let lock = NSLock()
+ private var items: [String] = []
+ private var limits: [String: TimeInterval] = [:]
+ private var slept: TimeInterval = 0
+ func add(_ item: String) {
+ lock.lock()
+ items.append(item)
+ lock.unlock()
+ }
+ func limit(_ name: String, _ value: TimeInterval) {
+ lock.lock()
+ limits[name] = value
+ lock.unlock()
+ }
+ func sleep(_ t: TimeInterval) {
+ lock.lock()
+ slept += t
+ lock.unlock()
+ }
+ var events: [String] {
+ lock.lock()
+ defer { lock.unlock() }
+ return items
+ }
+ var plain: [String] { events.filter { !$0.hasPrefix("log") } }
+ var timeouts: [String: TimeInterval] {
+ lock.lock()
+ defer { lock.unlock() }
+ return limits
+ }
+ var totalSlept: TimeInterval {
+ lock.lock()
+ defer { lock.unlock() }
+ return slept
+ }
+ func logged(_ needle: String) -> Bool {
+ events.contains { $0.hasPrefix("log") && $0.contains(needle) }
+ }
+ }
+
+ /// Statuses answered one per call, the last one repeats.
+ private final class Statuses: @unchecked Sendable {
+ private let lock = NSLock()
+ private var list: [String?]
+ init(_ list: [String?]) { self.list = list }
+ func next() -> String? {
+ lock.lock()
+ defer { lock.unlock() }
+ return list.count > 1 ? list.removeFirst() : list[0]
+ }
+ }
+
+ private func steps(
+ _ rec: Recorder, start: Bool = true, statuses: [String?] = ["Running"],
+ stop: Bool = true, quiesce: Bool = true, unreachable: Bool = false,
+ lastShutdown: String? = nil
+ ) -> VMHold.Steps {
+ let script = Statuses(statuses)
+ return VMHold.Steps(
+ start: {
+ rec.add("start")
+ return start
+ },
+ status: { timeout in
+ rec.add("status")
+ rec.limit("status", timeout)
+ if unreachable { throw ProcessRunnerError.launchFailed("x") }
+ return script.next()
+ },
+ quiesce: { timeout in
+ rec.add("quiesce")
+ rec.limit("quiesce", timeout)
+ return quiesce
+ },
+ stop: { timeout in
+ rec.add("stop")
+ rec.limit("stop", timeout)
+ return stop
+ },
+ lastShutdown: {
+ rec.add("lastShutdown")
+ return lastShutdown
+ },
+ log: { rec.add("log: \($0)") },
+ sleep: { rec.sleep($0) })
+ }
+
+ /// Runs `VMHold.run` and fires the termination latch after `delay` seconds.
+ private func run(
+ _ steps: VMHold.Steps, terminateAfter delay: TimeInterval = 0.05
+ ) async -> Int32 {
+ let latch = TerminationLatch()
+ Task {
+ try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000))
+ latch.fire()
+ }
+ return await VMHold.run(steps: steps, termination: latch)
+ }
+
+ // MARK: - The normal life of the job
+
+ func testStartsThenWaitsThenQuiescesAndStopsOnTermination() async {
+ let rec = Recorder()
+ let code = await run(steps(rec))
+ XCTAssertEqual(code, 0)
+ XCTAssertEqual(rec.plain, ["start", "lastShutdown", "status", "quiesce", "stop"])
+ }
+
+ /// After a hard stop the next start says so in the log (journal of the
+ /// previous boot), and a clean one is confirmed.
+ func testReportsHowTheLastShutdownWent() async {
+ let rec = Recorder()
+ _ = await run(steps(rec, lastShutdown: "previous boot ended cleanly"))
+ XCTAssertTrue(rec.logged("previous boot ended cleanly"))
+ }
+
+ /// The machine was already stopped (the app's Stop, prepare-shutdown):
+ /// `limactl stop` on it would only fail.
+ func testDoesNotStopAMachineThatIsStoppedOrMissing() async {
+ for status in ["Stopped", nil] as [String?] {
+ let rec = Recorder()
+ let code = await run(steps(rec, statuses: [status]))
+ XCTAssertEqual(code, 0)
+ XCTAssertFalse(rec.plain.contains("stop"), "status \(String(describing: status))")
+ XCTAssertFalse(rec.plain.contains("quiesce"))
+ }
+ }
+
+ /// Everything that is not Stopped is stopped at termination: a machine in a
+ /// transitional state would otherwise die hard with the Mac. Only a running
+ /// guest can be asked to quiesce.
+ func testStopsEveryStateThatIsNotStopped() async {
+ for status in ["Running", "Broken", "Installing", "Uninitialized", ""] {
+ let rec = Recorder()
+ let code = await run(steps(rec, statuses: [status]))
+ XCTAssertEqual(code, 0, status)
+ XCTAssertTrue(rec.plain.contains("stop"), "status '\(status)' must be stopped")
+ XCTAssertEqual(rec.plain.contains("quiesce"), status == "Running", status)
+ }
+ }
+
+ /// When Lima does not answer about the status we cannot tell, and a machine
+ /// left running is worse than a failed `limactl stop`.
+ func testUnknownStatusStillQuiescesAndStops() async {
+ let rec = Recorder()
+ let code = await run(steps(rec, unreachable: true))
+ XCTAssertEqual(code, 0)
+ XCTAssertTrue(rec.plain.contains("quiesce"))
+ XCTAssertTrue(rec.plain.contains("stop"))
+ }
+
+ /// `systemctl stop smbd && sync` in the guest is a courtesy: when it fails
+ /// (guest slow, Lima's shell busy) the stop still has to happen.
+ func testFailedQuiesceDoesNotPreventTheStop() async {
+ let rec = Recorder()
+ let code = await run(steps(rec, quiesce: false))
+ XCTAssertEqual(code, 0)
+ XCTAssertTrue(rec.plain.contains("stop"))
+ XCTAssertTrue(rec.logged("quiesce"))
+ XCTAssertLessThan(
+ rec.plain.firstIndex(of: "quiesce")!, rec.plain.firstIndex(of: "stop")!)
+ }
+
+ /// A stop that fails must be visible (exit code and log), not swallowed.
+ func testFailedStopIsReported() async {
+ let rec = Recorder()
+ let code = await run(steps(rec, stop: false))
+ XCTAssertEqual(code, 1)
+ XCTAssertTrue(rec.logged("stop"))
+ }
+
+ // MARK: - Time budget after SIGTERM
+
+ /// launchd sends SIGKILL ExitTimeOut seconds after SIGTERM. Status, quiesce
+ /// and stop run one after another, so the sum of their limits (each can give
+ /// up `ProcessRunner.giveUpSlack` seconds after its limit) must fit.
+ func testWorstCaseOfTheStopSequenceFitsInsideExitTimeout() async throws {
+ // A cancelled `limactl start` may take `cancelKillDelay` to end before the
+ // stop sequence even begins.
+ XCTAssertEqual(
+ VMHold.shutdownWorstCase, ProcessRunner.cancelKillDelay + VMHold.stopSequenceWorstCase)
+ XCTAssertLessThanOrEqual(
+ VMHold.shutdownWorstCase, VMHold.exitTimeOut - 5, "keep a margin to the SIGKILL")
+ let rec = Recorder()
+ _ = await run(steps(rec))
+ XCTAssertEqual(rec.timeouts["status"], VMHold.statusTimeout)
+ XCTAssertEqual(rec.timeouts["quiesce"], VMHold.quiesceTimeout)
+ XCTAssertEqual(rec.timeouts["stop"], VMHold.stopTimeout)
+
+ let root = URL(fileURLWithPath: #filePath)
+ .deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
+ .deletingLastPathComponent()
+ let plist = try String(
+ contentsOf: root.appendingPathComponent(
+ "launchd/com.renacode.cloudmachine.vm-start.plist.template"), encoding: .utf8)
+ XCTAssertTrue(plist.contains("vm-hold "), "the job runs vm-hold")
+ XCTAssertTrue(plist.contains("__CM_AGENT_BIN__"))
+ XCTAssertFalse(plist.contains("limactl "), "no bare limactl start")
+ XCTAssertTrue(
+ plist.contains("ExitTimeOut \n \(Int(VMHold.exitTimeOut)) "),
+ "ExitTimeOut must equal VMHold.exitTimeOut")
+ }
+
+ // MARK: - A start that does not finish in time (Mac under load)
+
+ /// 12.11: `limactl start` gave up after 10 min ("did not receive an event
+ /// with the `running` status") although the machine came up. A failed start
+ /// is not a failed machine: keep asking Lima, do not stop or restart it.
+ func testFailedStartIsToleratedWhileTheMachineComesUp() async {
+ let rec = Recorder()
+ let code = await run(
+ steps(rec, start: false, statuses: ["Installing", "Installing", "Broken", "Running"]),
+ terminateAfter: 0.3)
+ XCTAssertEqual(code, 0)
+ XCTAssertGreaterThan(rec.totalSlept, 0, "it waited between checks")
+ XCTAssertEqual(rec.plain.filter { $0 == "stop" }.count, 1, "stopped once, at termination")
+ XCTAssertTrue(rec.plain.contains("quiesce"))
+ }
+
+ func testWaitsLongerThanTenMinutes() async {
+ let rec = Recorder()
+ let code = await run(
+ steps(rec, start: false, statuses: Array(repeating: "Installing", count: 100) + ["Running"]),
+ terminateAfter: 1)
+ XCTAssertEqual(code, 0)
+ XCTAssertGreaterThan(rec.totalSlept, 20 * 60)
+ }
+
+ /// A machine that never comes up within the grace period: the job does NOT
+ /// exit (nobody would hold the machine if it came up at minute 61); it keeps
+ /// holding, and at termination stops whatever is not Stopped. The exit code
+ /// still says that the start failed.
+ func testNeverRunningKeepsHoldingAfterGraceAndStopsAtTermination() async {
+ let rec = Recorder()
+ let latch = TerminationLatch()
+ let finished = FlagBox()
+ let task = Task { () -> Int32 in
+ let code = await VMHold.run(
+ steps: self.steps(rec, start: false, statuses: ["Installing"]), termination: latch)
+ finished.set()
+ return code
+ }
+ while !rec.logged("holding") { try? await Task.sleep(nanoseconds: 20_000_000) }
+ try? await Task.sleep(nanoseconds: 100_000_000)
+ XCTAssertFalse(finished.value, "the job must not exit after the grace period")
+ XCTAssertGreaterThanOrEqual(rec.totalSlept, VMHold.startGrace)
+ XCTAssertFalse(rec.plain.contains("stop"), "nothing is stopped before termination")
+ latch.fire()
+ let code = await task.value
+ XCTAssertEqual(code, 1, "the machine never came up")
+ XCTAssertTrue(rec.plain.contains("stop"), "Installing is not Stopped: stopped at termination")
+ }
+
+ /// A machine that is simply Stopped after a failed start is not coming up:
+ /// no point polling for an hour, but the job still holds.
+ func testStoppedAfterFailedStartStopsPollingButKeepsHolding() async {
+ let rec = Recorder()
+ let code = await run(steps(rec, start: false, statuses: ["Stopped"]), terminateAfter: 0.2)
+ XCTAssertEqual(code, 1)
+ XCTAssertEqual(rec.totalSlept, 0)
+ XCTAssertFalse(rec.plain.contains("stop"), "nothing to stop")
+ XCTAssertTrue(rec.logged("holding"))
+ }
+
+ // MARK: - SIGTERM during the start and the waiting
+
+ /// SIGTERM while `limactl start` runs (up to 50 minutes under load): the
+ /// start is cancelled at once and the machine is stopped, not left to boot
+ /// into a dying session.
+ func testTerminationDuringStartCancelsTheStartAndStops() async {
+ let rec = Recorder()
+ let cancelled = FlagBox()
+ let base = steps(rec, statuses: ["Running"])
+ let hanging = VMHold.Steps(
+ start: {
+ rec.add("start")
+ while !Task.isCancelled { try? await Task.sleep(nanoseconds: 10_000_000) }
+ cancelled.set()
+ return false
+ },
+ status: base.status, quiesce: base.quiesce, stop: base.stop,
+ lastShutdown: base.lastShutdown, log: base.log, sleep: base.sleep)
+ let code = await run(hanging, terminateAfter: 0.1)
+ XCTAssertTrue(cancelled.value, "the start was cancelled")
+ XCTAssertEqual(code, 0)
+ XCTAssertEqual(rec.plain, ["start", "status", "quiesce", "stop"])
+ }
+
+ /// The waiting between status checks must not delay the stop by its length.
+ func testTerminationInterruptsTheWaitBetweenChecks() async {
+ let rec = Recorder()
+ let base = steps(rec, start: false, statuses: ["Installing", "Running"])
+ let longSleep = VMHold.Steps(
+ start: base.start, status: base.status, quiesce: base.quiesce, stop: base.stop,
+ lastShutdown: base.lastShutdown, log: base.log,
+ sleep: { _ in try? await Task.sleep(nanoseconds: 3_600_000_000_000) })
+ let began = Date()
+ let code = await run(longSleep, terminateAfter: 0.1)
+ XCTAssertLessThan(Date().timeIntervalSince(began), 5, "not a whole check interval")
+ XCTAssertEqual(code, 0)
+ XCTAssertTrue(rec.plain.contains("stop"))
+ }
+
+ /// A termination that arrived BEFORE `run` looked (the latch is a state,
+ /// not an event) still counts.
+ func testTerminationBeforeRunIsNotLost() async {
+ let rec = Recorder()
+ let latch = TerminationLatch()
+ latch.fire()
+ let code = await VMHold.run(steps: steps(rec), termination: latch)
+ XCTAssertEqual(code, 0)
+ XCTAssertTrue(rec.plain.contains("stop"))
+ }
+
+ /// The guest-side commands (their effect was tried on a throwaway machine).
+ func testGuestScripts() {
+ XCTAssertEqual(VMLayout.quiesceScript, "systemctl stop smbd; sync")
+ // The kernel's "recovery required" decides first: a boot killed early leaves
+ // no journal, and the previous journal alone then calls it clean.
+ let script = VMLayout.lastShutdownScript
+ XCTAssertTrue(script.contains("recovery required"))
+ XCTAssertTrue(script.contains("Journal stopped"))
+ XCTAssertLessThan(
+ script.range(of: "recovery required")!.lowerBound,
+ script.range(of: "Journal stopped")!.lowerBound)
+ }
+
+ private final class FlagBox: @unchecked Sendable {
+ private let lock = NSLock()
+ private var flag = false
+ func set() {
+ lock.lock()
+ flag = true
+ lock.unlock()
+ }
+ var value: Bool {
+ lock.lock()
+ defer { lock.unlock() }
+ return flag
+ }
+ }
+}
diff --git a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift
index 3391630..99b37ee 100644
--- a/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift
+++ b/mac-app/Tests/CloudMachineAppTests/VMLayoutTests.swift
@@ -18,6 +18,25 @@ final class VMLayoutTests: XCTestCase {
XCTAssertNil(VMLayout.instanceStatus(listJSON: ""))
}
+ /// `limactl list --json` of ALL instances fails when any one of them cannot
+ /// be read ("bufio.Scanner: token too long", seen on 12.11 on a throwaway
+ /// machine during its first boot) and then blinds everything that asks about
+ /// `cmsmb`. Ask for the one instance; "unmatched instances" means it is not
+ /// there, any other failure is an unanswered question.
+ func testStatusOfOneInstanceDistinguishesMissingFromUnanswered() throws {
+ let one = #"{"name":"cmsmb","status":"Running"}"#
+ XCTAssertEqual(
+ try VMLayout.instanceStatus(exitCode: 0, stdout: one, stderr: ""), "Running")
+ XCTAssertNil(
+ try VMLayout.instanceStatus(
+ exitCode: 1, stdout: "",
+ stderr: "level=warning msg=\"No instance matching cmsmb found.\"\n"
+ + "level=fatal msg=\"unmatched instances\""))
+ XCTAssertThrowsError(
+ try VMLayout.instanceStatus(
+ exitCode: 1, stdout: "", stderr: "level=fatal msg=\"bufio.Scanner: token too long\""))
+ }
+
/// Lima's user-mode network is not reachable by Time Machine; the vzNAT
/// address is.
func testAddressSkipsLimaUserNetwork() {
diff --git a/vm/README.md b/vm/README.md
index 761f094..493bdf0 100644
--- a/vm/README.md
+++ b/vm/README.md
@@ -16,7 +16,9 @@ tm-meta-sync (every 15 min): /srv/tm-local minus bands/ -> gdrive:CloudMachine/<
Each layer is there for a reason:
- Samba straight on rclone fails with `NT_STATUS_SHARING_VIOLATION`: rclone
- derives inodes from a hash of the path, and they change under a rename.
+ derives inodes from a hash of the path (`mount`) or from a counter of the
+ process (`mount2`, which production uses), and they change under a rename
+ (and, with `mount2`, at every restart of rclone).
So the image's small files stay on local ext4 and only `bands/` goes to Drive.
- macOS frees bands with a hole punch and keeps Samba's sparse flag and Apple
streams in extended attributes; rclone supports neither. `punchfs` fills
@@ -50,10 +52,68 @@ Each layer is there for a reason:
cloudmachine-agent vm-update # after an app upgrade: move the machine onto its template
`install-launchd` (also run by `--install-agents`) installs four agents:
-`app`, `backup-health`, `vm-start` (boots the machine at login) and `vm-guard`
+`app`, `backup-health`, `vm-start` (runs `vm-hold`: boots the machine at login and stays alive, so launchd's SIGTERM at logout/shutdown becomes a clean stop: `systemctl stop smbd; sync` in the guest, then `limactl stop`) and `vm-guard`
(pauses Time Machine when more than 110 GiB waits to be uploaded or the
machine has less than 40 GiB free, resumes below 40 GiB - only what it paused).
+### What `vm-hold` does, and does not
+
+- It starts the machine and stays alive even when `limactl start` fails or
+ times out (a loaded Mac needs longer than Lima's 10 minutes; the machine may
+ come up later). It never exits on its own, so nobody leaves a machine
+ running unattended.
+- On SIGTERM (logout, shutdown, `launchctl bootout`) it cancels a start in
+ progress and stops everything Lima does not report as `Stopped` (also
+ `Broken`, `Installing`, ...): status, quiesce, `limactl stop`, with limits
+ that together fit launchd's `ExitTimeOut` of 120 s. Lima's vz driver gives
+ the guest only 30 s to power off, which is why smbd is stopped and the guest
+ synced first. At the next start it logs how the guest's previous boot ended.
+- `install-launchd` does NOT reload a running `vm-start`: `unload` would stop
+ the machine mid-backup. A new plist takes effect at the next login or boot.
+- Not covered: power loss or a crash of the Mac itself. Lima's vz driver
+ flushes the guest disk with `fsync`, not `F_FULLFSYNC`, so the host's cache
+ can still lose the last seconds. `rclone`'s cache metadata is written
+ non-atomically (`Item._save`); a hard stop in that instant can lose it,
+ which `punchfs`'s `syncfs` narrows to the instant of the write.
+- `VMDurabilityTests` check the templates by text. The machine test
+ (`punchfs/test-punchfs.sh`) and the hard-kill proof are what exercise the
+ behaviour.
+
+## The hard-kill proof (`vm/tests/crash-loop.sh`)
+
+A sparsebundle APFS image (32 MB bands) on the SMB share of a throwaway
+machine, written with `F_FULLFSYNC` after every operation, the machine killed
+with `kill -9` (Lima host agent and Virtualization process) at a random moment,
+then restarted, the image re-attached, `fsck_apfs -n` run and every file that
+was durable before the kill compared with a ledger. Setup and use are in the
+header of the script. Result of 12.11.2026 (the machine was a vz machine with a
+local backend, so this tests the guest side of the chain, not Drive):
+
+| configuration | iterations | `fsck_apfs -n` clean | files equal to the ledger |
+|---|---|---|---|
+| `rclone mount`, no `syncfs` (before #58) | 8 | 8 | 0 (every run lost durable files, up to 100+ files and tens of MB each) |
+| `rclone mount2` + `syncfs` (this PR) | 39 | 39 | 38 (one run: 2 files with wrong content, `fsck` clean, not reproduced in 25 further runs) |
+
+The kills landed with dirty guest pages in 24 of 39 runs of the new
+configuration (median dirty 2 MB before the kill, against 150 MB in the old
+one). `punchfs -d` during a 25 s write showed macOS sending 1000 `FSYNC`
+requests for 14 bands (about 7 per `F_FULLFSYNC`) and every written band had an
+`FSYNC` after its last `WRITE`; nothing in the chain needs a "fsync all fds"
+barrier of its own.
+
+## Deploying a change to the fsync chain
+
+Verify the PROCESSES, not the unit files (a unit file on disk says nothing
+about the binary that runs):
+
+ limactl shell cmsmb -- pgrep -a rclone # "rclone mount2 ...", not "rclone mount ..."
+ limactl shell cmsmb -- pgrep -a punchfs # "-o source=/srv/tm,syncfs=/var/cache/rclone,..."
+ limactl shell cmsmb -- sudo systemctl is-failed punchfs # a binary without -o syncfs ends once with 78
+
+Restart `punchfs` and `rclone-tm` only when no backup runs. smbd stays
+masked (`systemctl mask smbd`, permanent) until all of the above is verified
+and a fresh image has been created.
+
## Updating the machine
Lima copies the template's files into `~/.lima/cmsmb/lima.yaml` when the
diff --git a/vm/cmsmb.yaml b/vm/cmsmb.yaml
index b4d1461..fe4c81c 100644
--- a/vm/cmsmb.yaml
+++ b/vm/cmsmb.yaml
@@ -58,12 +58,11 @@ provision:
- mode: data
path: /etc/systemd/system/smbd.service.d/limits.conf
file: systemd/smbd.service.d/limits.conf
+# punchfs.c, passthrough_helpers.h and test-punchfs.sh as ONE compressed archive
+# (vm/punchfs/pack.sh): Lima's `list --json` breaks past 64 KiB of embedded files.
- mode: data
- path: /opt/cloudmachine/punchfs/punchfs.c
- file: punchfs/punchfs.c
-- mode: data
- path: /opt/cloudmachine/punchfs/passthrough_helpers.h
- file: punchfs/passthrough_helpers.h
+ path: /opt/cloudmachine/punchfs/src.tar.gz.b64
+ file: punchfs/src.tar.gz.b64
- mode: data
path: /usr/local/sbin/smb-throttle
file: smb-throttle.sh
@@ -77,10 +76,6 @@ provision:
- mode: data
path: /etc/systemd/system/tm-meta-sync.timer
file: systemd/tm-meta-sync.timer
-- mode: data
- path: /opt/cloudmachine/punchfs/test-punchfs.sh
- file: punchfs/test-punchfs.sh
- permissions: 755
- mode: system
script: |
#!/bin/bash
@@ -119,7 +114,9 @@ provision:
chown tm:tm /srv/tm-local
printf 'TM_UID=%s\nTM_GID=%s\n' "$(id -u tm)" "$(id -g tm)" > /etc/cloudmachine/tm-ids.env
- # 5. punchfs, rebuilt only when its source changed.
+ # 5. punchfs, rebuilt only when its source changed. The sources arrive as
+ # one archive (see the `file:` entry above); unpack it first.
+ base64 -d /opt/cloudmachine/punchfs/src.tar.gz.b64 | tar -xz --no-same-owner -C /opt/cloudmachine/punchfs
src=/opt/cloudmachine/punchfs/punchfs.c
built=/var/lib/cloudmachine/punchfs.built.c
if [ ! -x /usr/local/bin/punchfs ] || ! cmp -s "$src" "$built"; then
diff --git a/vm/punchfs/README.md b/vm/punchfs/README.md
index 6d71e9d..7d6a270 100644
--- a/vm/punchfs/README.md
+++ b/vm/punchfs/README.md
@@ -17,21 +17,59 @@ between rclone's Google Drive mount (`/srv/tm`) and Samba (`/srv/tm-punch`).
and the Apple metadata streams (`streams_xattr`) in extended attributes,
and rclone has none. Without them macOS treats the band files as not
sparse and refuses to erase a band. punchfs stores each attribute as a
- small file `/srv/tm-xattr//` on the machine's local disk.
- rclone's inode is a hash of the path, so it stays the same for a band.
+ small file `/srv/tm-xattr/p//#xattr#/` on the
+ machine's local disk. The key is the PATH, not the inode: `rclone mount2`
+ hands out inodes from a counter of its process, so they change when rclone
+ restarts (measured: the same file got another number after a restart that
+ looked other files up first) and the sparse flag of a band would be lost
+ or land on another file. `unlink`/`rmdir` remove a path's attributes,
+ `rename` moves them. The old layout (`/srv/tm-xattr/`) cannot be
+ used and is removed when punchfs starts.
Everything else is upstream code, unchanged, apart from a header comment
saying what was modified and when (GPLv2 section 2a).
## Building and testing
+The sources reach the machine as ONE archive, `src.tar.gz.b64`, which the
+provisioning unpacks into `/opt/cloudmachine/punchfs/`: Lima embeds every file
+of the template in the machine's `lima.yaml`, and `limactl list --json` of a
+running machine fails ("bufio.Scanner: token too long") past 64 KiB, which blinds
+`vm-status`, `vm-guard`, `backup-health` and `vm-hold`. After changing
+`punchfs.c`, `passthrough_helpers.h` or `test-punchfs.sh` run `./pack.sh`; a test
+fails while the archive is stale.
+
The machine's provisioning builds it (`vm/cmsmb.yaml`):
gcc -Wall -O2 -D_FILE_OFFSET_BITS=64 -DHAVE_FALLOCATE punchfs.c \
$(pkg-config fuse3 --cflags --libs) -o /usr/local/bin/punchfs
-`test-punchfs.sh` checks both changes on a throwaway rclone mount of a local
-directory, without touching Google Drive:
+### Durable fsync (`-o syncfs=`)
+
+`rclone mount` answers fsync with OK and does nothing, and rclone writes its
+cache metadata (`vfsMeta`, with the list of cached ranges `Rs`) without fsync,
+so APFS barriers inside the Time Machine image were not durable and a hard stop
+of the machine lost writes APFS considered safe. Production therefore uses
+`rclone mount2` (passes fsync to the cache item) and `-o syncfs=/var/cache/rclone`:
+after a successful fsync/fsyncdir punchfs also calls `syncfs()` on that
+directory's filesystem and replies with its errno on failure. Without the
+option nothing changes. (From Linux 5.8 `syncfs` can also return a write error
+of ANOTHER writer on that filesystem, e.g. an upload buffer file; punchfs
+passes it on as EIO, which is the safe direction for Time Machine.)
+
+`punchfs.service` starts punchfs only when the installed binary knows
+`-o syncfs`; an old binary (the unit is copied before the binary is rebuilt, and
+a failed rebuild keeps the old one) ends once with status 78 instead of
+restarting in a loop. `cloudmachine-agent vm-update` rebuilds it.
+
+`test-punchfs.sh` checks all of this: both changes above (attributes also across
+a restart of rclone, a rename and an unlink) and the fsync on a throwaway rclone
+mount of a local directory, without touching Google Drive. The fsync check reads
+the dirty pages of rclone's cache data file from the kernel and the kernel's own
+record (ftrace `ext4:ext4_sync_fs`) that punchfs called syncfs, so neither another
+writer on a live machine nor the page cache can make it pass or fail wrongly. Run
+it once with `PUNCHFS_SYNCFS=0` (must FAIL the syncfs check) and with
+`RCLONE_MOUNT_CMD=mount PUNCHFS_SYNCFS=0` (must FAIL both) after changing it:
limactl shell cmsmb sudo /opt/cloudmachine/punchfs/test-punchfs.sh
diff --git a/vm/punchfs/pack.sh b/vm/punchfs/pack.sh
new file mode 100755
index 0000000..e65f50b
--- /dev/null
+++ b/vm/punchfs/pack.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+# Regenerates vm/punchfs/src.tar.gz.b64 from the punchfs sources.
+#
+# Lima embeds every `file:` of vm/cmsmb.yaml in the machine's lima.yaml, and
+# `limactl list --json` of a running machine fails ("bufio.Scanner: token too
+# long") once that JSON passes 64 KiB, which blinds vm-status, vm-guard,
+# backup-health and vm-hold. The sources are most of it, so they ship as one
+# compressed archive that the machine's provisioning unpacks into
+# /opt/cloudmachine/punchfs/ before it builds. A test keeps the archive equal
+# to the files: run this after changing any of them.
+set -euo pipefail
+cd "$(dirname "$0")"
+COPYFILE_DISABLE=1 tar --no-xattrs -cz punchfs.c passthrough_helpers.h test-punchfs.sh | base64 > src.tar.gz.b64
+echo "src.tar.gz.b64: $(wc -c < src.tar.gz.b64) bytes"
diff --git a/vm/punchfs/punchfs.c b/vm/punchfs/punchfs.c
index 78ec009..03a488e 100644
--- a/vm/punchfs/punchfs.c
+++ b/vm/punchfs/punchfs.c
@@ -54,6 +54,8 @@
#include
#include
#include
+#include
+#include
#include
#include
#include
@@ -98,6 +100,11 @@ struct lo_data {
int flock;
int xattr;
const char *source;
+ char *source_real; /* realpath(source), to turn an fd's path into a path below source */
+ /* -o syncfs=: after fsync, also syncfs() the filesystem of
+ * (rclone's cache dir, whose metadata files are written without fsync) */
+ const char *syncfs;
+ int syncfs_fd;
double timeout;
int cache;
int timeout_set;
@@ -111,6 +118,8 @@ static const struct fuse_opt lo_opts[] = {
offsetof(struct lo_data, writeback), 0 },
{ "source=%s",
offsetof(struct lo_data, source), 0 },
+ { "syncfs=%s",
+ offsetof(struct lo_data, syncfs), 0 },
{ "flock",
offsetof(struct lo_data, flock), 1 },
{ "no_flock",
@@ -136,6 +145,7 @@ static const struct fuse_opt lo_opts[] = {
static void passthrough_ll_help(void)
{
printf(
+" -o syncfs= After fsync also syncfs() the filesystem of \n"
" -o writeback Enable writeback\n"
" -o no_writeback Disable write back\n"
" -o source=/home/dir Source directory to be mounted\n"
@@ -482,11 +492,23 @@ static void lo_link(fuse_req_t req, fuse_ino_t ino, fuse_ino_t parent,
fuse_reply_err(req, saverr);
}
+/* extended attributes by path, see below */
+static int xmirror_child(struct lo_data *lo, struct lo_inode *parent, const char *name,
+ char *buf, size_t n);
+static void xattr_forget(const char *mirror);
+static void xattr_move(const char *from, const char *to);
+
static void lo_rmdir(fuse_req_t req, fuse_ino_t parent, const char *name)
{
int res;
+ char mirror[PATH_MAX + 320];
+ int have = xmirror_child(lo_data(req), lo_inode(req, parent), name,
+ mirror, sizeof(mirror)) == 0;
+
res = unlinkat(lo_fd(req, parent), name, AT_REMOVEDIR);
+ if (res == 0 && have)
+ xattr_forget(mirror);
fuse_reply_err(req, res == -1 ? errno : 0);
}
@@ -502,8 +524,16 @@ static void lo_rename(fuse_req_t req, fuse_ino_t parent, const char *name,
return;
}
+ char from[PATH_MAX + 320], to[PATH_MAX + 320];
+ int have = xmirror_child(lo_data(req), lo_inode(req, parent), name,
+ from, sizeof(from)) == 0
+ && xmirror_child(lo_data(req), lo_inode(req, newparent), newname,
+ to, sizeof(to)) == 0;
+
res = renameat(lo_fd(req, parent), name,
lo_fd(req, newparent), newname);
+ if (res == 0 && have)
+ xattr_move(from, to);
fuse_reply_err(req, res == -1 ? errno : 0);
}
@@ -512,7 +542,13 @@ static void lo_unlink(fuse_req_t req, fuse_ino_t parent, const char *name)
{
int res;
+ char mirror[PATH_MAX + 320];
+ int have = xmirror_child(lo_data(req), lo_inode(req, parent), name,
+ mirror, sizeof(mirror)) == 0;
+
res = unlinkat(lo_fd(req, parent), name, 0);
+ if (res == 0 && have)
+ xattr_forget(mirror);
fuse_reply_err(req, res == -1 ? errno : 0);
}
@@ -790,6 +826,16 @@ static void lo_create(fuse_req_t req, fuse_ino_t parent, const char *name,
fuse_reply_create(req, &e, fi);
}
+/* 0 or an errno. Without -o syncfs this does nothing. */
+static int lo_syncfs(fuse_req_t req)
+{
+ struct lo_data *lo = lo_data(req);
+
+ if (lo->syncfs_fd < 0)
+ return 0;
+ return syncfs(lo->syncfs_fd) == -1 ? errno : 0;
+}
+
static void lo_fsyncdir(fuse_req_t req, fuse_ino_t ino, int datasync,
struct fuse_file_info *fi)
{
@@ -800,7 +846,11 @@ static void lo_fsyncdir(fuse_req_t req, fuse_ino_t ino, int datasync,
res = fdatasync(fd);
else
res = fsync(fd);
- fuse_reply_err(req, res == -1 ? errno : 0);
+ if (res == -1) {
+ fuse_reply_err(req, errno);
+ return;
+ }
+ fuse_reply_err(req, lo_syncfs(req));
}
static void lo_open(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi)
@@ -867,7 +917,11 @@ static void lo_fsync(fuse_req_t req, fuse_ino_t ino, int datasync,
res = fdatasync(fi->fh);
else
res = fsync(fi->fh);
- fuse_reply_err(req, res == -1 ? errno : 0);
+ if (res == -1) {
+ fuse_reply_err(req, errno);
+ return;
+ }
+ fuse_reply_err(req, lo_syncfs(req));
}
static void lo_read(fuse_req_t req, fuse_ino_t ino, size_t size,
@@ -976,13 +1030,169 @@ static void lo_flock(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi,
/* CloudMachine: rclone has no extended attributes, but Samba keeps the
* sparse flag (user.DOSATTRIB) and the Apple metadata streams in them; without
* them macOS treats the band files as non-sparse and refuses to erase a band.
- * Attributes live in side files on local disk: XDIR//. rclone's
- * inode is a hash of the path, so it is stable for a given band. */
+ * Attributes live in side files on local disk, keyed by the PATH of the file
+ * relative to -o source: XROOT//#xattr#/.
+ *
+ * Not by inode: rclone's mount2 (go-fuse) gives inodes from a counter of the
+ * process (vfs inodeCount), so they change at every restart of rclone and the
+ * attributes of a band would be lost or end up on another file. The path is
+ * what stays. unlink/rmdir remove the attributes of the path, rename moves
+ * them (the mirror tree makes a directory rename move its children's too).
+ * The old layout (XDIR/) is wrong-keyed and is removed at start. */
#define XDIR "/srv/tm-xattr"
+#define XROOT XDIR "/p"
+#define XLEAF "#xattr#"
-static void xdir_of(char *buf, size_t n, struct lo_inode *in)
+/* 0 or an errno. rel gets the path of the inode below the source directory
+ * ("" for the root), from the kernel's idea of the O_PATH descriptor. */
+static int rel_path_of(struct lo_data *lo, struct lo_inode *in, char *rel, size_t n)
{
- snprintf(buf, n, XDIR "/%llx", (unsigned long long)in->ino);
+ char link[64], full[PATH_MAX + 16];
+ size_t sl = strlen(lo->source_real);
+ const char *rest;
+ ssize_t r;
+
+ snprintf(link, sizeof(link), "/proc/self/fd/%d", in->fd);
+ r = readlink(link, full, sizeof(full) - 1);
+ if (r < 0)
+ return errno;
+ full[r] = 0;
+ /* The kernel appends " (deleted)" for an unlinked file; a file that is
+ * really called that exists under that very path. */
+ if (r >= 10 && !strcmp(full + r - 10, " (deleted)")) {
+ struct stat st;
+ if (lstat(full, &st) == -1)
+ return ENOENT;
+ }
+ if (sl == 1)
+ sl = 0; /* source is "/" */
+ if (strncmp(full, lo->source_real, sl) || (full[sl] && full[sl] != '/'))
+ return EIO;
+ rest = full + sl;
+ while (*rest == '/')
+ rest++;
+ if (strlen(rest) >= n)
+ return ENAMETOOLONG;
+ strcpy(rel, rest);
+ return 0;
+}
+
+/* Mirror directory (no XLEAF) of rel, and of the name below a parent. */
+static int xmirror(char *buf, size_t n, const char *rel)
+{
+ int w = snprintf(buf, n, "%s%s%s", XROOT, *rel ? "/" : "", rel);
+ return (w < 0 || (size_t)w >= n) ? ENAMETOOLONG : 0;
+}
+
+static int xmirror_child(struct lo_data *lo, struct lo_inode *parent, const char *name,
+ char *buf, size_t n)
+{
+ char rel[PATH_MAX];
+ char full[PATH_MAX + 256];
+ int err = rel_path_of(lo, parent, rel, sizeof(rel));
+
+ if (err)
+ return err;
+ /* the leaf directory's name would mirror onto its parent's attributes */
+ if (!strcmp(name, XLEAF))
+ return EINVAL;
+ snprintf(full, sizeof(full), "%s%s%s", rel, *rel ? "/" : "", name);
+ return xmirror(buf, n, full);
+}
+
+/* The directory that holds the attributes of an inode. */
+static int xdir_of(struct lo_data *lo, struct lo_inode *in, char *buf, size_t n)
+{
+ char rel[PATH_MAX], mirror[PATH_MAX + 64];
+ int err = rel_path_of(lo, in, rel, sizeof(rel));
+
+ if (!err)
+ err = xmirror(mirror, sizeof(mirror), rel);
+ if (!err && snprintf(buf, n, "%s/" XLEAF, mirror) >= (int)n)
+ err = ENAMETOOLONG;
+ return err;
+}
+
+static int mkdirs(const char *path)
+{
+ char tmp[PATH_MAX + 64];
+ char *p;
+
+ if (strlen(path) >= sizeof(tmp))
+ return ENAMETOOLONG;
+ strcpy(tmp, path);
+ for (p = tmp + 1; *p; p++) {
+ if (*p != '/')
+ continue;
+ *p = 0;
+ if (mkdir(tmp, 0700) == -1 && errno != EEXIST)
+ return errno;
+ *p = '/';
+ }
+ if (mkdir(tmp, 0700) == -1 && errno != EEXIST)
+ return errno;
+ return 0;
+}
+
+static int rm_node(const char *path, const struct stat *sb, int flag, struct FTW *ftw)
+{
+ (void) sb; (void) flag; (void) ftw;
+ return remove(path);
+}
+
+/* Best effort: attributes of a path that is gone. */
+static void xattr_forget(const char *mirror)
+{
+ nftw(mirror, rm_node, 16, FTW_DEPTH | FTW_PHYS);
+}
+
+/* Best effort: after a rename the attributes follow the path. */
+static void xattr_move(const char *from, const char *to)
+{
+ char parent[PATH_MAX + 320], *slash;
+ struct stat st;
+
+ /* whatever the target had is gone, also when the source has none */
+ xattr_forget(to);
+ if (lstat(from, &st) == -1)
+ return;
+ snprintf(parent, sizeof(parent), "%s", to);
+ slash = strrchr(parent, '/');
+ if (slash && slash != parent) {
+ *slash = 0;
+ if (mkdirs(parent))
+ return;
+ }
+ rename(from, to);
+}
+
+/* At start: the pre-path layout keyed by inode (XDIR/) cannot be used and
+ * would only be read by a file that happens to get the same number. */
+static void xattr_drop_old_layout(void)
+{
+ DIR *d = opendir(XDIR);
+ struct dirent *e;
+ int dropped = 0;
+
+ if (!d)
+ return;
+ while ((e = readdir(d))) {
+ char path[PATH_MAX];
+ const char *c;
+ if (e->d_name[0] == '.' || !strcmp(e->d_name, "p"))
+ continue;
+ for (c = e->d_name; *c && isxdigit((unsigned char)*c); c++)
+ ;
+ if (*c)
+ continue;
+ snprintf(path, sizeof(path), XDIR "/%s", e->d_name);
+ xattr_forget(path);
+ dropped++;
+ }
+ closedir(d);
+ if (dropped)
+ fuse_log(FUSE_LOG_INFO, "removed %d extended attribute sets of the old inode-keyed layout\n",
+ dropped);
}
static int xname_ok(const char *name)
@@ -993,14 +1203,15 @@ static int xname_ok(const char *name)
static void lo_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name,
size_t size)
{
- char dir[256], path[1024];
+ char dir[PATH_MAX + 64], path[PATH_MAX + 512];
struct stat st;
char *value;
ssize_t r;
- int fd;
+ int fd, err;
if (!xname_ok(name)) { fuse_reply_err(req, ENODATA); return; }
- xdir_of(dir, sizeof(dir), lo_inode(req, ino));
+ err = xdir_of(lo_data(req), lo_inode(req, ino), dir, sizeof(dir));
+ if (err) { fuse_reply_err(req, err); return; }
snprintf(path, sizeof(path), "%s/%s", dir, name);
fd = open(path, O_RDONLY);
if (fd == -1) { fuse_reply_err(req, errno == ENOENT ? ENODATA : errno); return; }
@@ -1018,13 +1229,14 @@ static void lo_getxattr(fuse_req_t req, fuse_ino_t ino, const char *name,
static void lo_listxattr(fuse_req_t req, fuse_ino_t ino, size_t size)
{
- char dir[256];
+ char dir[PATH_MAX + 64];
DIR *d;
struct dirent *e;
char *buf = NULL;
size_t len = 0;
+ int err = xdir_of(lo_data(req), lo_inode(req, ino), dir, sizeof(dir));
- xdir_of(dir, sizeof(dir), lo_inode(req, ino));
+ if (err) { fuse_reply_err(req, err); return; }
d = opendir(dir);
if (d) {
while ((e = readdir(d))) {
@@ -1050,13 +1262,16 @@ static void lo_listxattr(fuse_req_t req, fuse_ino_t ino, size_t size)
static void lo_setxattr(fuse_req_t req, fuse_ino_t ino, const char *name,
const char *value, size_t size, int flags)
{
- char dir[256], path[1024], tmp[1100];
- int fd, exists;
+ char dir[PATH_MAX + 64], path[PATH_MAX + 512], tmp[PATH_MAX + 640];
+ int fd, exists, err;
if (!xname_ok(name)) { fuse_reply_err(req, EINVAL); return; }
- xdir_of(dir, sizeof(dir), lo_inode(req, ino));
- if (mkdir(XDIR, 0700) == -1 && errno != EEXIST) { fuse_reply_err(req, errno); return; }
- if (mkdir(dir, 0700) == -1 && errno != EEXIST) { fuse_reply_err(req, errno); return; }
+ err = xdir_of(lo_data(req), lo_inode(req, ino), dir, sizeof(dir));
+ if (err) { fuse_reply_err(req, err); return; }
+ /* a file named like the leaf directory would mix up with its attributes */
+ if (strstr(dir, "/" XLEAF "/")) { fuse_reply_err(req, EINVAL); return; }
+ err = mkdirs(dir);
+ if (err) { fuse_reply_err(req, err); return; }
snprintf(path, sizeof(path), "%s/%s", dir, name);
exists = access(path, F_OK) == 0;
if ((flags & XATTR_CREATE) && exists) { fuse_reply_err(req, EEXIST); return; }
@@ -1072,10 +1287,12 @@ static void lo_setxattr(fuse_req_t req, fuse_ino_t ino, const char *name,
static void lo_removexattr(fuse_req_t req, fuse_ino_t ino, const char *name)
{
- char dir[256], path[1024];
+ char dir[PATH_MAX + 64], path[PATH_MAX + 512];
+ int err;
if (!xname_ok(name)) { fuse_reply_err(req, ENODATA); return; }
- xdir_of(dir, sizeof(dir), lo_inode(req, ino));
+ err = xdir_of(lo_data(req), lo_inode(req, ino), dir, sizeof(dir));
+ if (err) { fuse_reply_err(req, err); return; }
snprintf(path, sizeof(path), "%s/%s", dir, name);
if (unlink(path) == -1) { fuse_reply_err(req, errno == ENOENT ? ENODATA : errno); return; }
fuse_reply_err(req, 0);
@@ -1167,7 +1384,8 @@ int main(int argc, char *argv[])
struct fuse_cmdline_opts opts;
struct fuse_loop_config config;
struct lo_data lo = { .debug = 0,
- .writeback = 0 };
+ .writeback = 0,
+ .syncfs_fd = -1 };
int ret = -1;
/* Don't mask creation mode, kernel already did that */
@@ -1224,6 +1442,12 @@ int main(int argc, char *argv[])
} else {
lo.source = "/";
}
+ lo.source_real = realpath(lo.source, NULL);
+ if (!lo.source_real) {
+ fuse_log(FUSE_LOG_ERR, "realpath(\"%s\"): %m\n", lo.source);
+ exit(1);
+ }
+ xattr_drop_old_layout();
if (!lo.timeout_set) {
switch (lo.cache) {
case CACHE_NEVER:
@@ -1251,6 +1475,15 @@ int main(int argc, char *argv[])
exit(1);
}
+ if (lo.syncfs) {
+ lo.syncfs_fd = open(lo.syncfs, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
+ if (lo.syncfs_fd == -1) {
+ fuse_log(FUSE_LOG_ERR, "open(\"%s\") for syncfs: %m\n",
+ lo.syncfs);
+ exit(1);
+ }
+ }
+
se = fuse_session_new(&args, &lo_oper, sizeof(lo_oper), &lo);
if (se == NULL)
goto err_out1;
@@ -1283,6 +1516,8 @@ int main(int argc, char *argv[])
if (lo.root.fd >= 0)
close(lo.root.fd);
+ if (lo.syncfs_fd >= 0)
+ close(lo.syncfs_fd);
return ret ? 1 : 0;
}
diff --git a/vm/punchfs/src.tar.gz.b64 b/vm/punchfs/src.tar.gz.b64
new file mode 100644
index 0000000..feac1b9
--- /dev/null
+++ b/vm/punchfs/src.tar.gz.b64
@@ -0,0 +1 @@
+H4sIAMGS9WoAA+19bVsbR7Lo+Yp+RVu2QcJ6xS/JBYs9MghHTwBxEcTJtX2VQRqhWSSNdmYEZhOf337rrXt6RiMknOzus89d4oDU011dXV1VXdVdXTObT/ujYVjp/9c/7qdWq7159Urh3+/evKa/tR3+XqvV669ewsP6653vXr+s116+eqNq9Vc7O2/+S9X+gTiZn3kYOQGgMrly+3+/n4Y3XnY9qDYcPgCHBlNT5u+/yU91O6fU0WW3tauOvLEb3oeRO1HeVF2GbhDOnL4Lzw/82X3gXY8iVTgoqh2YtDL8+k6pE+9m7Ieq+3c3cAeeejuh7/8d8vfKaL6fg+YXIy9Us8C/DpyJ6jtTdeWqgRdGgXc1j9yBmk8HbqCikasiN5iEyh/Sl/enl+r92fHtTgVgdF2XCoeApDronP3SPn1fyW1XczkawYzZeFeNvavhPHTVy0r9VaWm3C/OZDZ2qzMnDKNR4M+vR73xuNIvqYk/8IYe9D70A3Uw9ueDE6c/8qY43sK5O3UO/IFbREp0+pF/BQju1HbeVNTByJleuwPoye8NnfHY7zsRoAWfQnXl9G9U5Ku7wIu86TVAAkL4YUk50wFh736JXBjsoOxEMnqgB7a8cd2ZMoUhdht6Ax5uiON/50cj1ae+Q0AZ+r1Tjur7k4k7jRTyMPao8vZIdvN76rzVPDxpVSYDBBnBRACsgRe4MKbgXoXOfajuRveaktvqv7HHnAKabvO8EcGFLSZeEPhBKEOBGcQu7Qojzw2coD+6lzlEKPyoFOPoRAQg8P3Iblzh/vAfThmOC2bn6h5h/BWkVOVxEhGATGReAelU4P5t7oZRiHRHsH0/CNxw5k8HWBVYISgLG28jc/TVEFgl8vxpWFHtKVSfRoED0KG5xSSVPs2ZzTZDKCwhlGhk4+ggLOyHyQLzUh67t+5YNc/a0AMgNnMD4LGJM+3DUEf+fDxAKFfMz2MXO79yBsqZ+DQ2LA9ct6SAExS0AloCBOomVE7gqqkfIQCLSjSSGZK3Px87QUl5EdIRKqqZDzS7AiLD+AJ34t8KT6mCHyAUzQueGxYBp76DwsN0BPabun03DB3gFGg+cIcgBU4/mjtjhuWMEcR8GnnjWDilY0B6CvMH+E98QPoOxw3DuXGFe0gu1WTeHyEMqgPcPBt7KE+DirDgh5E7JXFySbb6xNjX2AdIKAxroApl36ow8QEb6BupVUQ6EKVApUzH9wlaUBP8wNRAJXA38vojwo6gaPZRIydEKIELkwQzMfEACrGP5lebbElcXWJPIHGopeHGDabAHNAxMIAjyADEucsdGHZGjJC/ZbYMcgjEYkt+ztIHqhUpHwKvhMN75Q2RD6BwK2LY8RRNmY9dVr3jey3HDKioqX8AE4L177xotCtl+HPd76vyB8QurVfVr7Ob6zII1dADiECUl6pc7g/HznUIH0D+wl9V2U81E8hPn6quPw/6wntPn2LhJ2/aH8/ha7oneAgq6ykwJWg61YPFotftXJ4ftEwZrmo9/P+n1nm33TlVL19BAw3vLWLXA3Elaa2M9q1H8ynotkGyDAoA/YUyz18ogu4Xyq58f5wuDIDqybKxN/GiMFmGAjqNkmX96H7mJouG0V2yAMjlBql2bhBMU/h60wiBpTqdoQpy0hS4D6vIJIulX3DlwuK4PG/P18gdg+BAH3lcZNQHl/RY4JbnpM9nPmDhBqTCgQHUr0CbeT/CFdabAiv8SrpYSmH9wgdAltmvymUNCHoRl0yoqqVy4jpTlDlYa+5Qy6CONfKGIMIINc4cOp5FQS9SoMvmvO46xLfYMxUHHrZCsK6su7yyQ1UQLVH/IOn9m5BXhj4uPVQKnfdFgiJvAjCAY5+CVDJ/Dgo95NqDXq+oNjeV+ab21Sv1++/KfG80oABqSMFJ+7RzjtUa6g21fCLwoEJ/Np6H+H+u10XU+j1mg0Lo/d31h4V4ZEVsL6WGCsVSbkPxT94iQuT7KpyguMMEjXzQ42m6PckX93JP3XHo5mTueqaK30Py97BdLwbaG8wnk/ue1P6U2/gNdFHoXdM4VrTdzW1sbBTWHVNR/UXV1a4q14t76iuiCfMzzOVSTKZ+y22ki7anYLDtKWBZsF4jWCfJIIHn5f0JmGlfcEIXG80C93ZVI8BODQd7+AEpDL/h88C9hc/wGz4j/m9e9VBzD/u4HK2ACAPLudP5BIdx0Dz4odU7bYHeK5lvnfOT5rH52jz+0PylW6Jm8QAGTuQgABH/HkEHHOjvHmM9cK/m1/LZLHbyfQjWsP5MOgE+gziA8IHlGqjtkNQ7Flpfe9DVmEaHH2ZONCrwA1i/0aabByBJU6DWFphS8BSIBcUOf75y0RDm+kRYAAPrS3g/7Q/DBirP/V3lDEG7qCEWgik+DvXzQtEsibKCgtVKbUAMtlUh6I/9qQvd8lIOD0pgJPiw0k/cyCFi8eqM2gxpEaGxAmulD5YbdVcknBIkoJ6FRvylR4ww8OdXoiiguVSgjuWzPOmFLj5Ncx1a02uwSEhaQTFGAoMECEwmBAZ/wo+fVQO54DeVNxOcJ83gD4fQO0hXkmVKMSPAnNXV1xI1Bs7+lvY13Z5ntfE8XNHYcEvckqd/dUuqZ7ckDl7RiuqkBvqYdqYzEpEVjahOqrPHtDOdCfs0no+HK5pKzazG67VEFrVRJi5uTMHOWoU11YSmlgZLAnHmkf84GKT3UkDGd+D3PgoM60sEk9sgu7JzdtFrnR7aQnXre4OUmUqWTwEfFHOoV8Hki4aFXB5X2JSaknVXNWNltZ6u+jTNG4Cx9xH/tMhRih/Z9W0R1fUPvTBuoNItRCirI3/iVqF7LBarPd5WYO+GvCh3YLcmEVDJH8GPHqVwW6iucVuoTOyeDZgepQAvVNeAFyprzq9XaqbygXih8iyreq1aN9UvuAw9RWA1u7YlGAuY0LPFyigAMdpN/Mae8NIGxOw26mAHUxnUBavtq+HflB2wLR/YwAK/lIyRvxEjBy4ty4V0k6IyldF1pubYJrsbsZf0p1RHJdsOh9/UMVjPBfiMNjGJ4Xmnc9FrHxbBIhScNjXa2G95n9bF3AaZphsZeAsORWVZjGyOxRjj6ou7fYN1MJQuzKCoHj4s7+M6H0NFl5AohwbVA0ROjodqqycNVbNhkeqhLr2I1I3a1vQHLUc2vb3Yw/I/hbpDX23jR+ptcfbBCkhNMJBJgwW1B5NRwObl/b4zIxbc5Ek5aJ71Wj+fdc4vet3LM/yL88NV78BnUr83llVksKqAhkusmTY32S9Z2tuH8/ZF613z4Mceqesimi8GDpEMMdgQl/+6QA2PO+97h613l+9LKi+028XtLe/Woc0fW2MCBy8dQapzqPk1HgSrMPDSluJ+dNyBpvir++fgzV3ir3AF5lbXhPUiQ4FHHQX+fYqnvpVh7kboC9PgUC4r6F4hL2/qEiZASjq3FdVrqEQ7HFYSUIPqlff102HgugX8UlwyuGuw4kHdr5BqFCD2im0RwoVYRGjosWqaYvMwts2xN3U1H+4toZUt2EgdthNgiccvuGXYADMAYDhRgbWPUSUw8fmS2gTYJdUEO+Tk7OKX3lnz4gf1O37v/nJy3D79EWyfo87xcecDjh+ZimA20BG2VKF0yiSYje97bhBwT7RbRJhZT4lg9Ji7xynQBmOWRgrXo7GyabaNLbTeUor0760z9galNaYgdGA1DbSPCd5Qf+pM3I9vXn3O8Jq2+U8jQ12L1+UNB/CYnrECj+eZiUqIaXnuti56zYuL895J59DSQoAefdaz2h9NfFhMPIA4KtGxT3k/jHoTPHFCzv2qcL3iJqEYjXokMPdV/FwN3fGwOhxUn3vAC4AmtZQeuIO4SUYXglqCJzaufdqCi3pMwq/JIRaSY7xsHwK/Jcvew1rMY517A5hb+I1aIZtIAKCo/oL9Gvyw/i6uxQNch8t1RPWaIF0/BOn9IqRrhnQdQ8pZ9PfvpiBWQDUWpTnyFlSNZV2tK1iPpmIS8277/zzAKMCuUzwPWeQV3Ob6c3jF9JHBLnEvf5xdmhftk9YCw5xgaTGh91GdhDO3r6Lbjzufad7gU+1zJboFddJXZPtQWT2jjOtNufASofc6J+2LRIuMpzK+7Eki1GHWP9CoszuBp9gHTcYKSDEUaG1oDWpzsgKPkzQeGcNZD48TG496Ao+JjQcq1pgd5zg109BwY3RbNH19OwMKVJBHELGjw4MPhyUVA4huS6r2GB60DWe9wmutDuuOhyua1N+FFYGWC8CCVru9xFpnVkKutNKLGXrTwYJLNE4vbmGUMp/0tnHW+gRDAdxOL4+PcUqS+7Jo4RU2zRYf0ggPDQuzBVNJzRJWFn5vqBlbSrHygQJxrcKIOAG/geU6Q1P01iqHb6Km5HQBauhtarUvs7XBqM/057jKixdUdgVDuZF5/Zoe23yaMTqZ1wA3QBe8s4EPBPFv5rOHTI2Zg4dpJWXvyBKbkd63jQuoFtz3oL4zUduusS6m7l3SCEjaHGvYeUtsEJzeiTsBc6kATF8r6XMM6BsbgfGBjNwTU0tmON4shueM8ZIKuQ3CHIrxXD5pUTJRwKhkgevoBa+zYEAKDC17SdFL26xUWSxVQf/brdV0V5bdRlJn0zjuT4Oi2syy/bEfuowbsakmC5ka1lqaeapTkhMh1I5GcbROOyetE7IEBKk+hQUV6mYSU8CKxm54wt8X9ZiGZglWQ9X34mJCWnOjLiTxVTL4Ckuw9Zik2H5MR03wfKVe2cDRK1uFEOnI6cJHsdPFLphUF842CFhumlUqlenYTLoyVQ2ENdTDVxIDpkFqN4flix1r2XBBTkHCL/etlXo+HnvV56Eq79NH8KZJURTMSeUYQ3bwV9GoFhaizCqMXXEvXqJqD69Fscw90YKQ4l+BoxXQov/17TrRXqUWVKI+lNLi+DjKxnhx543neXV23p6/eaXyTMHG87CoyW1TlojHRErofEuR6SnYdLX449qt8Vn0cYvx9qBdAwcrjq6b6dlOboCveuH9ZOxNb9YgsHEtFhYfjEV08ZgV/pQUnwIH8Ce7CXaX2nBYsgaJ5sJd+rSHKzp/78EZ1hqdR3oXOLOZGxQAHKofTWZEpySoI9I0Q5mcnKnQE7XXn1KxyFKA/uUCXvEWd0zSnCQyy5K/u7bBmWC79djtcdxGPLXA1xmzEM91iSzTJWgCn3yrKUaIW2iujVq31z46bJ+DYcFI1h7AcJnkpmXN2ohaV2uui+rx6Y+EI4l0pvZeoVrYpVk295m4LWxTfoOxumzD7KHVIms7ztZbxvzdTNi/S0CyUVx52CaurDCJ1/RWtcHFsosKEadlibu61K7WXGRZvI+wd5OmtUFJm9eVP9e6XmkPpmxT8ujWsNBIP/7rrbOltR6vv791F6G6be4C2GH/V/cU1gR1XVdim7artqP7hUPwe/2RN35wr8FyXB70efk7beWjxIEOmQKGthqiI/He0A+uQThtIIxKZm2MM0/UHQb+JIlCRIRO6btg8o2rRlK1ibZhBD+iMPROmj+rF+rlTk3rnRHMBkxYkqBJNzLTZLJ4aEN6MMpKKFJEyarFYguywApjudMNknreOun81ILVKyWiNdyDQWxRHhKTYSYgm9+MiKu/MF+qXdqbWVxmABXA4g+t1nZl8E4y67t3ySZGBDl20LkO09NIW5D0gJzyrFG22qc/NY/JSdWCSZuA1CeyXXr+MarwH8kTzOo6OBW+CD/AU5jI9WEbKhbps9VD5Bv4IEVpbuPJfIjbEIr1MKOj1SxIEs5DjRZV5uNZkGXk/0fRr/2T5X0+hWVbRrfWAkJ/SspEJE9NJE68fWXWxNW2g+wZp7e39hu08ixse5WhfM/uLn6WGS2wuFNnb0bxbpO1e2XtTC1sX2XuSMVgV9s7sluTsCA5LIEJh3CsPce19rgWRYe5o+dPV0VQ2XPITvbyQI4/4gdk2XKx/l5izfEoyJLjX+WVRp0Ozlj20OaV5RU1KWTdsMUDxUG4P661bAK+jfjJ+YvPqSysLFmfSqUH0ADuGUfeAjJi3MW0SDhr0pTnn7/EazHFo9DJkkfnrMpTb5XcSfBevEDxzxiFQPnofa6wlxp//4axoVys4QwTzrQAgEVra/+61v2ySOilkqEui7AxZjEoePhc/MOBNFZjG+xaUFqnzZPWRadz3Dl9T8BwiAAMD2+3PtW2UsuCoRg1xm6ErkbYvWCGUon7JdsD6/SRb5ypbXJ8oNgfDoG6HCi9Zwc8p2Bty4fCw0E6GbGj1DoZgslHzFmsgAdYK7wEO6LogVAhF5dT+wTncZqQ0dangXgQk9sYPHz8g00MF9FJGm1PZbjgS87qhDcreTynOz/snB7/osEtO5lDfxQQK+8P8NR3ODD0GxhE+Fk2MqY5M4BEWgzkvNE6nOYZS3n3g704FJJv0UA3dkg9OfmePCSm5e3qelI5INKiV7wsH3zq79K5gG854bF7ToO0omys8xRZou21eSCHSUtE2U/uy5KGDHsDH3RfgH/gXyHTRhXGp/0vDP0Asa1s0dU/KqpzEUiy+v13tjGTT7gyleyYusVMdTnwNSlXiwkvDPhHzhtseX9AjEo0dLxsmLYiWC5EdFAh8JyZrQbzmZQO9x+4E8VKMXmyJArRYzaExgsCprn4Ceo4muP0uSyxM80cTyzKAaHBDYWxgSUMl4tl6bq0i03yURKKEJ8ssP9GQkK0rsTOJOa1LiB5tHhnlYcq2hWtSvhENmOKd0wszhPdqwRfsI0vkU82SnrqCW8Oxshqr8/GQKWo35SqVlULaUSPNpLbWRsbKSpuxIFn3HI6wAspwASuM+H6HN+huG5Ofn0VGxvGqojehBDg2ZPR44hTT5gIYj0y5/LGrhcPn+JXkBUlDumBjWiuvCCyJKaaMq4Vyryw9Sx1NirWKXsKZW2ZWpUmbC/btfDitXr7VtV3uPJXws4O6Ft61Mk2onUoFk9nMXu6bJLRPid3lsvxM2RHCvGCwQL39NgIwDEDVcVvRRdzYjvBqC5wu15mdDEe0Yp7CiO+VYg0eZhm61GLifUA5suRxr2nzTBK4S1cpCHuYzuZaykX+uHliyLzdYbda+qVVJ0nxg502kA3T71o2BoAVV/ZKlmtX2Jt8RXz6yjFbAKSQBO+S2UglW1KwhLgFVZnysxQwvvwmIOHMmKgN+SMKV3IEB/gtgOmi6Db8QMNBnOQIDoeWK73gEBZ+Qj2zoN5vnO3BvBwoOYzuvqq7gJ2rZBXNQByFjilBSX/EFh0PR/vzjpjVFn3nKLGRd08dIOK6voAXsOQ1fOOr/Rv3WKWiPGYb7rqDCROxLkEKtRI5AEXTVpdeHkp8lWS5cEARE+KB0jVA7SMMc1LD1ACmYTbkO1g2dkL/ss3rMZy7rjmiqwdSmv1j5UFgTQwcPlethdMLUnsH287/INQrS9BFbPnuH/cGVhtu9hmCJuMiQU2thuz+Cqb0H0YePQHN90TR/mrHR4KFnuEk/Pnh/3IoL8p7Get8EUhDYcoU9oZDGQ8OG81L4pqU/1PHNTIgQxZ3tOjrsZor2f4gKtDF5m1p8PXZHueL16OidRe6SNhkrCe1FD1f1YklEyZHICKA8ZHmDWFSYqmTI6K+iCpD8zNZk6IMvBdSgmFd2YrqfNMisOg+80ZFyDXujWlKWdyKai3IHDxLNb24ig57ilRu7i4ZZ65t4YN1lE0lCID8KNkE2lT9KFLYyyfaPN4wZCPI0T9aCWT0EHk0ko/PFoKENBF4tCaa68cPRA/SO5iLT1c03yeOFvLrBlPJOmBZRs3f8KuDQma2eNL3+taySmP0VaEMaCVVFSkVhrPB0ZTycUCUTjUF2Y5YmsokQ6spPN/TRwwvtBsSib9csH8IrvjDvOezXUKRp1MjaCVyXKjJDIZF2VtzbcJmq95cMB30WCmO70P57RjxPMd12ugXpSae4lHvzdon+nDueyOLx0XtD87a50e4i33kTMdjDkBCZpzPOQKjYuSM5FBDCZp5E+8vhfdg6vlYXI8k58MqcO5DsUkpLaZucow/QdbfnFXYO56A9fROQixrcv+qZWkDs1cp9+fB5i+UbLUFSmRHiktSVDHCZHAMqYMhFDfnUXUXUU1OWld4F37gT8P7dQMiD9etqXGlHhPZ7HhLSq0ydefQaJrsZg1ZfTIDmUi0zQriim1h0imil5PuZXd6/8sxv7/Oy6QD+wgLrUm/6CGSu5XyeYikeBx1uEQ7O/Rn6AteXFJLB1yRZRQG2BsOKO3DL/1z/BlgXnkCvmoq9WPWwMN3bPWQfPwX7EWotr/FmfwMe6V/RQk/NbtK94/paXu3eXRT62DXvu0fVGQbdRvWSJpIAtLJAJsPP87xifiApkHhBvPx3NaMW0fL3kgW4y3WGmrt4ILfO1zhbVSjHev3e0dmbvHWHB02Ou2Wj/uJZuhftPnSPaDmR9am7SJOdQGg77ebnronh23D1o9DNHKnFDS4LRFsGJWc+mbZjI12960R12aKV6s+rC62QjNXroV3mv3gSciS1hAV6MrvwXGpUgTIY2+ZTLSTRMTknpoJuXb+JDo/xAj2ixojLYUDkuZsmjF4GpC9f3ZfWFTICBX818J5yGNIp5Ilo4ow/Nsf4tHQpUKPKFFxZUzYtjh64LvlJ04Jys7xS24aGEkRyFWiPEteUmpQ8dNqpl5BP6gulwcoWDNYqahskeZSF+tTAI+nHHMwwtupDJJtytgZ/U7XVU49MKb9sS5dsOdIvQLizgl7AXTOQjd8tUcrVF1BUZpyFuUDiaxdDlnuPIx+VL35B0n6XYw3+I8dMa8capuPMzOLWl376iZhz1VVGsyH6PpGINCw7RKGb9VwHm6wQCWVOA6ETjnepo5ISUmbnWOVIHRAJNTMjX92GqdcVqAPcxyOHYwJykPGZOKkiHKscOhsWfTbjXB7F3d96jbArtNvPTyXlFyHeFvY3d6HY3spUM27olpOG8gul0E82Md9+I5po6bo4mttap6IdBKmHTZRHIilAIiArNeXDAn7btCtP+/qY6aYIIe9I6Oe4YqmBp14enZ5enBD70fOsdWLgXEZx+wr+g8BnxegEjGhbKI07Vpe1VAtEF8ofpe4uBuSgdDLJUF6qKMVYtQt0BkKMqZPxEJU4/alXYzK5FNoXX3HcadsQpAQmm2ITqaa6N3WrOkSbeBaL9oqLs9k43hiaZlIZOY2USkvLKpqVwgJk1pnKFiUEq3sKY4jWi8OZNhRmoBX8uUJH6mlXI5T68ZDLKXNt95e83CR/IeGBminvRgYU6S20h6yw3J0OqcnXYuMMsVMrAuPO102XvhnlJi+2B3S6z1ZdfGKCnUt/sTxlD3KL36A1a5WUi4QxmCP/uWsNUlC4JeCTJuUHDW/K4zuXLonQpGd/NiQHs3sMCHblA57HQxFUb7XdG8nqE5o5z0Orkrn2aHsgcx2dO5XSV5+kTWH6wVsS7GRUbnhEUUp2Xp16GtHhw+ZbmmRQq0O9bHZOuqGd8BGXu3bvL9D8rHZBZ9WJQGsNLhHtJ9vLtCV32srQ1OVQ9LE4KBrkzOxl31M2bLq77FGyb71acUWvy0+hb3iPd1xvdTP0LAFPRorb+UynFHFa79Mo6hqK4Buk65TRHv9DIKzPcYWK9+wG0INwxVAQ0NqnxAcbuyaeOaXR5YZnBhvEdmwNdEIAyZa5kbBGddlIHnTD15scAVvnkBFkZQ5HIYifm3p3ROKSskJvDm/MEhL+a0tsGKW5Eo7SrdPtFvSUB6JjuMBEBJQuwV1gsNMxTozQEUoo0s4dLLDoAPrLyYVjvlActQ5DkUbiFT+JR2n9DEFNtj5x730gs/H7bPq2853HbkftkvoqlAh6xl5gMkAhQx2igL/KYNzjYuqfARiMpXw+C2Gk3KnD82fohsoavMrPLjVvNI5YVR8lk7/sBoCqM6DW00nRhfvsGE38NUilAcaSGfN0fClPytxJy0bCNPZ6YAlusH3gzApM0eQKeHWPQW8rkuDW0vybkaNLVuQJkwUuQL3GNGVTkeJyJK33yOo5fCMVkVAehmPl6IM1tT5JN1gIccbvtqtHs3le07vkwt1gHdCl3czRuQC20uJFphrNIcMY0voMCXItgfdWO4p85H4htrML7gswTUVJkRZbcaL36juZlXhYE7diN3UOSpA05g2XFZ7aHFSvurtCMLgrbBL84Yj+8passd8BN6eUxoXvwDJST9OHcVszEa4J2AOl2HeAK07U9mNBqgfYADqpUS+CRzSMWWK/uSbHkSYZK2pyYDLMWt04vYbMIZhd6xCk1ujZJrCxuDuOWreYMo9DnV2HFGPGv+YSZgAmDNp8cfw/FnHJD5/KShtqpbdghwq93hTbqIvE0acDjeM5Fk2/yI2/GmFl/AFFTGtM8Zcjb8qQ3YCiBmP7A/wyiZMa3BUTFtlsF4T1idxRqsAMsu6YUiqWhs68Rb66TdWOgdOYNMi6hcuilk3DtMHnQDbGNloFFsZITaQOX88xD/A2kgBVaiJmBD4MzsUvg2gogHxVYzzYQY8HdMIWhjkyZ9BPhPu3ZplA7gbTTNZ33YldY/O6/f6Jh2Nh5t5YcYaQSMZvNxU2JcNBt9chgc6wGWe3kr0TCedVDDNLPyDh/mCR+z8OMixv1giFG8WmrJ0HLL59DMOAlWx7t7e5YCXNRd9kTTUBamWV9YE6CawzSfEBTN0BejRJ5q1Dz4cokwY7kH3UZTucDB0P4bFph15rqUcW8tvimfPc3Yw9IpfiJzzE01ZbKvrhl50Q1RT2VJHRCeZlJjS4oG9xiK07ivlLKxmSwpWpQoIkxETOPwYgpFk9kiPUz8cELtUUPrJSDQtLhSAUKlEq085FLpBGpQiov8HnaiZi9exPsK2zOttFGS8f1h3nROgXzbMzsclTNgEPTad7WaDjQAorKXA0BarZ/b3YtMR55gQSfxivQoeGu422QwTXp00SlN/VLyDRGSuO6qZG7qGlY/uvgA/mF0Z+9Gh1d75mwQ6sZfors9K4sbXSEVwrN0vsOVzR3CJGBG45S1T+al2BXqGlwDWzDXuqmOKE4BCSMAMvwSGHMlHEnvsHVGiRPw89kP4JpnY0Yp+h1tzqdUh7ycT1vES7Bc74a8EQJWsotXmbfDsROO9hZ37EiVo49DG470VjmHLtiNnIEmoLwMhWINLCOdvespn1AnaBr5Wj+IQUU4Z23m2TpdL0QilCZeK086nWHSMNiGDvqjwLRBMdOmDVVBnUQfgN0FEknmtoaQlL9Qd2cJGYuUvvMeX2HmmW6K/7TLMxi4ZWI8cceM6836Xbwz9svk9WvgioKLTH4Z+Zm0aFLMxpW84w2a22byiMxr2hfACaKZQL6azidXbrCEfwaBP+vha5EYsfgdE3ydS072UWP8LCkF0re7ZFVBQDN3oOTaMl8USEyk2J0F175aUBR7W7gzGiUsloTH09cz4upofuv6C5hj2kgwj4E1Zvnignol3dznDHZyKwCAI0d4ISzK115UiI9usOvidr+4p/p8RXFDY7HdX4BssSptFmpGBdVU0o4xMavpmbZiE7KhV5ANISiZ5F/tYFFz3YorZB9ptU+POjB+7c0/H2TscuFrJMyOBO4V8F1X5k3mB33Kpad34crSFxxFz79Z66qS9sBAMHl+UCxJFG3zLl/JZxVW8pkbkkCzL2tlBs80oK2j8VhJApVTlkIpyZpQ+Lq+8zlDXYpBQW8zS3rnG/rwxIp5fWKop2+NLLmx2TlsXjSBB0WW1Fe9p6yNyIeyDvCxG711SjgSPpvbg2jZLemX9oDtPh/kb7ToiLmpJ83ccXAQt1l623A5DpxLlh1r8rGIGmC0y9GgjeHS4yEAT7avuYyk4gt72R0vAuYbInRr47clzb/EaYKtQ44FSNpzjOvgqQhWXQo55obz5un7VhIm8Rtm2OMbbRbYv1h4AMnM9s0TarJOd3T9LdldwEkNMPQEqEyQ7PHK4bV9IzLeMVqcCAp6Z3TWmoqsqxSCQ2CC6AVeZi60cE2NsaZy2NOLZfbiaNy2+BKOAAZXQwyN2C37QwL9eIm2F3gEopcWXpQfXLHNUeYeLYOsV6/29D2nrNUYFfuShboizxbW9qfsHyXW2Y1pvE0aL6XkZ9HTK8aXZIHfLgGUfsF5QQS76ZXhfVlUrYs3awvChtwklXFP3Al6glhGZ5fWQl/SvRMqnIuELnIl1/VFTZOtXACKjongkEaAqjXIQ0pDmiy5jQTaAn/zdoh08eB1pPAPLb52oVYiVqhaKp3To5bm0qK7b9L20CJMG8fftBhLnqh/9VoMLoYY/4guzIZ342bsuZnNti/mZh1utmXssYE8wb8C4ZbXezP46RtIIT6TpVH+wWaGHAQ0KLQ6DKXlUa/zo86kxOuuDkz+md5eQNd5WmRsMoClI+UNkcV1PAnvvHV23DxggE9WQMyy6czAaXfG2nziYT99Hlbo2MYMvcT7ZWADz7xBIRWNTUB0rH58fYk+XZxfnh7AOv2mVnuUJbbEKILxxrEmligX0b0uhNraEQNnuSEm2btwyEtMATzcBwXVaneSqCwYG+yWx5tyS+3A1X0uNT9WxF6z8/VtCvKbvJHUlbr/D9wL7EImMDXLf4Iz8cBMP/WGA3eofmj+1OoddM5+6R21j1s9WmQX7mf6s3sOgqFovhWc0MN9eBOBhN9yG+b+7tKwmlS9FEjw422Y+HUtoNQutlXtRsmlOREw/E3XOVM0SsXfVocDjLmVKHATBk68t1a9OHyXL13Vvjz/YkfwEtWJjBJmxKSPqcfUiB9qGpKRF1/XkisR6RmPAW9qyDbATQ0xRWUN1wTKPhAH/ECYrBUIrON/43eXJ10jzFSyUlXFgeXEBrjt23fXvIHBbYVPeLPRvkwiCHiG0CUBb1PhSUa4MDe0xpiVij9Nq1hjL77Heuzfjd1bd9zzZ6FcOgw48UQFX5S4sSEp6zzkg4q83pAL5QuW811aqcxfsJjMIymlz1wIOtUUwmcslMTfXCpfCDAWClgpYjUohfwFiykYSUrpMxXS6qhLXbHIK7wFKcX8JS7mhHD2My7BGvJ+JH4oXwh9uzyMy3WkiUaAv+ETcUb5gXzRLUy5fLHKMc1A4hkW8HN9t18/1t9pZHIdV0Yl3/AJ31YWUvAXjd5GjJzVQwI8wcabV5qW+Nl0uGF1p8ewEWOPReYaCGuCRuJqCFGWYt+lFX8h+DrGVLrQXxkhv39jEILPMnNfElP3Rc+R2SfRbBbGz8JEq9BqZdk8ekSmoPTgsgk0T+pNHnWqsKR1FyCIMq+FAD+XKKkcHQY73pSi1Z3guq9PzuHz7cfPC1eaoDhU9EsuozTP33f5Mgu3xnbpZAYhOBkYRb8duqkn/QnyMlhdM9xbh1+p56AFZvji3KF3rfjP3sK9Y7p2/Juq8Et6wX3h66MZP5X4xueD1eJb7XTY+3VPh91GqiEvNQSH8tDH26wTJ7xRxPI4Qg4a1gFckmBl4EkAFjqPc2xQqBUXs7bSa4Q3x36Fvpp3KGxAif26Vf01fjlQRd4NpJ/Er1aCEn0x034xu7kagCSmUFk9EYVNnFtcZGfojVHSnfhguy7rCj6shCP/jt60zhtcYozm56Fz7e6q56H6CNWAJuFn9ZYiWSlJzP6nKV1HI/aqfeb8bzYn0Lvb42KzssTlWS971zcE9VGkzsOEhoL1jqkk8rdugGyZxJ8uvYy9q8AJ7pXUgNEQ0kyvm2spZhcyhad5tgojmgPGJiaPyQf4SJqmKKobCkNjuv4y0mmhIiNYX46gMAl0y4xiGASjYEiv48YQoZy+WUJyAhyo5ZKGSl8sXrVetLWjT7n9Cp+JZ8UWOvIOK2MWGVuI39grTek0w4nM3Y3UTdMM+7p1fg6W8NDxKFzSl/74bL7wCfypT/kiTMFE28EbKsaTtirdLyC+9URWqyegGbuH7fMCAtMJtoorUIijHfFc2wpk1q+VTvaUyGVsUAKK5Kt53iI1hRQZKZu96P/Z5NLKhtBOtliWRZjRNcBSNEqRJ8b5qw5vSB+m293L+0J6cVbAOy/qj4g5SKPJOTgG9FuX2XfJyzDNUfYqnKhP0oGl2pA2XGxUf6gRX35fbPT9m1e1VG/8lkVlXaU39flw5wG66orIB+41XywoPB8PjSOmYmgLFE6tBbTBZU02h3QXY3kzVVOXsTPwIlg81waQLRcPzHss3rS+Fg3LWsutQZXK4hNQ2ocDUWodXHTOZX/uuNP6uXVgZDwJaQ1ht4ZSpMBqbr4o5YxuhpTjmEKTB0/MnN7UvbPUJDlDcXg5fy/SIxNk41p5YFFhp3SxpYhBIHqcP67HCT+CsBC6ZqVON95JNWYEaemAZiWVWn6Wwnlp7hENHDBRp3hdmdoC2dxrWIunA52B5R0ay5Icrh8F4xd9vL2AbYMJ55afm110Xou96fXYZVOoaNakBMJoBuIw9YnMbxzsAuZghe6raN4BaPr7Xlxl4oBxBaTqcR+hrpou31veeW/C9NpkkJqfE/XmU03XIqUEJLrFrxpDwz5r6kzdnd0UQHGNE5Xqu3LIlJ65PWmMKzVWIEud2dBKMmAkfV8sO94Jjh9YSiEWpoXKcSIn67WKSLq/qLqO5P6v//ws/bFNWLTLgBUqoz+5j1qt9ubVK4V/v3vzmv7Wdvg7/rz67vVrVX+9893rl7WXO7WaqtXfvHld/y9V+5PxyPyZY7QfoDK5cvt/v5+GN152Pag2HD4Ah8eizN9/k58qXfvD5WhXHcUZjLyputTZp+Rq4Lk78EI+ffToct0A4xzppiIbelhy5U3RWwFdPIE1h44tQeVa9ybB7vSGHt9lL1HyT2C5iRdhGs9Z4N96A31HiC40Uiwt3p8HbTfwyOGgm4ABXdPcxc/1Sgo1Co0TnPoYpTnBK/OgExzJL+pc+ZQ+dHYfeNcjQgt+wMb10CSh9E+4a4Jg4m7NJdEYJ+i0P3bA8Ano5t7OIiLQoUURjQiMczCXfFd/Pi46ZRZDGvj9+cSdRo6etCre3aBrkRMncgPPGYcx4WnCIrojEQ9D3w29+KHdVd3O0cWH5nlLweez885P7cPWoXr3CzxsqeblxQ+dc9U8PVQHnVO8WnsJJlJX/fprswv1t7bwEd12Pf1FtX4+O291uwoatE/OjtsABuCeN08v2q1uSbVPD44vD9un70sKoKjTzoU6bp+0L6DaRaeE3SGgxZaqc6ROWucHP8DX5rv2cfviF0LoqH1xit0dIYLqrHl+0T64PG6eq7PL87NOl6DhsA7b3YPjZvukdVgBHKBfBdb86YXq/tA8PrZHCf8Sg3zXAgSb744JEvUCg2QzEUcTfzoAkgFuxyXVPWsdtPEDGI8wkOb5LyUB223970uoBA8R2mHzpPkehlZYQRSYj4PL89YJ4gtk6F6+6160Ly4vWup9p3PYRVAAvts6/6l90OruqeNOl+gFwl9SeJxG3QMUIBY8hs/vLrttIlv79AIs1cuzi3bntIiAfuh8QDcHPBFofUgk7pzSmIFGaBcDXKQHzUBJffihBeXnSFKiWhNp0QXqHVwgNKsm9Ar0vLAGq05b74/b71unBy182kFAH9rdVhEmrN3FCm3uGbwhGuMlDR/nCndQ6KPFuSWaUdU+Us3Dn9qIvFQGNui2hWU6Rwipe3nwg1A/h5ZijrXlAW0rh/HlbJSYVHY6FStS8PhnILR0G9vhrcqTH087JAlogZNo5lK3jJLv3KVAbcwLmTxwtq9qJF5YGif6SL3fNfVuNdwVOG+9L1i7AbyJIWlWpU8dAhrHIbR+PjimDxylEOdSNXsc2lJLpBuTMx7LEdX7EgsY0PlKGgXdSwrA8emPDAADGpAE6DzEm1cMUE5hcG+GrslagDNAHrWPOllIDb2hvwQr2STv9Y7A6H3XPez1FqB2Owc/2lD1jhL4Kc5gEIDhrsK53lbiN4o8dL8pnFfC+ZSug+m9HDkqT991MlthtBlL20LkqGDPblRoHvUuT9s/g0gCgj2Qy1bzRKJBdYzJfkPvE4BrRe71tvogayuscQO5QYLgkNEzREIaiWBAHT6esRtitRJlwVRoMVyPpM2duzUeS/IcXA7dKYsdrI7cssIVqxTfyFQZOhNvjBnUZXBEBbqqO7u3KadnMYumezED4xjjicf/C6nZ2y5uhnN5t5KGVYxDF/Xr3SQs0n7tiJ0zn3vT06QPTawa1ju5sxjRvIWb3ELjEIVLHCHQY1GZ8o4McS/6H2Jjot3/+vUy+/9lvfbdjtj/9drLV2/A/n/98rud/9j//4yfp0+qwNnVKycc5Z6msq30R27/JlTCHbAsUUoSSnPBVYCpQWM5Iadv2AUACsQTHkZjFYI3OVb9STi5Ap028FXVn0XVPnYgrasCuJriwdxTANSOOOkIJtBA//TOuXPudU4Q3r2hW4E6L4q5oM4rrJWNFYCRWWunDiMDOjuDjCrHN7CunRklcsGC975/DTrsqToMcAN0IolMZvMophDqJH/GV+KFeHTz7g71IajBUDLGTF13EDK5wHuxkpHZ6cOAav0+VCzJpTE8KwRcJCMUJrdhRCmLFYIC/yNrPPiyhyu6M8SQSqxBB5KThu4ZMYCX2QDQ2TL33kRXSvqWcB7cIjUcK3ELgYIfmavC3Qg0nc4gIxlj+j5QV9LGgH1T1oliJIUMZgLhW5SOgcZ3LRHnazCjkBmMs0KZJ6jiqwpn91SypRHzbqgG88C5Gru7GmIZGWDmAKFNspmtkNMZkzGsEwUjc0X36MZyuor45JbugWpwCt+iUaCpmknOmXQ6k11lpfLAnifOTL2IAfDTG3xCIUREZ/bVqNOAOMzhDEFaBOXWIR4XAQZ7MTREBZdecufwdJrTNVNYMbaIUD5p/DxmQhYTDBX3YhLFyINE3BlyYi4PneNd7F8CspVIgwwUBwRijGiuaHowbxjmtcaVnrLZvdrFX5Q2tTcMi5ytxyRiImJiEqETKCnGAOMblyRtmkOvOc9TuEuQ+TpnSOHTREZMkUPLLwlqDO1O7haHkQejm3ghbgdjAeL9Vx/WUdA0wLkTL8LwJg/kVUgAzj2wbyUGdWHpSFFjVeZXbt94WaPJdUw3AmnsOrcuJ62KR8k5koD01BZFF48I5jO0oV7WFHVMPZJaBIR/tfXkzq8cY867DrQNUPiVnvyqvOupj/YEzUpxD5UEwDo/AKsRc51enl70Dk4OGwwYZY9yxB11e91fTg+Ouo0anU+6rrmJeOWOnFsPaEUMCbAKvJsAElRmXmO1iDqF2kz1e2DIkIzwKvJRs328q3Bl8SKSgGkfqcG3rokYzNY4VZQIqIRnk25EK4AYiYDTlFDAqri9Eb+CoIjkOuXPgP4cMILZ4dRIKOWs6FnNV+zEXHLhElcrcf3GlHtpCvASqZWqs2p/Qova25k32C9vc2pJTBYeJ2niF+EMKjnMmFd256CLvJlLVMsJlRvPfpNPu+XqPAyqtNjRei2C+DV35wc3jWeFyQ0I3EyVB6p66wSAyUxXKSMilZ/pp5gjr0qVZyr/DFsC2v28/mx9pLbmG4l2Poe4zYFbGjXkt5gKODC+y2ClYode4vuqenn2eDkSmXXo+BdgkXgLdSfa+EhTlPOUIXcn0p/RFftc5Fw38kLzZ8/yuS9okDRSIHI5FoseTEojn89xtj35EkZ4tEKiWkA3R6mPqjwFCphaeUUXYX+zDpBe4glSkmQ7+1Uwv6tTzNvz+++pun9/uDK4E+6eunOA7RP9Zlb7GqMYD2sFjjjD6yGYUTOBnd3lUvRsAu+pJPGVTopfA7+kD2pvOp8J4a2pwGoTVQ6GgheyJM1t9RlMeb68DY1hJZkpgaBaP7cvcjlSMghNPWWFo97qpGGg/vbVW9SkmHqvUtmHLtiGRPXfeFYHJTjyhlGO3h+Vf/bfgDlpJbc/8lXev6Fl8RlWhifklPET1FrWEyMszwoF/RkvuhXxsoKHIaxISR6mO5CRo1rv4ZbRs0Lo/k3V1WtwwRXYzlZYTvlvgFU9z2+1kqweKhyjOVir7GDtKb4zi5FCSZMudhW22t/c0aHyqk4B8ao55Us0ZufKrOacCcLo0+SrFmawqHpINlpSKekTAJP1y9H5FvPnYV4VKPsBNsLK6Lyzgevp1IrcHSjmEZpP5T6Mv/DmFe7t1159//q7N8WiIhYj4xg5TOU/1fLqndq39FhVDLx8LrewgD37LV20W+bF8WsutaQZnSsFu+X6V1EVeNDZkKiYhpGT3EdAItkmj5EbLIp2y2fxlxKv942Egs2RpdJjQRGWkJU8/yw9gPwyFV4uS5RiLJXlMthOZeqlTNlgKa3YJ3ptWVnKcV1I6Ht4MvavyxRLplrn551ztWkQIjF+9gS+22xsoQG8xaOhMctgDJ0A+BAzU9okSS8+mwkNsrQvrm26Y/TsvnO5YcOuS6qjjLkj87kT711DeMzmPWQ9eLTIdvR21q3mFjHeMJ8Tw2vs3IOMcOIzyXank33GLqf91jwyXsiQSNomFVZdKi/TrkEyLMtRJLMK3NPYIsc81mExT7OK3jwOJf/E7p9d7DJBKZclV28Np5nz+z6DIastPYMxmbZsDZ/PaRQtbMRThSlc1R31keyRCCkw6VIaLfGzSAy+uJs8Uyz/jILYcHzPQ2oNoy6DurQnTgOkhsAsrHA45kI70UyrPppNoXpbGIAph5wyzKsrEExELbzxZo06e6XwF7udhw3MjVOEBoZhsKphFNM1+1wuhqWQjuPk4ajyIqcfre78Ub0m2dPgYO0mkPeBuwZlIrKeMspFjonIGSF7/rBxmRuXy3HLeC4LhZdGUOI5LRR2TKk9txYuGIaMS9N1IImScN35lulFmH9sdl8+dnYNq+I+N8ZBGzM3j+7TEL+iWUY5htkkVeVbhbthfoIetFWQhqBJcG0g0eueynQ3M0zDFeo0GLqNmZw6r8atAjURv9qXWq1Wr+3UXi4iuQgsoWRiXClDbVYHW8+GW8Ci1wFYKWDAbDXi7rZMX3TOHe875bN6gCmZqC0E/+mTdLCVhp7q3MC37w4bnepY1g96GAsKdHYfjfzpS/y25U1mPu5x4fId7sFfc6GiAAUVCmeufy6pT3kLB4xOvPqU/4J/oUXiBrPgntCvhvKUJjiTHvFcfknwAwHDDDzZz81UpDqxUwtJarC8TYLrJWy9iDush+Vyeb1dRKz58A9AS2xkrMo6TdkoeSdxIcM0XkeL4cluZagKlEJabyT6wYAh8ZUv3If6MZHwq8RbrmTbouOK2cShOueuzk5HTXsE6WTUuesMkwRVbN6YIa9q/+uNpd/R4LjO5xak2GQxRyHWRndmzQfkHarjPUFerBqWAvZii87C0ajfRQ//HhkoTvuM8yx5qTAlNgsXq8RhwpUjwCYneXJkxFbnsr9sZQXXpg85Fvg+Pdp1HPD7Jgvn7t/mHjpgxZJYEkR/2W4I1R2+3gPn2aV04UMvCCPj4MRsNaLXhmD488DDtxLjfiUnZgsr9qZBDn03D/2ZutpRL9Ur9Zo8N/bDnnn5pK9CYyaE4JE4beBvfsw09Bua/z+zL0o+smXssuOY0fnyedRdA1ZGfI3viLxAJFuHFZR42PkCzStM/wRccVCn4GMU9XRdO7fW6/20KHAOPZ4clF2Ryrx4JrLExZyZx2N9LqBG+RxdFk1RA7xr2/pfYFU5swgThxaCqMWgqxfhWPRQgmxFSGuyaAnpP72GroHG4xbY69ULrGykx0zvOsHYIyaXjI+kRBeWwAX9H498a0GL0dTWF9cFPsDZXZEtMze5ZWouqkdaqfKL88kQQpOP85vnMLvDFfOqN77pdIrfOCG+nBhzT3DWFlRdHje09aFWBwOpZHLwyrPTd8N0tpfI91WB32thqrAiu9LvFSjmSNkgNvsZw7n1+pE3WbmERM7YfaA1dTF17zK7oJcV5XkOs589ANmYJEQTei2T0ES/QJh3l8xZAz5LpJ3+Y4zLeKQY9ylNrBxRoODE+Xpkpzq439V4SoZ788YEPtOgszY8ZgBgi1mmcZrN4lnC8xAD5oudbZ9AIcqyhb9IRyhduurjzFI4VbS0JU3sl8xphedV8xaIYLKqhm1bolmdne4oFmBKfGRe1pJIO56e28e6R1kjXRBoIcxDFPDBfYwnI6VP+YCbuJbekoDqOK69RdoVARItljHrAxOHnWfoWRslsvqp/qf8p3XJswpm8RNSB8pQDDgxwe7yA3g5z0TVEJ/ngNAQXcTapl36bA5aot3D7M6W6lYB86CWSnW1BGV5x9CfpV2o02xvifatvTG+CBkw4APylb5Rprd0hu/Aywhn4IWCLdvAXRrQUFJ5fhT5sArkKzn61sNz21BOU6wS9RbB7qu3NLnwF5gfT1OMw0ynJ+rt262zX7Zy4jr3o/sZvqVpMnFm6A6X0GrPSVAJwinJe+QS/rSHd5l0wc7nYqrk5edibkZRq+CQ34e4/V3Idw96Z833LXrNGZipDHXiTfF9bOSI08khvrwGb5BxuF+OL2+FFSvxKXwzuU9zE4QBqFfwF7/yLAtQSdKZNahu8+Cg1e1S3oRi7sqhW3pMhwqGDUI9f1iQgn4P43gr6NtixghwOAoTwGs20RcS84shJHm8+3qVp0yA0ADUTKNWzN0MU03iuJLsBjz1DZ58vK9KLo0/HFIMMyXEYTO1zDOlnqtZSNNVgg/FXbLZe0igj9DoswJuGTrzsT4tMnEm9BFAFhzWB7TmebgSghcXmoTFBG42qVCCmEKB6PaC3sWpqlXsUG2r77kW+GQBYg5MIZS7B3ktQGPKdfo9ZqEC8YpAb3AD6A1XZW63v6/evFSbqs4joIgLyVlJBTdDwYGrb6pCgd59+Po1v1fHQkS/GzWFCEBYggjT+kVD1TVeDAJweoUo0X1FXfI9IYl0/fHsCG+gYrg8fvxw3r5ovWse/GgGwHNJcOkCfoEKStxfMXf2Cx79RUGjCuJS5fieKkbkYLgOReb0MG+APga/U8+ioMoRO5/xiNccIMo5Yw30rK4CIHoYzq52rUJzIlmXQgr8CasYn1O1I3+qYP1djReqW4C/agSHw9UY1tIYrug36yz7wUFah9h0Dr1lkNvTZ9JbfChNzN8D4BitYR3n4kFWYTDEpXoezeZRQ/L0J8/iABUM8CrXi/miQhaYOH/1g13QZ/geAxAbCaii2UbnITdI7jCZw1FaXhACvbAHt5/5BBdfQMr3YprHLJJTuq/w4JEwrDnf4448rjm4jNKZB6wyptBRb17RZzYR6CRdQsDA1gA5u23w1wb1yZ/j6JiinuybRm1Pac4kY0eX1+2Du++XHdzRefEBfB7gy4qANAPrHACku/E9IwNqAV9oWWJErCOBnFw0ReMcI2Bks0ajISe+8f6MXRsMMvLL+3IeNQwr2zbz7eItDfXsN8Mi1d1q6avKW0wtvFjE3Q3DYmzm5A9J2HkbRWjLb6oVDwVXS7x+RvdMQPx3gVLO3Y3aqv5farpbVb8R36hnO1+3JApx4k6wVVHd4P7bitEyHhLCRmOmTV7Zn+MwwDj6D/q3yWN2cRiKMLJOOnELbE+SWYDeAXUJH/KvwOC6dYXOeSSLfeInaGBI0mKQ4mKIIuNlTl8S2KnydQTybfaOtVG1q+LQRH0dLxWwSMEPDf0y4Bms9PgmCjCroAowxRBvTdhiXr0dYn9oCShQCGSGityqMs9Q+W9zLwI7xF0OAWMk14MiFGOixHhzAlShBgUY4TNjMEsXy+oianlmmLitxSt0olQOQBB7vErBB14G38KiirrQMjF184Row5f6G/MNc08T48ynCJqGWcsX84ahrOExUOqgChZIbJQAS2p8qs80Qvl4b5PFauwO0Z2X1VWHDsd8RX0UEjEuRcNUGj5yRE2VHS7ijoTHYibWND7XB/v8kkQ+yRD4crQsrPspZXnTNDxEt6Rd/tcQr+OiLY4NAUUsqIx9sFrIhrTMcTBHE5b6GsZ5QPevoB93UCgEH/NnfpgHs15/wjf6fcx3wXzOf+b0F3SJFTFBu5rkqKQ+Ys8TFxZH7PXjZzJPHTBiyT4Nd7UFJVX4bd5vG/L9Y7n+GZCPbby4GF/pbFXC1+pMnC+FRDvopijGJiirNJQKvyCxUHC4HttcgPYu4M1VgGhAEcrZUUMsnel9gdBjWxqxRXJCwZWyByatOdKsTqab1sY6rIyPx2taxoTtRQ/u4rvDzU0OJwzB9U3ErZl6Bh7XLoCFjTFtrcM8Rawh6nan/8mq8J+f//z85+c/P//WP/8P40c8hADiAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
diff --git a/vm/punchfs/test-punchfs.sh b/vm/punchfs/test-punchfs.sh
index 3ff1206..367e504 100755
--- a/vm/punchfs/test-punchfs.sh
+++ b/vm/punchfs/test-punchfs.sh
@@ -6,21 +6,49 @@
# with no fallocate and no extended attributes - the same gaps as the Google
# Drive mount), puts punchfs on top, and checks the two things macOS needs:
# 1. hole punch / zero range succeed, read back as zeros, keep the size;
-# 2. extended attributes can be set, read, listed and removed.
-# Nothing touches /srv/tm or Google Drive. Attributes of the test file land in
-# /srv/tm-xattr/ and are removed at the end.
+# 2. extended attributes can be set, read, listed and removed;
+# 3. extended attributes are keyed by path, so they survive a restart of
+# rclone (whose mount2 inodes come from a per-process counter), follow a
+# rename and go away with the file;
+# 4. fsync through punchfs is durable:
+# - no page of rclone's cache DATA file is dirty or under writeback after
+# it (read per file from the kernel: /proc/self/pagemap +
+# /proc/kpageflags, so other writers on a live machine cannot fail it;
+# reading the data back could not tell page cache from disk);
+# - the kernel saw punchfs call syncfs on the cache's filesystem during
+# the fsync (ftrace event ext4:ext4_sync_fs). The metadata file (vfsMeta)
+# cannot be checked by page flags: ext4 starts its write at close, and
+# what is still missing is the journal commit, which syncfs forces.
+# The machine mounts / with commit=30, so a missing syncfs leaves the
+# change uncommitted for up to 30 s.
+# The mount is `rclone mount2` like production (`mount` ignores fsync); set
+# RCLONE_MOUNT_CMD=mount and PUNCHFS_SYNCFS=0 to see the old behaviour fail
+# (the dirty-page checks are the ones that must then FAIL: run it so once
+# after changing this script, a detector never seen failing proves nothing).
+# Nothing touches /srv/tm or Google Drive. Attributes of the test files land in
+# /srv/tm-xattr/p/cmtest--* and are removed at the end.
set -euo pipefail
PUNCHFS=${PUNCHFS:-/usr/local/bin/punchfs}
work=$(mktemp -d /var/tmp/punchfs-test.XXXXXX)
mkdir -p "$work/src" "$work/rc" "$work/punch" "$work/cache"
failures=0
+# Test files are named so that their attribute directories cannot be a real
+# file of the machine: /srv/tm-xattr/p/ is keyed by the path.
+tag="cmtest-$$"
+xroot=/srv/tm-xattr/p
+rclone_pid=""
+punch_pid=""
+stop_mounts() {
+ [ -n "$punch_pid" ] && { fusermount3 -u "$work/punch" 2>/dev/null || fusermount3 -uz "$work/punch" 2>/dev/null || true; wait "$punch_pid" 2>/dev/null || true; }
+ [ -n "$rclone_pid" ] && { fusermount3 -u "$work/rc" 2>/dev/null || fusermount3 -uz "$work/rc" 2>/dev/null || true; wait "$rclone_pid" 2>/dev/null || true; }
+ punch_pid=""; rclone_pid=""
+ return 0
+}
cleanup() {
- fusermount3 -uz "$work/punch" 2>/dev/null || true
- fusermount3 -uz "$work/rc" 2>/dev/null || true
- sleep 1
- rm -rf "$work"
+ stop_mounts
+ rm -rf "$work" "$xroot/$tag"-*
}
trap cleanup EXIT
@@ -34,19 +62,36 @@ wait_mounted() {
echo "not mounted: $1" >&2; return 1
}
-rclone mount "$work/src" "$work/rc" --config /dev/null --vfs-cache-mode full \
- --cache-dir "$work/cache" --log-level ERROR &
-wait_mounted "$work/rc"
-"$PUNCHFS" -f -o source="$work/rc" "$work/punch" &
-wait_mounted "$work/punch"
+# An existing file the cache has never seen, so that the partial write below
+# leaves a sparse "Rs" (only the written range is in the cache).
+head -c $((64 * 1048576)) /dev/zero | tr "\0" B > "$work/src/durable"
+
+RCLONE_MOUNT_CMD=${RCLONE_MOUNT_CMD:-mount2}
+PUNCHFS_SYNCFS=${PUNCHFS_SYNCFS:-1}
+punch_opts="source=$work/rc"
+[ "$PUNCHFS_SYNCFS" = 1 ] && punch_opts="$punch_opts,syncfs=$work/cache"
-f="$work/punch/band"
+start_rclone() {
+ rclone "$RCLONE_MOUNT_CMD" "$work/src" "$work/rc" --config /dev/null --vfs-cache-mode full \
+ --cache-dir "$work/cache" --log-level ERROR &
+ rclone_pid=$!
+ wait_mounted "$work/rc"
+}
+start_punch() {
+ "$PUNCHFS" -f -o "$punch_opts" "$work/punch" &
+ punch_pid=$!
+ wait_mounted "$work/punch"
+}
+start_rclone
+start_punch
+
+f="$work/punch/$tag-band"
MiB=1048576
head -c $((4 * MiB)) /dev/zero | tr '\0' 'A' > "$f"
# The layer below really refuses fallocate - otherwise this test proves nothing.
check "rclone below refuses hole punch (the gap punchfs covers)" \
- bash -c "! fallocate --punch-hole --offset 0 --length $MiB '$work/rc/band' 2>/dev/null"
+ bash -c "! fallocate --punch-hole --offset 0 --length $MiB '$work/rc/$tag-band' 2>/dev/null"
check "hole punch succeeds" fallocate --punch-hole --offset $MiB --length $MiB "$f"
check "size is kept after hole punch" test "$(stat -c %s "$f")" -eq $((4 * MiB))
@@ -71,9 +116,158 @@ check "XATTR_CREATE refuses an existing name" \
check "remove attributes" bash -c "setfattr -x user.cmtest '$f' && setfattr -x user.cmtest.bin '$f'"
check "removed attribute is gone" bash -c "! getfattr -n user.cmtest '$f' 2>/dev/null"
-# The side directory is keyed by the inode punchfs sees, i.e. rclone's.
-ino=$(printf '%x' "$(stat -c %i "$work/rc/band")")
-rmdir --ignore-fail-on-non-empty "/srv/tm-xattr/$ino" 2>/dev/null || true
+# --- extended attributes are keyed by path ---------------------------------
+# rclone mount2 gives inodes from a counter of its process, so they change when
+# rclone restarts (and with the order of lookups). Keyed by inode, the sparse
+# flag of every band would be lost or land on another file.
+g="$work/punch/$tag-keep"
+head -c 4096 /dev/zero > "$g"
+setfattr -n user.DOSATTRIB -v sparse "$g"
+setfattr -n user.cmtest.bin -v 0x00010203 "$g"
+ino_before=$(stat -c %i "$work/rc/$tag-keep")
+check "attribute directory is below the path-keyed tree" \
+ test -f "$xroot/$tag-keep/#xattr#/user.DOSATTRIB"
+
+# Restart rclone and punchfs the way systemd does (Requires=), after other
+# files were looked up first so that a counter hands out different numbers.
+stop_mounts
+for i in 1 2 3 4 5; do echo "$i" > "$work/src/$tag-other$i"; done
+if [ "$RCLONE_MOUNT_CMD" = mount2 ]; then
+ start_rclone
+ for i in 1 2 3 4 5; do stat -c %i "$work/rc/$tag-other$i" > /dev/null; done
+ ino_after=$(stat -c %i "$work/rc/$tag-keep")
+ check "(test is meaningful) rclone gave the file another inode after its restart" \
+ test "$ino_before" != "$ino_after"
+else
+ start_rclone
+fi
+start_punch
+check "attribute survives a restart of rclone" \
+ test "$(getfattr --only-values -n user.DOSATTRIB "$g" 2>/dev/null)" = sparse
+check "binary attribute survives a restart of rclone" \
+ bash -c "getfattr -e hex -n user.cmtest.bin '$g' | grep -q '=0x00010203'"
+check "a file looked up earlier has none of it" \
+ bash -c "! getfattr -n user.DOSATTRIB '$work/punch/$tag-other1' 2>/dev/null"
+
+# rename: the attributes follow the path
+mv "$g" "$work/punch/$tag-moved"
+check "attribute follows a rename" \
+ test "$(getfattr --only-values -n user.DOSATTRIB "$work/punch/$tag-moved" 2>/dev/null)" = sparse
+check "the old name keeps nothing" test ! -e "$xroot/$tag-keep"
+# a rename OVER a file replaces its attributes too (Samba replaces files by rename)
+echo old > "$work/punch/$tag-victim"
+setfattr -n user.DOSATTRIB -v stale "$work/punch/$tag-victim"
+echo new > "$work/punch/$tag-plain"
+mv "$work/punch/$tag-plain" "$work/punch/$tag-victim"
+check "rename over a file does not leave the old file's attributes" \
+ bash -c "! getfattr -n user.DOSATTRIB '$work/punch/$tag-victim' 2>/dev/null"
+# names that look like the machinery: a file called "#xattr#" must not touch
+# the attributes of its directory, one called "x (deleted)" must work
+mkdir "$work/punch/$tag-dir"
+setfattr -n user.cmtest -v dirattr "$work/punch/$tag-dir"
+echo x > "$work/punch/$tag-dir/#xattr#"
+rm "$work/punch/$tag-dir/#xattr#"
+check "removing a file named like the attribute leaf keeps the directory's attributes" \
+ test "$(getfattr --only-values -n user.cmtest "$work/punch/$tag-dir" 2>/dev/null)" = dirattr
+echo x > "$work/punch/$tag-odd (deleted)"
+check "a file whose name ends in ' (deleted)' has working attributes" \
+ bash -c "setfattr -n user.cmtest -v odd '$work/punch/$tag-odd (deleted)' && test \"\$(getfattr --only-values -n user.cmtest '$work/punch/$tag-odd (deleted)')\" = odd"
+# unlink: and go away with the file, so a new file of that name starts clean
+rm "$work/punch/$tag-moved"
+check "attributes go away with the file" test ! -e "$xroot/$tag-moved"
+echo new > "$work/punch/$tag-moved"
+check "a new file of that name has no attributes" \
+ bash -c "! getfattr -n user.DOSATTRIB '$work/punch/$tag-moved' 2>/dev/null"
+
+# --- durability of fsync ---------------------------------------------------
+# Pages of rclone's cache files that are dirty or under writeback, "dirty total".
+dirty_pages() { # dirty_pages
+ python3 - "$@" <<'PY'
+import ctypes, mmap, os, sys
+path, start, end = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
+ps = os.sysconf("SC_PAGE_SIZE")
+end = min(end, os.path.getsize(path))
+fd = os.open(path, os.O_RDONLY)
+m = mmap.mmap(fd, os.path.getsize(path), access=mmap.ACCESS_COPY)
+base = ctypes.addressof(ctypes.c_char.from_buffer(m))
+pm = open("/proc/self/pagemap", "rb", buffering=0)
+kf = open("/proc/kpageflags", "rb", buffering=0)
+dirty = total = 0
+for off in range(start - start % ps, end, ps):
+ _ = m[off] # fault the page cache page in (a clean one if it was dropped)
+ pm.seek(((base + off) // ps) * 8)
+ entry = int.from_bytes(pm.read(8), "little")
+ if not entry >> 63 & 1:
+ continue
+ kf.seek((entry & ((1 << 55) - 1)) * 8)
+ flags = int.from_bytes(kf.read(8), "little")
+ total += 1
+ if flags >> 4 & 1 or flags >> 8 & 1: # KPF_DIRTY, KPF_WRITEBACK
+ dirty += 1
+print(dirty, total)
+PY
+}
+
+tr=/sys/kernel/tracing
+trace_on() {
+ [ -w $tr/trace ] || return 1
+ echo 0 > $tr/tracing_on
+ : > $tr/trace
+ echo 1 > $tr/events/ext4/ext4_sync_fs/enable
+ echo 1 > $tr/tracing_on
+}
+trace_off() {
+ [ -w $tr/trace ] || return 0
+ echo 0 > $tr/events/ext4/ext4_sync_fs/enable 2>/dev/null || true
+ echo 0 > $tr/tracing_on
+ return 0
+}
+trap 'trace_off; cleanup' EXIT
+cache_dev=$(mountpoint -d "$(df --output=target "$work/cache" | tail -1)") # major:minor, as ftrace prints it
+d="$work/punch/durable"
+sync # the next write is a PARTIAL one into a file the cache has never seen
+# 8 MiB of new data at 8 MiB of a 64 MiB file, then fsync (dd conv=fsync = one fsync at the end)
+trace_ok=0; trace_on && trace_ok=1
+head -c $((8 * MiB)) /dev/zero | tr "\0" C | dd of="$d" bs=$MiB seek=8 conv=notrunc,fsync status=none
+syncfs_calls=0
+if [ "$trace_ok" = 1 ]; then
+ syncfs_calls=$(grep -c "punchfs.*ext4_sync_fs: dev ${cache_dev/:/,} " $tr/trace || true)
+fi
+trace_off
+echo "Dirty after fsync (whole machine, informational): $(awk '/^Dirty:/ {print $2}' /proc/meminfo) kB"
+
+if [ "$trace_ok" = 1 ]; then
+ echo "syncfs calls by punchfs during the fsync: $syncfs_calls"
+else
+ echo "ftrace is not available ($tr): cannot observe syncfs"
+fi
+check "punchfs called syncfs on the cache filesystem during fsync" test "$syncfs_calls" -gt 0
+
+# rclone's cache: data file and the metadata file ("Rs" = ranges present).
+data=$(find "$work/cache/vfs" -type f -name durable -print -quit)
+meta=$(find "$work/cache/vfsMeta" -type f -name durable -print -quit)
+check "cache data file exists" test -n "$data"
+check "vfsMeta file exists" test -n "$meta"
+if [ -n "$data" ]; then
+ read -r dd_dirty dd_total <<< "$(dirty_pages "$data" $((8 * MiB)) $((16 * MiB)) || echo "unreadable 0")"
+ echo "cache data pages dirty/in page cache: $dd_dirty/$dd_total"
+ check "fsync left no dirty page of the cache data (written range)" test "$dd_dirty" = 0 -a "$dd_total" -gt 0
+fi
+check "vfsMeta Rs covers the range written before fsync" \
+ python3 - "$meta" $((8 * MiB)) $((16 * MiB)) <<'PY'
+import json, sys
+meta = json.load(open(sys.argv[1]))
+start, end = int(sys.argv[2]), int(sys.argv[3])
+rs = sorted((r["Pos"], r["Pos"] + r["Size"]) for r in meta.get("Rs", []))
+merged = []
+for a, b in rs:
+ if merged and a <= merged[-1][1]:
+ merged[-1] = (merged[-1][0], max(merged[-1][1], b))
+ else:
+ merged.append((a, b))
+print("Rs:", merged)
+sys.exit(0 if any(a <= start and end <= b for a, b in merged) else 1)
+PY
if [ "$failures" -eq 0 ]; then echo "punchfs: all checks passed"; else echo "punchfs: $failures check(s) FAILED"; fi
exit "$failures"
diff --git a/vm/systemd/punchfs.service b/vm/systemd/punchfs.service
index afb168e..8869348 100644
--- a/vm/systemd/punchfs.service
+++ b/vm/systemd/punchfs.service
@@ -12,7 +12,18 @@ Type=simple
# passthrough_ll keeps one descriptor open for every inode the kernel has seen;
# a full backup has tens of thousands of bands, and the default 1024 ran out (EMFILE).
LimitNOFILE=524288
-ExecStart=/usr/local/bin/punchfs -f -o source=/srv/tm,allow_other /srv/tm-punch
+# syncfs=: after fsync(fd) punchfs also syncfs() the filesystem of rclone's
+# --cache-dir, so the cache metadata (vfsMeta, written without fsync) is durable too.
+#
+# Partial deployment: this unit reaches the machine before the binary is rebuilt
+# (and a failed rebuild leaves the old binary). An old punchfs does not know
+# -o syncfs and would exit at once, over and over, with Restart=always. Instead
+# the start checks that the binary has it and otherwise ends with 78, which
+# RestartPreventExitStatus turns into ONE visible failure (`systemctl --failed`,
+# the units that Require punchfs do not start): no backup that is not durable,
+# no restart loop. `cloudmachine-agent vm-update` rebuilds the binary.
+ExecStart=/bin/sh -c '/usr/local/bin/punchfs --help 2>&1 | grep -q "syncfs=" || { echo "punchfs: /usr/local/bin/punchfs has no -o syncfs (old binary): not starting, rebuild it (cloudmachine-agent vm-update)" >&2; exit 78; }; exec /usr/local/bin/punchfs -f -o source=/srv/tm,syncfs=/var/cache/rclone,allow_other /srv/tm-punch'
+RestartPreventExitStatus=78
ExecStop=/bin/fusermount3 -uz /srv/tm-punch
Restart=always
RestartSec=5
diff --git a/vm/systemd/rclone-tm.service b/vm/systemd/rclone-tm.service
index a04b64c..060663f 100644
--- a/vm/systemd/rclone-tm.service
+++ b/vm/systemd/rclone-tm.service
@@ -10,13 +10,17 @@ ConditionPathExists=/etc/rclone/rclone.conf
ConditionPathExists=/etc/cloudmachine/vm.env
[Service]
+# mount2 (go-fuse), not mount: `rclone mount` answers fsync with OK and does
+# nothing, so APFS barriers inside the Time Machine image were never durable.
+# mount2 passes fsync to the VFS cache item (data + metadata with an up-to-date
+# range list); punchfs adds syncfs() for the metadata file (see punchfs.service).
Type=notify
# rclone scans the whole VFS cache (up to 120G) before it mounts; after an
# interrupted upload that takes minutes, and the default 90 s kills it in a loop.
TimeoutStartSec=45min
EnvironmentFile=/etc/cloudmachine/vm.env
EnvironmentFile=/etc/cloudmachine/tm-ids.env
-ExecStart=/usr/local/bin/rclone mount ${CM_REMOTE} /srv/tm \
+ExecStart=/usr/local/bin/rclone mount2 ${CM_REMOTE} /srv/tm \
--config /etc/rclone/rclone.conf --allow-other --uid ${TM_UID} --gid ${TM_GID} --umask 002 \
--vfs-cache-mode full --vfs-cache-max-size 120G --vfs-cache-max-age 9999h --vfs-write-back 600s \
--cache-dir /var/cache/rclone --dir-cache-time 9999h --poll-interval 0 --attr-timeout 5m \
diff --git a/vm/tests/crash-loop-workload.py b/vm/tests/crash-loop-workload.py
new file mode 100644
index 0000000..0f2c2d9
--- /dev/null
+++ b/vm/tests/crash-loop-workload.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""Write load and ledger check for vm/tests/crash-loop.sh (see its header).
+
+ crash-loop-workload.py run DIR LEDGER [MAXBYTES] # runs until killed
+ crash-loop-workload.py verify DIR LEDGER # exit 0 = consistent
+
+`run` creates, overwrites and deletes files under DIR. After each operation it
+calls fcntl(F_FULLFSYNC) on the file and on DIR (what Time Machine's APFS
+barrier boils down to) and only then appends "done" to LEDGER on the local
+disk. `verify` replays LEDGER: every file whose operation completed must have
+exactly the recorded SHA-256 (or be absent after a completed delete); the one
+operation that was in flight when the machine was killed may have either state.
+"""
+import fcntl
+import hashlib
+import json
+import os
+import random
+import signal
+import sys
+import time
+
+
+def fullfsync(fd):
+ fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
+
+
+def sha_of(path):
+ h = hashlib.sha256()
+ with open(path, "rb") as f:
+ while chunk := f.read(1 << 20):
+ h.update(chunk)
+ return h.hexdigest()
+
+
+def write_file(path, size, mode):
+ h = hashlib.sha256()
+ chunks = []
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | (os.O_TRUNC if mode == "w" else 0), 0o644)
+ try:
+ left = size
+ while left > 0:
+ n = min(left, 1 << 20)
+ chunk = os.urandom(n)
+ h.update(chunk)
+ chunks.append(hashlib.sha1(chunk).hexdigest()[:8])
+ os.write(fd, chunk)
+ left -= n
+ fullfsync(fd)
+ finally:
+ os.close(fd)
+ return h.hexdigest(), chunks
+
+
+def fsync_dir(d):
+ fd = os.open(d, os.O_RDONLY)
+ try:
+ fullfsync(fd)
+ finally:
+ os.close(fd)
+
+
+def run(d, ledger, maxbytes):
+ stop = []
+ signal.signal(signal.SIGTERM, lambda *_: stop.append(1)) # finish the current operation, then exit
+ rnd = random.Random()
+ state = {} # relative path -> (sha, size) of completed operations
+ led = open(ledger, "a", buffering=1)
+ n = 0
+
+ def log(**kw):
+ led.write(json.dumps(dict(kw, t=round(time.time(), 3))) + "\n")
+ led.flush()
+
+ while not stop:
+ total = sum(s for _, s in state.values())
+ r = rnd.random()
+ if state and (total > maxbytes or r < 0.2):
+ rel = rnd.choice(sorted(state))
+ log(op="begin", kind="delete", path=rel)
+ os.unlink(os.path.join(d, rel))
+ fsync_dir(d)
+ del state[rel]
+ log(op="done", sha=None)
+ elif state and r < 0.5:
+ rel = rnd.choice(sorted(state))
+ size = rnd.choice([4096, 65536, rnd.randrange(1 << 20, 48 << 20)])
+ log(op="begin", kind="overwrite", path=rel)
+ sha, chunks = write_file(os.path.join(d, rel), size, "w")
+ fsync_dir(d)
+ state[rel] = (sha, size)
+ log(op="done", sha=sha, size=size, chunks=chunks)
+ else:
+ rel = "f%06d_%d" % (n, rnd.randrange(10**6))
+ size = rnd.choice([4096, 16384, 65536, rnd.randrange(1 << 20, 64 << 20)])
+ log(op="begin", kind="create", path=rel)
+ sha, chunks = write_file(os.path.join(d, rel), size, "w")
+ fsync_dir(d)
+ state[rel] = (sha, size)
+ log(op="done", sha=sha, size=size, chunks=chunks)
+ n += 1
+
+
+def diagnose(path, size, chunks):
+ """Where does a file that should be intact differ: size, bad 1 MiB chunks, all-zero?"""
+ out = " (size %s, expected %s" % (os.path.getsize(path), size)
+ if chunks:
+ bad, zero = [], 0
+ with open(path, "rb") as f:
+ for i, want in enumerate(chunks):
+ c = f.read(1 << 20)
+ if hashlib.sha1(c).hexdigest()[:8] != want:
+ bad.append(i)
+ zero += c.count(0) == len(c) and len(c) > 0
+ out += "; %d of %d MiB chunks differ: %s; %d of them all zeros" % (len(bad), len(chunks), bad[:20], zero)
+ return out + ")"
+
+
+def verify(d, ledger):
+ state, inflight, chunkmap = {}, None, {}
+ with open(ledger) as f:
+ for line in f:
+ try:
+ e = json.loads(line)
+ except ValueError:
+ continue # last line cut short
+ if e["op"] == "begin":
+ inflight = e
+ else:
+ if inflight["kind"] == "delete":
+ state.pop(inflight["path"], None)
+ else:
+ state[inflight["path"]] = e["sha"]
+ chunkmap[inflight["path"]] = (e.get("size"), e.get("chunks"))
+ inflight = None
+ bad = []
+ skip = inflight["path"] if inflight else None
+ for rel, sha in sorted(state.items()):
+ if rel == skip:
+ continue
+ p = os.path.join(d, rel)
+ if not os.path.exists(p):
+ bad.append("missing " + rel)
+ elif sha_of(p) != sha:
+ bad.append("content differs " + rel + diagnose(p, *chunkmap.get(rel, (None, None))))
+ for rel in sorted(os.listdir(d)):
+ if rel not in state and rel != skip:
+ bad.append("unexpected " + rel)
+ done = sum(1 for _ in state)
+ print("verified %d files, in flight: %s" % (done, inflight and "%s %s" % (inflight["kind"], inflight["path"])))
+ for b in bad:
+ print("MISMATCH", b)
+ return 1 if bad else 0
+
+
+if __name__ == "__main__":
+ mode, d, ledger = sys.argv[1:4]
+ if mode == "run":
+ run(d, ledger, int(sys.argv[4]) if len(sys.argv) > 4 else 3 << 30)
+ else:
+ sys.exit(verify(d, ledger))
diff --git a/vm/tests/crash-loop.sh b/vm/tests/crash-loop.sh
new file mode 100755
index 0000000..ef6b5b6
--- /dev/null
+++ b/vm/tests/crash-loop.sh
@@ -0,0 +1,204 @@
+#!/bin/bash
+# Crash-consistency loop for the fsync chain (APFS image on SMB -> Samba ->
+# punchfs -> rclone mount/mount2 -> VM disk). Run on the Mac, against a
+# DISPOSABLE Lima machine, never against cmsmb:
+#
+# limactl create --name cmtest --tty=false --set '.disk="30GiB"' vm/cmsmb.yaml
+# limactl start cmtest
+# vm/tests/crash-loop.sh prepare # guest: local backend, SMB password, smbd unmasked
+# vm/tests/crash-loop.sh run new 20 OUTDIR # mount2 + syncfs (the shipped configuration)
+# vm/tests/crash-loop.sh run old 10 OUTDIR # rclone mount, no syncfs (the configuration before PR #58)
+# limactl delete -f cmtest; rm -rf ~/cmtest-smb
+#
+# One iteration: attach the sparsebundle APFS image (32 MB bands, as in
+# production) from the SMB share, write for a random 5-40 s with
+# crash-loop-workload.py (create/overwrite/delete, F_FULLFSYNC after every
+# operation, ledger on the local disk), kill -9 the Lima host agent and the
+# Virtualization process of the machine (a hard crash, nothing graceful), force
+# the Mac side loose, start the machine, put the configuration back, remount,
+# `fsck_apfs -n` on the unmounted volume, mount it and compare every file
+# with the ledger. Results: OUTDIR/results.tsv, details in OUTDIR/iter-N.log.
+#
+# Notes: the machine's provisioning rewrites the unit files on every boot, so
+# the "old" configuration is applied by editing the units after each boot and
+# restarting the services. A damaged image is replaced by a fresh one so the
+# loop can go on. Env: CM_LOOP_VM (cmtest), CM_LOOP_SMB_MOUNT (~/cmtest-smb),
+# CM_LOOP_SMB_PASSWORD (cmtest), CM_LOOP_SIZE (6g). Takes minutes per iteration.
+set -u
+VM=${CM_LOOP_VM:-cmtest}
+MNT=${CM_LOOP_SMB_MOUNT:-$HOME/cmtest-smb}
+PASS=${CM_LOOP_SMB_PASSWORD:-cmtest}
+SIZE=${CM_LOOP_SIZE:-6g}
+HERE=$(cd "$(dirname "$0")" && pwd)
+[ "$VM" = cmsmb ] && { echo "refusing to touch cmsmb" >&2; exit 2; }
+
+gsh() { limactl shell "$VM" sudo bash -c "$1"; }
+ts() { date +%H:%M:%S; }
+
+prepare() {
+ gsh "set -e
+mkdir -p /srv/tm-local-remote
+echo 'CM_REMOTE=:local:/srv/tm-local-remote' > /etc/cloudmachine/vm.env
+echo '[x]' > /etc/rclone/rclone.conf
+printf '$PASS\n$PASS\n' | smbpasswd -a -s tm
+systemctl unmask smbd 2>/dev/null || true
+systemctl start rclone-tm punchfs smbd
+ip -4 -o addr show lima0"
+}
+
+guest_ip() { limactl shell "$VM" ip -4 -o addr show lima0 | awk '{print $4}' | cut -d/ -f1; }
+
+# old: rclone mount (fsync is a no-op), punchfs without syncfs. new: as provisioned.
+apply_config() {
+ if [ "$1" = old ]; then
+ gsh "systemctl stop smbd tm-bands-bind punchfs rclone-tm
+sed -i 's#rclone mount2#rclone mount#' /etc/systemd/system/rclone-tm.service
+sed -i '/^ExecStart=\/bin\/sh/c\\ExecStart=/usr/local/bin/punchfs -f -o source=/srv/tm,allow_other /srv/tm-punch' /etc/systemd/system/punchfs.service
+systemctl daemon-reload
+systemctl start smbd"
+ else
+ gsh "systemctl is-active --quiet smbd || systemctl start smbd"
+ fi
+ sleep 2
+ local c
+ c=$(gsh "pgrep -a rclone | cut -c1-60; pgrep -a punchfs")
+ echo "config check: $c"
+ if [ "$1" = old ]; then
+ echo "$c" | grep -q 'rclone mount2' && { echo "config old not applied"; return 1; }
+ echo "$c" | grep -q 'syncfs=' && { echo "config old not applied"; return 1; }
+ else
+ echo "$c" | grep -q 'rclone mount2' || { echo "config new not applied"; return 1; }
+ echo "$c" | grep -q 'syncfs=' || { echo "config new not applied"; return 1; }
+ fi
+ return 0
+}
+
+mount_smb() {
+ mkdir -p "$MNT"
+ mount | grep -q " on $MNT " && return 0
+ timeout 60 mount_smbfs "//tm:$PASS@$(guest_ip)/CloudMachine" "$MNT"
+}
+
+# Creates a fresh image and moves its bands onto punchfs (what vm-prepare-bundle does).
+new_image() {
+ mount_smb || return 1
+ rm -rf "$MNT/loop.sparsebundle" 2>/dev/null
+ gsh "systemctl stop smbd; umount -l /srv/tm-local/loop.sparsebundle/bands 2>/dev/null; rm -rf /srv/tm-local/loop.sparsebundle /srv/tm-punch/bands/*; systemctl start smbd"
+ umount -f "$MNT" 2>/dev/null; sleep 1; mount_smb || return 1
+ (cd "$MNT" && hdiutil create -quiet -type SPARSEBUNDLE -fs APFS -size "$SIZE" -imagekey sparse-band-size=65536 -volname cmloop loop) || return 1
+ umount "$MNT" || return 1
+ gsh "set -e; systemctl stop smbd; mkdir -p /srv/tm-punch/bands; cp -a /srv/tm-local/loop.sparsebundle/bands/. /srv/tm-punch/bands/
+echo CM_BUNDLE=loop.sparsebundle > /etc/cloudmachine/bundle.env; systemctl start tm-bands-bind smbd"
+ mount_smb
+}
+
+DEV=""; VOLDEV=""
+attach_nomount() {
+ local out
+ out=$(timeout 180 hdiutil attach -nomount -noverify "$MNT/loop.sparsebundle" 2>&1) || { echo "$out"; return 1; }
+ echo "$out"
+ out=$(echo "$out" | grep '^/dev/')
+ DEV=$(echo "$out" | head -1 | awk '{print $1}')
+ VOLDEV=$(echo "$out" | tail -1 | awk '{print $1}')
+}
+
+vz_pid() {
+ local p
+ for p in $(pgrep -f com.apple.Virtualization.VirtualMachine); do
+ lsof -p "$p" 2>/dev/null | grep -q "/.lima/$VM/" && echo "$p"
+ done
+}
+
+start_vm() {
+ limactl stop -f "$VM" >/dev/null 2>&1
+ limactl start "$VM" 2>&1 | tail -2
+}
+
+run() {
+ local CONFIG=$1 N=$2 OUT=$3 i
+ mkdir -p "$OUT"
+ [ -f "$OUT/results.tsv" ] || printf 'iter\tconfig\ttime\trun_s\tdirty_kB\twriteback_kB\tops_done\tin_flight\tguest_unclean\tdetach\tattach\tfsck\tmount\tverify\tnotes\n' > "$OUT/results.tsv"
+ umount -f "$MNT" 2>/dev/null # a mount from an earlier run points at a dead machine
+ start_vm
+ # Clean slate: no bundle, empty backend/cache, so every run starts from the same state.
+ gsh "systemctl stop smbd tm-bands-bind punchfs rclone-tm; rm -f /etc/cloudmachine/bundle.env
+rm -rf /srv/tm-local-remote/* /var/cache/rclone/* /srv/tm-xattr/* /srv/tm-local/*; systemctl reset-failed"
+ apply_config "$CONFIG" || exit 1
+ # Always a fresh image per run. (Never delete band files behind rclone's back:
+ # its directory cache would still list them and a later boot loses the bands dir.)
+ mount_smb && new_image || { echo "image create failed"; exit 1; }
+ for i in $(seq 1 "$N"); do
+ local L="$OUT/iter-$i.log" ledger="$OUT/ledger-$i.jsonl" notes="" detach=ok attach=ok fsck=- mountr=- verify=- unclean=- dirty=- wb=- ops=0 infl=-
+ exec 3>&1 1>"$L" 2>&1
+ echo "== iteration $i ($CONFIG) $(ts)"
+ # 1. attach + mount, clean up the previous iteration
+ mount_smb || notes="$notes smbmount-failed"
+ attach_nomount || { attach=FAIL; notes="$notes attach-failed-before-run"; }
+ if [ "$attach" = ok ]; then
+ diskutil mount "$VOLDEV" || notes="$notes mount-failed-before-run"
+ V=/Volumes/cmloop
+ touch $V/.metadata_never_index 2>/dev/null
+ rm -rf $V/it_* 2>/dev/null
+ mkdir -p "$V/it_$i" && sync
+ # 2. load
+ : > "$ledger"
+ python3 "$HERE/crash-loop-workload.py" run "$V/it_$i" "$ledger" $((3<<30)) &
+ WL=$!
+ T=$((5 + RANDOM % 36)); t0=$(date +%s)
+ sleep "$T"
+ dirty=$(limactl shell "$VM" grep '^Dirty:' /proc/meminfo | awk '{print $2}')
+ wb=$(limactl shell "$VM" grep '^Writeback:' /proc/meminfo | awk '{print $2}')
+ # 3. hard kill: hostagent and VZ process, SIGKILL
+ HA=$(cat ~/.lima/$VM/ha.pid); VZ=$(vz_pid)
+ echo "killing ha=$HA vz=$VZ"
+ kill -9 $HA $VZ
+ run_s=$(( $(date +%s) - t0 ))
+ kill -9 $WL 2>/dev/null
+ ops=$(grep -c '"done"' "$ledger"); infl=$(tail -1 "$ledger" | grep -q '"begin"' && tail -1 "$ledger" | python3 -c 'import sys,json;e=json.loads(sys.stdin.read());print(e["kind"])' || echo none)
+ # 4. Mac side loose
+ d0=$(date +%s)
+ timeout 120 hdiutil detach -force "$DEV" || { detach=HUNG; notes="$notes detach-hung"; }
+ timeout 60 umount -f "$MNT" || timeout 60 diskutil unmount force "$MNT" || notes="$notes umount-failed"
+ echo "detach took $(( $(date +%s) - d0 )) s"
+ else
+ run_s=0
+ fi
+ # 5. restart, config, remount
+ start_vm; apply_config "$CONFIG" || notes="$notes config-failed"
+ unclean=$(gsh "dmesg | grep -c 'recovery required'")
+ timeout 60 umount -f "$MNT" 2>/dev/null # a mount left over from before the crash is stale
+ mount_smb || notes="$notes smbmount-failed-after"
+ # 6. check
+ if attach_nomount; then
+ echo "-- fsck_apfs -n /dev/r${VOLDEV#/dev/}"
+ fsck_apfs -n "/dev/r${VOLDEV#/dev/}"; fsck=$?
+ if diskutil mount "$VOLDEV"; then
+ mountr=ok
+ python3 "$HERE/crash-loop-workload.py" verify "/Volumes/cmloop/it_$i" "$ledger" | tee "$OUT/verify-$i.txt"; verify=${PIPESTATUS[0]}
+ if [ "$verify" != 0 ]; then # keep the evidence before the next iteration deletes it
+ mkdir -p "$OUT/evidence-$i"
+ awk '/^MISMATCH content differs/{print $4}' "$OUT/verify-$i.txt" | while read -r f; do cp "/Volumes/cmloop/it_$i/$f" "$OUT/evidence-$i/" 2>/dev/null; done
+ fi
+ else
+ mountr=FAIL
+ fi
+ else
+ attach=FAIL; notes="$notes attach-failed-after"
+ fi
+ exec 1>&3 3>&-
+ printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$i" "$CONFIG" "$(ts)" "$run_s" "$dirty" "$wb" "$ops" "$infl" "$unclean" "$detach" "$attach" "$fsck" "$mountr" "$verify" "$notes" | tee -a "$OUT/results.tsv"
+ if [ "$attach" != ok ] || [ "$fsck" != 0 ] || [ "$mountr" != ok ]; then
+ echo "damaged image at iteration $i: replacing it" | tee -a "$OUT/results.tsv"
+ timeout 120 hdiutil detach -force "$DEV" >>"$L" 2>&1
+ new_image >>"$L" 2>&1 && attach_nomount >>"$L" 2>&1 && timeout 120 hdiutil detach -force "$DEV" >>"$L" 2>&1
+ else
+ timeout 120 hdiutil detach "$DEV" >>"$L" 2>&1 || timeout 120 hdiutil detach -force "$DEV" >>"$L" 2>&1
+ fi
+ done
+}
+
+case "${1:-}" in
+ prepare) prepare ;;
+ run) shift; run "$@" ;;
+ *) sed -n '2,25p' "$0"; exit 2 ;;
+esac