diff --git a/README.md b/README.md
index e73df9f1..dd31ff9a 100644
--- a/README.md
+++ b/README.md
@@ -1617,6 +1617,76 @@ let response = try await session.respond(
)
```
+llama.cpp's Metal backend stops a process with an assertion
+if the process exits while a model has layers on the GPU,
+and a generation that is still running at exit can crash on the GPU or the CPU.
+So `LlamaLanguageModel` registers an exit handler when a load ends.
+The handler stops new loads and generations,
+waits up to three seconds for running work to stop at its next token,
+and then frees every loaded model, projector, and cached context.
+After the handler starts,
+requests throw `LanguageModelSession.GenerationError.assetsUnavailable`.
+
+The handler is best effort.
+The process can still stop with the assertion in these cases:
+
+- It exits during the first load,
+ because the handler is registered when a load ends.
+- It exits from inside a load, a generation, or the release of a model,
+ for example from a llama.cpp callback.
+ That work can't end while the handler waits,
+ so the handler returns at once and frees nothing.
+ An exit from a llama.cpp callback on a thread
+ that llama.cpp or another library owns
+ isn't recognized as inside that work,
+ so the handler waits the full three seconds and then frees nothing.
+- Work is still running after three seconds.
+ The handler then frees nothing.
+- It exits without running exit handlers,
+ for example with `_exit` or after a crash.
+
+If freeing the models takes more than ten seconds,
+for example because another library's log callback blocks,
+the handler aborts the process, so that the exit can't hang.
+A crash report for this abort shows
+`LlamaLifecycle.abortAfterExitCleanupTimeout()` in its backtrace,
+which first tries to write one line to standard error,
+but aborts whether or not the line can be written.
+If the cleanup ends just after the limit,
+the cleanup's own thread can abort first,
+without the line,
+and the backtrace shows `LlamaLifecycle.abortAfterLateExitCleanup()`.
+
+No method unloads a model that something still references.
+To be certain that no model is loaded when the process exits,
+wait until every `respond` call returns and every response stream ends,
+start no new requests,
+and then release every reference to the `LlamaLanguageModel`,
+including the sessions that use it.
+The model frees its weights when the last reference is released.
+A response stream's generation can keep its model for a short time
+after the stream ends,
+until the generation stops at its next token.
+A `prewarm` keeps its model until its download and load end,
+and you can't wait for it directly.
+A request that you await after a `prewarm` waits for that load,
+but the `prewarm` can keep the model for a short time after the request returns.
+
+To be certain that the assertion can't happen,
+set the environment variable `GGML_METAL_NO_RESIDENCY` to any value
+before the first model loads.
+llama.cpp's Metal backend then doesn't use residency sets,
+so a process exits normally even with a model loaded on the GPU.
+Set it in the process's environment,
+or call `setenv("GGML_METAL_NO_RESIDENCY", "1", 1)` before the first request or `prewarm`,
+because `LlamaLanguageModel` starts the llama.cpp backend at the first load.
+If another library in the process uses llama.cpp,
+set it before that library starts llama.cpp with `llama_backend_init`.
+The cost is that residency sets keep a model's GPU memory wired between requests;
+without them, the system can reclaim that memory while the GPU is idle,
+and the next request waits for it again.
+This package doesn't set the variable.
+
### LiteRT-LM
diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift
index cc269aa4..e34ee8eb 100644
--- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift
+++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift
@@ -102,6 +102,9 @@ import Foundation
text: UnsafePointer?,
userData: UnsafeMutableRawPointer?
) {
+ #if DEBUG
+ LlamaTestHooks.onLog.withLock { $0 }?()
+ #endif
let minimum = LlamaRuntime.logLevel.withLock { $0 }
let logger = LlamaRuntime.Logger(userData: userData)
guard LlamaRuntime.shouldWrite(level: level, logger: logger, minimum: minimum), let text else {
@@ -120,6 +123,57 @@ import Foundation
/// let model = LlamaLanguageModel(modelPath: "/path/to/model.gguf")
/// let hubModel = LlamaLanguageModel(modelId: "ggml-org/gemma-3-4b-it-GGUF:Q4_K_M")
/// ```
+ ///
+ /// llama.cpp's Metal backend stops a process with an assertion
+ /// if the process exits while a model has layers on the GPU.
+ /// So the model registers an exit handler when a load ends,
+ /// which stops new loads and generations,
+ /// waits up to three seconds for running work,
+ /// and then frees every loaded model.
+ /// After the handler starts, requests throw
+ /// ``LanguageModelSession/GenerationError/assetsUnavailable(_:)``.
+ /// The handler can't help in these cases:
+ ///
+ /// - An exit during the first load.
+ /// - An exit from inside a load, a generation, or the release of a model,
+ /// such as from a llama.cpp callback.
+ /// An exit from a llama.cpp callback on a thread
+ /// that runs no task and releases no model,
+ /// such as a thread that llama.cpp or another library owns,
+ /// makes the handler wait the full three seconds and then free nothing.
+ /// - Work that is still running after three seconds.
+ /// - An exit that runs no exit handlers, such as `_exit` or a crash.
+ ///
+ /// If freeing the models takes more than ten seconds,
+ /// the handler aborts the process, so that the exit can't hang.
+ /// A crash report shows `abortAfterExitCleanupTimeout()` in its backtrace,
+ /// which first tries to write one line to standard error.
+ /// If the cleanup ends just after the limit,
+ /// the cleanup's thread can abort first, without the line,
+ /// and the backtrace shows `abortAfterLateExitCleanup()`.
+ ///
+ /// No method unloads a model that something still references.
+ /// To be certain that no model is loaded when the process exits,
+ /// wait until every request returns and every response stream ends,
+ /// start no new requests,
+ /// and then release every reference to the model, including its sessions.
+ /// A response stream's generation can keep the model for a short time
+ /// after the stream ends, until the generation stops at its next token.
+ /// A `prewarm` keeps the model until its download and load end,
+ /// and you can't wait for it directly.
+ /// A request that you await after a `prewarm` waits for that load,
+ /// but the `prewarm` can keep the model for a short time after the request returns.
+ ///
+ /// To be certain that the assertion can't happen,
+ /// set the environment variable `GGML_METAL_NO_RESIDENCY` to any value
+ /// before the first model loads,
+ /// in the process's environment or with `setenv` before the first request or `prewarm`,
+ /// because the model starts the llama.cpp backend at the first load.
+ /// llama.cpp's Metal backend then doesn't use residency sets,
+ /// so a process exits normally even with a model loaded on the GPU.
+ /// The cost is that the system can reclaim a model's GPU memory
+ /// while the GPU is idle, and the next request waits for it again.
+ /// This package doesn't set the variable.
public final class LlamaLanguageModel: LanguageModel, @unchecked Sendable {
/// The reason the model is unavailable.
/// This model is always available.
@@ -565,7 +619,13 @@ import Foundation
/// frees a context that is still decoding: it runs on a transient context
/// instead and leaves the cache untouched.
private let sessionContextLock = NSLock()
- private var cachedSessionContext: CachedSessionContext?
+ private var cachedSessionContext: CachedSessionContext? {
+ // The exit handler frees the cached context through the resources.
+ didSet { resources?.setCachedContext(cachedSessionContext?.context) }
+ }
+
+ /// The loaded model's resources, which the exit handler can free.
+ private var resources: LlamaModelResources?
/// The number of prompt tokens reused from the cached context by the most
/// recent chat generation.
@@ -580,21 +640,30 @@ import Foundation
/// model. The next chat generation prefills its full prompt again. Call this
/// under memory pressure or when a session is discarded. If a generation is
/// running on the cached context, it is freed as soon as that generation ends.
+ ///
+ /// After the exit handler starts, this method does nothing,
+ /// because the handler frees the cached context.
public func clearCachedContext() {
- discardCachedSessionContext()
+ // After the exit handler starts, the context may be freed already.
+ try? LlamaLifecycle.shared.perform { discardCachedSessionContext() }
}
private func discardCachedSessionContext() {
- sessionContextLock.lock()
- defer { sessionContextLock.unlock() }
- guard var cached = cachedSessionContext else { return }
- if cached.isCheckedOut {
- cached.discardWhenReleased = true
- cachedSessionContext = cached
- return
+ let unused: OpaquePointer? = sessionContextLock.withLock {
+ guard var cached = cachedSessionContext else { return nil }
+ if cached.isCheckedOut {
+ cached.discardWhenReleased = true
+ cachedSessionContext = cached
+ return nil
+ }
+ cachedSessionContext = nil
+ return cached.context
+ }
+ // Free without the lock: llama.cpp logs while it frees a context,
+ // and a log callback that calls `exit` runs the exit handler on this thread.
+ if let unused {
+ llama_free(unused)
}
- llama_free(cached.context)
- cachedSessionContext = nil
}
private func recordCachedTokens(_ tokens: [llama_token], context: OpaquePointer) {
@@ -615,19 +684,22 @@ import Foundation
/// context is checked back in (or freed, if a discard was requested while
/// it was busy); a transient context is freed.
private func releaseSessionContext(_ context: OpaquePointer) {
- sessionContextLock.lock()
- defer { sessionContextLock.unlock() }
- if var cached = cachedSessionContext, cached.context == context {
- if cached.discardWhenReleased {
- llama_free(cached.context)
- cachedSessionContext = nil
- } else {
+ let unused: OpaquePointer? = sessionContextLock.withLock {
+ if var cached = cachedSessionContext, cached.context == context {
+ if cached.discardWhenReleased {
+ cachedSessionContext = nil
+ return context
+ }
cached.isCheckedOut = false
cachedSessionContext = cached
+ return nil
}
- return
+ return context
+ }
+ // Free without the lock, as in `discardCachedSessionContext()`.
+ if let unused {
+ llama_free(unused)
}
- llama_free(context)
}
/// Returns a context for the session along with the index of the first
@@ -818,14 +890,10 @@ import Foundation
}
deinit {
- if let cached = cachedSessionContext {
- llama_free(cached.context)
- }
- if let mtmdContext = mtmdContext {
- mtmd_free(mtmdContext)
- }
- if let model = model {
- llama_model_free(model)
+ // The resources free the cached context, the projector, and the model,
+ // unless the exit handler freed them already.
+ if let resources {
+ LlamaLifecycle.shared.free(resources)
}
}
@@ -968,6 +1036,11 @@ import Foundation
let runtimeOptions = resolvedOptions(from: options)
let structuredOptions = resolvedStructuredOptions(from: options)
+ // The exit handler waits for this request's llama.cpp work,
+ // but not for its tool calls.
+ let operation = try LlamaLifecycle.shared.begin()
+ defer { operation.end() }
+
// Structured output does not go through the projector yet.
// Without an image marker, a prompt with an image throws
// before any generation or tool call,
@@ -1071,6 +1144,7 @@ import Foundation
try session.recordToolRound(toolRounds, completed: allEntries)
+ operation.suspend()
let (callsEntry, resolution) = try await resolveToolCalls(
parsedCalls,
requestContext: requestContext,
@@ -1104,7 +1178,9 @@ import Foundation
pendingEntries.append(.prompt(prompt))
}
}
+ try operation.resume()
}
+ try operation.resume()
if type == String.self {
return LanguageModelSession.Response(
content: text as! Content,
@@ -1215,6 +1291,10 @@ import Foundation
let runtimeOptions = resolvedOptions(from: options)
let maxTokens = runtimeOptions.maximumResponseTokens ?? 100
+ // The exit handler waits for this request's llama.cpp work,
+ // but not for its tool calls.
+ let operation = try LlamaLifecycle.shared.begin()
+ defer { operation.end() }
let outputFormat = self.currentToolCallFormat()
let toolRounds = session.toolRoundLimit(for: "Llama", options: options)
var accumulatedEntries: [Transcript.Entry] = []
@@ -1327,6 +1407,7 @@ import Foundation
try session.recordToolRound(toolRounds, completed: accumulatedEntries)
+ operation.suspend()
let (callsEntry, resolution) = try await self.resolveToolCalls(
parsedCalls,
requestContext: requestContext,
@@ -1359,6 +1440,7 @@ import Foundation
emittedBase += roundVisible
yieldSnapshot(emittedBase)
}
+ try operation.resume()
}
if emittedBase != lastYieldedText {
@@ -1382,6 +1464,8 @@ import Foundation
/// Downloads a Hub model's files if needed, then loads the model.
func ensureModelLoaded() async throws {
+ // After the exit handler starts, don't list or download a Hub model's files.
+ try LlamaLifecycle.shared.stopIfExiting()
let files =
if let hubSource {
try await hubSource.files()
@@ -1399,6 +1483,11 @@ import Foundation
guard !isModelLoaded else { return }
+ // The exit handler waits for this load.
+ // After the handler starts, this throws before the path is checked.
+ let operation = try LlamaLifecycle.shared.begin()
+ defer { operation.end() }
+
// Check if model file exists
guard FileManager.default.fileExists(atPath: modelPath) else {
throw LlamaLanguageModelError.invalidModelPath
@@ -1423,6 +1512,7 @@ import Foundation
let modelParams = createModelParams()
guard let loadedModel = llama_model_load_from_file(modelPath, modelParams) else {
+ try LlamaLifecycle.shared.stopIfExiting()
throw LlamaLanguageModelError.modelLoadFailed
}
@@ -1446,6 +1536,10 @@ import Foundation
self.vocab = llama_model_get_vocab(loadedModel)
self.tokenCache = StructuredGenerationTokenCache()
self.isModelLoaded = true
+ let resources = LlamaModelResources(model: loadedModel, projector: mtmdContext)
+ self.resources = resources
+ LlamaLifecycle.shared.track(resources)
+ LlamaRuntime.registerExitHandler()
}
private func createModelParams() -> llama_model_params {
@@ -1454,6 +1548,8 @@ import Foundation
// Try to reduce memory usage
params.load_mode = LLAMA_LOAD_MODE_MMAP
+ // Returning false stops the load, so that the exit handler doesn't wait for all of it.
+ params.progress_callback = { _, _ in !LlamaLifecycle.shared.isStopping }
return params
}
@@ -1737,6 +1833,7 @@ import Foundation
}
mutating func decode(_ token: Int) async throws {
+ try LlamaLifecycle.shared.stopIfExiting()
let llamaToken = llama_token(token)
batch.pointee.n_tokens = 1
@@ -1871,9 +1968,14 @@ import Foundation
var decodedTokens = promptTokens
for _ in 0 ..< maxTokens {
+ #if DEBUG
+ LlamaTestHooks.beforeToken.withLock { $0 }?()
+ #endif
if Task.isCancelled {
break
}
+ // The exit handler waits for this generation, so stop it early.
+ try LlamaLifecycle.shared.stopIfExiting()
// Sample next token from logits of the last token we just decoded
let nextToken = llama_sampler_sample(sampler, context, batch.n_tokens - 1)
@@ -2044,6 +2146,8 @@ import Foundation
if Task.isCancelled {
break
}
+ // The exit handler waits for this generation, so stop it early.
+ try LlamaLifecycle.shared.stopIfExiting()
let nextToken = llama_sampler_sample(samplerPtr, context, sampleIndex)
llama_sampler_accept(samplerPtr, nextToken)
@@ -2187,6 +2291,7 @@ import Foundation
let capacity = Int(batchSize)
var start = startIndex
while start < promptTokens.count {
+ try LlamaLifecycle.shared.stopIfExiting()
let count = min(capacity, promptTokens.count - start)
batch.n_tokens = Int32(count)
for i in 0 ..< count {
diff --git a/Sources/AnyLanguageModel/Models/LlamaLifecycle.swift b/Sources/AnyLanguageModel/Models/LlamaLifecycle.swift
new file mode 100644
index 00000000..98142e2b
--- /dev/null
+++ b/Sources/AnyLanguageModel/Models/LlamaLifecycle.swift
@@ -0,0 +1,521 @@
+import Foundation
+
+#if Llama
+ import LlamaSwift
+
+ extension LlamaRuntime {
+ /// Registers ``LlamaLifecycle/shutDown(deadline:)`` to run when the process exits.
+ ///
+ /// ggml creates static objects as it needs them,
+ /// such as its Metal device and buffer types during a model's first load,
+ /// and the C runtime destroys static objects and runs exit handlers
+ /// in the reverse order of their creation.
+ /// A handler that was registered before the first load
+ /// would run after ggml's objects are destroyed,
+ /// and freeing a model then would crash.
+ /// ggml doesn't report when it creates such an object,
+ /// so every load registers the handler again when it ends.
+ /// The last registration runs first,
+ /// and the handler does nothing after its first run.
+ ///
+ /// If a registration fails,
+ /// an earlier one could run after objects that the models use are destroyed,
+ /// so the handler then frees nothing.
+ static func registerExitHandler() {
+ if atexit({ _ = LlamaLifecycle.shared.shutDown() }) != 0 {
+ LlamaLifecycle.shared.exitHandlerRegistrationFailed()
+ }
+ }
+ }
+
+ /// Tracks the llama.cpp work in this process,
+ /// and tries to free every loaded model when the process exits.
+ ///
+ /// ggml's Metal backend frees its device in a C++ static destructor at exit,
+ /// and it stops the process with an assertion
+ /// if a model or a context still has buffers on the device then.
+ /// A generation that is still running at exit can also crash,
+ /// on the GPU or the CPU,
+ /// because the static objects that it uses are destroyed under it.
+ ///
+ /// The exit handler is best effort.
+ /// It must never make an exit worse than an exit without it,
+ /// so it never waits without a limit,
+ /// and it never frees a model or a context that running work could use.
+ /// It stops new loads and generations,
+ /// and it waits up to ``exitDeadline`` for the ones that are running:
+ /// a load stops at its next progress report,
+ /// and a generation stops before its next token or batch of prompt tokens.
+ /// If no work is running then,
+ /// it frees the cached session context, the projector, and the model
+ /// of every loaded ``LlamaLanguageModel``,
+ /// including models that Swift objects still hold.
+ /// Otherwise it frees nothing.
+ /// Freeing only the idle models wouldn't help,
+ /// because the busy model's buffers would still cause the assertion.
+ ///
+ /// The handler can't help, and the assertion can still happen,
+ /// in these cases:
+ ///
+ /// - An exit during the first load,
+ /// because the handler is registered when a load ends.
+ /// An earlier registration would run after ggml's objects are destroyed.
+ /// - An exit from inside a load, a generation, or a cleanup,
+ /// such as from a llama.cpp callback.
+ /// The handler sees that its task or thread is running that work,
+ /// which can't end while the handler waits,
+ /// and returns at once without freeing anything.
+ /// On a thread that runs no task and has no cleanup mark,
+ /// such as a thread that llama.cpp or another library owns,
+ /// the handler can't see this,
+ /// so it waits until the deadline and then frees nothing.
+ /// - Work that is still running at the deadline.
+ /// - An exit that runs no exit handlers, such as `_exit` or a crash.
+ ///
+ /// After the handler starts, every load and generation throws
+ /// ``LanguageModelSession/GenerationError/assetsUnavailable(_:)``.
+ ///
+ /// Nothing here calls llama.cpp with a lock held.
+ /// llama.cpp logs while it frees a model,
+ /// and runs the log callback on the same thread,
+ /// so a callback that calls `exit` would run the exit handler on that thread,
+ /// and a lock that isn't recursive would then never be released.
+ final class LlamaLifecycle: @unchecked Sendable {
+ static let shared = LlamaLifecycle()
+
+ /// How long the exit handler waits for running loads and generations.
+ ///
+ /// A running generation stops before its next token,
+ /// or before its next batch of prompt tokens,
+ /// so the wait is about one call to llama.cpp,
+ /// which takes milliseconds to a few hundred milliseconds for a 4B model.
+ /// Three seconds leaves room for a much larger model or a slow CPU,
+ /// and still ends a process that can't finish its work
+ /// soon enough that the delay looks like a slow quit, not a hang.
+ static let exitDeadline: TimeInterval = 3
+
+ /// How long the exit handler's cleanup may take before the process aborts.
+ ///
+ /// The cleanup frees every model after running work has ended.
+ /// llama.cpp calls the installed log callback while it frees,
+ /// and another library can replace that callback with one that blocks,
+ /// for example on a lock that the exiting thread holds,
+ /// and a GPU driver can stall while it releases buffers.
+ /// Without a limit the process would then hang,
+ /// where without the handler it would have stopped with ggml's assertion.
+ /// So the cleanup runs under a watchdog,
+ /// which calls `abort()` if the cleanup is still running after this time.
+ /// It tries to write one line to standard error first,
+ /// but it aborts whether or not the line can be written.
+ ///
+ /// Freeing Qwen3-4B (2.5 GB) and its context took about 35 milliseconds
+ /// on an Apple silicon Mac, on the GPU and on the CPU.
+ /// The work grows with the number and size of the buffers,
+ /// so a 70B model would take under a second by that measure.
+ /// Ten seconds leaves a margin of more than ten times that
+ /// for a slow but working free.
+ static let cleanupDeadline: TimeInterval = 10
+
+ /// What ``shutDown(deadline:)`` did.
+ enum ShutdownResult: Equatable {
+ /// It freed every model.
+ case freed
+ /// It had already run, so it did nothing.
+ case alreadyShutDown
+ /// It ran inside an operation, so it freed nothing without waiting.
+ case insideOperation
+ /// Work was still running at the deadline, so it freed nothing.
+ case timedOut
+ /// A registration of the exit handler failed, so it freed nothing.
+ case registrationFailed
+ }
+
+ /// The key in a thread's dictionary for the number of operations that the thread runs.
+ private static let depthKey = "AnyLanguageModel.LlamaLifecycle.operationDepth"
+
+ /// Guards the properties below, and signals when no operation is running.
+ private let condition = NSCondition()
+ private var isShuttingDown = false
+ private var registrationFailed = false
+ private var activeOperations = 0
+ /// The number of active operations in each task.
+ ///
+ /// A key is used only to compare tasks,
+ /// and only while its operation is active,
+ /// so its task is still running.
+ private var activeTasks: [UnsafeCurrentTask: Int] = [:]
+ private var resources: [ObjectIdentifier: LlamaModelResources] = [:]
+
+ /// Whether the process is exiting.
+ ///
+ /// Generations read this before each token,
+ /// which costs one uncontended lock and unlock.
+ var isStopping: Bool {
+ condition.withLock { isShuttingDown }
+ }
+
+ /// The number of loads, generations, and cleanups that are running.
+ var activeOperationCount: Int {
+ condition.withLock { activeOperations }
+ }
+
+ /// The error that work throws after the exit handler starts.
+ static var exiting: LanguageModelSession.GenerationError {
+ .assetsUnavailable(
+ .init(debugDescription: "The process is exiting, so llama.cpp work has stopped.")
+ )
+ }
+
+ /// Throws ``exiting`` if the exit handler has started.
+ func stopIfExiting() throws {
+ if isStopping { throw Self.exiting }
+ }
+
+ // MARK: Requests
+
+ /// A load or a generation, which the exit handler waits for.
+ ///
+ /// A request can await other code in the middle of its llama.cpp work,
+ /// so an operation belongs to the task that began it, not to a thread.
+ /// A request suspends its operation while it waits for other code,
+ /// such as a tool call,
+ /// so that the exit handler doesn't wait for that code.
+ /// Use an operation only in the task that began it.
+ final class Operation {
+ private let lifecycle: LlamaLifecycle
+ private var task: UnsafeCurrentTask?
+ private var isActive = true
+
+ fileprivate init(lifecycle: LlamaLifecycle, task: UnsafeCurrentTask?) {
+ self.lifecycle = lifecycle
+ self.task = task
+ }
+
+ /// Ends the operation for now, if it is active.
+ func suspend() {
+ guard isActive else { return }
+ isActive = false
+ lifecycle.endOperation(in: task)
+ }
+
+ /// Starts the operation again, if it is suspended.
+ ///
+ /// - Throws: ``LlamaLifecycle/exiting`` if the exit handler has started.
+ func resume() throws {
+ guard !isActive else { return }
+ task = try lifecycle.beginOperation()
+ isActive = true
+ }
+
+ /// Ends the operation.
+ func end() {
+ suspend()
+ }
+ }
+
+ /// Starts an operation in the current task, unless the process is exiting.
+ ///
+ /// End the operation when its llama.cpp work ends.
+ ///
+ /// - Throws: ``exiting`` if the exit handler has started.
+ func begin() throws -> Operation {
+ let task = try beginOperation()
+ return Operation(lifecycle: self, task: task)
+ }
+
+ private func beginOperation() throws -> UnsafeCurrentTask? {
+ // The task outlives the operation, which ends in the same task.
+ let task = withUnsafeCurrentTask { $0 }
+ try condition.withLock {
+ guard !isShuttingDown else { throw Self.exiting }
+ activeOperations += 1
+ if let task { activeTasks[task, default: 0] += 1 }
+ }
+ return task
+ }
+
+ private func endOperation(in task: UnsafeCurrentTask?) {
+ condition.withLock {
+ if let task {
+ activeTasks[task, default: 1] -= 1
+ if activeTasks[task] == 0 { activeTasks[task] = nil }
+ }
+ activeOperations -= 1
+ if activeOperations == 0 { condition.broadcast() }
+ }
+ }
+
+ // MARK: Cleanups
+
+ /// Runs synchronous llama.cpp work, unless the process is exiting.
+ func perform(_ body: () throws -> Result) throws -> Result {
+ try condition.withLock {
+ guard !isShuttingDown else { throw Self.exiting }
+ activeOperations += 1
+ }
+ return try runCounted(body)
+ }
+
+ /// Runs `body` as work that the caller has already counted,
+ /// marks the current thread as running it,
+ /// and stops counting it when it returns.
+ private func runCounted(_ body: () throws -> Result) rethrows -> Result {
+ // The body is synchronous, so it stays on this thread.
+ let thread = Thread.current.threadDictionary
+ let depth = thread[Self.depthKey] as? Int ?? 0
+ thread[Self.depthKey] = depth + 1
+ defer {
+ thread[Self.depthKey] = depth
+ condition.withLock {
+ activeOperations -= 1
+ if activeOperations == 0 { condition.broadcast() }
+ }
+ }
+ return try body()
+ }
+
+ /// Whether the current thread or task is running an operation.
+ private var isInsideOperation: Bool {
+ if (Thread.current.threadDictionary[Self.depthKey] as? Int ?? 0) > 0 {
+ return true
+ }
+ guard let task = withUnsafeCurrentTask(body: { $0 }) else { return false }
+ return condition.withLock { activeTasks[task] != nil }
+ }
+
+ /// Records that a registration of the exit handler failed.
+ func exitHandlerRegistrationFailed() {
+ condition.withLock { registrationFailed = true }
+ }
+
+ /// Records the resources of a model, so that ``shutDown(deadline:)`` can free them.
+ func track(_ model: LlamaModelResources) {
+ condition.withLock { resources[ObjectIdentifier(model)] = model }
+ }
+
+ /// Frees the resources of a model, unless ``shutDown(deadline:)`` freed them already.
+ ///
+ /// The cleanup runs even while the process is exiting.
+ /// It counts as running work,
+ /// so that the exit handler's thread check and deadline apply to it,
+ /// and it calls llama.cpp without the lock.
+ func free(_ model: LlamaModelResources) {
+ condition.withLock {
+ resources[ObjectIdentifier(model)] = nil
+ activeOperations += 1
+ }
+ runCounted { model.free() }
+ }
+
+ /// Stops new work, waits up to `deadline` for the work that is running,
+ /// and frees the resources of every model if no work is running then.
+ @discardableResult
+ func shutDown(deadline: TimeInterval = exitDeadline) -> ShutdownResult {
+ let insideOperation = isInsideOperation
+ var models: [LlamaModelResources] = []
+ let result: ShutdownResult = condition.withLock {
+ guard !isShuttingDown else { return .alreadyShutDown }
+ isShuttingDown = true
+ // The operation on this thread or task can't end while it waits here.
+ guard !insideOperation else { return .insideOperation }
+ guard !registrationFailed else { return .registrationFailed }
+ // Measure the deadline with a monotonic clock,
+ // and wait in short steps that start from the current wall-clock time,
+ // so that a change to the wall clock can't extend the wait.
+ let clock = ContinuousClock()
+ let end = clock.now + .seconds(deadline)
+ while activeOperations > 0 {
+ let remaining = clock.now.duration(to: end)
+ guard remaining > .zero else { break }
+ let step = min(remaining, .milliseconds(100))
+ _ = condition.wait(until: Date(timeIntervalSinceNow: step / .seconds(1)))
+ }
+ guard activeOperations == 0 else { return .timedOut }
+ models = Array(resources.values)
+ resources = [:]
+ return .freed
+ }
+ // Free without the lock, as in `free(_:)`.
+ // If a log callback calls `exit` here, this handler runs again and does nothing.
+ guard !models.isEmpty else { return result }
+ Self.withAbortDeadline(Self.cleanupDeadline) {
+ for model in models {
+ model.free()
+ }
+ }
+ return result
+ }
+
+ /// The state of a cleanup that runs under ``withAbortDeadline(_:_:)``.
+ private enum CleanupState: Sendable {
+ case running
+ case completed
+ case aborting
+ }
+
+ /// Runs `body`, and aborts the process if `body` is still running after `deadline`.
+ ///
+ /// A separate thread measures the time on a monotonic clock,
+ /// so the exiting thread can't stop it.
+ /// The thread ends as soon as `body` returns,
+ /// and doesn't keep the process alive.
+ ///
+ /// Completion and the timeout are decided in one step under a lock.
+ /// Once `body` has returned and completion is recorded,
+ /// the watchdog never aborts.
+ /// Once the watchdog has recorded the timeout,
+ /// this function aborts instead of returning,
+ /// even if `body` returns after that.
+ static func withAbortDeadline(_ deadline: TimeInterval, _ body: () -> Result) -> Result {
+ let state = Locked(CleanupState.running)
+ let finished = DispatchSemaphore(value: 0)
+ let watchdog = Thread {
+ guard finished.wait(timeout: .now() + deadline) == .timedOut else { return }
+ #if DEBUG
+ LlamaTestHooks.cleanupWatchdog.withLock { $0 }?(.beforeDecision)
+ #endif
+ let aborts = state.withLock { state in
+ guard state == .running else { return false }
+ state = .aborting
+ return true
+ }
+ #if DEBUG
+ LlamaTestHooks.cleanupWatchdog.withLock { $0 }?(.decided(aborts: aborts))
+ #endif
+ if aborts {
+ abortAfterExitCleanupTimeout()
+ }
+ }
+ watchdog.start()
+ let result = body()
+ let completed = state.withLock { state in
+ guard state == .running else { return false }
+ state = .completed
+ return true
+ }
+ // The watchdog has decided to abort, and is about to.
+ guard completed else { abortAfterLateExitCleanup() }
+ finished.signal()
+ return result
+ }
+
+ /// Aborts the process because the exit handler's cleanup returned
+ /// after the watchdog had decided to abort.
+ ///
+ /// A crash report for this abort shows this function
+ /// in the backtrace of the thread that called `abort()`.
+ /// It writes no message and doesn't wait for the watchdog.
+ @inline(never)
+ private static func abortAfterLateExitCleanup() -> Never {
+ abort()
+ }
+
+ /// Aborts the process because the exit handler's cleanup took too long.
+ ///
+ /// A crash report for this abort shows this function
+ /// in the backtrace of the thread that called `abort()`,
+ /// which identifies it.
+ /// The function also tries to write one line to standard error,
+ /// from another thread and with `write(2)`,
+ /// but waits at most 100 milliseconds for it:
+ /// standard error can be a full pipe that nobody reads,
+ /// and its lock can be held by the blocked thread,
+ /// so the abort must not depend on the message.
+ @inline(never)
+ private static func abortAfterExitCleanupTimeout() -> Never {
+ let written = DispatchSemaphore(value: 0)
+ let writer = Thread {
+ let message = "AnyLanguageModel: freeing llama.cpp models at exit didn't finish in time; aborting.\n"
+ _ = message.withCString { write(STDERR_FILENO, $0, strlen($0)) }
+ written.signal()
+ }
+ writer.start()
+ _ = written.wait(timeout: .now() + .milliseconds(100))
+ abort()
+ }
+ }
+
+ /// A loaded llama.cpp model, its projector, and its cached session context.
+ ///
+ /// ``LlamaLifecycle`` holds these resources until it frees them,
+ /// separately from the ``LlamaLanguageModel`` that uses them,
+ /// so that the exit handler can free a model that a Swift object still holds,
+ /// and so that the model's deinit frees nothing twice.
+ final class LlamaModelResources: @unchecked Sendable {
+ let model: OpaquePointer
+ let projector: OpaquePointer?
+
+ /// Guards `cachedContext` and `isFreed`.
+ private let lock = NSLock()
+ private var cachedContext: OpaquePointer?
+ private var isFreed = false
+
+ init(model: OpaquePointer, projector: OpaquePointer?) {
+ self.model = model
+ self.projector = projector
+ }
+
+ /// Records the model's cached session context, or `nil` if it has none.
+ ///
+ /// Any other context exists only during a load or a generation,
+ /// which frees it before it ends.
+ func setCachedContext(_ context: OpaquePointer?) {
+ lock.withLock { cachedContext = context }
+ }
+
+ /// Frees the cached context, the projector, and the model, once.
+ ///
+ /// Call it only through ``LlamaLifecycle``,
+ /// when no load or generation uses the model.
+ func free() {
+ // Take the context under the lock, and call llama.cpp without it.
+ let (isFirst, context): (Bool, OpaquePointer?) = lock.withLock {
+ guard !isFreed else { return (false, nil) }
+ isFreed = true
+ defer { cachedContext = nil }
+ return (true, cachedContext)
+ }
+ guard isFirst else { return }
+ if let context {
+ llama_free(context)
+ }
+ if let projector {
+ mtmd_free(projector)
+ }
+ llama_model_free(model)
+ }
+ }
+
+ #if DEBUG
+ /// Functions that tests run at points inside llama.cpp work.
+ ///
+ /// Only debug builds, which `swift test` uses, compile these hooks,
+ /// so they cost nothing in a release build.
+ enum LlamaTestHooks {
+ /// Runs at the start of each step of a chat generation's token loop,
+ /// before its checks for cancellation and exit,
+ /// on the generation's thread.
+ ///
+ /// After the hook returns,
+ /// the generation does no llama.cpp work before those checks,
+ /// so a generation that the hook holds until the exit handler starts
+ /// stops at once.
+ static let beforeToken = Locked<(@Sendable () -> Void)?>(nil)
+
+ /// Runs for each llama.cpp log message, before the level filter,
+ /// on the thread that logs it.
+ static let onLog = Locked<(@Sendable () -> Void)?>(nil)
+
+ /// A point in the cleanup watchdog after its deadline passes.
+ enum CleanupWatchdogEvent: Sendable, Equatable {
+ /// The watchdog is about to decide between completion and the timeout.
+ case beforeDecision
+ /// The watchdog has decided, and aborts if `aborts` is true.
+ case decided(aborts: Bool)
+ }
+
+ /// Runs on the cleanup watchdog's thread after its deadline passes.
+ static let cleanupWatchdog = Locked<(@Sendable (CleanupWatchdogEvent) -> Void)?>(nil)
+ }
+ #endif
+#endif // Llama
diff --git a/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift b/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift
new file mode 100644
index 00000000..fa52d288
--- /dev/null
+++ b/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift
@@ -0,0 +1,651 @@
+import Foundation
+import Testing
+
+@testable import AnyLanguageModel
+
+#if Llama
+ /// A model that a global variable holds for the life of the process.
+ nonisolated(unsafe) private var globalExitTestModel: LlamaLanguageModel?
+
+ /// Tests that exit a child process with a model loaded.
+ ///
+ /// The tests that use ``LlamaTestHooks`` run only in debug builds,
+ /// which compile the hooks.
+ ///
+ /// Without the exit handler,
+ /// llama.cpp's Metal backend stops such a process with an assertion
+ /// when the model has layers on the GPU,
+ /// and an exit during a generation crashes on the GPU and on the CPU.
+ @Suite(
+ "LlamaLanguageModel at exit",
+ .serialized,
+ .enabled(if: ProcessInfo.processInfo.environment["LLAMA_MODEL_PATH"] != nil)
+ )
+ struct LlamaLanguageModelExitTests {
+ static var modelPath: String {
+ ProcessInfo.processInfo.environment["LLAMA_MODEL_PATH"] ?? ""
+ }
+
+ /// Returns a model and a session after a first response,
+ /// so that the model and its cached context are loaded.
+ static func loadedModel(gpuLayers: Int32) async throws -> (LlamaLanguageModel, LanguageModelSession) {
+ let model = LlamaLanguageModel(modelPath: modelPath, gpuLayers: gpuLayers)
+ let session = LanguageModelSession(model: model)
+ _ = try await session.respond(to: "Say hello", options: GenerationOptions(maximumResponseTokens: 2))
+ return (model, session)
+ }
+
+ #if DEBUG
+ /// Holds a generation until the exit handler has started,
+ /// so that the handler must wait for it.
+ static func holdUntilShutdownStarts() {
+ let limit = ExitTestTimer.now() + 10_000_000_000
+ while !LlamaLifecycle.shared.isStopping {
+ // A distinct status shows that the exit handler never started.
+ if ExitTestTimer.now() > limit { _exit(9) }
+ usleep(1_000)
+ }
+ }
+
+ /// Starts a generation in another task,
+ /// and returns when its first token step starts.
+ ///
+ /// - Parameter whileActive: What the generation does at the start of that step,
+ /// while it is known to be running.
+ static func startGeneration(
+ _ session: LanguageModelSession,
+ whileActive: @escaping @Sendable () -> Void
+ ) {
+ let started = DispatchSemaphore(value: 0)
+ let once = Locked(false)
+ LlamaTestHooks.beforeToken.withLock {
+ $0 = {
+ let isFirst = once.withLock { done in
+ defer { done = true }
+ return !done
+ }
+ guard isFirst else { return }
+ started.signal()
+ whileActive()
+ }
+ }
+ Task.detached {
+ _ = try? await session.respond(
+ to: "Count from 1 to 1000.",
+ options: GenerationOptions(maximumResponseTokens: 2048)
+ )
+ }
+ // A distinct status shows that the generation never started.
+ if started.wait(timeout: .now() + 30) == .timedOut { _exit(3) }
+ }
+ #endif
+
+ @Test("A process exits normally with a loaded model", arguments: [Int32(-1), 0])
+ func idleExit(gpuLayers: Int32) async throws {
+ let name = "idleExit\(gpuLayers)"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .success) { [name = name as String, gpuLayers = gpuLayers as Int32] in
+ ExitTestTimer.childStarted(name)
+ let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: gpuLayers)
+ ExitTestTimer.stepStarted(name)
+ withExtendedLifetime((model, session)) { exit(0) }
+ }
+ await ExitTestTimer.finish(name, watchdog, measuring: false)
+ }
+
+ @Test("A process exits normally with a model in a global variable", arguments: [Int32(-1), 0])
+ func exitWithGlobalModel(gpuLayers: Int32) async throws {
+ let name = "exitWithGlobalModel\(gpuLayers)"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .success) { [name = name as String, gpuLayers = gpuLayers as Int32] in
+ ExitTestTimer.childStarted(name)
+ globalExitTestModel = LlamaLanguageModel(
+ modelPath: LlamaLanguageModelExitTests.modelPath,
+ gpuLayers: gpuLayers
+ )
+ _ = try await LanguageModelSession(model: globalExitTestModel!).respond(
+ to: "Say hello",
+ options: GenerationOptions(maximumResponseTokens: 2)
+ )
+ ExitTestTimer.stepStarted(name)
+ exit(0)
+ }
+ await ExitTestTimer.finish(name, watchdog, measuring: false)
+ }
+
+ #if DEBUG
+ @Test(
+ "The exit handler waits for a generation that is running, and the process exits normally",
+ arguments: [Int32(-1), 0]
+ )
+ func exitDuringGeneration(gpuLayers: Int32) async throws {
+ let name = "exitDuringGeneration\(gpuLayers)"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .success) { [name = name as String, gpuLayers = gpuLayers as Int32] in
+ ExitTestTimer.childStarted(name)
+ let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: gpuLayers)
+ LlamaLanguageModelExitTests.startGeneration(session) {
+ LlamaLanguageModelExitTests.holdUntilShutdownStarts()
+ }
+ ExitTestTimer.stepStarted(name)
+ withExtendedLifetime(model) { exit(0) }
+ }
+ await ExitTestTimer.finish(name, watchdog, measuring: false)
+ }
+
+ @Test("A generation that runs past the deadline doesn't keep the process alive")
+ func exitPastDeadline() async throws {
+ let name = "exitPastDeadline"
+ let watchdog = ExitTestTimer.watch(name)
+ // The handler frees nothing, so ggml's assertion ends the process,
+ // as it would without the handler.
+ await #expect(processExitsWith: .signal(SIGABRT)) {
+ ExitTestTimer.childStarted("exitPastDeadline")
+ let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1)
+ LlamaLanguageModelExitTests.startGeneration(session) {
+ while true { sleep(60) }
+ }
+ ExitTestTimer.stepStarted("exitPastDeadline")
+ withExtendedLifetime(model) { exit(0) }
+ }
+ let elapsed = try #require(await ExitTestTimer.finish(name, watchdog))
+ #expect(elapsed >= .seconds(LlamaLifecycle.exitDeadline))
+ }
+
+ @Test("An exit from inside a generation doesn't wait for that generation")
+ func exitInsideOperation() async throws {
+ let name = "exitInsideOperation"
+ let watchdog = ExitTestTimer.watch(name)
+ // The handler returns at once and frees nothing,
+ // so ggml's assertion ends the process before the deadline.
+ // The exit status can't show how long the handler took;
+ // `shutDownInsideTaskOperation()` checks that it returns at once.
+ await #expect(processExitsWith: .signal(SIGABRT)) {
+ ExitTestTimer.childStarted("exitInsideOperation")
+ let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1)
+ LlamaLanguageModelExitTests.startGeneration(session) {
+ ExitTestTimer.stepStarted("exitInsideOperation")
+ exit(0)
+ }
+ // The generation's thread exits the process.
+ withExtendedLifetime(model) {
+ while true { sleep(60) }
+ }
+ }
+ await ExitTestTimer.finish(name, watchdog, measuring: false)
+ }
+
+ @Test("A log callback that exits while a model is freed doesn't hang the process")
+ func exitFromLogDuringCleanup() async throws {
+ // The handler sees that the cleanup's thread is exiting, and returns at once,
+ // which `shutDownInsideThreadOperation()` checks without a model.
+ // The model is only partly freed, so ggml's assertion ends the process,
+ // as it would without the handler.
+ let name = "exitFromLogDuringCleanup"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .signal(SIGABRT)) {
+ ExitTestTimer.childStarted("exitFromLogDuringCleanup")
+ do {
+ let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1)
+ let once = Locked(false)
+ LlamaTestHooks.onLog.withLock {
+ $0 = {
+ let isFirst = once.withLock { done in
+ defer { done = true }
+ return !done
+ }
+ if isFirst { exit(0) }
+ }
+ }
+ ExitTestTimer.stepStarted("exitFromLogDuringCleanup")
+ // Releasing the last references frees the model,
+ // and llama.cpp logs while it frees.
+ _ = (model, session)
+ }
+ // A distinct status shows that llama.cpp logged nothing.
+ exit(4)
+ }
+ await ExitTestTimer.finish(name, watchdog, measuring: false)
+ }
+
+ @Test("A cleanup that blocks at exit aborts the process instead of hanging")
+ func exitWithBlockedCleanup() async throws {
+ // A log callback blocks while the exit handler frees the model,
+ // so the cleanup watchdog aborts the process.
+ let name = "exitWithBlockedCleanup"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .signal(SIGABRT)) {
+ ExitTestTimer.childStarted("exitWithBlockedCleanup")
+ let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1)
+ LlamaTestHooks.onLog.withLock {
+ $0 = {
+ if LlamaLifecycle.shared.isStopping {
+ while true { sleep(60) }
+ }
+ }
+ }
+ ExitTestTimer.stepStarted("exitWithBlockedCleanup")
+ withExtendedLifetime((model, session)) { exit(0) }
+ }
+ let elapsed = try #require(await ExitTestTimer.finish(name, watchdog))
+ #expect(elapsed >= .seconds(LlamaLifecycle.cleanupDeadline))
+ }
+ #endif
+
+ @Test("A process exits normally after the model is released", arguments: [Int32(-1), 0])
+ func exitAfterRelease(gpuLayers: Int32) async {
+ let name = "exitAfterRelease\(gpuLayers)"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .success) { [name = name as String, gpuLayers = gpuLayers as Int32] in
+ ExitTestTimer.childStarted(name)
+ weak var released: LlamaLanguageModel?
+ do {
+ let (model, _) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: gpuLayers)
+ released = model
+ }
+ // A distinct status shows that something still holds the model.
+ guard released == nil else { exit(4) }
+ exit(0)
+ }
+ await ExitTestTimer.finish(name, watchdog, measuring: false)
+ }
+
+ @Test("After the exit handler runs, a request throws, and nothing is freed twice")
+ func requestAfterShutdown() async {
+ let name = "requestAfterShutdown"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .success) {
+ ExitTestTimer.childStarted("requestAfterShutdown")
+ weak var released: LlamaLanguageModel?
+ do {
+ let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1)
+ released = model
+ guard LlamaLifecycle.shared.shutDown() == .freed else { exit(5) }
+ // The cached context is freed already, so this frees nothing.
+ model.clearCachedContext()
+ do {
+ _ = try await session.respond(to: "Say hello")
+ exit(6)
+ } catch LanguageModelSession.GenerationError.assetsUnavailable {}
+ do {
+ for try await _ in session.streamResponse(to: "Say hello") {}
+ exit(7)
+ } catch LanguageModelSession.GenerationError.assetsUnavailable {}
+ }
+ // The model's deinit ran after the handler freed the model,
+ // and freed nothing again.
+ guard released == nil else { exit(4) }
+ exit(0)
+ }
+ await ExitTestTimer.finish(name, watchdog, measuring: false)
+ }
+
+ @Test("After the exit handler runs, a model that isn't loaded fails to load")
+ func loadAfterShutdown() async {
+ let name = "loadAfterShutdown"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .success) {
+ ExitTestTimer.childStarted("loadAfterShutdown")
+ _ = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1)
+ LlamaLifecycle.shared.shutDown()
+ let other = LlamaLanguageModel(modelPath: LlamaLanguageModelExitTests.modelPath)
+ do {
+ _ = try await LanguageModelSession(model: other).respond(to: "Say hello")
+ exit(5)
+ } catch LanguageModelSession.GenerationError.assetsUnavailable {}
+ // A missing file throws the exit error, not an invalid path.
+ let missing = LlamaLanguageModel(modelPath: "/nonexistent/model.gguf")
+ do {
+ _ = try await LanguageModelSession(model: missing).respond(to: "Say hello")
+ exit(6)
+ } catch LanguageModelSession.GenerationError.assetsUnavailable {}
+ exit(0)
+ }
+ await ExitTestTimer.finish(name, watchdog, measuring: false)
+ }
+ }
+
+ /// Lets the parent of an exit test kill a child that hangs,
+ /// and measure how long the child takes to end after the step under test.
+ ///
+ /// Swift Testing's time limit doesn't end a child process,
+ /// so a child that hangs would keep the test run waiting.
+ /// The child writes its process ID when it starts,
+ /// and the monotonic time when the step under test starts,
+ /// to files that both processes name from the parent's process ID.
+ /// The measurement starts after the child's setup,
+ /// so a slow load doesn't change it.
+ ///
+ /// The measurement ends when the parent sees that the child has ended,
+ /// which a busy machine can delay by seconds.
+ /// So a test checks only that a time is at least a bound,
+ /// which a delay can't break,
+ /// and never that a time is less than a bound.
+ enum ExitTestTimer {
+ /// How long the parent waits for a child before it kills the child.
+ static let limit: Duration = .seconds(120)
+
+ /// The current time of a monotonic clock that every process on the system shares.
+ static func now() -> UInt64 {
+ clock_gettime_nsec_np(CLOCK_UPTIME_RAW)
+ }
+
+ private static func file(_ name: String, _ suffix: String, parent: pid_t) -> URL {
+ URL(fileURLWithPath: NSTemporaryDirectory())
+ .appendingPathComponent("llama-exit-test-\(parent)-\(name).\(suffix)")
+ }
+
+ /// The exit status of a child that couldn't write a timer file.
+ ///
+ /// The child also writes the error to standard error.
+ static let writeFailureStatus: CInt = 70
+
+ /// In the child: writes `text` to a timer file,
+ /// or ends the child with ``writeFailureStatus`` if it can't.
+ private static func write(_ text: String, to url: URL) {
+ do {
+ try text.write(to: url, atomically: true, encoding: .utf8)
+ } catch {
+ fputs("ExitTestTimer couldn't write \(url.path): \(error)\n", stderr)
+ _exit(writeFailureStatus)
+ }
+ }
+
+ /// In the child: records its process ID.
+ static func childStarted(_ name: String) {
+ write(String(getpid()), to: file(name, "pid", parent: getppid()))
+ }
+
+ /// In the child: records when the step under test starts.
+ static func stepStarted(_ name: String) {
+ write(String(now()), to: file(name, "start", parent: getppid()))
+ }
+
+ private static func read(_ name: String, _ suffix: String) -> String? {
+ try? String(contentsOf: file(name, suffix, parent: getpid()), encoding: .utf8)
+ }
+
+ /// In the parent: starts a task that kills the child if it hasn't ended after ``limit``,
+ /// and returns whether it did.
+ static func watch(_ name: String) -> Task {
+ for suffix in ["pid", "start"] {
+ try? FileManager.default.removeItem(at: file(name, suffix, parent: getpid()))
+ }
+ return Task.detached {
+ guard (try? await Task.sleep(for: limit)) != nil else { return false }
+ guard let pid = read(name, "pid").flatMap({ pid_t($0) }) else { return false }
+ kill(pid, SIGKILL)
+ return true
+ }
+ }
+
+ /// In the parent, after the child ends: stops the watchdog,
+ /// records an issue if it killed the child,
+ /// and, if `measuring`, returns the time from the step's start to now.
+ ///
+ /// If the start time can't be read,
+ /// this records an issue with the error and returns `nil`.
+ /// A child that couldn't write the file
+ /// exits with ``writeFailureStatus`` instead.
+ @discardableResult
+ static func finish(
+ _ name: String,
+ _ watchdog: Task,
+ measuring: Bool = true
+ ) async -> Duration? {
+ watchdog.cancel()
+ let killed = await watchdog.value
+ #expect(!killed, "The child process didn't end within \(limit).")
+ guard measuring else { return nil }
+ let url = file(name, "start", parent: getpid())
+ let text: String
+ do {
+ text = try String(contentsOf: url, encoding: .utf8)
+ } catch {
+ Issue.record("Couldn't read the step's start time from \(url.path): \(error)")
+ return nil
+ }
+ guard let start = UInt64(text) else {
+ Issue.record("The step's start time in \(url.path) isn't a number: \(text.debugDescription)")
+ return nil
+ }
+ return .nanoseconds(Int64(now() - start))
+ }
+ }
+
+ /// Tests of the exit handler's bookkeeping, without a model.
+ @Suite("LlamaLifecycle")
+ struct LlamaLifecycleTests {
+ @Test("Shutting down with no work frees, and a second shutdown does nothing")
+ func shutDownIdle() throws {
+ let lifecycle = LlamaLifecycle()
+ #expect(lifecycle.shutDown(deadline: 0) == .freed)
+ #expect(lifecycle.shutDown(deadline: 0) == .alreadyShutDown)
+ #expect(lifecycle.isStopping)
+ #expect(throws: LanguageModelSession.GenerationError.self) { try lifecycle.begin() }
+ #expect(throws: LanguageModelSession.GenerationError.self) { try lifecycle.perform {} }
+ }
+
+ @Test("Shutting down waits for running work, and frees nothing at the deadline")
+ func shutDownTimesOut() async throws {
+ let lifecycle = LlamaLifecycle()
+ nonisolated(unsafe) let operation = try lifecycle.begin()
+ // Shut down from a thread that runs no task, so the operation isn't its own.
+ let result = await withCheckedContinuation { continuation in
+ Thread.detachNewThread {
+ let start = ContinuousClock.now
+ let result = lifecycle.shutDown(deadline: 0.2)
+ continuation.resume(returning: (result, ContinuousClock.now - start))
+ }
+ }
+ #expect(result.0 == .timedOut)
+ #expect(result.1 >= .milliseconds(200))
+ operation.end()
+ #expect(lifecycle.activeOperationCount == 0)
+ }
+
+ @Test("Shutting down returns when running work ends")
+ func shutDownWaitsForWork() async throws {
+ let lifecycle = LlamaLifecycle()
+ nonisolated(unsafe) let operation = try lifecycle.begin()
+ let result = await withCheckedContinuation { continuation in
+ Thread.detachNewThread {
+ continuation.resume(returning: lifecycle.shutDown(deadline: 10))
+ }
+ usleep(100_000)
+ operation.end()
+ }
+ #expect(result == .freed)
+ }
+
+ @Test("Shutting down inside an operation of the current task returns at once")
+ func shutDownInsideTaskOperation() throws {
+ let lifecycle = LlamaLifecycle()
+ let operation = try lifecycle.begin()
+ #expect(lifecycle.shutDown(deadline: 10) == .insideOperation)
+ operation.end()
+ }
+
+ @Test("Shutting down inside synchronous work on the current thread returns at once")
+ func shutDownInsideThreadOperation() throws {
+ let lifecycle = LlamaLifecycle()
+ let result = try lifecycle.perform { lifecycle.shutDown(deadline: 10) }
+ #expect(result == .insideOperation)
+ }
+
+ @Test("A suspended operation doesn't hold up a shutdown, and can't resume after it")
+ func suspendedOperation() throws {
+ let lifecycle = LlamaLifecycle()
+ let operation = try lifecycle.begin()
+ operation.suspend()
+ #expect(lifecycle.activeOperationCount == 0)
+ #expect(lifecycle.shutDown(deadline: 0) == .freed)
+ #expect(throws: LanguageModelSession.GenerationError.self) { try operation.resume() }
+ operation.end()
+ #expect(lifecycle.activeOperationCount == 0)
+ }
+
+ @Test("The cleanup deadline does nothing after the cleanup ends")
+ func cleanupDeadlineAfterCleanup() async throws {
+ let name = "cleanupDeadlineAfterCleanup"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .success) {
+ ExitTestTimer.childStarted("cleanupDeadlineAfterCleanup")
+ ExitTestTimer.stepStarted("cleanupDeadlineAfterCleanup")
+ let value = LlamaLifecycle.withAbortDeadline(0.1) { 42 }
+ // Outlive the deadline; the watchdog must not abort.
+ usleep(500_000)
+ exit(value == 42 ? 0 : 5)
+ }
+ await ExitTestTimer.finish(name, watchdog, measuring: false)
+ }
+
+ @Test("A cleanup that runs past its deadline aborts the process")
+ func cleanupDeadlinePassed() async throws {
+ let name = "cleanupDeadlinePassed"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .signal(SIGABRT)) {
+ ExitTestTimer.childStarted("cleanupDeadlinePassed")
+ ExitTestTimer.stepStarted("cleanupDeadlinePassed")
+ LlamaLifecycle.withAbortDeadline(0.2) {
+ while true { sleep(60) }
+ }
+ }
+ let elapsed = try #require(await ExitTestTimer.finish(name, watchdog))
+ #expect(elapsed >= .milliseconds(200))
+ }
+
+ @Test("A cleanup that runs past its deadline aborts even if standard error is a full pipe")
+ func cleanupDeadlineWithFullStandardError() async throws {
+ let name = "cleanupDeadlineWithFullStandardError"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .signal(SIGABRT)) {
+ ExitTestTimer.childStarted("cleanupDeadlineWithFullStandardError")
+ // Make standard error a pipe that is full and that nobody reads.
+ // The read end stays open, so a write blocks instead of failing.
+ var fds: [CInt] = [0, 0]
+ guard pipe(&fds) == 0 else { _exit(3) }
+ let flags = fcntl(fds[1], F_GETFL)
+ _ = fcntl(fds[1], F_SETFL, flags | O_NONBLOCK)
+ let chunk = [UInt8](repeating: 0x2E, count: 4096)
+ while chunk.withUnsafeBytes({ Darwin.write(fds[1], $0.baseAddress, $0.count) }) > 0 {}
+ guard errno == EAGAIN else { _exit(4) }
+ _ = fcntl(fds[1], F_SETFL, flags)
+ guard dup2(fds[1], STDERR_FILENO) == STDERR_FILENO else { _exit(6) }
+ ExitTestTimer.stepStarted("cleanupDeadlineWithFullStandardError")
+ LlamaLifecycle.withAbortDeadline(0.2) {
+ while true { sleep(60) }
+ }
+ }
+ let elapsed = try #require(await ExitTestTimer.finish(name, watchdog))
+ #expect(elapsed >= .milliseconds(200))
+ }
+
+ #if DEBUG
+ @Test("Once the cleanup completes, a watchdog that timed out doesn't abort")
+ func cleanupCompletionWinsRace() async throws {
+ let name = "cleanupCompletionWinsRace"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .success) {
+ ExitTestTimer.childStarted("cleanupCompletionWinsRace")
+ let entered = DispatchSemaphore(value: 0)
+ let proceed = DispatchSemaphore(value: 0)
+ let decided = DispatchSemaphore(value: 0)
+ let aborts = Locked(nil)
+ LlamaTestHooks.cleanupWatchdog.withLock {
+ $0 = { event in
+ switch event {
+ case .beforeDecision:
+ // Hold the watchdog after its timeout
+ // until the cleanup has completed.
+ entered.signal()
+ proceed.wait()
+ case .decided(let value):
+ aborts.withLock { $0 = value }
+ decided.signal()
+ }
+ }
+ }
+ ExitTestTimer.stepStarted("cleanupCompletionWinsRace")
+ // The cleanup returns only after the watchdog's deadline has passed.
+ let value = LlamaLifecycle.withAbortDeadline(0.05) {
+ entered.wait()
+ return 42
+ }
+ // The child does nothing else, so it can block a thread here,
+ // in a synchronous function.
+ func waitForDecision() { decided.wait() }
+ proceed.signal()
+ waitForDecision()
+ exit(value == 42 && aborts.withLock({ $0 }) == false ? 0 : 5)
+ }
+ await ExitTestTimer.finish(name, watchdog, measuring: false)
+ }
+
+ @Test("Once a watchdog decides to abort, the cleanup doesn't return")
+ func cleanupTimeoutWinsRace() async throws {
+ let name = "cleanupTimeoutWinsRace"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .signal(SIGABRT)) {
+ ExitTestTimer.childStarted("cleanupTimeoutWinsRace")
+ let decided = DispatchSemaphore(value: 0)
+ LlamaTestHooks.cleanupWatchdog.withLock {
+ $0 = { event in
+ guard event == .decided(aborts: true) else { return }
+ decided.signal()
+ // Keep the watchdog from aborting,
+ // so that only the cleanup's own check can abort.
+ while true { sleep(60) }
+ }
+ }
+ ExitTestTimer.stepStarted("cleanupTimeoutWinsRace")
+ _ = LlamaLifecycle.withAbortDeadline(0.05) {
+ decided.wait()
+ return 42
+ }
+ // A distinct status shows that the cleanup returned.
+ exit(5)
+ }
+ await ExitTestTimer.finish(name, watchdog, measuring: false)
+ }
+ #endif
+
+ @Test("After the exit handler runs, a Hub model neither lists nor downloads its files")
+ func hubLoadAfterShutdown() async {
+ let name = "hubLoadAfterShutdown"
+ let watchdog = ExitTestTimer.watch(name)
+ await #expect(processExitsWith: .success) {
+ ExitTestTimer.childStarted("hubLoadAfterShutdown")
+ let repository = FakeHubRepository()
+ let model = LlamaLanguageModel(
+ givenModelPath: "",
+ givenMMProjPath: nil,
+ hubSource: LlamaHubModelSource(
+ modelId: "example/model-GGUF",
+ file: nil,
+ mmproj: .disabled,
+ hub: nil,
+ repository: repository
+ ),
+ gpuLayers: 0
+ )
+ guard LlamaLifecycle.shared.shutDown() == .freed else { exit(5) }
+ do {
+ _ = try await LanguageModelSession(model: model).respond(to: "Say hello")
+ exit(6)
+ } catch LanguageModelSession.GenerationError.assetsUnavailable {}
+ // A distinct status shows that the model reached the Hub.
+ guard repository.listingCalls == 0, repository.downloadStarts == 0 else { exit(7) }
+ exit(0)
+ }
+ await ExitTestTimer.finish(name, watchdog, measuring: false)
+ }
+
+ @Test("A failed registration of the exit handler makes the handler free nothing")
+ func registrationFailure() {
+ let lifecycle = LlamaLifecycle()
+ lifecycle.exitHandlerRegistrationFailed()
+ #expect(lifecycle.shutDown(deadline: 0) == .registrationFailed)
+ }
+ }
+#endif // Llama