From dcde9935dda688d5d766334a53499cee53abe6ec Mon Sep 17 00:00:00 2001 From: mattt Date: Mon, 16 Nov 2026 08:57:41 -0800 Subject: [PATCH 1/5] Free llama.cpp models before the process exits --- README.md | 39 ++ .../Models/LlamaLanguageModel.swift | 136 +++-- .../Models/LlamaLifecycle.swift | 397 +++++++++++++++ .../LlamaLanguageModelExitTests.swift | 470 ++++++++++++++++++ 4 files changed, 1014 insertions(+), 28 deletions(-) create mode 100644 Sources/AnyLanguageModel/Models/LlamaLifecycle.swift create mode 100644 Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift diff --git a/README.md b/README.md index 8349bcef..4d91d8f0 100644 --- a/README.md +++ b/README.md @@ -1590,6 +1590,45 @@ let response = try await session.respond( ) ``` +llama.cpp's Metal backend stops a process with an assertion +if the process exits while a model has layers on the GPU, +and a generation that is still running at exit can crash on the GPU or the CPU. +So `LlamaLanguageModel` registers an exit handler when a load ends. +The handler stops new loads and generations, +waits up to three seconds for running work to stop at its next token, +and then frees every loaded model, projector, and cached context. +After the handler starts, +requests throw `LanguageModelSession.GenerationError.assetsUnavailable`. + +The handler is best effort. +The process can still stop with the assertion in these cases: + +- It exits during the first load, + because the handler is registered when a load ends. +- It exits from inside a load, a generation, or the release of a model, + for example from a llama.cpp callback. + That work can't end while the handler waits, + so the handler returns at once and frees nothing. + An exit from a llama.cpp callback on a thread + that llama.cpp or another library owns + isn't recognized as inside that work, + so the handler waits the full three seconds and then frees nothing. +- Work is still running after three seconds. + The handler then frees nothing. +- It exits without running exit handlers, + for example with `_exit` or after a crash. + +No method unloads a model that something still references. +To be certain that no model is loaded when the process exits, +wait until every `respond` call returns and every response stream ends, +start no new requests, +and then release every reference to the `LlamaLanguageModel`, +including the sessions that use it. +The model frees its weights when the last reference is released. +A response stream's generation can keep its model for a short time +after the stream ends, +until the generation stops at its next token. + ### LiteRT-LM diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index ff0febdb..fcf69e71 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -102,6 +102,9 @@ import Foundation text: UnsafePointer?, userData: UnsafeMutableRawPointer? ) { + #if DEBUG + LlamaTestHooks.onLog.withLock { $0 }?() + #endif let minimum = LlamaRuntime.logLevel.withLock { $0 } let logger = LlamaRuntime.Logger(userData: userData) guard LlamaRuntime.shouldWrite(level: level, logger: logger, minimum: minimum), let text else { @@ -117,6 +120,34 @@ import Foundation /// ```swift /// let model = LlamaLanguageModel(modelPath: "/path/to/model.gguf") /// ``` + /// + /// llama.cpp's Metal backend stops a process with an assertion + /// if the process exits while a model has layers on the GPU. + /// So the model registers an exit handler when a load ends, + /// which stops new loads and generations, + /// waits up to three seconds for running work, + /// and then frees every loaded model. + /// After the handler starts, requests throw + /// ``LanguageModelSession/GenerationError/assetsUnavailable(_:)``. + /// The handler can't help in these cases: + /// + /// - An exit during the first load. + /// - An exit from inside a load, a generation, or the release of a model, + /// such as from a llama.cpp callback. + /// An exit from a llama.cpp callback on a thread + /// that runs no task and releases no model, + /// such as a thread that llama.cpp or another library owns, + /// makes the handler wait the full three seconds and then free nothing. + /// - Work that is still running after three seconds. + /// - An exit that runs no exit handlers, such as `_exit` or a crash. + /// + /// No method unloads a model that something still references. + /// To be certain that no model is loaded when the process exits, + /// wait until every request returns and every response stream ends, + /// start no new requests, + /// and then release every reference to the model, including its sessions. + /// A response stream's generation can keep the model for a short time + /// after the stream ends, until the generation stops at its next token. public final class LlamaLanguageModel: LanguageModel, @unchecked Sendable { /// The reason the model is unavailable. /// This model is always available. @@ -495,7 +526,13 @@ import Foundation /// frees a context that is still decoding: it runs on a transient context /// instead and leaves the cache untouched. private let sessionContextLock = NSLock() - private var cachedSessionContext: CachedSessionContext? + private var cachedSessionContext: CachedSessionContext? { + // The exit handler frees the cached context through the resources. + didSet { resources?.setCachedContext(cachedSessionContext?.context) } + } + + /// The loaded model's resources, which the exit handler can free. + private var resources: LlamaModelResources? /// The number of prompt tokens reused from the cached context by the most /// recent chat generation. @@ -510,21 +547,30 @@ import Foundation /// model. The next chat generation prefills its full prompt again. Call this /// under memory pressure or when a session is discarded. If a generation is /// running on the cached context, it is freed as soon as that generation ends. + /// + /// After the exit handler starts, this method does nothing, + /// because the handler frees the cached context. public func clearCachedContext() { - discardCachedSessionContext() + // After the exit handler starts, the context may be freed already. + try? LlamaLifecycle.shared.perform { discardCachedSessionContext() } } private func discardCachedSessionContext() { - sessionContextLock.lock() - defer { sessionContextLock.unlock() } - guard var cached = cachedSessionContext else { return } - if cached.isCheckedOut { - cached.discardWhenReleased = true - cachedSessionContext = cached - return + let unused: OpaquePointer? = sessionContextLock.withLock { + guard var cached = cachedSessionContext else { return nil } + if cached.isCheckedOut { + cached.discardWhenReleased = true + cachedSessionContext = cached + return nil + } + cachedSessionContext = nil + return cached.context + } + // Free without the lock: llama.cpp logs while it frees a context, + // and a log callback that calls `exit` runs the exit handler on this thread. + if let unused { + llama_free(unused) } - llama_free(cached.context) - cachedSessionContext = nil } private func recordCachedTokens(_ tokens: [llama_token], context: OpaquePointer) { @@ -545,19 +591,22 @@ import Foundation /// context is checked back in (or freed, if a discard was requested while /// it was busy); a transient context is freed. private func releaseSessionContext(_ context: OpaquePointer) { - sessionContextLock.lock() - defer { sessionContextLock.unlock() } - if var cached = cachedSessionContext, cached.context == context { - if cached.discardWhenReleased { - llama_free(cached.context) - cachedSessionContext = nil - } else { + let unused: OpaquePointer? = sessionContextLock.withLock { + if var cached = cachedSessionContext, cached.context == context { + if cached.discardWhenReleased { + cachedSessionContext = nil + return context + } cached.isCheckedOut = false cachedSessionContext = cached + return nil } - return + return context + } + // Free without the lock, as in `discardCachedSessionContext()`. + if let unused { + llama_free(unused) } - llama_free(context) } /// Returns a context for the session along with the index of the first @@ -734,14 +783,10 @@ import Foundation } deinit { - if let cached = cachedSessionContext { - llama_free(cached.context) - } - if let mtmdContext = mtmdContext { - mtmd_free(mtmdContext) - } - if let model = model { - llama_model_free(model) + // The resources free the cached context, the projector, and the model, + // unless the exit handler freed them already. + if let resources { + LlamaLifecycle.shared.free(resources) } } @@ -884,6 +929,11 @@ import Foundation let runtimeOptions = resolvedOptions(from: options) let structuredOptions = resolvedStructuredOptions(from: options) + // The exit handler waits for this request's llama.cpp work, + // but not for its tool calls. + let operation = try LlamaLifecycle.shared.begin() + defer { operation.end() } + // Structured output does not go through the projector yet. // Without an image marker, a prompt with an image throws // before any generation or tool call, @@ -987,6 +1037,7 @@ import Foundation try session.recordToolRound(toolRounds, completed: allEntries) + operation.suspend() let (callsEntry, resolution) = try await resolveToolCalls( parsedCalls, requestContext: requestContext, @@ -1020,7 +1071,9 @@ import Foundation pendingEntries.append(.prompt(prompt)) } } + try operation.resume() } + try operation.resume() if type == String.self { return LanguageModelSession.Response( content: text as! Content, @@ -1131,6 +1184,10 @@ import Foundation let runtimeOptions = resolvedOptions(from: options) let maxTokens = runtimeOptions.maximumResponseTokens ?? 100 + // The exit handler waits for this request's llama.cpp work, + // but not for its tool calls. + let operation = try LlamaLifecycle.shared.begin() + defer { operation.end() } let outputFormat = self.currentToolCallFormat() let toolRounds = session.toolRoundLimit(for: "Llama", options: options) var accumulatedEntries: [Transcript.Entry] = [] @@ -1243,6 +1300,7 @@ import Foundation try session.recordToolRound(toolRounds, completed: accumulatedEntries) + operation.suspend() let (callsEntry, resolution) = try await self.resolveToolCalls( parsedCalls, requestContext: requestContext, @@ -1275,6 +1333,7 @@ import Foundation emittedBase += roundVisible yieldSnapshot(emittedBase) } + try operation.resume() } if emittedBase != lastYieldedText { @@ -1302,6 +1361,11 @@ import Foundation guard !isModelLoaded else { return } + // The exit handler waits for this load. + // After the handler starts, this throws before the path is checked. + let operation = try LlamaLifecycle.shared.begin() + defer { operation.end() } + // Check if model file exists guard FileManager.default.fileExists(atPath: modelPath) else { throw LlamaLanguageModelError.invalidModelPath @@ -1326,6 +1390,7 @@ import Foundation let modelParams = createModelParams() guard let loadedModel = llama_model_load_from_file(modelPath, modelParams) else { + try LlamaLifecycle.shared.stopIfExiting() throw LlamaLanguageModelError.modelLoadFailed } @@ -1349,6 +1414,10 @@ import Foundation self.vocab = llama_model_get_vocab(loadedModel) self.tokenCache = StructuredGenerationTokenCache() self.isModelLoaded = true + let resources = LlamaModelResources(model: loadedModel, projector: mtmdContext) + self.resources = resources + LlamaLifecycle.shared.track(resources) + LlamaRuntime.registerExitHandler() } private func createModelParams() -> llama_model_params { @@ -1357,6 +1426,8 @@ import Foundation // Try to reduce memory usage params.load_mode = LLAMA_LOAD_MODE_MMAP + // Returning false stops the load, so that the exit handler doesn't wait for all of it. + params.progress_callback = { _, _ in !LlamaLifecycle.shared.isStopping } return params } @@ -1640,6 +1711,7 @@ import Foundation } mutating func decode(_ token: Int) async throws { + try LlamaLifecycle.shared.stopIfExiting() let llamaToken = llama_token(token) batch.pointee.n_tokens = 1 @@ -1774,9 +1846,14 @@ import Foundation var decodedTokens = promptTokens for _ in 0 ..< maxTokens { + #if DEBUG + LlamaTestHooks.beforeToken.withLock { $0 }?() + #endif if Task.isCancelled { break } + // The exit handler waits for this generation, so stop it early. + try LlamaLifecycle.shared.stopIfExiting() // Sample next token from logits of the last token we just decoded let nextToken = llama_sampler_sample(sampler, context, batch.n_tokens - 1) @@ -1947,6 +2024,8 @@ import Foundation if Task.isCancelled { break } + // The exit handler waits for this generation, so stop it early. + try LlamaLifecycle.shared.stopIfExiting() let nextToken = llama_sampler_sample(samplerPtr, context, sampleIndex) llama_sampler_accept(samplerPtr, nextToken) @@ -2090,6 +2169,7 @@ import Foundation let capacity = Int(batchSize) var start = startIndex while start < promptTokens.count { + try LlamaLifecycle.shared.stopIfExiting() let count = min(capacity, promptTokens.count - start) batch.n_tokens = Int32(count) for i in 0 ..< count { diff --git a/Sources/AnyLanguageModel/Models/LlamaLifecycle.swift b/Sources/AnyLanguageModel/Models/LlamaLifecycle.swift new file mode 100644 index 00000000..b234a7c6 --- /dev/null +++ b/Sources/AnyLanguageModel/Models/LlamaLifecycle.swift @@ -0,0 +1,397 @@ +import Foundation + +#if Llama + import LlamaSwift + + extension LlamaRuntime { + /// Registers ``LlamaLifecycle/shutDown(deadline:)`` to run when the process exits. + /// + /// ggml creates static objects as it needs them, + /// such as its Metal device and buffer types during a model's first load, + /// and the C runtime destroys static objects and runs exit handlers + /// in the reverse order of their creation. + /// A handler that was registered before the first load + /// would run after ggml's objects are destroyed, + /// and freeing a model then would crash. + /// ggml doesn't report when it creates such an object, + /// so every load registers the handler again when it ends. + /// The last registration runs first, + /// and the handler does nothing after its first run. + /// + /// If a registration fails, + /// an earlier one could run after objects that the models use are destroyed, + /// so the handler then frees nothing. + static func registerExitHandler() { + if atexit({ _ = LlamaLifecycle.shared.shutDown() }) != 0 { + LlamaLifecycle.shared.exitHandlerRegistrationFailed() + } + } + } + + /// Tracks the llama.cpp work in this process, + /// and tries to free every loaded model when the process exits. + /// + /// ggml's Metal backend frees its device in a C++ static destructor at exit, + /// and it stops the process with an assertion + /// if a model or a context still has buffers on the device then. + /// A generation that is still running at exit can also crash, + /// on the GPU or the CPU, + /// because the static objects that it uses are destroyed under it. + /// + /// The exit handler is best effort. + /// It must never make an exit worse than an exit without it, + /// so it never waits without a limit, + /// and it never frees a model or a context that running work could use. + /// It stops new loads and generations, + /// and it waits up to ``exitDeadline`` for the ones that are running: + /// a load stops at its next progress report, + /// and a generation stops before its next token or batch of prompt tokens. + /// If no work is running then, + /// it frees the cached session context, the projector, and the model + /// of every loaded ``LlamaLanguageModel``, + /// including models that Swift objects still hold. + /// Otherwise it frees nothing. + /// Freeing only the idle models wouldn't help, + /// because the busy model's buffers would still cause the assertion. + /// + /// The handler can't help, and the assertion can still happen, + /// in these cases: + /// + /// - An exit during the first load, + /// because the handler is registered when a load ends. + /// An earlier registration would run after ggml's objects are destroyed. + /// - An exit from inside a load, a generation, or a cleanup, + /// such as from a llama.cpp callback. + /// The handler sees that its task or thread is running that work, + /// which can't end while the handler waits, + /// and returns at once without freeing anything. + /// On a thread that runs no task and has no cleanup mark, + /// such as a thread that llama.cpp or another library owns, + /// the handler can't see this, + /// so it waits until the deadline and then frees nothing. + /// - Work that is still running at the deadline. + /// - An exit that runs no exit handlers, such as `_exit` or a crash. + /// + /// After the handler starts, every load and generation throws + /// ``LanguageModelSession/GenerationError/assetsUnavailable(_:)``. + /// + /// Nothing here calls llama.cpp with a lock held. + /// llama.cpp logs while it frees a model, + /// and runs the log callback on the same thread, + /// so a callback that calls `exit` would run the exit handler on that thread, + /// and a lock that isn't recursive would then never be released. + final class LlamaLifecycle: @unchecked Sendable { + static let shared = LlamaLifecycle() + + /// How long the exit handler waits for running loads and generations. + /// + /// A running generation stops before its next token, + /// or before its next batch of prompt tokens, + /// so the wait is about one call to llama.cpp, + /// which takes milliseconds to a few hundred milliseconds for a 4B model. + /// Three seconds leaves room for a much larger model or a slow CPU, + /// and still ends a process that can't finish its work + /// soon enough that the delay looks like a slow quit, not a hang. + static let exitDeadline: TimeInterval = 3 + + /// What ``shutDown(deadline:)`` did. + enum ShutdownResult: Equatable { + /// It freed every model. + case freed + /// It had already run, so it did nothing. + case alreadyShutDown + /// It ran inside an operation, so it freed nothing without waiting. + case insideOperation + /// Work was still running at the deadline, so it freed nothing. + case timedOut + /// A registration of the exit handler failed, so it freed nothing. + case registrationFailed + } + + /// The key in a thread's dictionary for the number of operations that the thread runs. + private static let depthKey = "AnyLanguageModel.LlamaLifecycle.operationDepth" + + /// Guards the properties below, and signals when no operation is running. + private let condition = NSCondition() + private var isShuttingDown = false + private var registrationFailed = false + private var activeOperations = 0 + /// The number of active operations in each task. + /// + /// A key is used only to compare tasks, + /// and only while its operation is active, + /// so its task is still running. + private var activeTasks: [UnsafeCurrentTask: Int] = [:] + private var resources: [ObjectIdentifier: LlamaModelResources] = [:] + + /// Whether the process is exiting. + /// + /// Generations read this before each token, + /// which costs one uncontended lock and unlock. + var isStopping: Bool { + condition.withLock { isShuttingDown } + } + + /// The number of loads, generations, and cleanups that are running. + var activeOperationCount: Int { + condition.withLock { activeOperations } + } + + /// The error that work throws after the exit handler starts. + static var exiting: LanguageModelSession.GenerationError { + .assetsUnavailable( + .init(debugDescription: "The process is exiting, so llama.cpp work has stopped.") + ) + } + + /// Throws ``exiting`` if the exit handler has started. + func stopIfExiting() throws { + if isStopping { throw Self.exiting } + } + + // MARK: Requests + + /// A load or a generation, which the exit handler waits for. + /// + /// A request can await other code in the middle of its llama.cpp work, + /// so an operation belongs to the task that began it, not to a thread. + /// A request suspends its operation while it waits for other code, + /// such as a tool call, + /// so that the exit handler doesn't wait for that code. + /// Use an operation only in the task that began it. + final class Operation { + private let lifecycle: LlamaLifecycle + private var task: UnsafeCurrentTask? + private var isActive = true + + fileprivate init(lifecycle: LlamaLifecycle, task: UnsafeCurrentTask?) { + self.lifecycle = lifecycle + self.task = task + } + + /// Ends the operation for now, if it is active. + func suspend() { + guard isActive else { return } + isActive = false + lifecycle.endOperation(in: task) + } + + /// Starts the operation again, if it is suspended. + /// + /// - Throws: ``LlamaLifecycle/exiting`` if the exit handler has started. + func resume() throws { + guard !isActive else { return } + task = try lifecycle.beginOperation() + isActive = true + } + + /// Ends the operation. + func end() { + suspend() + } + } + + /// Starts an operation in the current task, unless the process is exiting. + /// + /// End the operation when its llama.cpp work ends. + /// + /// - Throws: ``exiting`` if the exit handler has started. + func begin() throws -> Operation { + let task = try beginOperation() + return Operation(lifecycle: self, task: task) + } + + private func beginOperation() throws -> UnsafeCurrentTask? { + // The task outlives the operation, which ends in the same task. + let task = withUnsafeCurrentTask { $0 } + try condition.withLock { + guard !isShuttingDown else { throw Self.exiting } + activeOperations += 1 + if let task { activeTasks[task, default: 0] += 1 } + } + return task + } + + private func endOperation(in task: UnsafeCurrentTask?) { + condition.withLock { + if let task { + activeTasks[task, default: 1] -= 1 + if activeTasks[task] == 0 { activeTasks[task] = nil } + } + activeOperations -= 1 + if activeOperations == 0 { condition.broadcast() } + } + } + + // MARK: Cleanups + + /// Runs synchronous llama.cpp work, unless the process is exiting. + func perform(_ body: () throws -> Result) throws -> Result { + try condition.withLock { + guard !isShuttingDown else { throw Self.exiting } + activeOperations += 1 + } + return try runCounted(body) + } + + /// Runs `body` as work that the caller has already counted, + /// marks the current thread as running it, + /// and stops counting it when it returns. + private func runCounted(_ body: () throws -> Result) rethrows -> Result { + // The body is synchronous, so it stays on this thread. + let thread = Thread.current.threadDictionary + let depth = thread[Self.depthKey] as? Int ?? 0 + thread[Self.depthKey] = depth + 1 + defer { + thread[Self.depthKey] = depth + condition.withLock { + activeOperations -= 1 + if activeOperations == 0 { condition.broadcast() } + } + } + return try body() + } + + /// Whether the current thread or task is running an operation. + private var isInsideOperation: Bool { + if (Thread.current.threadDictionary[Self.depthKey] as? Int ?? 0) > 0 { + return true + } + guard let task = withUnsafeCurrentTask(body: { $0 }) else { return false } + return condition.withLock { activeTasks[task] != nil } + } + + /// Records that a registration of the exit handler failed. + func exitHandlerRegistrationFailed() { + condition.withLock { registrationFailed = true } + } + + /// Records the resources of a model, so that ``shutDown(deadline:)`` can free them. + func track(_ model: LlamaModelResources) { + condition.withLock { resources[ObjectIdentifier(model)] = model } + } + + /// Frees the resources of a model, unless ``shutDown(deadline:)`` freed them already. + /// + /// The cleanup runs even while the process is exiting. + /// It counts as running work, + /// so that the exit handler's thread check and deadline apply to it, + /// and it calls llama.cpp without the lock. + func free(_ model: LlamaModelResources) { + condition.withLock { + resources[ObjectIdentifier(model)] = nil + activeOperations += 1 + } + runCounted { model.free() } + } + + /// Stops new work, waits up to `deadline` for the work that is running, + /// and frees the resources of every model if no work is running then. + @discardableResult + func shutDown(deadline: TimeInterval = exitDeadline) -> ShutdownResult { + let insideOperation = isInsideOperation + var models: [LlamaModelResources] = [] + let result: ShutdownResult = condition.withLock { + guard !isShuttingDown else { return .alreadyShutDown } + isShuttingDown = true + // The operation on this thread or task can't end while it waits here. + guard !insideOperation else { return .insideOperation } + guard !registrationFailed else { return .registrationFailed } + // Measure the deadline with a monotonic clock, + // and wait in short steps that start from the current wall-clock time, + // so that a change to the wall clock can't extend the wait. + let clock = ContinuousClock() + let end = clock.now + .seconds(deadline) + while activeOperations > 0 { + let remaining = clock.now.duration(to: end) + guard remaining > .zero else { break } + let step = min(remaining, .milliseconds(100)) + _ = condition.wait(until: Date(timeIntervalSinceNow: step / .seconds(1))) + } + guard activeOperations == 0 else { return .timedOut } + models = Array(resources.values) + resources = [:] + return .freed + } + // Free without the lock, as in `free(_:)`. + // If a log callback calls `exit` here, this handler runs again and does nothing. + for model in models { + model.free() + } + return result + } + } + + /// A loaded llama.cpp model, its projector, and its cached session context. + /// + /// ``LlamaLifecycle`` holds these resources until it frees them, + /// separately from the ``LlamaLanguageModel`` that uses them, + /// so that the exit handler can free a model that a Swift object still holds, + /// and so that the model's deinit frees nothing twice. + final class LlamaModelResources: @unchecked Sendable { + let model: OpaquePointer + let projector: OpaquePointer? + + /// Guards `cachedContext` and `isFreed`. + private let lock = NSLock() + private var cachedContext: OpaquePointer? + private var isFreed = false + + init(model: OpaquePointer, projector: OpaquePointer?) { + self.model = model + self.projector = projector + } + + /// Records the model's cached session context, or `nil` if it has none. + /// + /// Any other context exists only during a load or a generation, + /// which frees it before it ends. + func setCachedContext(_ context: OpaquePointer?) { + lock.withLock { cachedContext = context } + } + + /// Frees the cached context, the projector, and the model, once. + /// + /// Call it only through ``LlamaLifecycle``, + /// when no load or generation uses the model. + func free() { + // Take the context under the lock, and call llama.cpp without it. + let (isFirst, context): (Bool, OpaquePointer?) = lock.withLock { + guard !isFreed else { return (false, nil) } + isFreed = true + defer { cachedContext = nil } + return (true, cachedContext) + } + guard isFirst else { return } + if let context { + llama_free(context) + } + if let projector { + mtmd_free(projector) + } + llama_model_free(model) + } + } + + #if DEBUG + /// Functions that tests run at points inside llama.cpp work. + /// + /// Only debug builds, which `swift test` uses, compile these hooks, + /// so they cost nothing in a release build. + enum LlamaTestHooks { + /// Runs at the start of each step of a chat generation's token loop, + /// before its checks for cancellation and exit, + /// on the generation's thread. + /// + /// After the hook returns, + /// the generation does no llama.cpp work before those checks, + /// so a generation that the hook holds until the exit handler starts + /// stops at once. + static let beforeToken = Locked<(@Sendable () -> Void)?>(nil) + + /// Runs for each llama.cpp log message, before the level filter, + /// on the thread that logs it. + static let onLog = Locked<(@Sendable () -> Void)?>(nil) + } + #endif +#endif // Llama diff --git a/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift b/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift new file mode 100644 index 00000000..54bf2fa5 --- /dev/null +++ b/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift @@ -0,0 +1,470 @@ +import Foundation +import Testing + +@testable import AnyLanguageModel + +#if Llama + /// A model that a global variable holds for the life of the process. + nonisolated(unsafe) private var globalExitTestModel: LlamaLanguageModel? + + /// Tests that exit a child process with a model loaded. + /// + /// The tests that use ``LlamaTestHooks`` run only in debug builds, + /// which compile the hooks. + /// + /// Without the exit handler, + /// llama.cpp's Metal backend stops such a process with an assertion + /// when the model has layers on the GPU, + /// and an exit during a generation crashes on the GPU and on the CPU. + @Suite( + "LlamaLanguageModel at exit", + .serialized, + .enabled(if: ProcessInfo.processInfo.environment["LLAMA_MODEL_PATH"] != nil) + ) + struct LlamaLanguageModelExitTests { + static var modelPath: String { + ProcessInfo.processInfo.environment["LLAMA_MODEL_PATH"] ?? "" + } + + /// Returns a model and a session after a first response, + /// so that the model and its cached context are loaded. + static func loadedModel(gpuLayers: Int32) async throws -> (LlamaLanguageModel, LanguageModelSession) { + let model = LlamaLanguageModel(modelPath: modelPath, gpuLayers: gpuLayers) + let session = LanguageModelSession(model: model) + _ = try await session.respond(to: "Say hello", options: GenerationOptions(maximumResponseTokens: 2)) + return (model, session) + } + + #if DEBUG + /// Holds a generation until the exit handler has started, + /// so that the handler must wait for it. + static func holdUntilShutdownStarts() { + let limit = ExitTestTimer.now() + 10_000_000_000 + while !LlamaLifecycle.shared.isStopping { + // A distinct status shows that the exit handler never started. + if ExitTestTimer.now() > limit { _exit(9) } + usleep(1_000) + } + } + + /// Starts a generation in another task, + /// and returns when its first token step starts. + /// + /// - Parameter whileActive: What the generation does at the start of that step, + /// while it is known to be running. + static func startGeneration( + _ session: LanguageModelSession, + whileActive: @escaping @Sendable () -> Void + ) { + let started = DispatchSemaphore(value: 0) + let once = Locked(false) + LlamaTestHooks.beforeToken.withLock { + $0 = { + let isFirst = once.withLock { done in + defer { done = true } + return !done + } + guard isFirst else { return } + started.signal() + whileActive() + } + } + nonisolated(unsafe) let session = session + Task.detached { + _ = try? await session.respond( + to: "Count from 1 to 1000.", + options: GenerationOptions(maximumResponseTokens: 2048) + ) + } + // A distinct status shows that the generation never started. + if started.wait(timeout: .now() + 30) == .timedOut { _exit(3) } + } + #endif + + @Test("A process exits normally with a loaded model", arguments: [Int32(-1), 0]) + func idleExit(gpuLayers: Int32) async throws { + let name = "idleExit\(gpuLayers)" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { [name = name as String, gpuLayers = gpuLayers as Int32] in + ExitTestTimer.childStarted(name) + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: gpuLayers) + ExitTestTimer.stepStarted(name) + withExtendedLifetime((model, session)) { exit(0) } + } + let elapsed = await ExitTestTimer.finish(name, watchdog) + #expect(try #require(elapsed) < .seconds(LlamaLifecycle.exitDeadline)) + } + + @Test("A process exits normally with a model in a global variable", arguments: [Int32(-1), 0]) + func exitWithGlobalModel(gpuLayers: Int32) async throws { + let name = "exitWithGlobalModel\(gpuLayers)" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { [name = name as String, gpuLayers = gpuLayers as Int32] in + ExitTestTimer.childStarted(name) + globalExitTestModel = LlamaLanguageModel( + modelPath: LlamaLanguageModelExitTests.modelPath, + gpuLayers: gpuLayers + ) + _ = try await LanguageModelSession(model: globalExitTestModel!).respond( + to: "Say hello", + options: GenerationOptions(maximumResponseTokens: 2) + ) + ExitTestTimer.stepStarted(name) + exit(0) + } + let elapsed = await ExitTestTimer.finish(name, watchdog) + #expect(try #require(elapsed) < .seconds(LlamaLifecycle.exitDeadline)) + } + + #if DEBUG + @Test( + "The exit handler waits for a generation that is running, and the process exits normally", + arguments: [Int32(-1), 0] + ) + func exitDuringGeneration(gpuLayers: Int32) async throws { + let name = "exitDuringGeneration\(gpuLayers)" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { [name = name as String, gpuLayers = gpuLayers as Int32] in + ExitTestTimer.childStarted(name) + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: gpuLayers) + LlamaLanguageModelExitTests.startGeneration(session) { + LlamaLanguageModelExitTests.holdUntilShutdownStarts() + } + ExitTestTimer.stepStarted(name) + withExtendedLifetime(model) { exit(0) } + } + let elapsed = await ExitTestTimer.finish(name, watchdog) + #expect(try #require(elapsed) < .seconds(LlamaLifecycle.exitDeadline)) + } + + @Test("A generation that runs past the deadline doesn't keep the process alive") + func exitPastDeadline() async throws { + let name = "exitPastDeadline" + let watchdog = ExitTestTimer.watch(name) + // The handler frees nothing, so ggml's assertion ends the process, + // as it would without the handler. + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("exitPastDeadline") + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1) + LlamaLanguageModelExitTests.startGeneration(session) { + while true { sleep(60) } + } + ExitTestTimer.stepStarted("exitPastDeadline") + withExtendedLifetime(model) { exit(0) } + } + let elapsed = try #require(await ExitTestTimer.finish(name, watchdog)) + #expect(elapsed >= .seconds(LlamaLifecycle.exitDeadline)) + #expect(elapsed < .seconds(LlamaLifecycle.exitDeadline + 5)) + } + + @Test("An exit from inside a generation doesn't wait for that generation") + func exitInsideOperation() async throws { + let name = "exitInsideOperation" + let watchdog = ExitTestTimer.watch(name) + // The handler returns at once and frees nothing, + // so ggml's assertion ends the process before the deadline. + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("exitInsideOperation") + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1) + LlamaLanguageModelExitTests.startGeneration(session) { + ExitTestTimer.stepStarted("exitInsideOperation") + exit(0) + } + // The generation's thread exits the process. + withExtendedLifetime(model) { + while true { sleep(60) } + } + } + let elapsed = await ExitTestTimer.finish(name, watchdog) + #expect(try #require(elapsed) < .seconds(LlamaLifecycle.exitDeadline)) + } + + @Test("A log callback that exits while a model is freed doesn't hang the process") + func exitFromLogDuringCleanup() async throws { + // The handler sees that the cleanup's thread is exiting, and returns at once. + // The model is only partly freed, so ggml's assertion ends the process, + // as it would without the handler. + let name = "exitFromLogDuringCleanup" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("exitFromLogDuringCleanup") + do { + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1) + let once = Locked(false) + LlamaTestHooks.onLog.withLock { + $0 = { + let isFirst = once.withLock { done in + defer { done = true } + return !done + } + if isFirst { exit(0) } + } + } + ExitTestTimer.stepStarted("exitFromLogDuringCleanup") + // Releasing the last references frees the model, + // and llama.cpp logs while it frees. + _ = (model, session) + } + // A distinct status shows that llama.cpp logged nothing. + exit(4) + } + let elapsed = await ExitTestTimer.finish(name, watchdog) + #expect(try #require(elapsed) < .seconds(LlamaLifecycle.exitDeadline)) + } + #endif + + @Test("A process exits normally after the model is released", arguments: [Int32(-1), 0]) + func exitAfterRelease(gpuLayers: Int32) async { + let name = "exitAfterRelease\(gpuLayers)" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { [name = name as String, gpuLayers = gpuLayers as Int32] in + ExitTestTimer.childStarted(name) + weak var released: LlamaLanguageModel? + do { + let (model, _) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: gpuLayers) + released = model + } + // A distinct status shows that something still holds the model. + guard released == nil else { exit(4) } + exit(0) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + @Test("After the exit handler runs, a request throws, and nothing is freed twice") + func requestAfterShutdown() async { + let name = "requestAfterShutdown" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { + ExitTestTimer.childStarted("requestAfterShutdown") + weak var released: LlamaLanguageModel? + do { + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1) + released = model + guard LlamaLifecycle.shared.shutDown() == .freed else { exit(5) } + // The cached context is freed already, so this frees nothing. + model.clearCachedContext() + do { + _ = try await session.respond(to: "Say hello") + exit(6) + } catch LanguageModelSession.GenerationError.assetsUnavailable {} + do { + for try await _ in session.streamResponse(to: "Say hello") {} + exit(7) + } catch LanguageModelSession.GenerationError.assetsUnavailable {} + } + // The model's deinit ran after the handler freed the model, + // and freed nothing again. + guard released == nil else { exit(4) } + exit(0) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + @Test("After the exit handler runs, a model that isn't loaded fails to load") + func loadAfterShutdown() async { + let name = "loadAfterShutdown" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { + ExitTestTimer.childStarted("loadAfterShutdown") + _ = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1) + LlamaLifecycle.shared.shutDown() + let other = LlamaLanguageModel(modelPath: LlamaLanguageModelExitTests.modelPath) + do { + _ = try await LanguageModelSession(model: other).respond(to: "Say hello") + exit(5) + } catch LanguageModelSession.GenerationError.assetsUnavailable {} + // A missing file throws the exit error, not an invalid path. + let missing = LlamaLanguageModel(modelPath: "/nonexistent/model.gguf") + do { + _ = try await LanguageModelSession(model: missing).respond(to: "Say hello") + exit(6) + } catch LanguageModelSession.GenerationError.assetsUnavailable {} + exit(0) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + } + + /// Lets the parent of an exit test kill a child that hangs, + /// and measure how long the child takes to end after the step under test. + /// + /// Swift Testing's time limit doesn't end a child process, + /// so a child that hangs would keep the test run waiting. + /// The child writes its process ID when it starts, + /// and the monotonic time when the step under test starts, + /// to files that both processes name from the parent's process ID. + /// The measurement starts after the child's setup, + /// so a slow load doesn't change it. + enum ExitTestTimer { + /// How long the parent waits for a child before it kills the child. + static let limit: Duration = .seconds(120) + + /// The current time of a monotonic clock that every process on the system shares. + static func now() -> UInt64 { + clock_gettime_nsec_np(CLOCK_UPTIME_RAW) + } + + private static func file(_ name: String, _ suffix: String, parent: pid_t) -> URL { + URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("llama-exit-test-\(parent)-\(name).\(suffix)") + } + + /// The exit status of a child that couldn't write a timer file. + /// + /// The child also writes the error to standard error. + static let writeFailureStatus: CInt = 70 + + /// In the child: writes `text` to a timer file, + /// or ends the child with ``writeFailureStatus`` if it can't. + private static func write(_ text: String, to url: URL) { + do { + try text.write(to: url, atomically: true, encoding: .utf8) + } catch { + fputs("ExitTestTimer couldn't write \(url.path): \(error)\n", stderr) + _exit(writeFailureStatus) + } + } + + /// In the child: records its process ID. + static func childStarted(_ name: String) { + write(String(getpid()), to: file(name, "pid", parent: getppid())) + } + + /// In the child: records when the step under test starts. + static func stepStarted(_ name: String) { + write(String(now()), to: file(name, "start", parent: getppid())) + } + + private static func read(_ name: String, _ suffix: String) -> String? { + try? String(contentsOf: file(name, suffix, parent: getpid()), encoding: .utf8) + } + + /// In the parent: starts a task that kills the child if it hasn't ended after ``limit``, + /// and returns whether it did. + static func watch(_ name: String) -> Task { + for suffix in ["pid", "start"] { + try? FileManager.default.removeItem(at: file(name, suffix, parent: getpid())) + } + return Task.detached { + guard (try? await Task.sleep(for: limit)) != nil else { return false } + guard let pid = read(name, "pid").flatMap({ pid_t($0) }) else { return false } + kill(pid, SIGKILL) + return true + } + } + + /// In the parent, after the child ends: stops the watchdog, + /// records an issue if it killed the child, + /// and, if `measuring`, returns the time from the step's start to now. + /// + /// If the start time can't be read, + /// this records an issue with the error and returns `nil`. + /// A child that couldn't write the file + /// exits with ``writeFailureStatus`` instead. + @discardableResult + static func finish( + _ name: String, + _ watchdog: Task, + measuring: Bool = true + ) async -> Duration? { + watchdog.cancel() + let killed = await watchdog.value + #expect(!killed, "The child process didn't end within \(limit).") + guard measuring else { return nil } + let url = file(name, "start", parent: getpid()) + let text: String + do { + text = try String(contentsOf: url, encoding: .utf8) + } catch { + Issue.record("Couldn't read the step's start time from \(url.path): \(error)") + return nil + } + guard let start = UInt64(text) else { + Issue.record("The step's start time in \(url.path) isn't a number: \(text.debugDescription)") + return nil + } + return .nanoseconds(Int64(now() - start)) + } + } + + /// Tests of the exit handler's bookkeeping, without a model. + @Suite("LlamaLifecycle") + struct LlamaLifecycleTests { + @Test("Shutting down with no work frees, and a second shutdown does nothing") + func shutDownIdle() throws { + let lifecycle = LlamaLifecycle() + #expect(lifecycle.shutDown(deadline: 0) == .freed) + #expect(lifecycle.shutDown(deadline: 0) == .alreadyShutDown) + #expect(lifecycle.isStopping) + #expect(throws: LanguageModelSession.GenerationError.self) { try lifecycle.begin() } + #expect(throws: LanguageModelSession.GenerationError.self) { try lifecycle.perform {} } + } + + @Test("Shutting down waits for running work, and frees nothing at the deadline") + func shutDownTimesOut() async throws { + let lifecycle = LlamaLifecycle() + nonisolated(unsafe) let operation = try lifecycle.begin() + // Shut down from a thread that runs no task, so the operation isn't its own. + let result = await withCheckedContinuation { continuation in + Thread.detachNewThread { + let start = ContinuousClock.now + let result = lifecycle.shutDown(deadline: 0.2) + continuation.resume(returning: (result, ContinuousClock.now - start)) + } + } + #expect(result.0 == .timedOut) + #expect(result.1 >= .milliseconds(200)) + operation.end() + #expect(lifecycle.activeOperationCount == 0) + } + + @Test("Shutting down returns when running work ends") + func shutDownWaitsForWork() async throws { + let lifecycle = LlamaLifecycle() + nonisolated(unsafe) let operation = try lifecycle.begin() + let result = await withCheckedContinuation { continuation in + Thread.detachNewThread { + continuation.resume(returning: lifecycle.shutDown(deadline: 10)) + } + usleep(100_000) + operation.end() + } + #expect(result == .freed) + } + + @Test("Shutting down inside an operation of the current task returns at once") + func shutDownInsideTaskOperation() throws { + let lifecycle = LlamaLifecycle() + let operation = try lifecycle.begin() + #expect(lifecycle.shutDown(deadline: 10) == .insideOperation) + operation.end() + } + + @Test("Shutting down inside synchronous work on the current thread returns at once") + func shutDownInsideThreadOperation() throws { + let lifecycle = LlamaLifecycle() + let result = try lifecycle.perform { lifecycle.shutDown(deadline: 10) } + #expect(result == .insideOperation) + } + + @Test("A suspended operation doesn't hold up a shutdown, and can't resume after it") + func suspendedOperation() throws { + let lifecycle = LlamaLifecycle() + let operation = try lifecycle.begin() + operation.suspend() + #expect(lifecycle.activeOperationCount == 0) + #expect(lifecycle.shutDown(deadline: 0) == .freed) + #expect(throws: LanguageModelSession.GenerationError.self) { try operation.resume() } + operation.end() + #expect(lifecycle.activeOperationCount == 0) + } + + @Test("A failed registration of the exit handler makes the handler free nothing") + func registrationFailure() { + let lifecycle = LlamaLifecycle() + lifecycle.exitHandlerRegistrationFailed() + #expect(lifecycle.shutDown(deadline: 0) == .registrationFailed) + } + } +#endif // Llama From 94298f573b7043c945160a4e3c7b6b7fe0ae7b35 Mon Sep 17 00:00:00 2001 From: mattt Date: Mon, 16 Nov 2026 09:53:30 -0800 Subject: [PATCH 2/5] Abort if freeing llama.cpp models at exit takes too long --- README.md | 12 ++ .../Models/LlamaLanguageModel.swift | 8 + .../Models/LlamaLifecycle.swift | 128 ++++++++++++++- .../LlamaLanguageModelExitTests.swift | 148 ++++++++++++++++++ 4 files changed, 294 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4d91d8f0..6e071814 100644 --- a/README.md +++ b/README.md @@ -1618,6 +1618,18 @@ The process can still stop with the assertion in these cases: - It exits without running exit handlers, for example with `_exit` or after a crash. +If freeing the models takes more than ten seconds, +for example because another library's log callback blocks, +the handler aborts the process, so that the exit can't hang. +A crash report for this abort shows +`LlamaLifecycle.abortAfterExitCleanupTimeout()` in its backtrace, +which first tries to write one line to standard error, +but aborts whether or not the line can be written. +If the cleanup ends just after the limit, +the cleanup's own thread can abort first, +without the line, +and the backtrace shows `LlamaLifecycle.abortAfterLateExitCleanup()`. + No method unloads a model that something still references. To be certain that no model is loaded when the process exits, wait until every `respond` call returns and every response stream ends, diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index fcf69e71..39c7a68f 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -141,6 +141,14 @@ import Foundation /// - Work that is still running after three seconds. /// - An exit that runs no exit handlers, such as `_exit` or a crash. /// + /// If freeing the models takes more than ten seconds, + /// the handler aborts the process, so that the exit can't hang. + /// A crash report shows `abortAfterExitCleanupTimeout()` in its backtrace, + /// which first tries to write one line to standard error. + /// If the cleanup ends just after the limit, + /// the cleanup's thread can abort first, without the line, + /// and the backtrace shows `abortAfterLateExitCleanup()`. + /// /// No method unloads a model that something still references. /// To be certain that no model is loaded when the process exits, /// wait until every request returns and every response stream ends, diff --git a/Sources/AnyLanguageModel/Models/LlamaLifecycle.swift b/Sources/AnyLanguageModel/Models/LlamaLifecycle.swift index b234a7c6..98142e2b 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLifecycle.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLifecycle.swift @@ -94,6 +94,28 @@ import Foundation /// soon enough that the delay looks like a slow quit, not a hang. static let exitDeadline: TimeInterval = 3 + /// How long the exit handler's cleanup may take before the process aborts. + /// + /// The cleanup frees every model after running work has ended. + /// llama.cpp calls the installed log callback while it frees, + /// and another library can replace that callback with one that blocks, + /// for example on a lock that the exiting thread holds, + /// and a GPU driver can stall while it releases buffers. + /// Without a limit the process would then hang, + /// where without the handler it would have stopped with ggml's assertion. + /// So the cleanup runs under a watchdog, + /// which calls `abort()` if the cleanup is still running after this time. + /// It tries to write one line to standard error first, + /// but it aborts whether or not the line can be written. + /// + /// Freeing Qwen3-4B (2.5 GB) and its context took about 35 milliseconds + /// on an Apple silicon Mac, on the GPU and on the CPU. + /// The work grows with the number and size of the buffers, + /// so a 70B model would take under a second by that measure. + /// Ten seconds leaves a margin of more than ten times that + /// for a slow but working free. + static let cleanupDeadline: TimeInterval = 10 + /// What ``shutDown(deadline:)`` did. enum ShutdownResult: Equatable { /// It freed every model. @@ -315,11 +337,102 @@ import Foundation } // Free without the lock, as in `free(_:)`. // If a log callback calls `exit` here, this handler runs again and does nothing. - for model in models { - model.free() + guard !models.isEmpty else { return result } + Self.withAbortDeadline(Self.cleanupDeadline) { + for model in models { + model.free() + } + } + return result + } + + /// The state of a cleanup that runs under ``withAbortDeadline(_:_:)``. + private enum CleanupState: Sendable { + case running + case completed + case aborting + } + + /// Runs `body`, and aborts the process if `body` is still running after `deadline`. + /// + /// A separate thread measures the time on a monotonic clock, + /// so the exiting thread can't stop it. + /// The thread ends as soon as `body` returns, + /// and doesn't keep the process alive. + /// + /// Completion and the timeout are decided in one step under a lock. + /// Once `body` has returned and completion is recorded, + /// the watchdog never aborts. + /// Once the watchdog has recorded the timeout, + /// this function aborts instead of returning, + /// even if `body` returns after that. + static func withAbortDeadline(_ deadline: TimeInterval, _ body: () -> Result) -> Result { + let state = Locked(CleanupState.running) + let finished = DispatchSemaphore(value: 0) + let watchdog = Thread { + guard finished.wait(timeout: .now() + deadline) == .timedOut else { return } + #if DEBUG + LlamaTestHooks.cleanupWatchdog.withLock { $0 }?(.beforeDecision) + #endif + let aborts = state.withLock { state in + guard state == .running else { return false } + state = .aborting + return true + } + #if DEBUG + LlamaTestHooks.cleanupWatchdog.withLock { $0 }?(.decided(aborts: aborts)) + #endif + if aborts { + abortAfterExitCleanupTimeout() + } + } + watchdog.start() + let result = body() + let completed = state.withLock { state in + guard state == .running else { return false } + state = .completed + return true } + // The watchdog has decided to abort, and is about to. + guard completed else { abortAfterLateExitCleanup() } + finished.signal() return result } + + /// Aborts the process because the exit handler's cleanup returned + /// after the watchdog had decided to abort. + /// + /// A crash report for this abort shows this function + /// in the backtrace of the thread that called `abort()`. + /// It writes no message and doesn't wait for the watchdog. + @inline(never) + private static func abortAfterLateExitCleanup() -> Never { + abort() + } + + /// Aborts the process because the exit handler's cleanup took too long. + /// + /// A crash report for this abort shows this function + /// in the backtrace of the thread that called `abort()`, + /// which identifies it. + /// The function also tries to write one line to standard error, + /// from another thread and with `write(2)`, + /// but waits at most 100 milliseconds for it: + /// standard error can be a full pipe that nobody reads, + /// and its lock can be held by the blocked thread, + /// so the abort must not depend on the message. + @inline(never) + private static func abortAfterExitCleanupTimeout() -> Never { + let written = DispatchSemaphore(value: 0) + let writer = Thread { + let message = "AnyLanguageModel: freeing llama.cpp models at exit didn't finish in time; aborting.\n" + _ = message.withCString { write(STDERR_FILENO, $0, strlen($0)) } + written.signal() + } + writer.start() + _ = written.wait(timeout: .now() + .milliseconds(100)) + abort() + } } /// A loaded llama.cpp model, its projector, and its cached session context. @@ -392,6 +505,17 @@ import Foundation /// Runs for each llama.cpp log message, before the level filter, /// on the thread that logs it. static let onLog = Locked<(@Sendable () -> Void)?>(nil) + + /// A point in the cleanup watchdog after its deadline passes. + enum CleanupWatchdogEvent: Sendable, Equatable { + /// The watchdog is about to decide between completion and the timeout. + case beforeDecision + /// The watchdog has decided, and aborts if `aborts` is true. + case decided(aborts: Bool) + } + + /// Runs on the cleanup watchdog's thread after its deadline passes. + static let cleanupWatchdog = Locked<(@Sendable (CleanupWatchdogEvent) -> Void)?>(nil) } #endif #endif // Llama diff --git a/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift b/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift index 54bf2fa5..0edd00ad 100644 --- a/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift +++ b/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift @@ -211,6 +211,30 @@ import Testing let elapsed = await ExitTestTimer.finish(name, watchdog) #expect(try #require(elapsed) < .seconds(LlamaLifecycle.exitDeadline)) } + + @Test("A cleanup that blocks at exit aborts the process instead of hanging") + func exitWithBlockedCleanup() async throws { + // A log callback blocks while the exit handler frees the model, + // so the cleanup watchdog aborts the process. + let name = "exitWithBlockedCleanup" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("exitWithBlockedCleanup") + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1) + LlamaTestHooks.onLog.withLock { + $0 = { + if LlamaLifecycle.shared.isStopping { + while true { sleep(60) } + } + } + } + ExitTestTimer.stepStarted("exitWithBlockedCleanup") + withExtendedLifetime((model, session)) { exit(0) } + } + let elapsed = try #require(await ExitTestTimer.finish(name, watchdog)) + #expect(elapsed >= .seconds(LlamaLifecycle.cleanupDeadline)) + #expect(elapsed < .seconds(LlamaLifecycle.cleanupDeadline + 5)) + } #endif @Test("A process exits normally after the model is released", arguments: [Int32(-1), 0]) @@ -460,6 +484,130 @@ import Testing #expect(lifecycle.activeOperationCount == 0) } + @Test("The cleanup deadline does nothing after the cleanup ends") + func cleanupDeadlineAfterCleanup() async throws { + let name = "cleanupDeadlineAfterCleanup" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { + ExitTestTimer.childStarted("cleanupDeadlineAfterCleanup") + ExitTestTimer.stepStarted("cleanupDeadlineAfterCleanup") + let value = LlamaLifecycle.withAbortDeadline(0.1) { 42 } + // Outlive the deadline; the watchdog must not abort. + usleep(500_000) + exit(value == 42 ? 0 : 5) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + @Test("A cleanup that runs past its deadline aborts the process") + func cleanupDeadlinePassed() async throws { + let name = "cleanupDeadlinePassed" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("cleanupDeadlinePassed") + ExitTestTimer.stepStarted("cleanupDeadlinePassed") + LlamaLifecycle.withAbortDeadline(0.2) { + while true { sleep(60) } + } + } + let elapsed = try #require(await ExitTestTimer.finish(name, watchdog)) + #expect(elapsed >= .milliseconds(200)) + #expect(elapsed < .seconds(5)) + } + + @Test("A cleanup that runs past its deadline aborts even if standard error is a full pipe") + func cleanupDeadlineWithFullStandardError() async throws { + let name = "cleanupDeadlineWithFullStandardError" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("cleanupDeadlineWithFullStandardError") + // Make standard error a pipe that is full and that nobody reads. + // The read end stays open, so a write blocks instead of failing. + var fds: [CInt] = [0, 0] + guard pipe(&fds) == 0 else { _exit(3) } + let flags = fcntl(fds[1], F_GETFL) + _ = fcntl(fds[1], F_SETFL, flags | O_NONBLOCK) + let chunk = [UInt8](repeating: 0x2E, count: 4096) + while chunk.withUnsafeBytes({ Darwin.write(fds[1], $0.baseAddress, $0.count) }) > 0 {} + guard errno == EAGAIN else { _exit(4) } + _ = fcntl(fds[1], F_SETFL, flags) + guard dup2(fds[1], STDERR_FILENO) == STDERR_FILENO else { _exit(6) } + ExitTestTimer.stepStarted("cleanupDeadlineWithFullStandardError") + LlamaLifecycle.withAbortDeadline(0.2) { + while true { sleep(60) } + } + } + let elapsed = try #require(await ExitTestTimer.finish(name, watchdog)) + #expect(elapsed >= .milliseconds(200)) + #expect(elapsed < .seconds(5)) + } + + #if DEBUG + @Test("Once the cleanup completes, a watchdog that timed out doesn't abort") + func cleanupCompletionWinsRace() async throws { + let name = "cleanupCompletionWinsRace" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { + ExitTestTimer.childStarted("cleanupCompletionWinsRace") + let entered = DispatchSemaphore(value: 0) + let proceed = DispatchSemaphore(value: 0) + let decided = DispatchSemaphore(value: 0) + let aborts = Locked(nil) + LlamaTestHooks.cleanupWatchdog.withLock { + $0 = { event in + switch event { + case .beforeDecision: + // Hold the watchdog after its timeout + // until the cleanup has completed. + entered.signal() + proceed.wait() + case .decided(let value): + aborts.withLock { $0 = value } + decided.signal() + } + } + } + ExitTestTimer.stepStarted("cleanupCompletionWinsRace") + // The cleanup returns only after the watchdog's deadline has passed. + let value = LlamaLifecycle.withAbortDeadline(0.05) { + entered.wait() + return 42 + } + proceed.signal() + decided.wait() + exit(value == 42 && aborts.withLock({ $0 }) == false ? 0 : 5) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + @Test("Once a watchdog decides to abort, the cleanup doesn't return") + func cleanupTimeoutWinsRace() async throws { + let name = "cleanupTimeoutWinsRace" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("cleanupTimeoutWinsRace") + let decided = DispatchSemaphore(value: 0) + LlamaTestHooks.cleanupWatchdog.withLock { + $0 = { event in + guard event == .decided(aborts: true) else { return } + decided.signal() + // Keep the watchdog from aborting, + // so that only the cleanup's own check can abort. + while true { sleep(60) } + } + } + ExitTestTimer.stepStarted("cleanupTimeoutWinsRace") + _ = LlamaLifecycle.withAbortDeadline(0.05) { + decided.wait() + return 42 + } + // A distinct status shows that the cleanup returned. + exit(5) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + #endif + @Test("A failed registration of the exit handler makes the handler free nothing") func registrationFailure() { let lifecycle = LlamaLifecycle() From 9ea628c20678081ce4e77585eb24cbf55383b6a1 Mon Sep 17 00:00:00 2001 From: mattt Date: Mon, 16 Nov 2026 16:56:44 -0800 Subject: [PATCH 3/5] Check only lower time bounds in the exit tests The parent measures a child's time until it sees the child end, and a busy CI runner delayed that by about seven seconds, so upper bounds failed although every child ended as expected. Exact exit statuses, the lower bounds, the parent's kill timer, and the lifecycle unit tests still catch a broken handler or watchdog. --- .../LlamaLanguageModelExitTests.swift | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift b/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift index 0edd00ad..c68283b3 100644 --- a/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift +++ b/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift @@ -91,8 +91,7 @@ import Testing ExitTestTimer.stepStarted(name) withExtendedLifetime((model, session)) { exit(0) } } - let elapsed = await ExitTestTimer.finish(name, watchdog) - #expect(try #require(elapsed) < .seconds(LlamaLifecycle.exitDeadline)) + await ExitTestTimer.finish(name, watchdog, measuring: false) } @Test("A process exits normally with a model in a global variable", arguments: [Int32(-1), 0]) @@ -112,8 +111,7 @@ import Testing ExitTestTimer.stepStarted(name) exit(0) } - let elapsed = await ExitTestTimer.finish(name, watchdog) - #expect(try #require(elapsed) < .seconds(LlamaLifecycle.exitDeadline)) + await ExitTestTimer.finish(name, watchdog, measuring: false) } #if DEBUG @@ -133,8 +131,7 @@ import Testing ExitTestTimer.stepStarted(name) withExtendedLifetime(model) { exit(0) } } - let elapsed = await ExitTestTimer.finish(name, watchdog) - #expect(try #require(elapsed) < .seconds(LlamaLifecycle.exitDeadline)) + await ExitTestTimer.finish(name, watchdog, measuring: false) } @Test("A generation that runs past the deadline doesn't keep the process alive") @@ -154,7 +151,6 @@ import Testing } let elapsed = try #require(await ExitTestTimer.finish(name, watchdog)) #expect(elapsed >= .seconds(LlamaLifecycle.exitDeadline)) - #expect(elapsed < .seconds(LlamaLifecycle.exitDeadline + 5)) } @Test("An exit from inside a generation doesn't wait for that generation") @@ -163,6 +159,8 @@ import Testing let watchdog = ExitTestTimer.watch(name) // The handler returns at once and frees nothing, // so ggml's assertion ends the process before the deadline. + // The exit status can't show how long the handler took; + // `shutDownInsideTaskOperation()` checks that it returns at once. await #expect(processExitsWith: .signal(SIGABRT)) { ExitTestTimer.childStarted("exitInsideOperation") let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1) @@ -175,13 +173,13 @@ import Testing while true { sleep(60) } } } - let elapsed = await ExitTestTimer.finish(name, watchdog) - #expect(try #require(elapsed) < .seconds(LlamaLifecycle.exitDeadline)) + await ExitTestTimer.finish(name, watchdog, measuring: false) } @Test("A log callback that exits while a model is freed doesn't hang the process") func exitFromLogDuringCleanup() async throws { - // The handler sees that the cleanup's thread is exiting, and returns at once. + // The handler sees that the cleanup's thread is exiting, and returns at once, + // which `shutDownInsideThreadOperation()` checks without a model. // The model is only partly freed, so ggml's assertion ends the process, // as it would without the handler. let name = "exitFromLogDuringCleanup" @@ -208,8 +206,7 @@ import Testing // A distinct status shows that llama.cpp logged nothing. exit(4) } - let elapsed = await ExitTestTimer.finish(name, watchdog) - #expect(try #require(elapsed) < .seconds(LlamaLifecycle.exitDeadline)) + await ExitTestTimer.finish(name, watchdog, measuring: false) } @Test("A cleanup that blocks at exit aborts the process instead of hanging") @@ -233,7 +230,6 @@ import Testing } let elapsed = try #require(await ExitTestTimer.finish(name, watchdog)) #expect(elapsed >= .seconds(LlamaLifecycle.cleanupDeadline)) - #expect(elapsed < .seconds(LlamaLifecycle.cleanupDeadline + 5)) } #endif @@ -320,6 +316,12 @@ import Testing /// to files that both processes name from the parent's process ID. /// The measurement starts after the child's setup, /// so a slow load doesn't change it. + /// + /// The measurement ends when the parent sees that the child has ended, + /// which a busy machine can delay by seconds. + /// So a test checks only that a time is at least a bound, + /// which a delay can't break, + /// and never that a time is less than a bound. enum ExitTestTimer { /// How long the parent waits for a child before it kills the child. static let limit: Duration = .seconds(120) @@ -512,7 +514,6 @@ import Testing } let elapsed = try #require(await ExitTestTimer.finish(name, watchdog)) #expect(elapsed >= .milliseconds(200)) - #expect(elapsed < .seconds(5)) } @Test("A cleanup that runs past its deadline aborts even if standard error is a full pipe") @@ -539,7 +540,6 @@ import Testing } let elapsed = try #require(await ExitTestTimer.finish(name, watchdog)) #expect(elapsed >= .milliseconds(200)) - #expect(elapsed < .seconds(5)) } #if DEBUG From 1ad446e4b8919db4af3428fa9c177a1571ddf00e Mon Sep 17 00:00:00 2001 From: mattt Date: Tue, 17 Nov 2026 06:19:33 -0800 Subject: [PATCH 4/5] Document GGML_METAL_NO_RESIDENCY as the certain way to avoid the exit assertion --- README.md | 15 +++++++++++++++ .../Models/LlamaLanguageModel.swift | 11 +++++++++++ 2 files changed, 26 insertions(+) diff --git a/README.md b/README.md index 6e071814..689f40e0 100644 --- a/README.md +++ b/README.md @@ -1641,6 +1641,21 @@ A response stream's generation can keep its model for a short time after the stream ends, until the generation stops at its next token. +To be certain that the assertion can't happen, +set the environment variable `GGML_METAL_NO_RESIDENCY` to any value +before the first model loads. +llama.cpp's Metal backend then doesn't use residency sets, +so a process exits normally even with a model loaded on the GPU. +Set it in the process's environment, +or call `setenv("GGML_METAL_NO_RESIDENCY", "1", 1)` before the first request, +because `LlamaLanguageModel` starts the llama.cpp backend at the first load. +If another library in the process uses llama.cpp, +set it before that library starts llama.cpp with `llama_backend_init`. +The cost is that residency sets keep a model's GPU memory wired between requests; +without them, the system can reclaim that memory while the GPU is idle, +and the next request waits for it again. +This package doesn't set the variable. + ### LiteRT-LM diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index 39c7a68f..1c24dd1f 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -156,6 +156,17 @@ import Foundation /// and then release every reference to the model, including its sessions. /// A response stream's generation can keep the model for a short time /// after the stream ends, until the generation stops at its next token. + /// + /// To be certain that the assertion can't happen, + /// set the environment variable `GGML_METAL_NO_RESIDENCY` to any value + /// before the first model loads, + /// in the process's environment or with `setenv` before the first request, + /// because the model starts the llama.cpp backend at the first load. + /// llama.cpp's Metal backend then doesn't use residency sets, + /// so a process exits normally even with a model loaded on the GPU. + /// The cost is that the system can reclaim a model's GPU memory + /// while the GPU is idle, and the next request waits for it again. + /// This package doesn't set the variable. public final class LlamaLanguageModel: LanguageModel, @unchecked Sendable { /// The reason the model is unavailable. /// This model is always available. From 8efee9334a1e9461b97c360d8ed0b82734e16825 Mon Sep 17 00:00:00 2001 From: mattt Date: Tue, 17 Nov 2026 13:50:44 -0800 Subject: [PATCH 5/5] Stop Hub loads after the exit handler starts, and document prewarm --- README.md | 6 ++- .../Models/LlamaLanguageModel.swift | 8 +++- .../LlamaLanguageModelExitTests.swift | 37 ++++++++++++++++++- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 35217f30..dd31ff9a 100644 --- a/README.md +++ b/README.md @@ -1667,6 +1667,10 @@ The model frees its weights when the last reference is released. A response stream's generation can keep its model for a short time after the stream ends, until the generation stops at its next token. +A `prewarm` keeps its model until its download and load end, +and you can't wait for it directly. +A request that you await after a `prewarm` waits for that load, +but the `prewarm` can keep the model for a short time after the request returns. To be certain that the assertion can't happen, set the environment variable `GGML_METAL_NO_RESIDENCY` to any value @@ -1674,7 +1678,7 @@ before the first model loads. llama.cpp's Metal backend then doesn't use residency sets, so a process exits normally even with a model loaded on the GPU. Set it in the process's environment, -or call `setenv("GGML_METAL_NO_RESIDENCY", "1", 1)` before the first request, +or call `setenv("GGML_METAL_NO_RESIDENCY", "1", 1)` before the first request or `prewarm`, because `LlamaLanguageModel` starts the llama.cpp backend at the first load. If another library in the process uses llama.cpp, set it before that library starts llama.cpp with `llama_backend_init`. diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index f4de2ed3..e34ee8eb 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -159,11 +159,15 @@ import Foundation /// and then release every reference to the model, including its sessions. /// A response stream's generation can keep the model for a short time /// after the stream ends, until the generation stops at its next token. + /// A `prewarm` keeps the model until its download and load end, + /// and you can't wait for it directly. + /// A request that you await after a `prewarm` waits for that load, + /// but the `prewarm` can keep the model for a short time after the request returns. /// /// To be certain that the assertion can't happen, /// set the environment variable `GGML_METAL_NO_RESIDENCY` to any value /// before the first model loads, - /// in the process's environment or with `setenv` before the first request, + /// in the process's environment or with `setenv` before the first request or `prewarm`, /// because the model starts the llama.cpp backend at the first load. /// llama.cpp's Metal backend then doesn't use residency sets, /// so a process exits normally even with a model loaded on the GPU. @@ -1460,6 +1464,8 @@ import Foundation /// Downloads a Hub model's files if needed, then loads the model. func ensureModelLoaded() async throws { + // After the exit handler starts, don't list or download a Hub model's files. + try LlamaLifecycle.shared.stopIfExiting() let files = if let hubSource { try await hubSource.files() diff --git a/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift b/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift index c68283b3..fa52d288 100644 --- a/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift +++ b/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift @@ -69,7 +69,6 @@ import Testing whileActive() } } - nonisolated(unsafe) let session = session Task.detached { _ = try? await session.respond( to: "Count from 1 to 1000.", @@ -573,8 +572,11 @@ import Testing entered.wait() return 42 } + // The child does nothing else, so it can block a thread here, + // in a synchronous function. + func waitForDecision() { decided.wait() } proceed.signal() - decided.wait() + waitForDecision() exit(value == 42 && aborts.withLock({ $0 }) == false ? 0 : 5) } await ExitTestTimer.finish(name, watchdog, measuring: false) @@ -608,6 +610,37 @@ import Testing } #endif + @Test("After the exit handler runs, a Hub model neither lists nor downloads its files") + func hubLoadAfterShutdown() async { + let name = "hubLoadAfterShutdown" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { + ExitTestTimer.childStarted("hubLoadAfterShutdown") + let repository = FakeHubRepository() + let model = LlamaLanguageModel( + givenModelPath: "", + givenMMProjPath: nil, + hubSource: LlamaHubModelSource( + modelId: "example/model-GGUF", + file: nil, + mmproj: .disabled, + hub: nil, + repository: repository + ), + gpuLayers: 0 + ) + guard LlamaLifecycle.shared.shutDown() == .freed else { exit(5) } + do { + _ = try await LanguageModelSession(model: model).respond(to: "Say hello") + exit(6) + } catch LanguageModelSession.GenerationError.assetsUnavailable {} + // A distinct status shows that the model reached the Hub. + guard repository.listingCalls == 0, repository.downloadStarts == 0 else { exit(7) } + exit(0) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + @Test("A failed registration of the exit handler makes the handler free nothing") func registrationFailure() { let lifecycle = LlamaLifecycle()