diff --git a/README.md b/README.md index e73df9f1..dd31ff9a 100644 --- a/README.md +++ b/README.md @@ -1617,6 +1617,76 @@ let response = try await session.respond( ) ``` +llama.cpp's Metal backend stops a process with an assertion +if the process exits while a model has layers on the GPU, +and a generation that is still running at exit can crash on the GPU or the CPU. +So `LlamaLanguageModel` registers an exit handler when a load ends. +The handler stops new loads and generations, +waits up to three seconds for running work to stop at its next token, +and then frees every loaded model, projector, and cached context. +After the handler starts, +requests throw `LanguageModelSession.GenerationError.assetsUnavailable`. + +The handler is best effort. +The process can still stop with the assertion in these cases: + +- It exits during the first load, + because the handler is registered when a load ends. +- It exits from inside a load, a generation, or the release of a model, + for example from a llama.cpp callback. + That work can't end while the handler waits, + so the handler returns at once and frees nothing. + An exit from a llama.cpp callback on a thread + that llama.cpp or another library owns + isn't recognized as inside that work, + so the handler waits the full three seconds and then frees nothing. +- Work is still running after three seconds. + The handler then frees nothing. +- It exits without running exit handlers, + for example with `_exit` or after a crash. + +If freeing the models takes more than ten seconds, +for example because another library's log callback blocks, +the handler aborts the process, so that the exit can't hang. +A crash report for this abort shows +`LlamaLifecycle.abortAfterExitCleanupTimeout()` in its backtrace, +which first tries to write one line to standard error, +but aborts whether or not the line can be written. +If the cleanup ends just after the limit, +the cleanup's own thread can abort first, +without the line, +and the backtrace shows `LlamaLifecycle.abortAfterLateExitCleanup()`. + +No method unloads a model that something still references. +To be certain that no model is loaded when the process exits, +wait until every `respond` call returns and every response stream ends, +start no new requests, +and then release every reference to the `LlamaLanguageModel`, +including the sessions that use it. +The model frees its weights when the last reference is released. +A response stream's generation can keep its model for a short time +after the stream ends, +until the generation stops at its next token. +A `prewarm` keeps its model until its download and load end, +and you can't wait for it directly. +A request that you await after a `prewarm` waits for that load, +but the `prewarm` can keep the model for a short time after the request returns. + +To be certain that the assertion can't happen, +set the environment variable `GGML_METAL_NO_RESIDENCY` to any value +before the first model loads. +llama.cpp's Metal backend then doesn't use residency sets, +so a process exits normally even with a model loaded on the GPU. +Set it in the process's environment, +or call `setenv("GGML_METAL_NO_RESIDENCY", "1", 1)` before the first request or `prewarm`, +because `LlamaLanguageModel` starts the llama.cpp backend at the first load. +If another library in the process uses llama.cpp, +set it before that library starts llama.cpp with `llama_backend_init`. +The cost is that residency sets keep a model's GPU memory wired between requests; +without them, the system can reclaim that memory while the GPU is idle, +and the next request waits for it again. +This package doesn't set the variable. + ### LiteRT-LM diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index cc269aa4..e34ee8eb 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -102,6 +102,9 @@ import Foundation text: UnsafePointer?, userData: UnsafeMutableRawPointer? ) { + #if DEBUG + LlamaTestHooks.onLog.withLock { $0 }?() + #endif let minimum = LlamaRuntime.logLevel.withLock { $0 } let logger = LlamaRuntime.Logger(userData: userData) guard LlamaRuntime.shouldWrite(level: level, logger: logger, minimum: minimum), let text else { @@ -120,6 +123,57 @@ import Foundation /// let model = LlamaLanguageModel(modelPath: "/path/to/model.gguf") /// let hubModel = LlamaLanguageModel(modelId: "ggml-org/gemma-3-4b-it-GGUF:Q4_K_M") /// ``` + /// + /// llama.cpp's Metal backend stops a process with an assertion + /// if the process exits while a model has layers on the GPU. + /// So the model registers an exit handler when a load ends, + /// which stops new loads and generations, + /// waits up to three seconds for running work, + /// and then frees every loaded model. + /// After the handler starts, requests throw + /// ``LanguageModelSession/GenerationError/assetsUnavailable(_:)``. + /// The handler can't help in these cases: + /// + /// - An exit during the first load. + /// - An exit from inside a load, a generation, or the release of a model, + /// such as from a llama.cpp callback. + /// An exit from a llama.cpp callback on a thread + /// that runs no task and releases no model, + /// such as a thread that llama.cpp or another library owns, + /// makes the handler wait the full three seconds and then free nothing. + /// - Work that is still running after three seconds. + /// - An exit that runs no exit handlers, such as `_exit` or a crash. + /// + /// If freeing the models takes more than ten seconds, + /// the handler aborts the process, so that the exit can't hang. + /// A crash report shows `abortAfterExitCleanupTimeout()` in its backtrace, + /// which first tries to write one line to standard error. + /// If the cleanup ends just after the limit, + /// the cleanup's thread can abort first, without the line, + /// and the backtrace shows `abortAfterLateExitCleanup()`. + /// + /// No method unloads a model that something still references. + /// To be certain that no model is loaded when the process exits, + /// wait until every request returns and every response stream ends, + /// start no new requests, + /// and then release every reference to the model, including its sessions. + /// A response stream's generation can keep the model for a short time + /// after the stream ends, until the generation stops at its next token. + /// A `prewarm` keeps the model until its download and load end, + /// and you can't wait for it directly. + /// A request that you await after a `prewarm` waits for that load, + /// but the `prewarm` can keep the model for a short time after the request returns. + /// + /// To be certain that the assertion can't happen, + /// set the environment variable `GGML_METAL_NO_RESIDENCY` to any value + /// before the first model loads, + /// in the process's environment or with `setenv` before the first request or `prewarm`, + /// because the model starts the llama.cpp backend at the first load. + /// llama.cpp's Metal backend then doesn't use residency sets, + /// so a process exits normally even with a model loaded on the GPU. + /// The cost is that the system can reclaim a model's GPU memory + /// while the GPU is idle, and the next request waits for it again. + /// This package doesn't set the variable. public final class LlamaLanguageModel: LanguageModel, @unchecked Sendable { /// The reason the model is unavailable. /// This model is always available. @@ -565,7 +619,13 @@ import Foundation /// frees a context that is still decoding: it runs on a transient context /// instead and leaves the cache untouched. private let sessionContextLock = NSLock() - private var cachedSessionContext: CachedSessionContext? + private var cachedSessionContext: CachedSessionContext? { + // The exit handler frees the cached context through the resources. + didSet { resources?.setCachedContext(cachedSessionContext?.context) } + } + + /// The loaded model's resources, which the exit handler can free. + private var resources: LlamaModelResources? /// The number of prompt tokens reused from the cached context by the most /// recent chat generation. @@ -580,21 +640,30 @@ import Foundation /// model. The next chat generation prefills its full prompt again. Call this /// under memory pressure or when a session is discarded. If a generation is /// running on the cached context, it is freed as soon as that generation ends. + /// + /// After the exit handler starts, this method does nothing, + /// because the handler frees the cached context. public func clearCachedContext() { - discardCachedSessionContext() + // After the exit handler starts, the context may be freed already. + try? LlamaLifecycle.shared.perform { discardCachedSessionContext() } } private func discardCachedSessionContext() { - sessionContextLock.lock() - defer { sessionContextLock.unlock() } - guard var cached = cachedSessionContext else { return } - if cached.isCheckedOut { - cached.discardWhenReleased = true - cachedSessionContext = cached - return + let unused: OpaquePointer? = sessionContextLock.withLock { + guard var cached = cachedSessionContext else { return nil } + if cached.isCheckedOut { + cached.discardWhenReleased = true + cachedSessionContext = cached + return nil + } + cachedSessionContext = nil + return cached.context + } + // Free without the lock: llama.cpp logs while it frees a context, + // and a log callback that calls `exit` runs the exit handler on this thread. + if let unused { + llama_free(unused) } - llama_free(cached.context) - cachedSessionContext = nil } private func recordCachedTokens(_ tokens: [llama_token], context: OpaquePointer) { @@ -615,19 +684,22 @@ import Foundation /// context is checked back in (or freed, if a discard was requested while /// it was busy); a transient context is freed. private func releaseSessionContext(_ context: OpaquePointer) { - sessionContextLock.lock() - defer { sessionContextLock.unlock() } - if var cached = cachedSessionContext, cached.context == context { - if cached.discardWhenReleased { - llama_free(cached.context) - cachedSessionContext = nil - } else { + let unused: OpaquePointer? = sessionContextLock.withLock { + if var cached = cachedSessionContext, cached.context == context { + if cached.discardWhenReleased { + cachedSessionContext = nil + return context + } cached.isCheckedOut = false cachedSessionContext = cached + return nil } - return + return context + } + // Free without the lock, as in `discardCachedSessionContext()`. + if let unused { + llama_free(unused) } - llama_free(context) } /// Returns a context for the session along with the index of the first @@ -818,14 +890,10 @@ import Foundation } deinit { - if let cached = cachedSessionContext { - llama_free(cached.context) - } - if let mtmdContext = mtmdContext { - mtmd_free(mtmdContext) - } - if let model = model { - llama_model_free(model) + // The resources free the cached context, the projector, and the model, + // unless the exit handler freed them already. + if let resources { + LlamaLifecycle.shared.free(resources) } } @@ -968,6 +1036,11 @@ import Foundation let runtimeOptions = resolvedOptions(from: options) let structuredOptions = resolvedStructuredOptions(from: options) + // The exit handler waits for this request's llama.cpp work, + // but not for its tool calls. + let operation = try LlamaLifecycle.shared.begin() + defer { operation.end() } + // Structured output does not go through the projector yet. // Without an image marker, a prompt with an image throws // before any generation or tool call, @@ -1071,6 +1144,7 @@ import Foundation try session.recordToolRound(toolRounds, completed: allEntries) + operation.suspend() let (callsEntry, resolution) = try await resolveToolCalls( parsedCalls, requestContext: requestContext, @@ -1104,7 +1178,9 @@ import Foundation pendingEntries.append(.prompt(prompt)) } } + try operation.resume() } + try operation.resume() if type == String.self { return LanguageModelSession.Response( content: text as! Content, @@ -1215,6 +1291,10 @@ import Foundation let runtimeOptions = resolvedOptions(from: options) let maxTokens = runtimeOptions.maximumResponseTokens ?? 100 + // The exit handler waits for this request's llama.cpp work, + // but not for its tool calls. + let operation = try LlamaLifecycle.shared.begin() + defer { operation.end() } let outputFormat = self.currentToolCallFormat() let toolRounds = session.toolRoundLimit(for: "Llama", options: options) var accumulatedEntries: [Transcript.Entry] = [] @@ -1327,6 +1407,7 @@ import Foundation try session.recordToolRound(toolRounds, completed: accumulatedEntries) + operation.suspend() let (callsEntry, resolution) = try await self.resolveToolCalls( parsedCalls, requestContext: requestContext, @@ -1359,6 +1440,7 @@ import Foundation emittedBase += roundVisible yieldSnapshot(emittedBase) } + try operation.resume() } if emittedBase != lastYieldedText { @@ -1382,6 +1464,8 @@ import Foundation /// Downloads a Hub model's files if needed, then loads the model. func ensureModelLoaded() async throws { + // After the exit handler starts, don't list or download a Hub model's files. + try LlamaLifecycle.shared.stopIfExiting() let files = if let hubSource { try await hubSource.files() @@ -1399,6 +1483,11 @@ import Foundation guard !isModelLoaded else { return } + // The exit handler waits for this load. + // After the handler starts, this throws before the path is checked. + let operation = try LlamaLifecycle.shared.begin() + defer { operation.end() } + // Check if model file exists guard FileManager.default.fileExists(atPath: modelPath) else { throw LlamaLanguageModelError.invalidModelPath @@ -1423,6 +1512,7 @@ import Foundation let modelParams = createModelParams() guard let loadedModel = llama_model_load_from_file(modelPath, modelParams) else { + try LlamaLifecycle.shared.stopIfExiting() throw LlamaLanguageModelError.modelLoadFailed } @@ -1446,6 +1536,10 @@ import Foundation self.vocab = llama_model_get_vocab(loadedModel) self.tokenCache = StructuredGenerationTokenCache() self.isModelLoaded = true + let resources = LlamaModelResources(model: loadedModel, projector: mtmdContext) + self.resources = resources + LlamaLifecycle.shared.track(resources) + LlamaRuntime.registerExitHandler() } private func createModelParams() -> llama_model_params { @@ -1454,6 +1548,8 @@ import Foundation // Try to reduce memory usage params.load_mode = LLAMA_LOAD_MODE_MMAP + // Returning false stops the load, so that the exit handler doesn't wait for all of it. + params.progress_callback = { _, _ in !LlamaLifecycle.shared.isStopping } return params } @@ -1737,6 +1833,7 @@ import Foundation } mutating func decode(_ token: Int) async throws { + try LlamaLifecycle.shared.stopIfExiting() let llamaToken = llama_token(token) batch.pointee.n_tokens = 1 @@ -1871,9 +1968,14 @@ import Foundation var decodedTokens = promptTokens for _ in 0 ..< maxTokens { + #if DEBUG + LlamaTestHooks.beforeToken.withLock { $0 }?() + #endif if Task.isCancelled { break } + // The exit handler waits for this generation, so stop it early. + try LlamaLifecycle.shared.stopIfExiting() // Sample next token from logits of the last token we just decoded let nextToken = llama_sampler_sample(sampler, context, batch.n_tokens - 1) @@ -2044,6 +2146,8 @@ import Foundation if Task.isCancelled { break } + // The exit handler waits for this generation, so stop it early. + try LlamaLifecycle.shared.stopIfExiting() let nextToken = llama_sampler_sample(samplerPtr, context, sampleIndex) llama_sampler_accept(samplerPtr, nextToken) @@ -2187,6 +2291,7 @@ import Foundation let capacity = Int(batchSize) var start = startIndex while start < promptTokens.count { + try LlamaLifecycle.shared.stopIfExiting() let count = min(capacity, promptTokens.count - start) batch.n_tokens = Int32(count) for i in 0 ..< count { diff --git a/Sources/AnyLanguageModel/Models/LlamaLifecycle.swift b/Sources/AnyLanguageModel/Models/LlamaLifecycle.swift new file mode 100644 index 00000000..98142e2b --- /dev/null +++ b/Sources/AnyLanguageModel/Models/LlamaLifecycle.swift @@ -0,0 +1,521 @@ +import Foundation + +#if Llama + import LlamaSwift + + extension LlamaRuntime { + /// Registers ``LlamaLifecycle/shutDown(deadline:)`` to run when the process exits. + /// + /// ggml creates static objects as it needs them, + /// such as its Metal device and buffer types during a model's first load, + /// and the C runtime destroys static objects and runs exit handlers + /// in the reverse order of their creation. + /// A handler that was registered before the first load + /// would run after ggml's objects are destroyed, + /// and freeing a model then would crash. + /// ggml doesn't report when it creates such an object, + /// so every load registers the handler again when it ends. + /// The last registration runs first, + /// and the handler does nothing after its first run. + /// + /// If a registration fails, + /// an earlier one could run after objects that the models use are destroyed, + /// so the handler then frees nothing. + static func registerExitHandler() { + if atexit({ _ = LlamaLifecycle.shared.shutDown() }) != 0 { + LlamaLifecycle.shared.exitHandlerRegistrationFailed() + } + } + } + + /// Tracks the llama.cpp work in this process, + /// and tries to free every loaded model when the process exits. + /// + /// ggml's Metal backend frees its device in a C++ static destructor at exit, + /// and it stops the process with an assertion + /// if a model or a context still has buffers on the device then. + /// A generation that is still running at exit can also crash, + /// on the GPU or the CPU, + /// because the static objects that it uses are destroyed under it. + /// + /// The exit handler is best effort. + /// It must never make an exit worse than an exit without it, + /// so it never waits without a limit, + /// and it never frees a model or a context that running work could use. + /// It stops new loads and generations, + /// and it waits up to ``exitDeadline`` for the ones that are running: + /// a load stops at its next progress report, + /// and a generation stops before its next token or batch of prompt tokens. + /// If no work is running then, + /// it frees the cached session context, the projector, and the model + /// of every loaded ``LlamaLanguageModel``, + /// including models that Swift objects still hold. + /// Otherwise it frees nothing. + /// Freeing only the idle models wouldn't help, + /// because the busy model's buffers would still cause the assertion. + /// + /// The handler can't help, and the assertion can still happen, + /// in these cases: + /// + /// - An exit during the first load, + /// because the handler is registered when a load ends. + /// An earlier registration would run after ggml's objects are destroyed. + /// - An exit from inside a load, a generation, or a cleanup, + /// such as from a llama.cpp callback. + /// The handler sees that its task or thread is running that work, + /// which can't end while the handler waits, + /// and returns at once without freeing anything. + /// On a thread that runs no task and has no cleanup mark, + /// such as a thread that llama.cpp or another library owns, + /// the handler can't see this, + /// so it waits until the deadline and then frees nothing. + /// - Work that is still running at the deadline. + /// - An exit that runs no exit handlers, such as `_exit` or a crash. + /// + /// After the handler starts, every load and generation throws + /// ``LanguageModelSession/GenerationError/assetsUnavailable(_:)``. + /// + /// Nothing here calls llama.cpp with a lock held. + /// llama.cpp logs while it frees a model, + /// and runs the log callback on the same thread, + /// so a callback that calls `exit` would run the exit handler on that thread, + /// and a lock that isn't recursive would then never be released. + final class LlamaLifecycle: @unchecked Sendable { + static let shared = LlamaLifecycle() + + /// How long the exit handler waits for running loads and generations. + /// + /// A running generation stops before its next token, + /// or before its next batch of prompt tokens, + /// so the wait is about one call to llama.cpp, + /// which takes milliseconds to a few hundred milliseconds for a 4B model. + /// Three seconds leaves room for a much larger model or a slow CPU, + /// and still ends a process that can't finish its work + /// soon enough that the delay looks like a slow quit, not a hang. + static let exitDeadline: TimeInterval = 3 + + /// How long the exit handler's cleanup may take before the process aborts. + /// + /// The cleanup frees every model after running work has ended. + /// llama.cpp calls the installed log callback while it frees, + /// and another library can replace that callback with one that blocks, + /// for example on a lock that the exiting thread holds, + /// and a GPU driver can stall while it releases buffers. + /// Without a limit the process would then hang, + /// where without the handler it would have stopped with ggml's assertion. + /// So the cleanup runs under a watchdog, + /// which calls `abort()` if the cleanup is still running after this time. + /// It tries to write one line to standard error first, + /// but it aborts whether or not the line can be written. + /// + /// Freeing Qwen3-4B (2.5 GB) and its context took about 35 milliseconds + /// on an Apple silicon Mac, on the GPU and on the CPU. + /// The work grows with the number and size of the buffers, + /// so a 70B model would take under a second by that measure. + /// Ten seconds leaves a margin of more than ten times that + /// for a slow but working free. + static let cleanupDeadline: TimeInterval = 10 + + /// What ``shutDown(deadline:)`` did. + enum ShutdownResult: Equatable { + /// It freed every model. + case freed + /// It had already run, so it did nothing. + case alreadyShutDown + /// It ran inside an operation, so it freed nothing without waiting. + case insideOperation + /// Work was still running at the deadline, so it freed nothing. + case timedOut + /// A registration of the exit handler failed, so it freed nothing. + case registrationFailed + } + + /// The key in a thread's dictionary for the number of operations that the thread runs. + private static let depthKey = "AnyLanguageModel.LlamaLifecycle.operationDepth" + + /// Guards the properties below, and signals when no operation is running. + private let condition = NSCondition() + private var isShuttingDown = false + private var registrationFailed = false + private var activeOperations = 0 + /// The number of active operations in each task. + /// + /// A key is used only to compare tasks, + /// and only while its operation is active, + /// so its task is still running. + private var activeTasks: [UnsafeCurrentTask: Int] = [:] + private var resources: [ObjectIdentifier: LlamaModelResources] = [:] + + /// Whether the process is exiting. + /// + /// Generations read this before each token, + /// which costs one uncontended lock and unlock. + var isStopping: Bool { + condition.withLock { isShuttingDown } + } + + /// The number of loads, generations, and cleanups that are running. + var activeOperationCount: Int { + condition.withLock { activeOperations } + } + + /// The error that work throws after the exit handler starts. + static var exiting: LanguageModelSession.GenerationError { + .assetsUnavailable( + .init(debugDescription: "The process is exiting, so llama.cpp work has stopped.") + ) + } + + /// Throws ``exiting`` if the exit handler has started. + func stopIfExiting() throws { + if isStopping { throw Self.exiting } + } + + // MARK: Requests + + /// A load or a generation, which the exit handler waits for. + /// + /// A request can await other code in the middle of its llama.cpp work, + /// so an operation belongs to the task that began it, not to a thread. + /// A request suspends its operation while it waits for other code, + /// such as a tool call, + /// so that the exit handler doesn't wait for that code. + /// Use an operation only in the task that began it. + final class Operation { + private let lifecycle: LlamaLifecycle + private var task: UnsafeCurrentTask? + private var isActive = true + + fileprivate init(lifecycle: LlamaLifecycle, task: UnsafeCurrentTask?) { + self.lifecycle = lifecycle + self.task = task + } + + /// Ends the operation for now, if it is active. + func suspend() { + guard isActive else { return } + isActive = false + lifecycle.endOperation(in: task) + } + + /// Starts the operation again, if it is suspended. + /// + /// - Throws: ``LlamaLifecycle/exiting`` if the exit handler has started. + func resume() throws { + guard !isActive else { return } + task = try lifecycle.beginOperation() + isActive = true + } + + /// Ends the operation. + func end() { + suspend() + } + } + + /// Starts an operation in the current task, unless the process is exiting. + /// + /// End the operation when its llama.cpp work ends. + /// + /// - Throws: ``exiting`` if the exit handler has started. + func begin() throws -> Operation { + let task = try beginOperation() + return Operation(lifecycle: self, task: task) + } + + private func beginOperation() throws -> UnsafeCurrentTask? { + // The task outlives the operation, which ends in the same task. + let task = withUnsafeCurrentTask { $0 } + try condition.withLock { + guard !isShuttingDown else { throw Self.exiting } + activeOperations += 1 + if let task { activeTasks[task, default: 0] += 1 } + } + return task + } + + private func endOperation(in task: UnsafeCurrentTask?) { + condition.withLock { + if let task { + activeTasks[task, default: 1] -= 1 + if activeTasks[task] == 0 { activeTasks[task] = nil } + } + activeOperations -= 1 + if activeOperations == 0 { condition.broadcast() } + } + } + + // MARK: Cleanups + + /// Runs synchronous llama.cpp work, unless the process is exiting. + func perform(_ body: () throws -> Result) throws -> Result { + try condition.withLock { + guard !isShuttingDown else { throw Self.exiting } + activeOperations += 1 + } + return try runCounted(body) + } + + /// Runs `body` as work that the caller has already counted, + /// marks the current thread as running it, + /// and stops counting it when it returns. + private func runCounted(_ body: () throws -> Result) rethrows -> Result { + // The body is synchronous, so it stays on this thread. + let thread = Thread.current.threadDictionary + let depth = thread[Self.depthKey] as? Int ?? 0 + thread[Self.depthKey] = depth + 1 + defer { + thread[Self.depthKey] = depth + condition.withLock { + activeOperations -= 1 + if activeOperations == 0 { condition.broadcast() } + } + } + return try body() + } + + /// Whether the current thread or task is running an operation. + private var isInsideOperation: Bool { + if (Thread.current.threadDictionary[Self.depthKey] as? Int ?? 0) > 0 { + return true + } + guard let task = withUnsafeCurrentTask(body: { $0 }) else { return false } + return condition.withLock { activeTasks[task] != nil } + } + + /// Records that a registration of the exit handler failed. + func exitHandlerRegistrationFailed() { + condition.withLock { registrationFailed = true } + } + + /// Records the resources of a model, so that ``shutDown(deadline:)`` can free them. + func track(_ model: LlamaModelResources) { + condition.withLock { resources[ObjectIdentifier(model)] = model } + } + + /// Frees the resources of a model, unless ``shutDown(deadline:)`` freed them already. + /// + /// The cleanup runs even while the process is exiting. + /// It counts as running work, + /// so that the exit handler's thread check and deadline apply to it, + /// and it calls llama.cpp without the lock. + func free(_ model: LlamaModelResources) { + condition.withLock { + resources[ObjectIdentifier(model)] = nil + activeOperations += 1 + } + runCounted { model.free() } + } + + /// Stops new work, waits up to `deadline` for the work that is running, + /// and frees the resources of every model if no work is running then. + @discardableResult + func shutDown(deadline: TimeInterval = exitDeadline) -> ShutdownResult { + let insideOperation = isInsideOperation + var models: [LlamaModelResources] = [] + let result: ShutdownResult = condition.withLock { + guard !isShuttingDown else { return .alreadyShutDown } + isShuttingDown = true + // The operation on this thread or task can't end while it waits here. + guard !insideOperation else { return .insideOperation } + guard !registrationFailed else { return .registrationFailed } + // Measure the deadline with a monotonic clock, + // and wait in short steps that start from the current wall-clock time, + // so that a change to the wall clock can't extend the wait. + let clock = ContinuousClock() + let end = clock.now + .seconds(deadline) + while activeOperations > 0 { + let remaining = clock.now.duration(to: end) + guard remaining > .zero else { break } + let step = min(remaining, .milliseconds(100)) + _ = condition.wait(until: Date(timeIntervalSinceNow: step / .seconds(1))) + } + guard activeOperations == 0 else { return .timedOut } + models = Array(resources.values) + resources = [:] + return .freed + } + // Free without the lock, as in `free(_:)`. + // If a log callback calls `exit` here, this handler runs again and does nothing. + guard !models.isEmpty else { return result } + Self.withAbortDeadline(Self.cleanupDeadline) { + for model in models { + model.free() + } + } + return result + } + + /// The state of a cleanup that runs under ``withAbortDeadline(_:_:)``. + private enum CleanupState: Sendable { + case running + case completed + case aborting + } + + /// Runs `body`, and aborts the process if `body` is still running after `deadline`. + /// + /// A separate thread measures the time on a monotonic clock, + /// so the exiting thread can't stop it. + /// The thread ends as soon as `body` returns, + /// and doesn't keep the process alive. + /// + /// Completion and the timeout are decided in one step under a lock. + /// Once `body` has returned and completion is recorded, + /// the watchdog never aborts. + /// Once the watchdog has recorded the timeout, + /// this function aborts instead of returning, + /// even if `body` returns after that. + static func withAbortDeadline(_ deadline: TimeInterval, _ body: () -> Result) -> Result { + let state = Locked(CleanupState.running) + let finished = DispatchSemaphore(value: 0) + let watchdog = Thread { + guard finished.wait(timeout: .now() + deadline) == .timedOut else { return } + #if DEBUG + LlamaTestHooks.cleanupWatchdog.withLock { $0 }?(.beforeDecision) + #endif + let aborts = state.withLock { state in + guard state == .running else { return false } + state = .aborting + return true + } + #if DEBUG + LlamaTestHooks.cleanupWatchdog.withLock { $0 }?(.decided(aborts: aborts)) + #endif + if aborts { + abortAfterExitCleanupTimeout() + } + } + watchdog.start() + let result = body() + let completed = state.withLock { state in + guard state == .running else { return false } + state = .completed + return true + } + // The watchdog has decided to abort, and is about to. + guard completed else { abortAfterLateExitCleanup() } + finished.signal() + return result + } + + /// Aborts the process because the exit handler's cleanup returned + /// after the watchdog had decided to abort. + /// + /// A crash report for this abort shows this function + /// in the backtrace of the thread that called `abort()`. + /// It writes no message and doesn't wait for the watchdog. + @inline(never) + private static func abortAfterLateExitCleanup() -> Never { + abort() + } + + /// Aborts the process because the exit handler's cleanup took too long. + /// + /// A crash report for this abort shows this function + /// in the backtrace of the thread that called `abort()`, + /// which identifies it. + /// The function also tries to write one line to standard error, + /// from another thread and with `write(2)`, + /// but waits at most 100 milliseconds for it: + /// standard error can be a full pipe that nobody reads, + /// and its lock can be held by the blocked thread, + /// so the abort must not depend on the message. + @inline(never) + private static func abortAfterExitCleanupTimeout() -> Never { + let written = DispatchSemaphore(value: 0) + let writer = Thread { + let message = "AnyLanguageModel: freeing llama.cpp models at exit didn't finish in time; aborting.\n" + _ = message.withCString { write(STDERR_FILENO, $0, strlen($0)) } + written.signal() + } + writer.start() + _ = written.wait(timeout: .now() + .milliseconds(100)) + abort() + } + } + + /// A loaded llama.cpp model, its projector, and its cached session context. + /// + /// ``LlamaLifecycle`` holds these resources until it frees them, + /// separately from the ``LlamaLanguageModel`` that uses them, + /// so that the exit handler can free a model that a Swift object still holds, + /// and so that the model's deinit frees nothing twice. + final class LlamaModelResources: @unchecked Sendable { + let model: OpaquePointer + let projector: OpaquePointer? + + /// Guards `cachedContext` and `isFreed`. + private let lock = NSLock() + private var cachedContext: OpaquePointer? + private var isFreed = false + + init(model: OpaquePointer, projector: OpaquePointer?) { + self.model = model + self.projector = projector + } + + /// Records the model's cached session context, or `nil` if it has none. + /// + /// Any other context exists only during a load or a generation, + /// which frees it before it ends. + func setCachedContext(_ context: OpaquePointer?) { + lock.withLock { cachedContext = context } + } + + /// Frees the cached context, the projector, and the model, once. + /// + /// Call it only through ``LlamaLifecycle``, + /// when no load or generation uses the model. + func free() { + // Take the context under the lock, and call llama.cpp without it. + let (isFirst, context): (Bool, OpaquePointer?) = lock.withLock { + guard !isFreed else { return (false, nil) } + isFreed = true + defer { cachedContext = nil } + return (true, cachedContext) + } + guard isFirst else { return } + if let context { + llama_free(context) + } + if let projector { + mtmd_free(projector) + } + llama_model_free(model) + } + } + + #if DEBUG + /// Functions that tests run at points inside llama.cpp work. + /// + /// Only debug builds, which `swift test` uses, compile these hooks, + /// so they cost nothing in a release build. + enum LlamaTestHooks { + /// Runs at the start of each step of a chat generation's token loop, + /// before its checks for cancellation and exit, + /// on the generation's thread. + /// + /// After the hook returns, + /// the generation does no llama.cpp work before those checks, + /// so a generation that the hook holds until the exit handler starts + /// stops at once. + static let beforeToken = Locked<(@Sendable () -> Void)?>(nil) + + /// Runs for each llama.cpp log message, before the level filter, + /// on the thread that logs it. + static let onLog = Locked<(@Sendable () -> Void)?>(nil) + + /// A point in the cleanup watchdog after its deadline passes. + enum CleanupWatchdogEvent: Sendable, Equatable { + /// The watchdog is about to decide between completion and the timeout. + case beforeDecision + /// The watchdog has decided, and aborts if `aborts` is true. + case decided(aborts: Bool) + } + + /// Runs on the cleanup watchdog's thread after its deadline passes. + static let cleanupWatchdog = Locked<(@Sendable (CleanupWatchdogEvent) -> Void)?>(nil) + } + #endif +#endif // Llama diff --git a/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift b/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift new file mode 100644 index 00000000..fa52d288 --- /dev/null +++ b/Tests/AnyLanguageModelTests/LlamaLanguageModelExitTests.swift @@ -0,0 +1,651 @@ +import Foundation +import Testing + +@testable import AnyLanguageModel + +#if Llama + /// A model that a global variable holds for the life of the process. + nonisolated(unsafe) private var globalExitTestModel: LlamaLanguageModel? + + /// Tests that exit a child process with a model loaded. + /// + /// The tests that use ``LlamaTestHooks`` run only in debug builds, + /// which compile the hooks. + /// + /// Without the exit handler, + /// llama.cpp's Metal backend stops such a process with an assertion + /// when the model has layers on the GPU, + /// and an exit during a generation crashes on the GPU and on the CPU. + @Suite( + "LlamaLanguageModel at exit", + .serialized, + .enabled(if: ProcessInfo.processInfo.environment["LLAMA_MODEL_PATH"] != nil) + ) + struct LlamaLanguageModelExitTests { + static var modelPath: String { + ProcessInfo.processInfo.environment["LLAMA_MODEL_PATH"] ?? "" + } + + /// Returns a model and a session after a first response, + /// so that the model and its cached context are loaded. + static func loadedModel(gpuLayers: Int32) async throws -> (LlamaLanguageModel, LanguageModelSession) { + let model = LlamaLanguageModel(modelPath: modelPath, gpuLayers: gpuLayers) + let session = LanguageModelSession(model: model) + _ = try await session.respond(to: "Say hello", options: GenerationOptions(maximumResponseTokens: 2)) + return (model, session) + } + + #if DEBUG + /// Holds a generation until the exit handler has started, + /// so that the handler must wait for it. + static func holdUntilShutdownStarts() { + let limit = ExitTestTimer.now() + 10_000_000_000 + while !LlamaLifecycle.shared.isStopping { + // A distinct status shows that the exit handler never started. + if ExitTestTimer.now() > limit { _exit(9) } + usleep(1_000) + } + } + + /// Starts a generation in another task, + /// and returns when its first token step starts. + /// + /// - Parameter whileActive: What the generation does at the start of that step, + /// while it is known to be running. + static func startGeneration( + _ session: LanguageModelSession, + whileActive: @escaping @Sendable () -> Void + ) { + let started = DispatchSemaphore(value: 0) + let once = Locked(false) + LlamaTestHooks.beforeToken.withLock { + $0 = { + let isFirst = once.withLock { done in + defer { done = true } + return !done + } + guard isFirst else { return } + started.signal() + whileActive() + } + } + Task.detached { + _ = try? await session.respond( + to: "Count from 1 to 1000.", + options: GenerationOptions(maximumResponseTokens: 2048) + ) + } + // A distinct status shows that the generation never started. + if started.wait(timeout: .now() + 30) == .timedOut { _exit(3) } + } + #endif + + @Test("A process exits normally with a loaded model", arguments: [Int32(-1), 0]) + func idleExit(gpuLayers: Int32) async throws { + let name = "idleExit\(gpuLayers)" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { [name = name as String, gpuLayers = gpuLayers as Int32] in + ExitTestTimer.childStarted(name) + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: gpuLayers) + ExitTestTimer.stepStarted(name) + withExtendedLifetime((model, session)) { exit(0) } + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + @Test("A process exits normally with a model in a global variable", arguments: [Int32(-1), 0]) + func exitWithGlobalModel(gpuLayers: Int32) async throws { + let name = "exitWithGlobalModel\(gpuLayers)" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { [name = name as String, gpuLayers = gpuLayers as Int32] in + ExitTestTimer.childStarted(name) + globalExitTestModel = LlamaLanguageModel( + modelPath: LlamaLanguageModelExitTests.modelPath, + gpuLayers: gpuLayers + ) + _ = try await LanguageModelSession(model: globalExitTestModel!).respond( + to: "Say hello", + options: GenerationOptions(maximumResponseTokens: 2) + ) + ExitTestTimer.stepStarted(name) + exit(0) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + #if DEBUG + @Test( + "The exit handler waits for a generation that is running, and the process exits normally", + arguments: [Int32(-1), 0] + ) + func exitDuringGeneration(gpuLayers: Int32) async throws { + let name = "exitDuringGeneration\(gpuLayers)" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { [name = name as String, gpuLayers = gpuLayers as Int32] in + ExitTestTimer.childStarted(name) + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: gpuLayers) + LlamaLanguageModelExitTests.startGeneration(session) { + LlamaLanguageModelExitTests.holdUntilShutdownStarts() + } + ExitTestTimer.stepStarted(name) + withExtendedLifetime(model) { exit(0) } + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + @Test("A generation that runs past the deadline doesn't keep the process alive") + func exitPastDeadline() async throws { + let name = "exitPastDeadline" + let watchdog = ExitTestTimer.watch(name) + // The handler frees nothing, so ggml's assertion ends the process, + // as it would without the handler. + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("exitPastDeadline") + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1) + LlamaLanguageModelExitTests.startGeneration(session) { + while true { sleep(60) } + } + ExitTestTimer.stepStarted("exitPastDeadline") + withExtendedLifetime(model) { exit(0) } + } + let elapsed = try #require(await ExitTestTimer.finish(name, watchdog)) + #expect(elapsed >= .seconds(LlamaLifecycle.exitDeadline)) + } + + @Test("An exit from inside a generation doesn't wait for that generation") + func exitInsideOperation() async throws { + let name = "exitInsideOperation" + let watchdog = ExitTestTimer.watch(name) + // The handler returns at once and frees nothing, + // so ggml's assertion ends the process before the deadline. + // The exit status can't show how long the handler took; + // `shutDownInsideTaskOperation()` checks that it returns at once. + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("exitInsideOperation") + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1) + LlamaLanguageModelExitTests.startGeneration(session) { + ExitTestTimer.stepStarted("exitInsideOperation") + exit(0) + } + // The generation's thread exits the process. + withExtendedLifetime(model) { + while true { sleep(60) } + } + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + @Test("A log callback that exits while a model is freed doesn't hang the process") + func exitFromLogDuringCleanup() async throws { + // The handler sees that the cleanup's thread is exiting, and returns at once, + // which `shutDownInsideThreadOperation()` checks without a model. + // The model is only partly freed, so ggml's assertion ends the process, + // as it would without the handler. + let name = "exitFromLogDuringCleanup" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("exitFromLogDuringCleanup") + do { + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1) + let once = Locked(false) + LlamaTestHooks.onLog.withLock { + $0 = { + let isFirst = once.withLock { done in + defer { done = true } + return !done + } + if isFirst { exit(0) } + } + } + ExitTestTimer.stepStarted("exitFromLogDuringCleanup") + // Releasing the last references frees the model, + // and llama.cpp logs while it frees. + _ = (model, session) + } + // A distinct status shows that llama.cpp logged nothing. + exit(4) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + @Test("A cleanup that blocks at exit aborts the process instead of hanging") + func exitWithBlockedCleanup() async throws { + // A log callback blocks while the exit handler frees the model, + // so the cleanup watchdog aborts the process. + let name = "exitWithBlockedCleanup" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("exitWithBlockedCleanup") + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1) + LlamaTestHooks.onLog.withLock { + $0 = { + if LlamaLifecycle.shared.isStopping { + while true { sleep(60) } + } + } + } + ExitTestTimer.stepStarted("exitWithBlockedCleanup") + withExtendedLifetime((model, session)) { exit(0) } + } + let elapsed = try #require(await ExitTestTimer.finish(name, watchdog)) + #expect(elapsed >= .seconds(LlamaLifecycle.cleanupDeadline)) + } + #endif + + @Test("A process exits normally after the model is released", arguments: [Int32(-1), 0]) + func exitAfterRelease(gpuLayers: Int32) async { + let name = "exitAfterRelease\(gpuLayers)" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { [name = name as String, gpuLayers = gpuLayers as Int32] in + ExitTestTimer.childStarted(name) + weak var released: LlamaLanguageModel? + do { + let (model, _) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: gpuLayers) + released = model + } + // A distinct status shows that something still holds the model. + guard released == nil else { exit(4) } + exit(0) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + @Test("After the exit handler runs, a request throws, and nothing is freed twice") + func requestAfterShutdown() async { + let name = "requestAfterShutdown" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { + ExitTestTimer.childStarted("requestAfterShutdown") + weak var released: LlamaLanguageModel? + do { + let (model, session) = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1) + released = model + guard LlamaLifecycle.shared.shutDown() == .freed else { exit(5) } + // The cached context is freed already, so this frees nothing. + model.clearCachedContext() + do { + _ = try await session.respond(to: "Say hello") + exit(6) + } catch LanguageModelSession.GenerationError.assetsUnavailable {} + do { + for try await _ in session.streamResponse(to: "Say hello") {} + exit(7) + } catch LanguageModelSession.GenerationError.assetsUnavailable {} + } + // The model's deinit ran after the handler freed the model, + // and freed nothing again. + guard released == nil else { exit(4) } + exit(0) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + @Test("After the exit handler runs, a model that isn't loaded fails to load") + func loadAfterShutdown() async { + let name = "loadAfterShutdown" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { + ExitTestTimer.childStarted("loadAfterShutdown") + _ = try await LlamaLanguageModelExitTests.loadedModel(gpuLayers: -1) + LlamaLifecycle.shared.shutDown() + let other = LlamaLanguageModel(modelPath: LlamaLanguageModelExitTests.modelPath) + do { + _ = try await LanguageModelSession(model: other).respond(to: "Say hello") + exit(5) + } catch LanguageModelSession.GenerationError.assetsUnavailable {} + // A missing file throws the exit error, not an invalid path. + let missing = LlamaLanguageModel(modelPath: "/nonexistent/model.gguf") + do { + _ = try await LanguageModelSession(model: missing).respond(to: "Say hello") + exit(6) + } catch LanguageModelSession.GenerationError.assetsUnavailable {} + exit(0) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + } + + /// Lets the parent of an exit test kill a child that hangs, + /// and measure how long the child takes to end after the step under test. + /// + /// Swift Testing's time limit doesn't end a child process, + /// so a child that hangs would keep the test run waiting. + /// The child writes its process ID when it starts, + /// and the monotonic time when the step under test starts, + /// to files that both processes name from the parent's process ID. + /// The measurement starts after the child's setup, + /// so a slow load doesn't change it. + /// + /// The measurement ends when the parent sees that the child has ended, + /// which a busy machine can delay by seconds. + /// So a test checks only that a time is at least a bound, + /// which a delay can't break, + /// and never that a time is less than a bound. + enum ExitTestTimer { + /// How long the parent waits for a child before it kills the child. + static let limit: Duration = .seconds(120) + + /// The current time of a monotonic clock that every process on the system shares. + static func now() -> UInt64 { + clock_gettime_nsec_np(CLOCK_UPTIME_RAW) + } + + private static func file(_ name: String, _ suffix: String, parent: pid_t) -> URL { + URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("llama-exit-test-\(parent)-\(name).\(suffix)") + } + + /// The exit status of a child that couldn't write a timer file. + /// + /// The child also writes the error to standard error. + static let writeFailureStatus: CInt = 70 + + /// In the child: writes `text` to a timer file, + /// or ends the child with ``writeFailureStatus`` if it can't. + private static func write(_ text: String, to url: URL) { + do { + try text.write(to: url, atomically: true, encoding: .utf8) + } catch { + fputs("ExitTestTimer couldn't write \(url.path): \(error)\n", stderr) + _exit(writeFailureStatus) + } + } + + /// In the child: records its process ID. + static func childStarted(_ name: String) { + write(String(getpid()), to: file(name, "pid", parent: getppid())) + } + + /// In the child: records when the step under test starts. + static func stepStarted(_ name: String) { + write(String(now()), to: file(name, "start", parent: getppid())) + } + + private static func read(_ name: String, _ suffix: String) -> String? { + try? String(contentsOf: file(name, suffix, parent: getpid()), encoding: .utf8) + } + + /// In the parent: starts a task that kills the child if it hasn't ended after ``limit``, + /// and returns whether it did. + static func watch(_ name: String) -> Task { + for suffix in ["pid", "start"] { + try? FileManager.default.removeItem(at: file(name, suffix, parent: getpid())) + } + return Task.detached { + guard (try? await Task.sleep(for: limit)) != nil else { return false } + guard let pid = read(name, "pid").flatMap({ pid_t($0) }) else { return false } + kill(pid, SIGKILL) + return true + } + } + + /// In the parent, after the child ends: stops the watchdog, + /// records an issue if it killed the child, + /// and, if `measuring`, returns the time from the step's start to now. + /// + /// If the start time can't be read, + /// this records an issue with the error and returns `nil`. + /// A child that couldn't write the file + /// exits with ``writeFailureStatus`` instead. + @discardableResult + static func finish( + _ name: String, + _ watchdog: Task, + measuring: Bool = true + ) async -> Duration? { + watchdog.cancel() + let killed = await watchdog.value + #expect(!killed, "The child process didn't end within \(limit).") + guard measuring else { return nil } + let url = file(name, "start", parent: getpid()) + let text: String + do { + text = try String(contentsOf: url, encoding: .utf8) + } catch { + Issue.record("Couldn't read the step's start time from \(url.path): \(error)") + return nil + } + guard let start = UInt64(text) else { + Issue.record("The step's start time in \(url.path) isn't a number: \(text.debugDescription)") + return nil + } + return .nanoseconds(Int64(now() - start)) + } + } + + /// Tests of the exit handler's bookkeeping, without a model. + @Suite("LlamaLifecycle") + struct LlamaLifecycleTests { + @Test("Shutting down with no work frees, and a second shutdown does nothing") + func shutDownIdle() throws { + let lifecycle = LlamaLifecycle() + #expect(lifecycle.shutDown(deadline: 0) == .freed) + #expect(lifecycle.shutDown(deadline: 0) == .alreadyShutDown) + #expect(lifecycle.isStopping) + #expect(throws: LanguageModelSession.GenerationError.self) { try lifecycle.begin() } + #expect(throws: LanguageModelSession.GenerationError.self) { try lifecycle.perform {} } + } + + @Test("Shutting down waits for running work, and frees nothing at the deadline") + func shutDownTimesOut() async throws { + let lifecycle = LlamaLifecycle() + nonisolated(unsafe) let operation = try lifecycle.begin() + // Shut down from a thread that runs no task, so the operation isn't its own. + let result = await withCheckedContinuation { continuation in + Thread.detachNewThread { + let start = ContinuousClock.now + let result = lifecycle.shutDown(deadline: 0.2) + continuation.resume(returning: (result, ContinuousClock.now - start)) + } + } + #expect(result.0 == .timedOut) + #expect(result.1 >= .milliseconds(200)) + operation.end() + #expect(lifecycle.activeOperationCount == 0) + } + + @Test("Shutting down returns when running work ends") + func shutDownWaitsForWork() async throws { + let lifecycle = LlamaLifecycle() + nonisolated(unsafe) let operation = try lifecycle.begin() + let result = await withCheckedContinuation { continuation in + Thread.detachNewThread { + continuation.resume(returning: lifecycle.shutDown(deadline: 10)) + } + usleep(100_000) + operation.end() + } + #expect(result == .freed) + } + + @Test("Shutting down inside an operation of the current task returns at once") + func shutDownInsideTaskOperation() throws { + let lifecycle = LlamaLifecycle() + let operation = try lifecycle.begin() + #expect(lifecycle.shutDown(deadline: 10) == .insideOperation) + operation.end() + } + + @Test("Shutting down inside synchronous work on the current thread returns at once") + func shutDownInsideThreadOperation() throws { + let lifecycle = LlamaLifecycle() + let result = try lifecycle.perform { lifecycle.shutDown(deadline: 10) } + #expect(result == .insideOperation) + } + + @Test("A suspended operation doesn't hold up a shutdown, and can't resume after it") + func suspendedOperation() throws { + let lifecycle = LlamaLifecycle() + let operation = try lifecycle.begin() + operation.suspend() + #expect(lifecycle.activeOperationCount == 0) + #expect(lifecycle.shutDown(deadline: 0) == .freed) + #expect(throws: LanguageModelSession.GenerationError.self) { try operation.resume() } + operation.end() + #expect(lifecycle.activeOperationCount == 0) + } + + @Test("The cleanup deadline does nothing after the cleanup ends") + func cleanupDeadlineAfterCleanup() async throws { + let name = "cleanupDeadlineAfterCleanup" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { + ExitTestTimer.childStarted("cleanupDeadlineAfterCleanup") + ExitTestTimer.stepStarted("cleanupDeadlineAfterCleanup") + let value = LlamaLifecycle.withAbortDeadline(0.1) { 42 } + // Outlive the deadline; the watchdog must not abort. + usleep(500_000) + exit(value == 42 ? 0 : 5) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + @Test("A cleanup that runs past its deadline aborts the process") + func cleanupDeadlinePassed() async throws { + let name = "cleanupDeadlinePassed" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("cleanupDeadlinePassed") + ExitTestTimer.stepStarted("cleanupDeadlinePassed") + LlamaLifecycle.withAbortDeadline(0.2) { + while true { sleep(60) } + } + } + let elapsed = try #require(await ExitTestTimer.finish(name, watchdog)) + #expect(elapsed >= .milliseconds(200)) + } + + @Test("A cleanup that runs past its deadline aborts even if standard error is a full pipe") + func cleanupDeadlineWithFullStandardError() async throws { + let name = "cleanupDeadlineWithFullStandardError" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("cleanupDeadlineWithFullStandardError") + // Make standard error a pipe that is full and that nobody reads. + // The read end stays open, so a write blocks instead of failing. + var fds: [CInt] = [0, 0] + guard pipe(&fds) == 0 else { _exit(3) } + let flags = fcntl(fds[1], F_GETFL) + _ = fcntl(fds[1], F_SETFL, flags | O_NONBLOCK) + let chunk = [UInt8](repeating: 0x2E, count: 4096) + while chunk.withUnsafeBytes({ Darwin.write(fds[1], $0.baseAddress, $0.count) }) > 0 {} + guard errno == EAGAIN else { _exit(4) } + _ = fcntl(fds[1], F_SETFL, flags) + guard dup2(fds[1], STDERR_FILENO) == STDERR_FILENO else { _exit(6) } + ExitTestTimer.stepStarted("cleanupDeadlineWithFullStandardError") + LlamaLifecycle.withAbortDeadline(0.2) { + while true { sleep(60) } + } + } + let elapsed = try #require(await ExitTestTimer.finish(name, watchdog)) + #expect(elapsed >= .milliseconds(200)) + } + + #if DEBUG + @Test("Once the cleanup completes, a watchdog that timed out doesn't abort") + func cleanupCompletionWinsRace() async throws { + let name = "cleanupCompletionWinsRace" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { + ExitTestTimer.childStarted("cleanupCompletionWinsRace") + let entered = DispatchSemaphore(value: 0) + let proceed = DispatchSemaphore(value: 0) + let decided = DispatchSemaphore(value: 0) + let aborts = Locked(nil) + LlamaTestHooks.cleanupWatchdog.withLock { + $0 = { event in + switch event { + case .beforeDecision: + // Hold the watchdog after its timeout + // until the cleanup has completed. + entered.signal() + proceed.wait() + case .decided(let value): + aborts.withLock { $0 = value } + decided.signal() + } + } + } + ExitTestTimer.stepStarted("cleanupCompletionWinsRace") + // The cleanup returns only after the watchdog's deadline has passed. + let value = LlamaLifecycle.withAbortDeadline(0.05) { + entered.wait() + return 42 + } + // The child does nothing else, so it can block a thread here, + // in a synchronous function. + func waitForDecision() { decided.wait() } + proceed.signal() + waitForDecision() + exit(value == 42 && aborts.withLock({ $0 }) == false ? 0 : 5) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + @Test("Once a watchdog decides to abort, the cleanup doesn't return") + func cleanupTimeoutWinsRace() async throws { + let name = "cleanupTimeoutWinsRace" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .signal(SIGABRT)) { + ExitTestTimer.childStarted("cleanupTimeoutWinsRace") + let decided = DispatchSemaphore(value: 0) + LlamaTestHooks.cleanupWatchdog.withLock { + $0 = { event in + guard event == .decided(aborts: true) else { return } + decided.signal() + // Keep the watchdog from aborting, + // so that only the cleanup's own check can abort. + while true { sleep(60) } + } + } + ExitTestTimer.stepStarted("cleanupTimeoutWinsRace") + _ = LlamaLifecycle.withAbortDeadline(0.05) { + decided.wait() + return 42 + } + // A distinct status shows that the cleanup returned. + exit(5) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + #endif + + @Test("After the exit handler runs, a Hub model neither lists nor downloads its files") + func hubLoadAfterShutdown() async { + let name = "hubLoadAfterShutdown" + let watchdog = ExitTestTimer.watch(name) + await #expect(processExitsWith: .success) { + ExitTestTimer.childStarted("hubLoadAfterShutdown") + let repository = FakeHubRepository() + let model = LlamaLanguageModel( + givenModelPath: "", + givenMMProjPath: nil, + hubSource: LlamaHubModelSource( + modelId: "example/model-GGUF", + file: nil, + mmproj: .disabled, + hub: nil, + repository: repository + ), + gpuLayers: 0 + ) + guard LlamaLifecycle.shared.shutDown() == .freed else { exit(5) } + do { + _ = try await LanguageModelSession(model: model).respond(to: "Say hello") + exit(6) + } catch LanguageModelSession.GenerationError.assetsUnavailable {} + // A distinct status shows that the model reached the Hub. + guard repository.listingCalls == 0, repository.downloadStarts == 0 else { exit(7) } + exit(0) + } + await ExitTestTimer.finish(name, watchdog, measuring: false) + } + + @Test("A failed registration of the exit handler makes the handler free nothing") + func registrationFailure() { + let lifecycle = LlamaLifecycle() + lifecycle.exitHandlerRegistrationFailed() + #expect(lifecycle.shutDown(deadline: 0) == .registrationFailed) + } + } +#endif // Llama