From c57cf816bdd4c6a45a31b93e84846acd0e45b9f8 Mon Sep 17 00:00:00 2001 From: mattt Date: Mon, 16 Nov 2026 06:37:37 -0800 Subject: [PATCH] Match a cached llama.cpp context only to its live session --- .../Models/LlamaLanguageModel.swift | 26 ++++++++++++--- .../LlamaSessionContextOwnerTests.swift | 32 +++++++++++++++++++ 2 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 Tests/AnyLanguageModelTests/LlamaSessionContextOwnerTests.swift diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index cfa89c1e..84917b74 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -379,9 +379,28 @@ import Foundation /// Whether the model is currently loaded. Protected by `modelLoadLock`. private var isModelLoaded: Bool = false + /// The session that a cached context belongs to. + /// + /// A new session can get the address of a session that was freed, + /// so a session's `ObjectIdentifier` doesn't identify it after it's freed. + /// This holds the session weakly, + /// so that a context left by a freed session belongs to no session. + struct SessionContextOwner { + private weak var session: LanguageModelSession? + + init(_ session: LanguageModelSession) { + self.session = session + } + + /// Whether `session` is the live session that the context belongs to. + func isOwner(_ session: LanguageModelSession) -> Bool { + self.session === session + } + } + /// A context kept alive for one session so exchanges reuse its state. private struct CachedSessionContext { - let sessionID: ObjectIdentifier + let owner: SessionContextOwner let context: OpaquePointer var tokens: [llama_token] let contextSize: UInt32 @@ -480,13 +499,12 @@ import Foundation promptTokens: [llama_token], options: ResolvedGenerationOptions ) throws -> (context: OpaquePointer, startIndex: Int) { - let sessionID = ObjectIdentifier(session) sessionContextLock.lock() defer { sessionContextLock.unlock() } if var cached = cachedSessionContext, !cached.isCheckedOut, - cached.sessionID == sessionID, + cached.owner.isOwner(session), cached.contextSize == options.contextSize, cached.batchSize == options.batchSize { @@ -529,7 +547,7 @@ import Foundation throw LlamaLanguageModelError.encoderOnlyModel } cachedSessionContext = CachedSessionContext( - sessionID: sessionID, + owner: SessionContextOwner(session), context: context, tokens: [], contextSize: options.contextSize, diff --git a/Tests/AnyLanguageModelTests/LlamaSessionContextOwnerTests.swift b/Tests/AnyLanguageModelTests/LlamaSessionContextOwnerTests.swift new file mode 100644 index 00000000..50633f20 --- /dev/null +++ b/Tests/AnyLanguageModelTests/LlamaSessionContextOwnerTests.swift @@ -0,0 +1,32 @@ +import Testing + +@testable import AnyLanguageModel + +#if Llama + @Suite("LlamaLanguageModel.SessionContextOwner") + struct LlamaSessionContextOwnerTests { + let model = LlamaLanguageModel(modelPath: "/path/to/model.gguf") + + @Test func ownerIsTheSessionThatItHolds() { + let session = LanguageModelSession(model: model) + let other = LanguageModelSession(model: model) + let owner = LlamaLanguageModel.SessionContextOwner(session) + #expect(owner.isOwner(session)) + #expect(!owner.isOwner(other)) + } + + /// A session made after the owner is freed isn't the owner, + /// even when it gets the freed session's address, + /// as the allocator usually gives it. + @Test func sessionAtAFreedSessionsAddressIsNotTheOwner() { + var owner: LlamaLanguageModel.SessionContextOwner? + do { + owner = .init(LanguageModelSession(model: model)) + } + let sessions = (0 ..< 10).map { _ in LanguageModelSession(model: model) } + for session in sessions { + #expect(owner?.isOwner(session) == false) + } + } + } +#endif