Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -379,9 +379,28 @@ import Foundation
/// Whether the model is currently loaded. Protected by `modelLoadLock`.
private var isModelLoaded: Bool = false

/// The session that a cached context belongs to.
///
/// A new session can get the address of a session that was freed,
/// so a session's `ObjectIdentifier` doesn't identify it after it's freed.
/// This holds the session weakly,
/// so that a context left by a freed session belongs to no session.
struct SessionContextOwner {
private weak var session: LanguageModelSession?

init(_ session: LanguageModelSession) {
self.session = session
}

/// Whether `session` is the live session that the context belongs to.
func isOwner(_ session: LanguageModelSession) -> Bool {
self.session === session
}
}

/// A context kept alive for one session so exchanges reuse its state.
private struct CachedSessionContext {
let sessionID: ObjectIdentifier
let owner: SessionContextOwner
let context: OpaquePointer
var tokens: [llama_token]
let contextSize: UInt32
Expand Down Expand Up @@ -480,13 +499,12 @@ import Foundation
promptTokens: [llama_token],
options: ResolvedGenerationOptions
) throws -> (context: OpaquePointer, startIndex: Int) {
let sessionID = ObjectIdentifier(session)
sessionContextLock.lock()
defer { sessionContextLock.unlock() }

if var cached = cachedSessionContext,
!cached.isCheckedOut,
cached.sessionID == sessionID,
cached.owner.isOwner(session),
cached.contextSize == options.contextSize,
cached.batchSize == options.batchSize
{
Expand Down Expand Up @@ -529,7 +547,7 @@ import Foundation
throw LlamaLanguageModelError.encoderOnlyModel
}
cachedSessionContext = CachedSessionContext(
sessionID: sessionID,
owner: SessionContextOwner(session),
context: context,
tokens: [],
contextSize: options.contextSize,
Expand Down
32 changes: 32 additions & 0 deletions Tests/AnyLanguageModelTests/LlamaSessionContextOwnerTests.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import Testing

@testable import AnyLanguageModel

#if Llama
@Suite("LlamaLanguageModel.SessionContextOwner")
struct LlamaSessionContextOwnerTests {
let model = LlamaLanguageModel(modelPath: "/path/to/model.gguf")

@Test func ownerIsTheSessionThatItHolds() {
let session = LanguageModelSession(model: model)
let other = LanguageModelSession(model: model)
let owner = LlamaLanguageModel.SessionContextOwner(session)
#expect(owner.isOwner(session))
#expect(!owner.isOwner(other))
}

/// A session made after the owner is freed isn't the owner,
/// even when it gets the freed session's address,
/// as the allocator usually gives it.
@Test func sessionAtAFreedSessionsAddressIsNotTheOwner() {
var owner: LlamaLanguageModel.SessionContextOwner?
do {
owner = .init(LanguageModelSession(model: model))
}
let sessions = (0 ..< 10).map { _ in LanguageModelSession(model: model) }
for session in sessions {
#expect(owner?.isOwner(session) == false)
}
}
}
#endif
Loading