diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index e34ee8eb..6d69f115 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -555,7 +555,7 @@ import Foundation } /// The loaded model instance - private var model: OpaquePointer? + private(set) var model: OpaquePointer? /// The model's vocabulary private var vocab: OpaquePointer? @@ -1596,11 +1596,45 @@ import Foundation return params } - private func applySampling( - sampler: UnsafeMutablePointer, - effectiveTemperature: Float, + /// Creates the sampler chain for a generation. + /// + /// The chain keeps state between tokens, such as the recent tokens for the penalties. + /// Call `llama_sampler_sample`, which accepts the token that it returns, + /// or call `llama_sampler_apply` and then `llama_sampler_accept` once. + /// + /// - Parameters: + /// - vocabularySize: The number of tokens in the model's vocabulary. + /// - options: The options for the generation. + /// - Returns: The sampler chain, which the caller frees with `llama_sampler_free`. + static func makeSamplerChain( + vocabularySize: Int32, + options: ResolvedGenerationOptions + ) -> UnsafeMutablePointer? { + guard let sampler = llama_sampler_chain_init(llama_sampler_chain_default_params()) else { + return nil + } + if options.repeatPenalty != 1.0 || options.frequencyPenalty != 0.0 || options.presencePenalty != 0.0 { + llama_sampler_chain_add( + sampler, + llama_sampler_init_penalties( + vocabularySize, + options.repeatLastN, + options.repeatPenalty, + options.frequencyPenalty, + options.presencePenalty + ) + ) + } + addSamplers(to: sampler, vocabularySize: vocabularySize, options: options) + return sampler + } + + private static func addSamplers( + to sampler: UnsafeMutablePointer, + vocabularySize: Int32, options: ResolvedGenerationOptions ) { + let effectiveTemperature = options.temperature if let mirostat = options.mirostat { llama_sampler_chain_add(sampler, llama_sampler_init_temp(effectiveTemperature)) @@ -1609,7 +1643,7 @@ import Foundation llama_sampler_chain_add( sampler, llama_sampler_init_mirostat( - Int32(options.contextSize), + vocabularySize, options.seed, tau, eta, @@ -1689,25 +1723,15 @@ import Foundation contextSize: options.contextSize ) - guard let sampler = llama_sampler_chain_init(llama_sampler_chain_default_params()) else { - throw LlamaLanguageModelError.decodingFailed - } - defer { llama_sampler_free(sampler) } - let samplerPointer = UnsafeMutablePointer(sampler) - - if options.repeatPenalty != 1.0 || options.frequencyPenalty != 0.0 || options.presencePenalty != 0.0 { - llama_sampler_chain_add( - samplerPointer, - llama_sampler_init_penalties( - llama_vocab_n_tokens(vocab), - options.repeatLastN, - options.repeatPenalty, - options.frequencyPenalty, - options.presencePenalty - ) + guard + let samplerPointer = Self.makeSamplerChain( + vocabularySize: llama_vocab_n_tokens(vocab), + options: options ) + else { + throw LlamaLanguageModelError.decodingFailed } - applySampling(sampler: samplerPointer, effectiveTemperature: options.temperature, options: options) + defer { llama_sampler_free(samplerPointer) } let vocabSize = Int(llama_vocab_n_tokens(vocab)) let initialPosition: Int32 = hasEncoder ? 1 : Int32(promptTokens.count) @@ -1893,7 +1917,7 @@ import Foundation } } - private func performTokenGeneration( + func performTokenGeneration( context: OpaquePointer, vocab: OpaquePointer, promptTokens: [llama_token], @@ -1926,36 +1950,15 @@ import Foundation startIndex: startIndex ) - // Initialize sampler chain with options - guard let sampler = llama_sampler_chain_init(llama_sampler_chain_default_params()) else { + guard + let sampler = Self.makeSamplerChain( + vocabularySize: llama_vocab_n_tokens(vocab), + options: options + ) + else { throw LlamaLanguageModelError.decodingFailed } defer { llama_sampler_free(sampler) } - let samplerPtr = UnsafeMutablePointer(sampler) - - let effectiveTemperature = Float(options.temperature) - - // Apply repeat/frequency/presence penalties from custom options - let effectiveRepeatPenalty = options.repeatPenalty - let effectiveRepeatLastN = options.repeatLastN - let effectiveFrequencyPenalty = options.frequencyPenalty - let effectivePresencePenalty = options.presencePenalty - - if effectiveRepeatPenalty != 1.0 || effectiveFrequencyPenalty != 0.0 || effectivePresencePenalty != 0.0 { - llama_sampler_chain_add( - samplerPtr, - llama_sampler_init_penalties( - llama_vocab_n_tokens(vocab), - effectiveRepeatLastN, - effectiveRepeatPenalty, - effectiveFrequencyPenalty, - effectivePresencePenalty - ) - ) - } - - // Check for mirostat sampling (takes precedence over standard sampling) - applySampling(sampler: samplerPtr, effectiveTemperature: effectiveTemperature, options: options) // Generate tokens one by one // Track position - for encoder-decoder models, we start from position 1 (after decoder start token) @@ -1977,9 +1980,9 @@ import Foundation // The exit handler waits for this generation, so stop it early. try LlamaLifecycle.shared.stopIfExiting() - // Sample next token from logits of the last token we just decoded + // Sample the next token from the logits of the last decoded token. + // llama_sampler_sample also accepts the token into the sampler chain. let nextToken = llama_sampler_sample(sampler, context, batch.n_tokens - 1) - llama_sampler_accept(sampler, nextToken) // Check for end of sequence if !counts.acceptToken(isEndOfGeneration: llama_vocab_is_eog(vocab, nextToken)) { @@ -2115,25 +2118,15 @@ import Foundation } } - guard let sampler = llama_sampler_chain_init(llama_sampler_chain_default_params()) else { - throw LlamaLanguageModelError.decodingFailed - } - defer { llama_sampler_free(sampler) } - let samplerPtr = UnsafeMutablePointer(sampler) - - if options.repeatPenalty != 1.0 || options.frequencyPenalty != 0.0 || options.presencePenalty != 0.0 { - llama_sampler_chain_add( - samplerPtr, - llama_sampler_init_penalties( - llama_vocab_n_tokens(vocab), - options.repeatLastN, - options.repeatPenalty, - options.frequencyPenalty, - options.presencePenalty - ) + guard + let samplerPtr = Self.makeSamplerChain( + vocabularySize: llama_vocab_n_tokens(vocab), + options: options ) + else { + throw LlamaLanguageModelError.decodingFailed } - applySampling(sampler: samplerPtr, effectiveTemperature: options.temperature, options: options) + defer { llama_sampler_free(samplerPtr) } var batch = llama_batch_init(1, 0, 1) defer { llama_batch_free(batch) } @@ -2149,8 +2142,8 @@ import Foundation // The exit handler waits for this generation, so stop it early. try LlamaLifecycle.shared.stopIfExiting() + // llama_sampler_sample also accepts the token into the sampler chain. let nextToken = llama_sampler_sample(samplerPtr, context, sampleIndex) - llama_sampler_accept(samplerPtr, nextToken) if !counts.acceptToken(isEndOfGeneration: llama_vocab_is_eog(vocab, nextToken)) { break diff --git a/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift b/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift index 5188ec62..9d895cd6 100644 --- a/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift @@ -4,6 +4,8 @@ import Testing @testable import AnyLanguageModel #if Llama + import LlamaSwift + @Suite( "LlamaLanguageModel", .serialized, @@ -394,6 +396,122 @@ import Testing #expect(!response.content.isEmpty) } + /// The token loop accepts each sampled token into the sampler chain once, + /// so it makes the same tokens as a loop that calls `llama_sampler_sample` alone. + /// The penalties sampler counts accepted tokens, so a second accept changes them. + /// + /// The fixture depends on the model. + /// With Qwen3-4B Q4_K_M, a second accept makes the lists differ at index 15, + /// because the frequency penalty counts each earlier token twice. + /// If another model makes the same pieces either way, change the prompt or the options. + @Test func tokenLoopAcceptsEachTokenOnce() async throws { + try await model.ensureModelLoaded() + let llamaModel = try #require(model.model) + if llama_model_has_encoder(llamaModel) { + try Test.cancel("The reference loop supports only decoder-only models.") + } + let vocab = try #require(llama_model_get_vocab(llamaModel)) + let options = LlamaLanguageModel.ResolvedGenerationOptions( + seed: 42, + repeatPenalty: 1.3, + repeatLastN: 64, + frequencyPenalty: 0.5, + presencePenalty: 0.5 + ) + let maxTokens = 64 + let prompt = "List the colors of the rainbow, then list them again:" + let promptTokens = try #require(tokenize(prompt, vocab: vocab)) + + let packageContext = try #require(makeContext(llamaModel, options: options)) + defer { llama_free(packageContext) } + var packagePieces: [String] = [] + _ = try model.performTokenGeneration( + context: packageContext, + vocab: vocab, + promptTokens: promptTokens, + startIndex: 0, + maxTokens: maxTokens, + options: options + ) { piece in + packagePieces.append(piece) + return true + } + + let referenceContext = try #require(makeContext(llamaModel, options: options)) + defer { llama_free(referenceContext) } + let sampler = try #require( + LlamaLanguageModel.makeSamplerChain(vocabularySize: llama_vocab_n_tokens(vocab), options: options) + ) + defer { llama_sampler_free(sampler) } + var batch = llama_batch_init(Int32(promptTokens.count), 0, 1) + defer { llama_batch_free(batch) } + setBatch(&batch, tokens: promptTokens, startPosition: 0) + try #require(llama_decode(referenceContext, batch) == 0) + var referencePieces: [String] = [] + var position = Int32(promptTokens.count) + for _ in 0 ..< maxTokens { + let token = llama_sampler_sample(sampler, referenceContext, batch.n_tokens - 1) + if llama_vocab_is_eog(vocab, token) { break } + if let piece = piece(token, vocab: vocab) { + referencePieces.append(piece) + } + setBatch(&batch, tokens: [token], startPosition: position) + position += 1 + try #require(llama_decode(referenceContext, batch) == 0) + } + + #expect(!referencePieces.isEmpty) + let firstDifference = zip(packagePieces, referencePieces).enumerated().first { $1.0 != $1.1 }?.offset + #expect( + packagePieces == referencePieces, + "The pieces first differ at index \(firstDifference.map(String.init) ?? "end")" + ) + } + + private func makeContext( + _ llamaModel: OpaquePointer, + options: LlamaLanguageModel.ResolvedGenerationOptions + ) -> OpaquePointer? { + var params = llama_context_default_params() + params.n_ctx = options.contextSize + params.n_batch = options.batchSize + params.n_threads = options.threads + params.n_threads_batch = options.threads + return llama_init_from_model(llamaModel, params) + } + + private func tokenize(_ text: String, vocab: OpaquePointer) -> [llama_token]? { + let capacity = Int32(text.utf8.count + 8) + var tokens = [llama_token](repeating: 0, count: Int(capacity)) + let count = llama_tokenize(vocab, text, Int32(text.utf8.count), &tokens, capacity, true, true) + guard count > 0 else { return nil } + return Array(tokens.prefix(Int(count))) + } + + private func piece(_ token: llama_token, vocab: OpaquePointer) -> String? { + var buffer = [CChar](repeating: 0, count: 64) + var count = llama_token_to_piece(vocab, token, &buffer, Int32(buffer.count), 0, false) + if count < 0 { + buffer = [CChar](repeating: 0, count: Int(-count)) + count = llama_token_to_piece(vocab, token, &buffer, Int32(buffer.count), 0, false) + } + guard count > 0 else { return nil } + return String(decoding: buffer.prefix(Int(count)).map { UInt8(bitPattern: $0) }, as: UTF8.self) + } + + private func setBatch(_ batch: inout llama_batch, tokens: [llama_token], startPosition: Int32) { + batch.n_tokens = Int32(tokens.count) + for (index, token) in tokens.enumerated() { + batch.token[index] = token + batch.pos[index] = startPosition + Int32(index) + batch.n_seq_id[index] = 1 + if let seqIDs = batch.seq_id, let seqID = seqIDs[index] { + seqID[0] = 0 + } + batch.logits[index] = index == tokens.count - 1 ? 1 : 0 + } + } + @Test func multimodal_rejectsImageURL() async throws { let session = LanguageModelSession(model: model) let imageSegment = Transcript.ImageSegment(url: testImageURL) diff --git a/Tests/AnyLanguageModelTests/LlamaSamplerChainTests.swift b/Tests/AnyLanguageModelTests/LlamaSamplerChainTests.swift new file mode 100644 index 00000000..ac5c6520 --- /dev/null +++ b/Tests/AnyLanguageModelTests/LlamaSamplerChainTests.swift @@ -0,0 +1,59 @@ +import Foundation +import Testing + +@testable import AnyLanguageModel + +#if Llama + import LlamaSwift + + /// Tests of the sampler chain that `LlamaLanguageModel` builds, + /// applied to hand-built candidates without a model. + @Suite("Llama sampler chain") + struct LlamaSamplerChainTests { + /// Mirostat v1 keeps the most probable tokens, and the number that it keeps + /// depends on the vocabulary size that the sampler gets. + @Test("Mirostat v1 gets the vocabulary size") + func mirostatV1UsesVocabularySize() throws { + let vocabularySize: Int32 = 32_000 + let contextSize: UInt32 = 2048 + let options = LlamaLanguageModel.ResolvedGenerationOptions( + contextSize: contextSize, + seed: 1, + temperature: 1, + repeatPenalty: 1, + mirostat: .v1(tau: 5, eta: 0.1) + ) + + let chain = try #require( + LlamaLanguageModel.makeSamplerChain(vocabularySize: vocabularySize, options: options) + ) + defer { llama_sampler_free(chain) } + let expected = try #require(llama_sampler_init_mirostat(vocabularySize, 1, 5, 0.1, 100)) + defer { llama_sampler_free(expected) } + let withContextSize = try #require(llama_sampler_init_mirostat(Int32(contextSize), 1, 5, 0.1, 100)) + defer { llama_sampler_free(withContextSize) } + + let keptCount = candidateCount(after: chain, vocabularySize: vocabularySize) + #expect(keptCount == candidateCount(after: expected, vocabularySize: vocabularySize)) + #expect(keptCount != candidateCount(after: withContextSize, vocabularySize: vocabularySize)) + } + + /// Applies a sampler to Zipf-distributed candidates + /// and returns the number of candidates that it keeps. + private func candidateCount(after sampler: UnsafeMutablePointer, vocabularySize: Int32) -> Int { + var candidates = (0 ..< vocabularySize).map { id in + llama_token_data(id: id, logit: -1.1 * logf(Float(id + 1)), p: 0) + } + return candidates.withUnsafeMutableBufferPointer { buffer in + var array = llama_token_data_array( + data: buffer.baseAddress, + size: buffer.count, + selected: -1, + sorted: false + ) + llama_sampler_apply(sampler, &array) + return array.size + } + } + } +#endif