Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 63 additions & 70 deletions Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -555,7 +555,7 @@ import Foundation
}

/// The loaded model instance
private var model: OpaquePointer?
private(set) var model: OpaquePointer?

/// The model's vocabulary
private var vocab: OpaquePointer?
Expand Down Expand Up @@ -1596,11 +1596,45 @@ import Foundation
return params
}

private func applySampling(
sampler: UnsafeMutablePointer<llama_sampler>,
effectiveTemperature: Float,
/// Creates the sampler chain for a generation.
///
/// The chain keeps state between tokens, such as the recent tokens for the penalties.
/// Call `llama_sampler_sample`, which accepts the token that it returns,
/// or call `llama_sampler_apply` and then `llama_sampler_accept` once.
///
/// - Parameters:
/// - vocabularySize: The number of tokens in the model's vocabulary.
/// - options: The options for the generation.
/// - Returns: The sampler chain, which the caller frees with `llama_sampler_free`.
static func makeSamplerChain(
vocabularySize: Int32,
options: ResolvedGenerationOptions
) -> UnsafeMutablePointer<llama_sampler>? {
guard let sampler = llama_sampler_chain_init(llama_sampler_chain_default_params()) else {
return nil
}
if options.repeatPenalty != 1.0 || options.frequencyPenalty != 0.0 || options.presencePenalty != 0.0 {
llama_sampler_chain_add(
sampler,
llama_sampler_init_penalties(
vocabularySize,
options.repeatLastN,
options.repeatPenalty,
options.frequencyPenalty,
options.presencePenalty
)
)
}
addSamplers(to: sampler, vocabularySize: vocabularySize, options: options)
return sampler
}

private static func addSamplers(
to sampler: UnsafeMutablePointer<llama_sampler>,
vocabularySize: Int32,
options: ResolvedGenerationOptions
) {
let effectiveTemperature = options.temperature
if let mirostat = options.mirostat {
llama_sampler_chain_add(sampler, llama_sampler_init_temp(effectiveTemperature))

Expand All @@ -1609,7 +1643,7 @@ import Foundation
llama_sampler_chain_add(
sampler,
llama_sampler_init_mirostat(
Int32(options.contextSize),
vocabularySize,
options.seed,
tau,
eta,
Expand Down Expand Up @@ -1689,25 +1723,15 @@ import Foundation
contextSize: options.contextSize
)

guard let sampler = llama_sampler_chain_init(llama_sampler_chain_default_params()) else {
throw LlamaLanguageModelError.decodingFailed
}
defer { llama_sampler_free(sampler) }
let samplerPointer = UnsafeMutablePointer<llama_sampler>(sampler)

if options.repeatPenalty != 1.0 || options.frequencyPenalty != 0.0 || options.presencePenalty != 0.0 {
llama_sampler_chain_add(
samplerPointer,
llama_sampler_init_penalties(
llama_vocab_n_tokens(vocab),
options.repeatLastN,
options.repeatPenalty,
options.frequencyPenalty,
options.presencePenalty
)
guard
let samplerPointer = Self.makeSamplerChain(
vocabularySize: llama_vocab_n_tokens(vocab),
options: options
)
else {
throw LlamaLanguageModelError.decodingFailed
}
applySampling(sampler: samplerPointer, effectiveTemperature: options.temperature, options: options)
defer { llama_sampler_free(samplerPointer) }

let vocabSize = Int(llama_vocab_n_tokens(vocab))
let initialPosition: Int32 = hasEncoder ? 1 : Int32(promptTokens.count)
Expand Down Expand Up @@ -1893,7 +1917,7 @@ import Foundation
}
}

private func performTokenGeneration(
func performTokenGeneration(
context: OpaquePointer,
vocab: OpaquePointer,
promptTokens: [llama_token],
Expand Down Expand Up @@ -1926,36 +1950,15 @@ import Foundation
startIndex: startIndex
)

// Initialize sampler chain with options
guard let sampler = llama_sampler_chain_init(llama_sampler_chain_default_params()) else {
guard
let sampler = Self.makeSamplerChain(
vocabularySize: llama_vocab_n_tokens(vocab),
options: options
)
else {
throw LlamaLanguageModelError.decodingFailed
}
defer { llama_sampler_free(sampler) }
let samplerPtr = UnsafeMutablePointer<llama_sampler>(sampler)

let effectiveTemperature = Float(options.temperature)

// Apply repeat/frequency/presence penalties from custom options
let effectiveRepeatPenalty = options.repeatPenalty
let effectiveRepeatLastN = options.repeatLastN
let effectiveFrequencyPenalty = options.frequencyPenalty
let effectivePresencePenalty = options.presencePenalty

if effectiveRepeatPenalty != 1.0 || effectiveFrequencyPenalty != 0.0 || effectivePresencePenalty != 0.0 {
llama_sampler_chain_add(
samplerPtr,
llama_sampler_init_penalties(
llama_vocab_n_tokens(vocab),
effectiveRepeatLastN,
effectiveRepeatPenalty,
effectiveFrequencyPenalty,
effectivePresencePenalty
)
)
}

// Check for mirostat sampling (takes precedence over standard sampling)
applySampling(sampler: samplerPtr, effectiveTemperature: effectiveTemperature, options: options)

// Generate tokens one by one
// Track position - for encoder-decoder models, we start from position 1 (after decoder start token)
Expand All @@ -1977,9 +1980,9 @@ import Foundation
// The exit handler waits for this generation, so stop it early.
try LlamaLifecycle.shared.stopIfExiting()

// Sample next token from logits of the last token we just decoded
// Sample the next token from the logits of the last decoded token.
// llama_sampler_sample also accepts the token into the sampler chain.
let nextToken = llama_sampler_sample(sampler, context, batch.n_tokens - 1)
llama_sampler_accept(sampler, nextToken)

// Check for end of sequence
if !counts.acceptToken(isEndOfGeneration: llama_vocab_is_eog(vocab, nextToken)) {
Expand Down Expand Up @@ -2115,25 +2118,15 @@ import Foundation
}
}

guard let sampler = llama_sampler_chain_init(llama_sampler_chain_default_params()) else {
throw LlamaLanguageModelError.decodingFailed
}
defer { llama_sampler_free(sampler) }
let samplerPtr = UnsafeMutablePointer<llama_sampler>(sampler)

if options.repeatPenalty != 1.0 || options.frequencyPenalty != 0.0 || options.presencePenalty != 0.0 {
llama_sampler_chain_add(
samplerPtr,
llama_sampler_init_penalties(
llama_vocab_n_tokens(vocab),
options.repeatLastN,
options.repeatPenalty,
options.frequencyPenalty,
options.presencePenalty
)
guard
let samplerPtr = Self.makeSamplerChain(
vocabularySize: llama_vocab_n_tokens(vocab),
options: options
)
else {
throw LlamaLanguageModelError.decodingFailed
}
applySampling(sampler: samplerPtr, effectiveTemperature: options.temperature, options: options)
defer { llama_sampler_free(samplerPtr) }

var batch = llama_batch_init(1, 0, 1)
defer { llama_batch_free(batch) }
Expand All @@ -2149,8 +2142,8 @@ import Foundation
// The exit handler waits for this generation, so stop it early.
try LlamaLifecycle.shared.stopIfExiting()

// llama_sampler_sample also accepts the token into the sampler chain.
let nextToken = llama_sampler_sample(samplerPtr, context, sampleIndex)
llama_sampler_accept(samplerPtr, nextToken)

if !counts.acceptToken(isEndOfGeneration: llama_vocab_is_eog(vocab, nextToken)) {
break
Expand Down
118 changes: 118 additions & 0 deletions Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import Testing
@testable import AnyLanguageModel

#if Llama
import LlamaSwift

@Suite(
"LlamaLanguageModel",
.serialized,
Expand Down Expand Up @@ -394,6 +396,122 @@ import Testing
#expect(!response.content.isEmpty)
}

/// The token loop accepts each sampled token into the sampler chain once,
/// so it makes the same tokens as a loop that calls `llama_sampler_sample` alone.
/// The penalties sampler counts accepted tokens, so a second accept changes them.
///
/// The fixture depends on the model.
/// With Qwen3-4B Q4_K_M, a second accept makes the lists differ at index 15,
/// because the frequency penalty counts each earlier token twice.
/// If another model makes the same pieces either way, change the prompt or the options.
@Test func tokenLoopAcceptsEachTokenOnce() async throws {
try await model.ensureModelLoaded()
let llamaModel = try #require(model.model)
if llama_model_has_encoder(llamaModel) {
try Test.cancel("The reference loop supports only decoder-only models.")
}
let vocab = try #require(llama_model_get_vocab(llamaModel))
let options = LlamaLanguageModel.ResolvedGenerationOptions(
seed: 42,
repeatPenalty: 1.3,
repeatLastN: 64,
frequencyPenalty: 0.5,
presencePenalty: 0.5
)
let maxTokens = 64
let prompt = "List the colors of the rainbow, then list them again:"
let promptTokens = try #require(tokenize(prompt, vocab: vocab))

let packageContext = try #require(makeContext(llamaModel, options: options))
defer { llama_free(packageContext) }
var packagePieces: [String] = []
_ = try model.performTokenGeneration(
context: packageContext,
vocab: vocab,
promptTokens: promptTokens,
startIndex: 0,
maxTokens: maxTokens,
options: options
) { piece in
packagePieces.append(piece)
return true
}

let referenceContext = try #require(makeContext(llamaModel, options: options))
defer { llama_free(referenceContext) }
let sampler = try #require(
LlamaLanguageModel.makeSamplerChain(vocabularySize: llama_vocab_n_tokens(vocab), options: options)
)
defer { llama_sampler_free(sampler) }
var batch = llama_batch_init(Int32(promptTokens.count), 0, 1)
defer { llama_batch_free(batch) }
setBatch(&batch, tokens: promptTokens, startPosition: 0)
try #require(llama_decode(referenceContext, batch) == 0)
var referencePieces: [String] = []
var position = Int32(promptTokens.count)
for _ in 0 ..< maxTokens {
let token = llama_sampler_sample(sampler, referenceContext, batch.n_tokens - 1)
if llama_vocab_is_eog(vocab, token) { break }
if let piece = piece(token, vocab: vocab) {
referencePieces.append(piece)
}
setBatch(&batch, tokens: [token], startPosition: position)
position += 1
try #require(llama_decode(referenceContext, batch) == 0)
}

#expect(!referencePieces.isEmpty)
let firstDifference = zip(packagePieces, referencePieces).enumerated().first { $1.0 != $1.1 }?.offset
#expect(
packagePieces == referencePieces,
"The pieces first differ at index \(firstDifference.map(String.init) ?? "end")"
)
}

private func makeContext(
_ llamaModel: OpaquePointer,
options: LlamaLanguageModel.ResolvedGenerationOptions
) -> OpaquePointer? {
var params = llama_context_default_params()
params.n_ctx = options.contextSize
params.n_batch = options.batchSize
params.n_threads = options.threads
params.n_threads_batch = options.threads
return llama_init_from_model(llamaModel, params)
}

private func tokenize(_ text: String, vocab: OpaquePointer) -> [llama_token]? {
let capacity = Int32(text.utf8.count + 8)
var tokens = [llama_token](repeating: 0, count: Int(capacity))
let count = llama_tokenize(vocab, text, Int32(text.utf8.count), &tokens, capacity, true, true)
guard count > 0 else { return nil }
return Array(tokens.prefix(Int(count)))
}

private func piece(_ token: llama_token, vocab: OpaquePointer) -> String? {
var buffer = [CChar](repeating: 0, count: 64)
var count = llama_token_to_piece(vocab, token, &buffer, Int32(buffer.count), 0, false)
if count < 0 {
buffer = [CChar](repeating: 0, count: Int(-count))
count = llama_token_to_piece(vocab, token, &buffer, Int32(buffer.count), 0, false)
}
guard count > 0 else { return nil }
return String(decoding: buffer.prefix(Int(count)).map { UInt8(bitPattern: $0) }, as: UTF8.self)
}

private func setBatch(_ batch: inout llama_batch, tokens: [llama_token], startPosition: Int32) {
batch.n_tokens = Int32(tokens.count)
for (index, token) in tokens.enumerated() {
batch.token[index] = token
batch.pos[index] = startPosition + Int32(index)
batch.n_seq_id[index] = 1
if let seqIDs = batch.seq_id, let seqID = seqIDs[index] {
seqID[0] = 0
}
batch.logits[index] = index == tokens.count - 1 ? 1 : 0
}
}

@Test func multimodal_rejectsImageURL() async throws {
let session = LanguageModelSession(model: model)
let imageSegment = Transcript.ImageSegment(url: testImageURL)
Expand Down
59 changes: 59 additions & 0 deletions Tests/AnyLanguageModelTests/LlamaSamplerChainTests.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import Foundation
import Testing

@testable import AnyLanguageModel

#if Llama
import LlamaSwift

/// Tests of the sampler chain that `LlamaLanguageModel` builds,
/// applied to hand-built candidates without a model.
@Suite("Llama sampler chain")
struct LlamaSamplerChainTests {
/// Mirostat v1 keeps the most probable tokens, and the number that it keeps
/// depends on the vocabulary size that the sampler gets.
@Test("Mirostat v1 gets the vocabulary size")
func mirostatV1UsesVocabularySize() throws {
let vocabularySize: Int32 = 32_000
let contextSize: UInt32 = 2048
let options = LlamaLanguageModel.ResolvedGenerationOptions(
contextSize: contextSize,
seed: 1,
temperature: 1,
repeatPenalty: 1,
mirostat: .v1(tau: 5, eta: 0.1)
)

let chain = try #require(
LlamaLanguageModel.makeSamplerChain(vocabularySize: vocabularySize, options: options)
)
defer { llama_sampler_free(chain) }
let expected = try #require(llama_sampler_init_mirostat(vocabularySize, 1, 5, 0.1, 100))
defer { llama_sampler_free(expected) }
let withContextSize = try #require(llama_sampler_init_mirostat(Int32(contextSize), 1, 5, 0.1, 100))
defer { llama_sampler_free(withContextSize) }

let keptCount = candidateCount(after: chain, vocabularySize: vocabularySize)
#expect(keptCount == candidateCount(after: expected, vocabularySize: vocabularySize))
#expect(keptCount != candidateCount(after: withContextSize, vocabularySize: vocabularySize))
}

/// Applies a sampler to Zipf-distributed candidates
/// and returns the number of candidates that it keeps.
private func candidateCount(after sampler: UnsafeMutablePointer<llama_sampler>, vocabularySize: Int32) -> Int {
var candidates = (0 ..< vocabularySize).map { id in
llama_token_data(id: id, logit: -1.1 * logf(Float(id + 1)), p: 0)
}
return candidates.withUnsafeMutableBufferPointer { buffer in
var array = llama_token_data_array(
data: buffer.baseAddress,
size: buffer.count,
selected: -1,
sorted: false
)
llama_sampler_apply(sampler, &array)
return array.size
}
}
}
#endif
Loading