diff --git a/Package.swift b/Package.swift index 799976b..ba5ead5 100644 --- a/Package.swift +++ b/Package.swift @@ -67,6 +67,7 @@ let package = Package( .executableTarget(name: "ImageSortDemo", dependencies: ["ImageSort"], exclude: ["README.md"]), .executableTarget(name: "KevCheck", dependencies: ["FluidUse"]), .executableTarget(name: "ClefVisionCheck", dependencies: ["FluidUse"]), + .executableTarget(name: "ClefFlashDemo", dependencies: ["FluidUse"], exclude: ["README.md", "demo.sh"]), .executableTarget(name: "InternDecisionCheck", dependencies: ["FluidUse"]), .executableTarget(name: "Decision2Check", dependencies: ["FluidUse"]), .executableTarget( diff --git a/Sources/ClefFlashDemo/ClefFlashDemoApp.swift b/Sources/ClefFlashDemo/ClefFlashDemoApp.swift new file mode 100644 index 0000000..66c19fe --- /dev/null +++ b/Sources/ClefFlashDemo/ClefFlashDemoApp.swift @@ -0,0 +1,28 @@ +import AppKit +import SwiftUI + +@main +struct ClefFlashDemoApp: App { + @StateObject private var model = TriageModel() + + init() { + setvbuf(stdout, nil, _IOLBF, 0) + // Bare SwiftPM executables start as background processes; make this one a regular windowed app. + for key in UserDefaults.standard.dictionaryRepresentation().keys where key.hasPrefix("NSWindow Frame") { + UserDefaults.standard.removeObject(forKey: key) // open at the default size, not a remembered one + } + NSApplication.shared.setActivationPolicy(.regular) + NSApplication.shared.activate(ignoringOtherApps: true) + } + + var body: some Scene { + WindowGroup("clef-flash 9B — on this Mac") { + ContentView() + .environmentObject(model) + .frame(minWidth: 1000, minHeight: 640) + .task { await model.start() } + } + .defaultSize(width: 1280, height: 820) + .windowResizability(.contentMinSize) + } +} diff --git a/Sources/ClefFlashDemo/ContentView.swift b/Sources/ClefFlashDemo/ContentView.swift new file mode 100644 index 0000000..c78efb1 --- /dev/null +++ b/Sources/ClefFlashDemo/ContentView.swift @@ -0,0 +1,187 @@ +import SwiftUI + +struct ContentView: View { + @EnvironmentObject var model: TriageModel + + var body: some View { + VStack(spacing: 0) { + header + Divider() + if model.ready { + board + Divider() + composer + } else { + Spacer() + ProgressView(model.status).controlSize(.large) + Spacer() + } + } + .background(Color(nsColor: .windowBackgroundColor)) + } + + private var header: some View { + HStack(alignment: .firstTextBaseline, spacing: 18) { + VStack(alignment: .leading, spacing: 4) { + Text("Cloudflare clef-flash · 9B decision model") + .font(.system(size: 26, weight: .bold)) + Text("Running on this Mac · Core ML on the GPU · no network") + .font(.system(size: 15)).foregroundStyle(.secondary) + } + Spacer() + if let median = model.medianMilliseconds { + VStack(alignment: .trailing, spacing: 0) { + Text(String(format: "%.2f s", median / 1000)) + .font(.system(size: 44, weight: .heavy, design: .rounded)).monospacedDigit() + Text("per ticket · 3 decisions in one pass") + .font(.system(size: 13)).foregroundStyle(.secondary) + } + } + } + .padding(.horizontal, 28).padding(.vertical, 18) + } + + private var board: some View { + HStack(alignment: .top, spacing: 16) { + IncomingLane() + .frame(width: 340) + DecidedFeed() + .frame(maxWidth: .infinity) + } + .padding(18) + } + + private var composer: some View { + HStack(spacing: 12) { + TextField("Type your own ticket and press Return…", text: $model.draft) + .textFieldStyle(.roundedBorder).font(.system(size: 15)) + .onSubmit { model.submitDraft() } + Button(model.running ? "Pause stream" : "Resume stream") { model.toggleStream() } + Button("Restart") { model.restart() } + if model.busy { ProgressView().controlSize(.small) } + } + .padding(.horizontal, 28).padding(.vertical, 14) + } +} + +struct IncomingLane: View { + @EnvironmentObject var model: TriageModel + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + HStack { + Image(systemName: "tray.and.arrow.down.fill").foregroundStyle(.blue) + Text("Incoming").font(.system(size: 17, weight: .semibold)) + Spacer() + Text("\(model.triagedCount) triaged").font(.system(size: 13)).foregroundStyle(.secondary) + .monospacedDigit() + } + if let current = model.current { + VStack(alignment: .leading, spacing: 8) { + HStack(spacing: 6) { + ProgressView().controlSize(.small) + Text("clef-flash is reading…").font(.system(size: 12, weight: .semibold)).foregroundStyle(.blue) + } + Text(current.text).font(.system(size: 14)).fixedSize(horizontal: false, vertical: true) + } + .padding(12) + .frame(maxWidth: .infinity, alignment: .leading) + .background(RoundedRectangle(cornerRadius: 10).fill(Color.blue.opacity(0.12))) + .overlay(RoundedRectangle(cornerRadius: 10).stroke(Color.blue, lineWidth: 2)) + .id(current.id) + } + ForEach(model.incoming) { ticket in + Text(ticket.text).font(.system(size: 13)).foregroundStyle(.secondary).lineLimit(2) + .padding(10) + .frame(maxWidth: .infinity, alignment: .leading) + .background(RoundedRectangle(cornerRadius: 10).fill(Color(nsColor: .controlBackgroundColor))) + .overlay( + RoundedRectangle(cornerRadius: 10).stroke( + ticket.typed ? Color.accentColor : .clear, lineWidth: 2) + ) + .transition(.move(edge: .bottom).combined(with: .opacity)) + } + Spacer(minLength: 0) + } + .padding(14) + .frame(maxHeight: .infinity, alignment: .top) + .background(RoundedRectangle(cornerRadius: 14).fill(Color.blue.opacity(0.05))) + .clipped() + } +} + +struct DecidedFeed: View { + @EnvironmentObject var model: TriageModel + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + HStack(spacing: 10) { + Image(systemName: "checkmark.circle.fill").foregroundStyle(.green) + Text("Decided").font(.system(size: 17, weight: .semibold)) + Spacer() + ForEach(TriageModel.Team.allCases) { team in + HStack(spacing: 5) { + Circle().fill(team.color).frame(width: 8, height: 8) + Text(team.title).font(.system(size: 13)) + Text("\(model.count(for: team))").font(.system(size: 13, weight: .semibold)).monospacedDigit() + } + .padding(.horizontal, 9).padding(.vertical, 4) + .background(Capsule().fill(team.color.opacity(0.12))) + } + } + ScrollView { + LazyVStack(spacing: 8) { + ForEach(model.tickets.reversed()) { ticket in + TicketRow(ticket: ticket) + .transition(.move(edge: .top).combined(with: .opacity)) + } + } + } + } + .padding(14) + .frame(maxHeight: .infinity, alignment: .top) + .background(RoundedRectangle(cornerRadius: 14).fill(Color.green.opacity(0.04))) + } +} + +struct TicketRow: View { + let ticket: TriageModel.Ticket + + private var urgencyColor: Color { [Color.green, .blue, .orange, .red][ticket.urgency] } + + var body: some View { + HStack(spacing: 12) { + Text(ticket.team.title) + .font(.system(size: 13, weight: .semibold)).lineLimit(1) + .frame(width: 104) + .padding(.vertical, 6) + .background(RoundedRectangle(cornerRadius: 7).fill(ticket.team.color.opacity(0.2))) + .foregroundStyle(ticket.team.color) + Text(ticket.text).font(.system(size: 14)).lineLimit(2) + .frame(maxWidth: .infinity, alignment: .leading) + Pill(text: TriageModel.urgencyLabels[ticket.urgency], color: urgencyColor) + .frame(width: 70, alignment: .leading) + Pill(text: "Refund", color: .pink).opacity(ticket.refund > 0.5 ? 1 : 0) + Text(String(format: "%.0f%%", ticket.teamConfidence * 100)) + .font(.system(size: 12)).foregroundStyle(.secondary).monospacedDigit().frame( + width: 36, alignment: .trailing) + Text(String(format: "%.0f ms", ticket.milliseconds)) + .font(.system(size: 13, weight: .semibold)).monospacedDigit().frame(width: 60, alignment: .trailing) + } + .padding(.horizontal, 12).padding(.vertical, 9) + .background(RoundedRectangle(cornerRadius: 10).fill(Color(nsColor: .controlBackgroundColor))) + .overlay(RoundedRectangle(cornerRadius: 10).stroke(ticket.typed ? Color.accentColor : .clear, lineWidth: 2)) + } +} + +struct Pill: View { + let text: String + let color: Color + + var body: some View { + Text(text).font(.system(size: 11, weight: .semibold)).lineLimit(1).fixedSize() + .padding(.horizontal, 7).padding(.vertical, 3) + .background(Capsule().fill(color.opacity(0.18))) + .foregroundStyle(color) + } +} diff --git a/Sources/ClefFlashDemo/README.md b/Sources/ClefFlashDemo/README.md new file mode 100644 index 0000000..e7be844 --- /dev/null +++ b/Sources/ClefFlashDemo/README.md @@ -0,0 +1,18 @@ +# ClefFlashDemo + +Cloudflare [clef-flash](https://huggingface.co/Cloudflare/clef-flash) (Qwen3.5-9B decision model, Apache-2.0) triaging +support tickets on this Mac through Core ML. Each ticket gets three typed decisions in one forward pass: team (choice), +urgency (score), refund (noul). Tickets are fictional mock data; typed tickets jump the queue. + +```bash +Sources/ClefFlashDemo/demo.sh # downloads FluidInference/clef-flash-coreml (~11 GB) on first run +Sources/ClefFlashDemo/demo.sh /path/to/bundle # or a local bundle (CLEF_FLASH_BUNDLE) +``` + +- Runtime: `ClefFlashManager` — 8 decoder packages (4 layers each, 8-bit weights, 6.5 GB) chained on the GPU + the + joint schema head (fp32), 512-token bucket. ~0.50 s per ticket on an M5 Pro; ~40 s to load (first launch also + compiles the packages into `/compiled/`). +- Bundle: [FluidInference/clef-flash-coreml](https://huggingface.co/FluidInference/clef-flash-coreml), pinned in + `ClefFlashModelStore` with per-file SHA-256. +- GPU only: the Qwen3.5 decoder puts 0 ops on the Neural Engine (every op falls back to CPU under `CPU_AND_NE`). +- Same answers as the Python Core ML pipeline on the mock tickets; 8-bit vs fp32 reference: 1 flip / 611 ARC questions. diff --git a/Sources/ClefFlashDemo/TriageModel.swift b/Sources/ClefFlashDemo/TriageModel.swift new file mode 100644 index 0000000..8b1a96c --- /dev/null +++ b/Sources/ClefFlashDemo/TriageModel.swift @@ -0,0 +1,229 @@ +import Foundation +import FluidUse +import SwiftUI + +/// Support-ticket triage with Cloudflare clef-flash (9B) running locally: each ticket gets three typed decisions +/// (team, urgency, refund) from one forward pass. Tickets are fictional mock data. +@MainActor +final class TriageModel: ObservableObject { + enum Team: String, CaseIterable, Identifiable { + case billing, engineering, account, product + var id: String { rawValue } + var title: String { rawValue.capitalized } + var color: Color { + switch self { + case .billing: return .orange + case .engineering: return .red + case .account: return .purple + case .product: return .teal + } + } + } + + struct Ticket: Identifiable { + let id = UUID() + let text: String + let team: Team + let teamConfidence: Float + let urgency: Int + let refund: Float + let milliseconds: Double + let typed: Bool + } + + struct Incoming: Identifiable, Equatable { + let id = UUID() + let text: String + let typed: Bool + } + + static let urgencyLabels = ["Low", "Normal", "High", "Critical"] + + static let questions: [(id: String, question: ClefQuestion)] = [ + ( + "team", + .choice( + instructions: "Which team should handle this ticket?", + criteria: [ + "billing": "Charges, refunds, invoices", "engineering": "Bugs, outages, API errors", + "account": "Login, security, account changes", "product": "Feature requests and questions", + ]) + ), + ("urgency", .score(instructions: "How urgent is this ticket?", criteria: urgencyLabels)), + ("refund", .noul(instructions: "Is the customer asking for money back?")), + ] + + // Fictional tickets (mock data, not real customers). + static let mockTickets = [ + "Hi, I was charged twice for my Pro plan this month. Can you refund the duplicate?", + "The dashboard has been throwing 500 errors since this morning and none of our team can log in.", + "Would love a dark mode for the mobile app, the white screen is brutal at night.", + "I forgot my password and the reset email never arrives. Checked spam already.", + "Our checkout is down and customers can't pay. This is costing us thousands per hour!!", + "Can I get an invoice with our VAT number on it for last quarter?", + "Your latest update deleted all my saved drafts. Two weeks of work gone. I want my money back.", + "Quick question: do you support exporting reports to CSV?", + "Please cancel my subscription and close my account, I'm moving to another tool.", + "API requests started timing out after we upgraded to v3 of the SDK.", + "Someone logged into my account from another country. I didn't do that. Please lock it.", + "Is there a discount for nonprofits? We're a small animal shelter.", + "Webhooks stopped firing for new orders about an hour ago, nothing in the logs.", + "I was promised a refund three weeks ago and still haven't seen it on my card.", + "Can you add Spanish to the language options? Half our team would use it.", + "How do I change the email address on my account? I left my old job.", + "The iOS app crashes every time I open the calendar tab.", + "We were billed for 50 seats but we only have 12 users. Please fix and credit us.", + "Two-factor codes aren't arriving by SMS, I'm locked out before a client meeting.", + "Would be great if the search bar supported filters by date.", + ] + + @Published var status = "Loading clef-flash 9B…" + @Published var ready = false + @Published var tickets: [Ticket] = [] + @Published var running = false + @Published var draft = "" + @Published var busy = false + /// Tickets waiting to be read (front = next), and the one the model is reading right now. + @Published var incoming: [Incoming] = [] + @Published var current: Incoming? + + /// The loaded model as a closure (ClefFlashManager needs macOS 15; the package targets 14): answers + total ms. + private var answer: ((String) async throws -> ([ClefAnswer], Double))? + private var next = 0 + /// Bumped by `restart()`; a decision that finishes for an older generation is dropped. + private var generation = 0 + private var streamTask: Task? + + var medianMilliseconds: Double? { + let times = tickets.map(\.milliseconds).sorted() + return times.isEmpty ? nil : times[times.count / 2] + } + + func count(for team: Team) -> Int { tickets.reduce(0) { $0 + ($1.team == team ? 1 : 0) } } + + func start() async { + guard answer == nil else { return } + do { + guard #available(macOS 15.0, *) else { + status = "clef-flash's Core ML path needs macOS 15" + return + } + let directory: URL + if let path = ProcessInfo.processInfo.environment["CLEF_FLASH_BUNDLE"], !path.isEmpty { + directory = URL(fileURLWithPath: path) + } else { + status = "Downloading clef-flash Core ML (FluidInference/clef-flash-coreml, ~11 GB)…" + directory = try await ClefFlashModelStore.ensure() + } + let started = Date() + status = "Loading clef-flash 9B (first launch compiles the packages)…" + let manager = try await ClefFlashManager.load(from: directory, bucket: 512) + try await manager.warm() + // a couple of full-length tickets page every part's weights in before the board starts timing + for text in Self.mockTickets.prefix(2) { + _ = try await manager.answer(state: text, questions: Self.questions) + } + answer = { text in + let result = try await manager.answer(state: text, questions: Self.questions) + return (result.answers, result.totalMilliseconds) + } + print("loaded in \(String(format: "%.1f", Date().timeIntervalSince(started))) s") + ready = true + status = "" + toggleStream() + } catch { + status = "Failed to load: \(error.localizedDescription)" + } + } + + func toggleStream() { + if running { + streamTask?.cancel() + running = false + return + } + running = true + streamTask = Task { [weak self] in + // back to back: the next ticket starts the moment the previous one is decided + while let self, !Task.isCancelled { + self.refill() + guard !self.incoming.isEmpty else { break } + let ticket = withAnimation(.easeOut(duration: 0.2)) { self.incoming.removeFirst() } + withAnimation(.easeOut(duration: 0.2)) { self.current = ticket } + await self.triage(ticket.text, typed: ticket.typed) + await Task.yield() + } + } + } + + /// Keep the visible queue full of mock arrivals. + private func refill() { + while incoming.count < 8 { + incoming.append(Incoming(text: Self.mockTickets[next % Self.mockTickets.count], typed: false)) + next += 1 + } + } + + func submitDraft() { + let text = draft.trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { return } + draft = "" + // typed tickets jump the queue + withAnimation(.easeOut(duration: 0.2)) { incoming.insert(Incoming(text: text, typed: true), at: 0) } + if !running { + Task { + let ticket = incoming.removeFirst() + current = ticket + await triage(ticket.text, typed: true) + } + } + } + + /// Clear the board, put the mock queue back at its first ticket and stream again. + func restart() { + streamTask?.cancel() + running = false + generation += 1 + withAnimation(.easeOut(duration: 0.2)) { + tickets.removeAll() + incoming.removeAll() + current = nil + } + next = 0 + toggleStream() + } + + var triagedCount: Int { tickets.count } + + private func triage(_ text: String, typed: Bool) async { + guard let answer else { return } + let generation = generation + busy = true + defer { busy = false } + do { + let (answers, milliseconds) = try await answer(text) + guard generation == self.generation else { return } + let team = answers[0] + let urgency = answers[1] + let refund = answers[2] + let teamIndex = team.probabilities.indices.max { team.probabilities[$0] < team.probabilities[$1] } ?? 0 + let urgencyIndex = + urgency.probabilities.indices.max { urgency.probabilities[$0] < urgency.probabilities[$1] } ?? 0 + let ticket = Ticket( + text: text, team: Team(rawValue: team.optionIDs[teamIndex]) ?? .product, + teamConfidence: team.probabilities[teamIndex], urgency: urgencyIndex, refund: refund.noul ?? 0, + milliseconds: milliseconds, typed: typed) + print( + String( + format: "%5.0f ms %-11@ %-8@ refund %.2f %@", milliseconds, ticket.team.rawValue, + Self.urgencyLabels[urgencyIndex], ticket.refund, text)) + withAnimation(.spring(duration: 0.35)) { + tickets.append(ticket) + if tickets.count > 200 { tickets.removeFirst(tickets.count - 200) } + current = nil + } + } catch { + status = "Error: \(error.localizedDescription)" + } + } +} diff --git a/Sources/ClefFlashDemo/demo.sh b/Sources/ClefFlashDemo/demo.sh new file mode 100755 index 0000000..5410bae --- /dev/null +++ b/Sources/ClefFlashDemo/demo.sh @@ -0,0 +1,24 @@ +#!/bin/zsh +# clef-flash triage demo, plus one Ghostty window with macmon (GPU / power) on top and the live decision log below. +# Sources/ClefFlashDemo/demo.sh [bundle dir] (or CLEF_FLASH_BUNDLE; default: download FluidInference/clef-flash-coreml) +set -e +ROOT=$(cd "$(dirname "$0")/../.." && pwd) +BUNDLE=${1:-${CLEF_FLASH_BUNDLE:-}} +LOG=${TMPDIR:-/tmp}/clef-flash-demo.log + +[[ -z "$BUNDLE" || -f "$BUNDLE/config.json" ]] || { echo "no clef-flash bundle at $BUNDLE"; exit 1; } +# the 9B weights need the GPU and ~7 GB of RAM to themselves; other model jobs make every ticket page +if pgrep -f "reference.py|run_coreml.py|export.py" >/dev/null; then + echo "warning: a model-lab job is running and will slow the demo (pgrep -fl 'reference.py|run_coreml.py|export.py')" +fi +swift build -c release --product ClefFlashDemo --package-path "$ROOT" +: > "$LOG" +if ! tmux has-session -t clef-flash 2>/dev/null; then + tmux new-session -d -s clef-flash -x 160 -y 60 macmon + tmux set-option -t clef-flash remain-on-exit on + tmux split-window -v -l 60% -t clef-flash "tail -n 300 -F '$LOG'" + open -na /Applications/Ghostty.app --args -e "$(command -v tmux)" attach -t clef-flash +fi +pkill -x ClefFlashDemo 2>/dev/null || true +CLEF_FLASH_BUNDLE="$BUNDLE" "$ROOT/.build/release/ClefFlashDemo" > "$LOG" 2> "$LOG.stderr" & +echo "demo pid $! · log $LOG · first ticket after ~40 s of model loading (plus an ~11 GB download on first run)" diff --git a/Sources/FluidUse/ClefFlash/ClefFlashManager.swift b/Sources/FluidUse/ClefFlash/ClefFlashManager.swift new file mode 100644 index 0000000..fadaa04 --- /dev/null +++ b/Sources/FluidUse/ClefFlash/ClefFlashManager.swift @@ -0,0 +1,232 @@ +import CoreML +import Foundation + +/// Cloudflare clef-flash (Qwen3.5-9B decision model) on Core ML, text only: Clef-style typed decisions in one +/// prefill pass. The decoder ships as layer-group packages (one function per sequence bucket) chained on the GPU, +/// followed by the joint schema head. An actor: Core ML's synchronous `prediction` is not thread-safe. +@available(macOS 15.0, iOS 18.0, *) +public actor ClefFlashManager { + public struct Result: Sendable { + public let answers: [ClefAnswer] + public let inputTokens: Int + public let bucket: Int + public let decoderMilliseconds: Double + public let headMilliseconds: Double + public let totalMilliseconds: Double + } + + struct Config: Sendable { + let hidden: Int + let rotaryDim: Int + let ropeTheta: Double + let padID: Int + let vocab: Int + let buckets: [Int] + let maxQuestions: Int + let maxOptions: Int + let imageTokenID: Int + let visionStartID: Int + let visionEndID: Int + let parts: [String] + let head: String + + init(json: [String: Any]) throws { + func int(_ key: String) throws -> Int { + guard let v = (json[key] as? NSNumber)?.intValue else { + throw ClefVisionError.invalidAsset("config.json: \(key)") + } + return v + } + guard let buckets = json["buckets"] as? [NSNumber], let parts = json["parts"] as? [String], + let head = json["head"] as? String, let theta = (json["rope_theta"] as? NSNumber)?.doubleValue + else { throw ClefVisionError.invalidAsset("config.json is missing sections") } + hidden = try int("hidden_size") + rotaryDim = try int("rotary_dim") + ropeTheta = theta + padID = try int("pad_id") + vocab = try int("vocab_size") + self.buckets = buckets.map(\.intValue).sorted() + maxQuestions = try int("max_questions") + maxOptions = try int("max_options") + imageTokenID = try int("image_token_id") + visionStartID = try int("vision_start_token_id") + visionEndID = try int("vision_end_token_id") + self.parts = parts + self.head = head + } + } + + public nonisolated let bucket: Int + let config: Config + let tokenizer: QwenBPETokenizer + private let parts: [MLModel] + private let head: MLModel + private let embeddings: Data // fp16 [vocab, hidden], input gather + private let outputEmbeddings: Data // fp16 [vocab, hidden], the head's lexical option rows + private let cos: MLMultiArray + private let sin: MLMultiArray + + /// Loads one sequence bucket (records up to `bucket` tokens). Packages are compiled once into `compiled/`. + public static func load( + from directory: URL, bucket: Int = 512, computeUnits: MLComputeUnits = .cpuAndGPU + ) async throws + -> ClefFlashManager + { + let json = try JSONSerialization.jsonObject( + with: Data(contentsOf: directory.appendingPathComponent("config.json"))) + guard let dict = json as? [String: Any] else { throw ClefVisionError.invalidAsset("config.json") } + let config = try Config(json: dict) + guard config.buckets.contains(bucket) else { throw ClefVisionError.invalidInput("no \(bucket)-token bucket") } + let tokenizer = try QwenBPETokenizer(tokenizerJsonURL: directory.appendingPathComponent("tokenizer.json")) + let tableBytes = config.vocab * config.hidden * 2 + let embeddings = try Data( + contentsOf: directory.appendingPathComponent("embeddings.f16"), options: .alwaysMapped) + let outputEmbeddings = try Data( + contentsOf: directory.appendingPathComponent("output_embeddings.f16"), options: .alwaysMapped) + guard embeddings.count == tableBytes, outputEmbeddings.count == tableBytes else { + throw ClefVisionError.invalidAsset("embedding table size") + } + let configuration = MLModelConfiguration() + configuration.computeUnits = computeUnits + configuration.functionName = "L\(bucket)" + let compiledDir = directory.appendingPathComponent("compiled") + try FileManager.default.createDirectory(at: compiledDir, withIntermediateDirectories: true) + func load(_ path: String) async throws -> MLModel { + let source = directory.appendingPathComponent(path) + let target = compiledDir.appendingPathComponent( + source.deletingPathExtension().lastPathComponent + ".mlmodelc") + if !FileManager.default.fileExists(atPath: target.path) { + let temporary = try await MLModel.compileModel(at: source) + try FileManager.default.moveItem(at: temporary, to: target) + } + return try await MLModel.load(contentsOf: target, configuration: configuration) + } + var parts: [MLModel] = [] + for path in config.parts { parts.append(try await load(path)) } + let head = try await load(config.head) + let (cos, sin) = try ropeTables(length: bucket, config: config) + return ClefFlashManager( + bucket: bucket, config: config, tokenizer: tokenizer, parts: parts, head: head, embeddings: embeddings, + outputEmbeddings: outputEmbeddings, cos: cos, sin: sin) + } + + init( + bucket: Int, config: Config, tokenizer: QwenBPETokenizer, parts: [MLModel], head: MLModel, embeddings: Data, + outputEmbeddings: Data, cos: MLMultiArray, sin: MLMultiArray + ) { + self.bucket = bucket + self.config = config + self.tokenizer = tokenizer + self.parts = parts + self.head = head + self.embeddings = embeddings + self.outputEmbeddings = outputEmbeddings + self.cos = cos + self.sin = sin + } + + /// Text-only rows: position p on every M-RoPE axis, so the interleaved sections collapse to plain RoPE. fp16 [L, R]. + static func ropeTables(length: Int, config: Config) throws -> (MLMultiArray, MLMultiArray) { + let dim = config.rotaryDim + let half = dim / 2 + let cosArray = try MLMultiArray(shape: [length as NSNumber, dim as NSNumber], dataType: .float16) + let sinArray = try MLMultiArray(shape: [length as NSNumber, dim as NSNumber], dataType: .float16) + let c = cosArray.dataPointer.assumingMemoryBound(to: Float16.self) + let s = sinArray.dataPointer.assumingMemoryBound(to: Float16.self) + for position in 0.. ClefEncodedRecord { + try ClefRecordEncoder.encode( + state: state, questions: questions, imageTokenCounts: [], tokenizer: tokenizer, + imageTokenID: config.imageTokenID, visionStartID: config.visionStartID, visionEndID: config.visionEndID, + maxLength: bucket) + } + + /// Decide `questions` about `state` (text or JSON-like value). Answers follow the question order; option + /// probabilities follow Clef's option order (`ClefAnswer.optionIDs`). + public func answer(state: Any, questions: [(id: String, question: ClefQuestion)]) throws -> Result { + let started = Date() + let record = try encode(state: state, questions: questions) + let L = bucket + let D = config.hidden + // 1. embedding gather (fp16 table -> fp16 row) + let row = try MLMultiArray(shape: [1, L as NSNumber, D as NSNumber], dataType: .float16) + let padID = config.padID + embeddings.withUnsafeBytes { raw in + let table = raw.bindMemory(to: Float16.self).baseAddress! + let out = row.dataPointer.assumingMemoryBound(to: Float16.self) + for position in 0..