diff --git a/aspec/commands/evaluate_medusa_acceptance.py b/aspec/commands/evaluate_medusa_acceptance.py new file mode 100644 index 0000000..09dabe6 --- /dev/null +++ b/aspec/commands/evaluate_medusa_acceptance.py @@ -0,0 +1,180 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Measure Medusa greedy acceptance and decode equality with a Hugging Face Target.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import torch + + +def _choices(path, heads): + if path is not None: + choices = json.loads(Path(path).read_text()) + else: + from aspec.inference.medusa_consumer import DEFAULT_MEDUSA_CHOICES + + choices = DEFAULT_MEDUSA_CHOICES + filtered = [list(item) for item in choices if len(item) <= heads] + if not filtered: + raise ValueError("No medusa_choices path fits the draft's head depth") + return filtered + + +def _parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--draft", type=Path, required=True) + parser.add_argument("--target", required=True) + parser.add_argument("--prompts", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--limit", type=int, default=8) + parser.add_argument("--max-new-tokens", type=int, default=32) + parser.add_argument("--choices", type=Path, default=None) + parser.add_argument("--top-k", type=int, default=10) + parser.add_argument("--verify-batch-size", type=int, default=16) + parser.add_argument("--sampling", choices=("greedy", "typical", "nucleus"), + default="greedy") + parser.add_argument("--temperature", type=float, default=0.7) + parser.add_argument("--top-p", type=float, default=0.8, dest="top_p") + parser.add_argument("--posterior-threshold", type=float, default=0.3, + dest="posterior_threshold") + parser.add_argument("--posterior-alpha", type=float, default=0.09, + dest="posterior_alpha") + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--device", default=None) + parser.add_argument( + "--dtype", choices=("bfloat16", "float16", "float32"), default="bfloat16") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--enable-thinking", action="store_true") + return parser.parse_args() + + +def main(): + args = _parse_args() + from transformers import AutoModelForCausalLM, AutoTokenizer + + from aspec.commands.evaluate_draft import load_prompts + from aspec.draft_artifact import load_native_draft + from aspec.inference.medusa_consumer import HFCausalVerifier, MedusaConsumer + + device = args.device or ("cuda" if torch.cuda.is_available() else "cpu") + dtype = getattr(torch, args.dtype) + tokenizer = AutoTokenizer.from_pretrained( + args.target, trust_remote_code=args.trust_remote_code) + target = AutoModelForCausalLM.from_pretrained( + args.target, torch_dtype=dtype, trust_remote_code=args.trust_remote_code) + target.to(device).eval() + draft = load_native_draft(args.draft, device=device, dtype=dtype) + choices = _choices(args.choices, draft.medusa_num_heads) + verifier = HFCausalVerifier( + target, device=device, verify_batch_size=args.verify_batch_size) + consumer = MedusaConsumer( + draft, verifier, medusa_choices=choices, top_k=args.top_k) + prompts = load_prompts( + args.prompts, tokenizer, args.limit, + enable_thinking=True if args.enable_thinking else None) + + greedy = args.sampling == "greedy" + if greedy: + temperature = 0.0 + else: + if args.temperature <= 0: + raise ValueError("sampling acceptance requires --temperature > 0") + temperature = args.temperature + + def generator(): + value = torch.Generator(device=torch.device(device)) + value.manual_seed(args.seed) + return value + + eos = tokenizer.eos_token_id + results = [] + for ids in prompts: + prefix = torch.tensor(ids, dtype=torch.long) + if greedy: + baseline = verifier.greedy_generate( + prefix, max_new_tokens=args.max_new_tokens, eos_token_id=eos) + else: + baseline = None + speculative = consumer.generate( + prefix, max_new_tokens=args.max_new_tokens, eos_token_id=eos, + sampling=args.sampling, temperature=temperature, top_p=args.top_p, + posterior_threshold=args.posterior_threshold, + posterior_alpha=args.posterior_alpha, + generator=None if args.sampling != "nucleus" else generator()) + repeat = consumer.generate( + prefix, max_new_tokens=args.max_new_tokens, eos_token_id=eos, + sampling=args.sampling, temperature=temperature, top_p=args.top_p, + posterior_threshold=args.posterior_threshold, + posterior_alpha=args.posterior_alpha, + generator=None if args.sampling != "nucleus" else generator()) + start = len(ids) + equal = None + if baseline is not None: + equal = ( + speculative["token_ids"][start:].tolist() + == baseline["token_ids"][start:].tolist()) + results.append({ + "prompt_sha256": hashlib.sha256( + json.dumps(ids).encode("utf-8")).hexdigest(), + "prompt_tokens": len(ids), + "generated_tokens": speculative["generated_tokens"], + "verifier_calls": speculative["verifier_calls"], + "acceptance_length": speculative["acceptance_length"], + "decode_equal": equal, + "repeat_stable": ( + speculative["token_ids"].tolist() == repeat["token_ids"].tolist()), + }) + + total_tokens = sum(item["generated_tokens"] for item in results) + total_calls = sum(item["verifier_calls"] for item in results) + if total_calls < 1: + raise ValueError("Acceptance measurement produced no verifier calls") + choices_blob = json.dumps(choices, sort_keys=True, separators=(",", ":")) + report = { + "schema_version": "aspec.medusa-acceptance/v1", + "draft_manifest_sha256": hashlib.sha256( + (args.draft / "export_manifest.json").read_bytes()).hexdigest(), + "target": args.target, + "prompts": str(args.prompts), + "prompt_count": len(results), + "max_new_tokens": args.max_new_tokens, + "top_k": args.top_k, + "sampling": args.sampling, + "temperature": temperature, + "top_p": args.top_p if not greedy else None, + "posterior_threshold": (args.posterior_threshold if not greedy else None), + "posterior_alpha": args.posterior_alpha if not greedy else None, + "seed": args.seed if args.sampling == "nucleus" else None, + "choices_sha256": hashlib.sha256(choices_blob.encode("utf-8")).hexdigest(), + "choices_count": len(choices), + "device": device, + "device_name": ( + torch.cuda.get_device_name(torch.device(device)) + if device.startswith("cuda") else None), + "dtype": args.dtype, + "acceptance_length": total_tokens / total_calls, + "acceptance_convention": ( + "generated_tokens / verifier_calls; includes the bonus token"), + "decode_equal": (all(item["decode_equal"] for item in results) + if greedy else None), + "decode_equal_applicable": greedy, + "repeat_stable": all(item["repeat_stable"] for item in results), + "results": results, + } + try: + serialized = json.dumps(report, indent=2, sort_keys=True, allow_nan=False) + except ValueError as exc: + raise ValueError("Acceptance report contains non-finite metrics") from exc + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(serialized + "\n") + print(serialized) + + +if __name__ == "__main__": + main() diff --git a/aspec/inference/medusa_consumer.py b/aspec/inference/medusa_consumer.py new file mode 100644 index 0000000..68165bf --- /dev/null +++ b/aspec/inference/medusa_consumer.py @@ -0,0 +1,551 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +# Adapted from FasterDecoding/Medusa e2a5d20c048a9b0a4092e6933c34313687422518 +# (medusa/model/utils.py, medusa/model/medusa_choices.py, Apache-2.0). +# Tree buffers, candidate construction and greedy acceptance follow the +# reference; batched path verification and explicit evidence accounting are +# ASpec additions. +"""Medusa candidate-tree proposals, verification and acceptance accounting. + +The consumer is deliberately not a serving runtime: callers provide a verifier +that owns the Target forward pass, KV/cache policy and sampling. The reference +verifier here batches independent teacher-forced paths, which is exact for +greedy verification but makes no throughput claim. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol, Sequence + +import torch +from torch import Tensor + +DEFAULT_TOPK = 10 + +# Reference Medusa tree for Vicuna/7B-style heads (medusa/model/medusa_choices.py). +DEFAULT_MEDUSA_CHOICES = [ + [0], [0, 0], [1], [0, 1], [2], [0, 0, 0], [1, 0], [0, 2], [3], [0, 3], + [4], [0, 4], [2, 0], [0, 5], [0, 0, 1], [5], [0, 6], [6], [0, 7], + [0, 1, 0], [1, 1], [7], [0, 8], [0, 0, 2], [3, 0], [0, 9], [8], [9], + [1, 0, 0], [0, 2, 0], [1, 2], [0, 0, 3], [4, 0], [2, 1], [0, 0, 4], + [0, 0, 5], [0, 0, 0, 0], [0, 1, 1], [0, 0, 6], [0, 3, 0], [5, 0], + [1, 3], [0, 0, 7], [0, 0, 8], [0, 0, 9], [6, 0], [0, 4, 0], [1, 4], + [7, 0], [0, 1, 2], [2, 0, 0], [3, 1], [2, 2], [8, 0], [0, 5, 0], + [1, 5], [1, 0, 1], [0, 2, 1], [9, 0], [0, 6, 0], [0, 0, 0, 1], + [1, 6], [0, 7, 0], +] + + +@dataclass(frozen=True) +class TreeBuffers: + choices: tuple[tuple[int, ...], ...] + tree_indices: Tensor + position_ids: Tensor + retrieve_indices: Tensor + attention_mask: Tensor + + +@dataclass(frozen=True) +class TreeProposal: + bonus_token: Tensor + candidates: Tensor + lengths: Tensor + medusa_logits: Tensor + + +@dataclass(frozen=True) +class Acceptance: + path_index: int + accept_length: int + accepted_tokens: Tensor + + +def _validate_choices(medusa_choices: Sequence[Sequence[int]], top_k: int) -> list[tuple[int, ...]]: + if top_k < 1: + raise ValueError("top_k must be a positive integer") + choices = [tuple(int(value) for value in path) for path in medusa_choices] + if not choices: + raise ValueError("medusa_choices must not be empty") + if len(set(choices)) != len(choices): + raise ValueError("medusa_choices must not contain duplicates") + available = set(choices) + for path in choices: + if not path: + raise ValueError("medusa_choices paths must be non-empty") + if any(value < 0 or value >= top_k for value in path): + raise ValueError(f"medusa_choices values must be in [0, {top_k})") + for depth in range(1, len(path)): + if path[:depth] not in available: + raise ValueError(f"medusa_choices must be prefix-closed: {path}") + return choices + + +def generate_tree_buffers( + medusa_choices: Sequence[Sequence[int]] | None = None, + *, + top_k: int = DEFAULT_TOPK, + device: Any = "cpu", +) -> TreeBuffers: + """Port of the reference ``generate_medusa_buffers`` with an explicit top-k.""" + choices = _validate_choices(DEFAULT_MEDUSA_CHOICES if medusa_choices is None else medusa_choices, top_k) + sorted_choices = sorted(choices, key=lambda path: (len(path), path)) + depth_counts: list[int] = [] + previous_depth = 0 + for path in sorted_choices: + depth = len(path) + if depth != previous_depth: + depth_counts.append(0) + depth_counts[depth - 1] += 1 + previous_depth = depth + + tree_len = len(sorted_choices) + 1 + attention_mask = torch.eye(tree_len, dtype=torch.bool) + attention_mask[:, 0] = True + start = 0 + for depth, count in enumerate(depth_counts, start=1): + for offset in range(count): + choice = sorted_choices[start + offset] + if len(choice) > 1: + ancestors = [ + sorted_choices.index(choice[:prefix]) + 1 + for prefix in range(1, len(choice)) + ] + attention_mask[start + offset + 1, ancestors] = True + start += count + + tree_indices = torch.zeros(tree_len, dtype=torch.long) + start = 0 + for depth, count in enumerate(depth_counts): + for offset in range(count): + tree_indices[start + offset + 1] = ( + sorted_choices[start + offset][-1] + top_k * depth + 1) + start += count + + position_ids = torch.zeros(tree_len, dtype=torch.long) + start = 0 + for depth, count in enumerate(depth_counts): + position_ids[start + 1:start + count + 1] = depth + 1 + start += count + + retrieve_nest: list[list[int]] = [] + retrieve_paths: set[tuple[int, ...]] = set() + for index in range(len(sorted_choices)): + choice = sorted_choices[-index - 1] + if choice in retrieve_paths: + continue + retrieve_indices = [] + for depth in range(len(choice)): + prefix = choice[:depth + 1] + retrieve_indices.append(sorted_choices.index(prefix)) + retrieve_paths.add(prefix) + retrieve_nest.append(retrieve_indices) + max_length = max(len(path) for path in retrieve_nest) + padded = [path + [-2] * (max_length - len(path)) for path in retrieve_nest] + retrieve = torch.tensor(padded, dtype=torch.long) + 1 + retrieve = torch.cat([torch.zeros((retrieve.shape[0], 1), dtype=torch.long), retrieve], dim=1) + + return TreeBuffers( + choices=tuple(sorted_choices), + tree_indices=tree_indices.to(device), + position_ids=position_ids.to(device), + retrieve_indices=retrieve.to(device), + attention_mask=attention_mask.unsqueeze(0).unsqueeze(0).to(device), + ) + + +def _draft_dtype(model: torch.nn.Module) -> torch.dtype: + parameter = next(model.parameters(), None) + return parameter.dtype if parameter is not None else torch.float32 + + +class MedusaTreeProposer: + """Build a Medusa candidate tree from one Target final hidden state.""" + + def __init__( + self, + draft_model: torch.nn.Module, + *, + medusa_choices: Sequence[Sequence[int]] | None = None, + top_k: int = DEFAULT_TOPK, + device: Any = None, + ): + self.model = draft_model.eval() + heads = getattr(draft_model, "medusa_num_heads", None) + if not isinstance(heads, int) or heads < 1: + raise ValueError("MedusaTreeProposer requires a heads-only Medusa draft") + choices = DEFAULT_MEDUSA_CHOICES if medusa_choices is None else medusa_choices + if device is None: + tensor = next(draft_model.parameters(), None) + if tensor is None: + tensor = next(draft_model.buffers(), None) + device = tensor.device if tensor is not None else "cpu" + self.buffers = generate_tree_buffers( + [path for path in choices if len(path) <= heads], top_k=top_k, + device=device) + if not self.buffers.choices: + raise ValueError("medusa_choices has no path within the draft's head depth") + self.top_k = top_k + + @torch.no_grad() + def propose(self, last_hidden: Tensor, base_logits: Tensor) -> TreeProposal: + hidden = last_hidden.reshape(1, 1, -1).to( + device=self.buffers.tree_indices.device, dtype=_draft_dtype(self.model)) + logits = self.model(hidden) + if logits.ndim != 4 or logits.shape[0] != self.model.medusa_num_heads: + raise ValueError("Medusa draft must return [heads, batch, sequence, vocab] logits") + head_logits = logits[:, 0, 0] + topk = torch.topk(head_logits, self.top_k, dim=-1).indices.reshape(-1) + flat_base = base_logits.reshape(-1) + if flat_base.numel() != logits.shape[-1]: + raise ValueError("Base logits vocabulary differs from the draft head vocabulary") + bonus = flat_base.argmax() + candidates = torch.cat([bonus.reshape(1), topk]) + tree_tokens = candidates[self.buffers.tree_indices] + extended = torch.cat([tree_tokens, torch.zeros(1, dtype=torch.long, + device=tree_tokens.device)]) + cart = extended[self.buffers.retrieve_indices] + lengths = (self.buffers.retrieve_indices[:, 1:] >= 0).sum(dim=1) + 1 + return TreeProposal( + bonus_token=bonus, + candidates=cart, + lengths=lengths, + medusa_logits=head_logits, + ) + + +class GreedyVerifier(Protocol): + """Caller-owned Target forward. Returns post-norm hidden and next-token argmax.""" + + def prefill(self, prefix_ids: Tensor) -> tuple[Tensor, Tensor]: + ... + + def verify(self, prefix_ids: Tensor, candidates: Tensor, lengths: Tensor) -> Tensor: + ... + + +def _valid_steps(proposal: TreeProposal, width: int) -> Tensor: + steps = torch.arange(width - 1, device=proposal.candidates.device)[None, :] + return steps < (proposal.lengths - 1)[:, None] + + +def _pick_best(proposal: TreeProposal, mask: Tensor, *, likelihood: Tensor | None = None) -> Acceptance: + lengths = torch.cumprod(mask, dim=1).sum(dim=1) + best_length = int(lengths.max()) + candidates = (lengths == best_length).nonzero().flatten() + if likelihood is not None and best_length > 0: + scores = torch.log(likelihood[candidates, :best_length] + 1e-12).sum(dim=-1) + best = candidates[scores.argmax()] + else: + best = candidates[0] + accepted = proposal.candidates[best, :best_length + 1].clone() + return Acceptance(path_index=int(best), accept_length=int(best_length), + accepted_tokens=accepted) + + +def accept_typical( + proposal: TreeProposal, + logits: Tensor, + *, + temperature: float, + posterior_threshold: float = 0.3, + posterior_alpha: float = 0.09, +) -> Acceptance: + """Reference typical-acceptance fast path (deterministic).""" + if temperature <= 0: + raise ValueError("typical acceptance requires temperature > 0") + if not 0 <= posterior_threshold <= 1: + raise ValueError("posterior_threshold must be in [0, 1]") + if posterior_alpha < 0: + raise ValueError("posterior_alpha must be non-negative") + width = proposal.candidates.shape[1] + probs = torch.softmax(logits[:, :width - 1].float() / temperature, dim=-1) + tokens = proposal.candidates[:, 1:width].to(probs.device) + candidate_prob = probs.gather(-1, tokens.unsqueeze(-1)).squeeze(-1) + entropy = -(probs * torch.log(probs + 1e-5)).sum(dim=-1) + threshold = torch.minimum( + torch.full_like(entropy, posterior_threshold), + torch.exp(-entropy) * posterior_alpha, + ) + mask = (candidate_prob > threshold) & _valid_steps(proposal, width).to(probs.device) + return _pick_best(proposal, mask, likelihood=candidate_prob) + + +def accept_nucleus( + proposal: TreeProposal, + logits: Tensor, + *, + temperature: float, + top_p: float, + generator: torch.Generator, +) -> Acceptance: + """Reference nucleus sampling acceptance.""" + if temperature <= 0: + raise ValueError("nucleus acceptance requires temperature > 0") + if not 0 < top_p <= 1: + raise ValueError("top_p must be in (0, 1]") + width = proposal.candidates.shape[1] + probs = torch.softmax(logits[:, :width - 1].float() / temperature, dim=-1) + flat = probs.reshape(-1, probs.shape[-1]) + if top_p < 1: + sorted_probs, sorted_indices = torch.sort(flat, descending=True, dim=-1) + cumulative = torch.cumsum(sorted_probs, dim=-1) + remove = cumulative > top_p + remove[..., 1:] = remove[..., :-1].clone() + remove[..., 0] = False + filtered = torch.zeros_like(remove).scatter(1, sorted_indices, remove) + flat = flat.masked_fill(filtered, 0.0) + flat = flat / flat.sum(dim=-1, keepdim=True).clamp_min(1e-12) + sampled = torch.multinomial(flat, 1, generator=generator).view(-1, width - 1) + tokens = proposal.candidates[:, 1:width].to(probs.device) + mask = (tokens == sampled) & _valid_steps(proposal, width).to(probs.device) + return _pick_best(proposal, mask) + + +def accept_greedy(proposal: TreeProposal, next_tokens: Tensor) -> Acceptance: + """Reference greedy acceptance over verified path predictions.""" + candidates = proposal.candidates + lengths = proposal.lengths + if next_tokens.shape[0] != candidates.shape[0]: + raise ValueError("Verifier returned a different number of paths") + best_path = 0 + best_length = -1 + for path in range(candidates.shape[0]): + real = int(lengths[path]) + if real < 1: + continue + accepted = 0 + for depth in range(real - 1): + if int(next_tokens[path, depth]) != int(candidates[path, depth + 1]): + break + accepted += 1 + if accepted > best_length: + best_length = accepted + best_path = path + accepted_tokens = candidates[best_path, :best_length + 1].clone() + return Acceptance(path_index=best_path, accept_length=best_length, + accepted_tokens=accepted_tokens) + + +class MedusaConsumer: + """Greedy speculative steps with explicit acceptance accounting.""" + + def __init__( + self, + draft_model: torch.nn.Module, + verifier: GreedyVerifier, + *, + medusa_choices: Sequence[Sequence[int]] | None = None, + top_k: int = DEFAULT_TOPK, + ): + self.proposer = MedusaTreeProposer( + draft_model, medusa_choices=medusa_choices, top_k=top_k) + self.verifier = verifier + + @torch.no_grad() + def step( + self, + prefix_ids: Tensor, + *, + sampling: str = "greedy", + temperature: float = 0.0, + top_p: float = 0.8, + posterior_threshold: float = 0.3, + posterior_alpha: float = 0.09, + generator: torch.Generator | None = None, + ) -> Acceptance: + hidden, base_logits = self.verifier.prefill(prefix_ids) + proposal = self.proposer.propose(hidden, base_logits) + if sampling == "greedy": + next_tokens = self.verifier.verify( + prefix_ids, proposal.candidates, proposal.lengths) + return accept_greedy(proposal, next_tokens) + if not hasattr(self.verifier, "verify_logits"): + raise ValueError(f"Verifier does not implement {sampling} acceptance") + logits = self.verifier.verify_logits( + prefix_ids, proposal.candidates, proposal.lengths) + if sampling == "typical": + return accept_typical( + proposal, logits, temperature=temperature, + posterior_threshold=posterior_threshold, + posterior_alpha=posterior_alpha) + if sampling == "nucleus": + if generator is None: + raise ValueError("nucleus acceptance requires a torch.Generator") + if generator.device.type != logits.device.type or ( + generator.device.index is not None + and logits.device.index is not None + and generator.device.index != logits.device.index): + raise ValueError("nucleus generator must be on the logits device") + return accept_nucleus( + proposal, logits, temperature=temperature, top_p=top_p, + generator=generator) + raise ValueError(f"Unsupported sampling policy: {sampling!r}") + + @torch.no_grad() + def generate( + self, + prefix_ids: Tensor, + *, + max_new_tokens: int, + eos_token_id: int | None = None, + sampling: str = "greedy", + temperature: float = 0.0, + top_p: float = 0.8, + posterior_threshold: float = 0.3, + posterior_alpha: float = 0.09, + generator: torch.Generator | None = None, + ) -> dict[str, Any]: + if max_new_tokens < 1: + raise ValueError("max_new_tokens must be positive") + device = getattr(self.verifier, "device", prefix_ids.device) + prefix = prefix_ids.reshape(1, -1).long().to(device) + generated = prefix + steps = 0 + accepted_tokens = 0 + while accepted_tokens < max_new_tokens: + acceptance = self.step( + generated, sampling=sampling, temperature=temperature, + top_p=top_p, posterior_threshold=posterior_threshold, + posterior_alpha=posterior_alpha, generator=generator) + new = acceptance.accepted_tokens + remaining = max_new_tokens - accepted_tokens + if new.numel() > remaining: + new = new[:remaining] + generated = torch.cat([generated, new.reshape(1, -1)], dim=1) + accepted_tokens += new.numel() + steps += 1 + if eos_token_id is not None and int(eos_token_id) in new.tolist(): + stop = generated[0].tolist().index(int(eos_token_id)) + generated = generated[:, :stop + 1] + accepted_tokens = generated.shape[1] - prefix.shape[1] + break + return { + "token_ids": generated[0], + "generated_tokens": accepted_tokens, + "verifier_calls": steps, + "acceptance_length": accepted_tokens / steps if steps else None, + } + + +class HFCausalVerifier: + """Reference verifier: batched teacher-forced paths through a Hugging Face causal LM. + + Exact for greedy acceptance. It re-runs the prefix on every step and makes no + throughput, KV-cache or serving claim. + """ + + def __init__( + self, + model: torch.nn.Module, + *, + device: Any = "cpu", + verify_batch_size: int = 16, + ): + if verify_batch_size < 1: + raise ValueError("verify_batch_size must be positive") + self.model = model.eval() + self.device = torch.device(device) + self.verify_batch_size = verify_batch_size + self.base = getattr(model, "model", None) + if self.base is None or not hasattr(self.base, "forward"): + raise ValueError("HFCausalVerifier requires a causal LM exposing model.model") + self.lm_head = getattr(model, "lm_head", None) + if self.lm_head is None: + raise ValueError("HFCausalVerifier requires a causal LM exposing lm_head") + + def _forward(self, input_ids: Tensor, attention_mask: Tensor) -> Tensor: + outputs = self.base( + input_ids=input_ids.to(self.device), + attention_mask=attention_mask.to(self.device), + use_cache=False, + ) + return outputs.last_hidden_state + + @torch.no_grad() + def prefill(self, prefix_ids: Tensor) -> tuple[Tensor, Tensor]: + prefix = prefix_ids.reshape(1, -1).long() + hidden = self._forward(prefix, torch.ones_like(prefix))[0, -1] + logits = self.lm_head(hidden) + return hidden, logits + + @torch.no_grad() + def verify(self, prefix_ids: Tensor, candidates: Tensor, lengths: Tensor) -> Tensor: + prefix = prefix_ids.reshape(-1).long() + paths = candidates.to(self.device) + lengths = lengths.to(self.device) + width = int(lengths.max()) + outputs = torch.empty( + (paths.shape[0], width - 1), dtype=torch.long, device=self.device) + for start in range(0, paths.shape[0], self.verify_batch_size): + stop = min(start + self.verify_batch_size, paths.shape[0]) + rows = paths[start:stop, :width] + mask = (torch.arange(width, device=self.device)[None, :] + < lengths[start:stop, None]) + sequences = torch.cat( + [prefix[None, :].expand(rows.shape[0], -1), + torch.where(mask, rows, torch.zeros_like(rows))], dim=1) + attention = torch.cat( + [torch.ones((rows.shape[0], prefix.numel()), dtype=torch.long, + device=self.device), mask.long()], dim=1) + hidden = self._forward(sequences, attention) + path_hidden = hidden[:, prefix.numel():prefix.numel() + width - 1] + next_tokens = self.lm_head(path_hidden).argmax(dim=-1) + outputs[start:stop] = next_tokens + return outputs + + @torch.no_grad() + def verify_logits( + self, prefix_ids: Tensor, candidates: Tensor, lengths: Tensor) -> Tensor: + """Return next-token logits at candidate prediction positions [paths, width-1, vocab].""" + prefix = prefix_ids.reshape(-1).long() + paths = candidates.to(self.device) + lengths = lengths.to(self.device) + width = int(lengths.max()) + vocab = getattr(self.lm_head, "out_features", None) + if vocab is None: + vocab = int(self.model.config.vocab_size) + outputs = torch.empty( + (paths.shape[0], width - 1, vocab), dtype=torch.float32, device=self.device) + for start in range(0, paths.shape[0], self.verify_batch_size): + stop = min(start + self.verify_batch_size, paths.shape[0]) + rows = paths[start:stop, :width] + mask = (torch.arange(width, device=self.device)[None, :] + < lengths[start:stop, None]) + sequences = torch.cat( + [prefix[None, :].expand(rows.shape[0], -1), + torch.where(mask, rows, torch.zeros_like(rows))], dim=1) + attention = torch.cat( + [torch.ones((rows.shape[0], prefix.numel()), dtype=torch.long, + device=self.device), mask.long()], dim=1) + hidden = self._forward(sequences, attention) + path_hidden = hidden[:, prefix.numel():prefix.numel() + width - 1] + outputs[start:stop] = self.lm_head(path_hidden).float() + return outputs + + @torch.no_grad() + def greedy_generate( + self, + prefix_ids: Tensor, + *, + max_new_tokens: int, + eos_token_id: int | None = None, + ) -> dict[str, Any]: + if max_new_tokens < 1: + raise ValueError("max_new_tokens must be positive") + generated = prefix_ids.reshape(1, -1).long().to(self.device) + steps = 0 + while steps < max_new_tokens: + _, logits = self.prefill(generated) + token = logits.argmax().reshape(1, 1) + generated = torch.cat([generated, token], dim=1) + steps += 1 + if eos_token_id is not None and int(token) == int(eos_token_id): + break + return { + "token_ids": generated[0], + "generated_tokens": steps, + "verifier_calls": steps, + "acceptance_length": 1.0, + } diff --git a/docs/support.md b/docs/support.md index 6ee9d35..4e5518c 100644 --- a/docs/support.md +++ b/docs/support.md @@ -13,7 +13,7 @@ and tests do not imply CPU training support. | EAGLE3 | Recurrent TTT, full/pruned vocabulary, GQA/MHA or materialized MLA; FA4 packing/CP | No EAGLE3 MoE, anchored mode or whole-model compile | | P-EAGLE core | Qwen3 full-vocabulary parallel model/loss, deterministic dense slots, eager/SDPA | Model/loss prerequisite only; no public trainer, COD sampler, resume, export or consumer validation; see [scope](user-guide/peagle.md) | | Domino | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts; training, `lambda_base` curriculum, held-out corrected/backbone observations and exact recovery with FlexAttention, CP1, padded/non-packing | No packed/CP>1, MoE, sparse attention or MTP conditioning; native export and consumer acceptance pending; see [scope](user-guide/domino.md) | -| Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery and native export; SDPA CP1, MBS1, non-packing, REPLICATE, one node | No packing/CP, warmstart or candidate-tree acceptance yet; see [scope](user-guide/medusa.md) | +| Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery and native export; greedy candidate-tree acceptance via a standalone Hugging Face evaluator; SDPA CP1, MBS1, non-packing, REPLICATE, one node | No packing/CP or warmstart; no SGLang Medusa integration or production-serving claim; see [scope](user-guide/medusa.md) | | EAGLE4 | Dense and selected upstream MoE; Draft/candidate stages; packing/CP and native assets | Candidate Top-K forward-KL is explicitly implemented from public semantics, not an unpublished official recipe; no serving integration | | DFlash2 | Experimental full/causal/SWA training with Flex, cuDNN THD or FA4 packing/CP | Experimental objective; no claim to reproduce an official training recipe | | JetSpec core | Qwen3-style causal parallel Draft head and temperature-scaled forward-KL objective, reused DFlash block machinery; chunked full-vocabulary KL and explicit causal visibility | Model/loss prerequisite only; no trainer selection, tree builder, recovery, export or consumer validation; CPU causal packing and direct unmasked Draft calls fail closed; see [scope](user-guide/jetspec.md) | diff --git a/docs/user-guide/algorithms.md b/docs/user-guide/algorithms.md index 40281c9..0b89914 100644 --- a/docs/user-guide/algorithms.md +++ b/docs/user-guide/algorithms.md @@ -165,5 +165,6 @@ the sum of per-head mean cross-entropies (assistant tokens only): each head's CE within a microbatch, and microbatches are weighted equally by gradient accumulation rather than pooled by token count. The heads own trainable full-vocabulary projections. Start from `configs/recipes/medusa_qwen3_8b.yaml` (SDPA, CP1, MBS1, non-packing, REPLICATE, Store/TCP). -Native export, strict reload and per-head held-out reporting are implemented; candidate-tree -acceptance remains future work; see [details](medusa.md). +Native export, strict reload and per-head held-out reporting are implemented, as is greedy and +typical/nucleus sampling candidate-tree acceptance through a standalone Hugging Face evaluator; +SGLang integration and production serving remain future work; see [details](medusa.md). diff --git a/docs/user-guide/medusa.md b/docs/user-guide/medusa.md index 4608b29..f47d791 100644 --- a/docs/user-guide/medusa.md +++ b/docs/user-guide/medusa.md @@ -81,5 +81,23 @@ The report records the artifact and dataset identities, per-head token counts, t mean CE and token accuracy, and the sum of the per-head means. These are held-out training observations, not candidate-tree acceptance. -Candidate-tree acceptance and decode-equality under a Medusa evaluator are not part of this -scope; they require a verified consumer adapter. +## Candidate-tree acceptance + +`aspec.inference.medusa_consumer` builds the reference Medusa tree (one Target bonus token plus +per-head top-k paths), verifies it, and accounts for accepted tokens. Greedy acceptance and +decode equality can be measured against a Hugging Face Target: + +```bash +python -m tools.evaluate_medusa_acceptance \ + --draft outputs/medusa-export --target Qwen/Qwen3-8B \ + --prompts /path/to/prompts.jsonl --output outputs/medusa-acceptance.json +``` + +The report records `acceptance_length = generated_tokens / verifier_calls` (includes the bonus +token), greedy decode equality against Target-only decoding, and repeat stability, together with +the artifact and tree-config hashes. The bundled verifier batches teacher-forced candidate paths +through the Target and re-runs the prefix on every step: it is a reference evaluator, not a +serving runtime, and it makes no throughput or SGLang-compatibility claim. `--sampling` accepts +`greedy` (default), `typical` (fast deterministic threshold) or `nucleus` (seeded top-p); +`decode_equal` is only reported for greedy, while sampling reports acceptance length and +repeat stability with the same seed. diff --git a/tests/cpu/test_medusa_consumer.py b/tests/cpu/test_medusa_consumer.py new file mode 100644 index 0000000..dbdebca --- /dev/null +++ b/tests/cpu/test_medusa_consumer.py @@ -0,0 +1,199 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Medusa candidate-tree consumer: buffers, proposals, verification and acceptance.""" + +import pytest +import torch +from torch import nn +from transformers import LlamaConfig, LlamaForCausalLM + +from aspec.inference.medusa_consumer import ( + HFCausalVerifier, + MedusaConsumer, + TreeProposal, + accept_greedy, + accept_nucleus, + accept_typical, + generate_tree_buffers, +) + + +def _tiny_model(): + torch.manual_seed(5) + config = LlamaConfig( + vocab_size=32, hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=1, max_position_embeddings=128, + pad_token_id=0, + ) + return LlamaForCausalLM(config).eval() + + +class _FixedDraft(nn.Module): + def __init__(self, heads, vocab): + super().__init__() + torch.manual_seed(3) + self.register_buffer("logits", torch.randn(heads, vocab)) + self.medusa_num_heads = heads + + def forward(self, hidden): + return self.logits[:, None, None, :].expand( + -1, hidden.shape[0], hidden.shape[1], -1) + + +def test_tree_buffers_follow_the_reference_layout(): + buffers = generate_tree_buffers([[0], [1], [0, 0]], top_k=2) + assert buffers.choices == ((0,), (1,), (0, 0)) + assert buffers.tree_indices.tolist() == [0, 1, 2, 3] + assert buffers.position_ids.tolist() == [0, 1, 1, 2] + # The reference emits one retrieval row per maximal path: (0,0) and (1,). + assert buffers.retrieve_indices.shape == (2, 3) + assert buffers.retrieve_indices[:, 0].tolist() == [0, 0] + # Tree position 3 is (0,0); it attends to its ancestor (0,) at position 1. + assert buffers.attention_mask[0, 0, 3, 1].item() is True + assert buffers.attention_mask[0, 0, 3, 2].item() is False + with pytest.raises(ValueError, match="prefix-closed"): + generate_tree_buffers([[0, 0]], top_k=2) + with pytest.raises(ValueError, match="in \\[0, 2\\)"): + generate_tree_buffers([[2]], top_k=2) + + +def test_greedy_acceptance_chooses_the_longest_path(): + proposal = TreeProposal( + bonus_token=torch.tensor(10), + candidates=torch.tensor([[10, 1, 2], [10, 3, 4], [11, 5, 0]]), + lengths=torch.tensor([3, 3, 2]), + medusa_logits=torch.zeros(2, 12), + ) + acceptance = accept_greedy(proposal, torch.tensor([[1, 2], [1, 9], [5, 8]])) + assert acceptance.path_index == 0 + assert acceptance.accept_length == 2 + assert acceptance.accepted_tokens.tolist() == [10, 1, 2] + + +def test_hf_verifier_matches_sequential_teacher_forcing(): + model = _tiny_model() + verifier = HFCausalVerifier(model, verify_batch_size=1) + prefix = torch.tensor([[1, 2, 3, 4, 5]]) + candidates = torch.tensor([[6, 7, 8], [9, 10, 0]]) + lengths = torch.tensor([3, 2]) + actual = verifier.verify(prefix, candidates, lengths) + for row, length in enumerate(lengths.tolist()): + sequence = torch.cat([prefix[0], candidates[row, :length]]) + with torch.no_grad(): + hidden = model.model(input_ids=sequence[None]).last_hidden_state + logits = model.lm_head(hidden[0]) + expected = [int(logits[prefix.shape[1] + index].argmax()) + for index in range(length - 1)] + assert actual[row, :len(expected)].tolist() == expected + + +def test_greedy_speculation_matches_target_and_is_stable(): + model = _tiny_model() + verifier = HFCausalVerifier(model) + draft = _FixedDraft(heads=2, vocab=32) + consumer = MedusaConsumer( + draft, verifier, medusa_choices=[[0], [1], [0, 0], [0, 1]], top_k=2) + prefix = torch.tensor([[1, 2, 3, 4]]) + speculative = consumer.generate(prefix, max_new_tokens=6) + baseline = verifier.greedy_generate(prefix, max_new_tokens=6) + assert speculative["token_ids"].tolist() == baseline["token_ids"].tolist() + assert speculative["verifier_calls"] >= 1 + assert speculative["acceptance_length"] >= 1.0 + repeat = consumer.generate(prefix, max_new_tokens=6) + assert repeat["token_ids"].tolist() == speculative["token_ids"].tolist() + +def test_default_choices_filter_to_the_draft_depth(): + from aspec.commands.evaluate_medusa_acceptance import _choices + + choices = _choices(None, 2) + assert choices and max(len(path) for path in choices) <= 2 + with pytest.raises(ValueError, match="head depth"): + _choices(None, 0) + + +def test_default_tree_matches_the_reference_buffers(): + buffers = generate_tree_buffers(top_k=10) + assert buffers.tree_indices.shape == (64,) + assert buffers.tree_indices[:11].tolist() == list(range(11)) + assert buffers.position_ids[:11].tolist() == [0] + [1] * 10 + assert buffers.retrieve_indices.shape == (42, 5) + assert int(buffers.attention_mask.sum()) == 207 + + +def test_eos_stops_both_paths_at_the_same_token(): + model = _tiny_model() + verifier = HFCausalVerifier(model) + draft = _FixedDraft(heads=2, vocab=32) + consumer = MedusaConsumer( + draft, verifier, medusa_choices=[[0], [1], [0, 0], [0, 1]], top_k=2) + prefix = torch.tensor([[1, 2, 3, 4]]) + probe = verifier.greedy_generate(prefix, max_new_tokens=6) + eos = int(probe["token_ids"][prefix.shape[1]]) + baseline = verifier.greedy_generate( + prefix, max_new_tokens=6, eos_token_id=eos) + speculative = consumer.generate(prefix, max_new_tokens=6, eos_token_id=eos) + assert speculative["token_ids"].tolist() == baseline["token_ids"].tolist() + assert speculative["token_ids"][-1].item() == eos + + + +def _single_path_proposal(): + return TreeProposal( + bonus_token=torch.tensor(10), + candidates=torch.tensor([[10, 1, 2]]), + lengths=torch.tensor([3]), + medusa_logits=torch.zeros(2, 5), + ) + + +def test_typical_acceptance_stops_at_the_first_low_probability_token(): + logits = torch.zeros(1, 2, 5) + logits[0, 0, 1] = 5.0 + logits[0, 1, 3] = 5.0 + acceptance = accept_typical( + _single_path_proposal(), logits, temperature=1.0, + posterior_threshold=0.3, posterior_alpha=0.09) + assert acceptance.accept_length == 1 + assert acceptance.accepted_tokens.tolist() == [10, 1] + + +def test_nucleus_acceptance_is_seeded_and_masks_mismatches(): + logits = torch.zeros(1, 2, 5) + logits[0, 0, 1] = 5.0 + logits[0, 1, 3] = 5.0 + first = accept_nucleus( + _single_path_proposal(), logits, temperature=1.0, top_p=1.0, + generator=torch.Generator().manual_seed(7)) + second = accept_nucleus( + _single_path_proposal(), logits, temperature=1.0, top_p=1.0, + generator=torch.Generator().manual_seed(7)) + assert first.accept_length == 1 + assert first.accepted_tokens.tolist() == [10, 1] + assert second.accepted_tokens.tolist() == first.accepted_tokens.tolist() + + +def test_verify_logits_argmax_matches_verify(): + model = _tiny_model() + verifier = HFCausalVerifier(model) + prefix = torch.tensor([[1, 2, 3, 4]]) + candidates = torch.tensor([[6, 7, 8], [9, 10, 0]]) + lengths = torch.tensor([3, 2]) + logits = verifier.verify_logits(prefix, candidates, lengths) + assert logits.shape == (2, 2, 32) + assert torch.equal(logits.argmax(-1), verifier.verify(prefix, candidates, lengths)) + + +def test_sampling_generation_is_repeatable(): + model = _tiny_model() + verifier = HFCausalVerifier(model) + draft = _FixedDraft(heads=2, vocab=32) + consumer = MedusaConsumer( + draft, verifier, medusa_choices=[[0], [1], [0, 0], [0, 1]], top_k=2) + prefix = torch.tensor([[1, 2, 3, 4]]) + first = consumer.generate( + prefix, max_new_tokens=3, sampling="typical", temperature=1.0) + second = consumer.generate( + prefix, max_new_tokens=3, sampling="typical", temperature=1.0) + assert first["token_ids"].tolist() == second["token_ids"].tolist() + assert first["verifier_calls"] >= 1 diff --git a/tools/evaluate_medusa_acceptance.py b/tools/evaluate_medusa_acceptance.py new file mode 100644 index 0000000..886bf61 --- /dev/null +++ b/tools/evaluate_medusa_acceptance.py @@ -0,0 +1,13 @@ +# Copyright 2026 Ant Group Co., Ltd. +# SPDX-License-Identifier: Apache-2.0 + +"""Compatibility entry point for the installed evaluate_medusa_acceptance command.""" + +import sys +from importlib import import_module + +_command = import_module("aspec.commands.evaluate_medusa_acceptance") +if __name__ == "__main__": + _command.main() +else: + sys.modules[__name__] = _command