Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions aspec/commands/convert_to_hf.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,12 @@ def _serving_config(raw_config: dict) -> dict:

if raw_config.get("mtp_conditioning"):
raise NotImplementedError("MTP-conditioned Drafts require native --format aspec export")
if raw_config.get("architectures") == ["DominoDraftModel"]:
raise NotImplementedError(
"Domino SGLang export is not validated in ASpec yet; use --format aspec. "
"Upstream SGLang 0.5.21 includes a Domino loader and an end-to-end test, "
"but ASpec has not validated its exported artifact against it"
)

if raw_config.get("architectures") == ["MedusaDraftModel"]:
raise NotImplementedError(
Expand Down Expand Up @@ -259,7 +265,7 @@ def _serving_config(raw_config: dict) -> dict:
def _checkpoint_training_contract(model_dir: Path, raw_config: dict) -> dict:
"""Recover authoritative training identity from the checkpoint/run artifacts."""
algorithm = resolve_draft_algorithm(raw_config)
family = "DSpark" if algorithm == "dspark" else "DFlash"
family = "DSpark" if algorithm == "dspark" else "Domino" if algorithm == "domino" else "DFlash"
if algorithm == "dflash2":
raw_config = AutoDraftModelConfig.from_dict(raw_config).to_dict()
contract = {"algorithm": algorithm}
Expand All @@ -271,7 +277,7 @@ def _checkpoint_training_contract(model_dir: Path, raw_config: dict) -> dict:
raise ValueError("Export algorithm conflicts with the checkpoint algorithm")
if topology.get("training_objective") is not None:
contract["objective"] = topology["training_objective"]
if algorithm in {"dspark", "dflash", "dflash2"} and "block_size" in contract["objective"]:
if algorithm in {"dspark", "dflash", "dflash2", "domino"} and "block_size" in contract["objective"]:
contract["block_size"] = contract["objective"]["block_size"]
if algorithm == "eagle4":
from aspec.config.eagle4 import eagle4_training_policy
Expand All @@ -294,7 +300,7 @@ def _checkpoint_training_contract(model_dir: Path, raw_config: dict) -> dict:
# A real run's snapshot records block size even for legacy checkpoints whose
# standalone model config did not persist the training-only field.
checkpoint_dir = model_dir.parent
if algorithm in {"dspark", "dflash", "dflash2"} and checkpoint_dir.parent.name == "checkpoints":
if algorithm in {"dspark", "dflash", "dflash2", "domino"} and checkpoint_dir.parent.name == "checkpoints":
snapshot = checkpoint_dir.parent.parent / "config.yaml"
if snapshot.is_file():
from omegaconf import OmegaConf
Expand All @@ -309,7 +315,7 @@ def _checkpoint_training_contract(model_dir: Path, raw_config: dict) -> dict:
if declared is not None and declared != size:
raise ValueError("Export block_size conflicts with the training snapshot")
contract["block_size"] = size
if algorithm in {"dspark", "dflash", "dflash2"}:
if algorithm in {"dspark", "dflash", "dflash2", "domino"}:
size = contract.get("block_size", raw_config.get("block_size"))
if size is not None:
if type(size) is not int or size < 2:
Expand Down Expand Up @@ -379,6 +385,11 @@ def convert_checkpoint(
raise ValueError("Export output directory must not overlap the source checkpoint or config")

raw_config = json.loads(config_path.read_text(encoding="utf-8"))
nested = raw_config.get("dflash_config")
if isinstance(nested, dict) and nested.get("projector_type") == "domino":
from aspec.models.draft.domino import normalize_official_domino_config

raw_config = normalize_official_domino_config(raw_config)
if artifact_format not in {"sglang", "aspec"}:
raise ValueError("artifact_format must be sglang or aspec")
if verify and artifact_format != "aspec":
Expand Down Expand Up @@ -505,8 +516,10 @@ def convert_checkpoint(
manifest["draft_structure"]["attention"]["dsa"] = policy["dsa"]
algorithm = training_contract["algorithm"]
block_proposal = algorithm in {"dspark", "dflash"} and not raw_config.get("mtp_conditioning")
domino_proposal = algorithm == "domino" and not raw_config.get("mtp_conditioning")
mla_proposal = raw_config.get("architectures") == ["Eagle3DeepseekV2ForCausalLM"]
proposal = ("aspec.draft_artifact.MLAEagleConsumer" if mla_proposal else
"aspec.draft_artifact.DominoConsumer" if domino_proposal else
"aspec.draft_artifact.BlockDraftConsumer" if block_proposal else None)
manifest["draft_structure"]["loader_contract"] = {
"runtime": "aspec", "entrypoint": "aspec.draft_artifact.load_native_draft",
Expand All @@ -518,7 +531,7 @@ def convert_checkpoint(
"heads-only asset; strict reload and per-head held-out metrics; "
"no candidate-tree acceptance" if algorithm == "medusa" else
"asset loading and weights-only warmstart; no native proposal component"),
"borrowed_tensors": ["Target output head"] if algorithm in {"dspark", "dflash"} else [],
"borrowed_tensors": (["Target output head"] if algorithm in {"dspark", "dflash", "domino"} else []),
}
(output_dir / "export_manifest.json").write_text(
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
Expand Down
93 changes: 93 additions & 0 deletions aspec/draft_artifact.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,99 @@ def propose(self, input_ids, target_features, *, lm_head_weight):
return result


class DominoConsumer:
"""One Domino block from verifier features and its frozen output head.

The backbone stays block-parallel. Only the suffix is rolled out
sequentially through the GRU-conditioned correction head, matching the
published Domino inference path. The caller owns verification/acceptance.
"""

def __init__(self, draft_model, *, block_size):
from aspec.models.dflash import DFlashModel

self.model = draft_model.eval()
self.algorithm = resolve_draft_algorithm(draft_model.config.to_dict())
if self.algorithm != "domino":
raise ValueError("DominoConsumer requires a Domino asset")
self.config = draft_model.config
if type(block_size) is not int or block_size < 2:
raise ValueError("block_size must be an integer >= 2")
if self.config.block_size != block_size:
raise ValueError("Proposal block size disagrees with the trained asset")
self.block_size = block_size
self.shift_label = bool(self.config.shift_label)
self.suffix_start = int(self.config.pure_draft_prefix_len)
if self.shift_label:
self.k_draft = block_size
else:
self.k_draft = block_size - 1
if not 0 <= self.suffix_start < self.k_draft:
raise ValueError("Domino prefix leaves no corrected position in the block")
backend = "fa4" if next(draft_model.parameters()).device.type == "cuda" else "sdpa"
self.backbone = DFlashModel(draft_model, block_size=block_size, num_anchors=1,

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we use the same CPU attention mask here that Domino uses during training? The plain DFlashModel wrapper skips the dense mask for this block on CPU, so the draft can attend to the anchor's target feature and the zero-padded rows. Those rows still count in the softmax. The test builds its reference with this same backbone, so it would agree either way. One small check I'd love to see for both label layouts: keep the token IDs fixed, change only the anchor's target feature, and check that the backbone logits stay the same.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in a4f9b1a: DominoConsumer now enables _materialize_cpu_block_mask and uses the same dense block mask as training. Added a regression for both label layouts that changes only the anchor target feature and asserts the backbone hidden states/logits stay identical.

attention_backend=backend)
# Match Domino training on CPU: hide context rows at/after the anchor and
# other blocks with the same dense block mask instead of plain SDPA.
self.backbone._materialize_cpu_block_mask = True

@torch.no_grad()
def propose(self, input_ids, target_features, *, lm_head_weight):
"""IDs/features describe one known prefix, including its last anchor token."""
if input_ids.ndim != 2 or input_ids.shape[0] != 1 or input_ids.shape[1] < 1:
raise ValueError("Block proposal currently consumes one nonempty prefix")
length = input_ids.shape[1]
config = self.model.config
width = config.target_hidden_size * config.num_target_layers
if target_features.shape != (1, length, width):
raise ValueError("Target features must match the asset's ordered layers and prefix")
if lm_head_weight.shape != (config.vocab_size, config.hidden_size):
raise ValueError("Frozen Target output head shape does not match the Draft")
if input_ids.device != target_features.device or input_ids.device != lm_head_weight.device:
raise ValueError("Proposal IDs, features and output head must share a device")
if length + self.block_size - 1 > config.max_position_embeddings:
raise ValueError("Proposal exceeds the artifact's supported position range")

anchor = torch.tensor([[length - 1]], device=input_ids.device)
padded = F.pad(input_ids, (0, self.block_size), value=config.mask_token_id)
mask = torch.ones((1, length + self.block_size), device=input_ids.device)
if self.backbone.attention_backend == "fa4":
features = target_features
packed_positions = torch.arange(length, device=input_ids.device)[None]
else:
# The non-direct-CP context path expects one feature row per context
# token. Anchor visibility excludes these padded future rows.
features = F.pad(target_features, (0, 0, 0, self.block_size))
packed_positions = None
hidden = self.backbone._draft_backbone(
padded, features, mask,
packed_context_positions=packed_positions,
injected_anchors=anchor, injected_keep_mask=torch.ones_like(anchor, dtype=torch.bool),
)[0][0].unsqueeze(0)
if not self.shift_label:
hidden = hidden[:, 1:, :]
base_logits = F.linear(hidden, lm_head_weight)

prefix_ids = base_logits[:, : self.suffix_start].argmax(-1)
realized = torch.cat((input_ids[:, -1:], prefix_ids), dim=1)
prefix_embeds = self.model.embed_tokens(realized).to(self.model.prefix_gru.weight_ih_l0.dtype)
_, gru_hidden = self.model.prefix_gru(prefix_embeds)

tokens = [prefix_ids]
for position in range(self.suffix_start, self.k_draft):
z = hidden[:, position : position + 1, :]
state = gru_hidden.transpose(0, 1)
bias = self.model.embed_proj(torch.cat((z, state), dim=-1))
current = (base_logits[:, position : position + 1, :] + bias).argmax(-1)
tokens.append(current)
if position + 1 < self.k_draft:
new_embed = self.model.embed_tokens(current).to(
self.model.prefix_gru.weight_ih_l0.dtype
)
_, gru_hidden = self.model.prefix_gru(new_embed, gru_hidden)
return {"token_ids": torch.cat(tokens, dim=1), "algorithm": "domino"}


class MLAEagleConsumer:
"""Greedy recurrent MLA proposals with an owner-local materialized KV cache.

Expand Down
6 changes: 6 additions & 0 deletions aspec/models/draft/auto.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,12 @@ class AutoDraftModelConfig:
def from_dict(cls, config: dict):
config = dict(config)

nested = config.get("dflash_config")
if isinstance(nested, dict) and nested.get("projector_type") == "domino":
from aspec.models.draft.domino import normalize_official_domino_config

config = normalize_official_domino_config(config)

if "tie_word_embeddings" in config and config.get("architectures") != ["MedusaDraftModel"]:
logger.info("Set draft model tie_word_embeddings to False")
config["tie_word_embeddings"] = False
Expand Down
74 changes: 74 additions & 0 deletions aspec/models/draft/domino.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,3 +177,77 @@ def correct_logits(
dim=2,
)
return corrected.reshape(batch, draft_len, logits.size(-1))


def normalize_official_domino_config(raw: dict) -> dict:
"""Translate the published Domino config layout into ASpec's flat layout.

The reference checkpoint publishes ``architectures=["DFlashDraftModel"]``,
``model_type="qwen3"`` and nests the Domino fields under ``dflash_config``.
In that layout ``num_target_layers`` is the Target decoder depth, while
ASpec's flat config uses it as the number of captured feature layers.
"""
raw = dict(raw)
nested = raw.pop("dflash_config", None)
if not isinstance(nested, dict) or nested.get("projector_type") != "domino":
raise ValueError("Not an official Domino draft config")
layer_ids = nested.get("target_layer_ids")
if not isinstance(layer_ids, list) or not layer_ids:
raise ValueError("Official Domino config requires target_layer_ids")
target_depth = raw.get("num_target_layers")
if isinstance(target_depth, bool) or not isinstance(target_depth, int) or target_depth < 1:
raise ValueError("Official Domino config requires the Target decoder depth")
raw["architectures"] = ["DominoDraftModel"]
raw["model_type"] = "domino"
raw["projector_type"] = "domino"
for key in (
"mask_token_id",
"shift_label",
"target_layer_ids",
"pure_draft_prefix_len",
"gru_hidden_dim",
"emb_dim",
):
if key in nested:
raw[key] = nested[key]
raw["num_target_layers"] = len(layer_ids)
raw["target_num_hidden_layers"] = target_depth
raw.pop("auto_map", None)
raw.pop("max_window_layers", None)
return raw


def official_domino_config(raw: dict) -> dict:
"""Translate ASpec's flat Domino config into the published HF layout."""
from aspec.config.draft_algorithm import resolve_draft_algorithm

raw = dict(raw)
if resolve_draft_algorithm(raw) != "domino":
raise ValueError("Not an ASpec Domino draft config")
target_depth = raw.get("target_num_hidden_layers")
layer_ids = raw.get("target_layer_ids")
if isinstance(target_depth, bool) or not isinstance(target_depth, int) or target_depth < 1:
raise ValueError("Domino export requires the Target decoder depth")
if not isinstance(layer_ids, list) or not layer_ids:
raise ValueError("Domino export requires target_layer_ids")
nested = {
key: raw[key]
for key in (
"mask_token_id",
"shift_label",
"target_layer_ids",
"pure_draft_prefix_len",
"gru_hidden_dim",
"emb_dim",
)
if key in raw
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}
# DominoConfig defaults the flat field; the published layout must carry the
# discriminator explicitly so the normalizer can recognize it.
nested["projector_type"] = "domino"
raw["architectures"] = ["DFlashDraftModel"]
raw["model_type"] = "qwen3"
raw["num_target_layers"] = target_depth
raw["dflash_config"] = nested
raw.pop("auto_map", None)
return raw
93 changes: 93 additions & 0 deletions configs/recipes/domino_qwen3_8b.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
# Formal single-node Domino recipe: training, lambda_base curriculum, held-out
# backbone/corrected observations and exact recovery. Hyperparameters are a
# starting point, not a convergence or acceptance claim.
model:
target_model_path: ${oc.env:ASPEC_TARGET_MODEL,Qwen/Qwen3-8B}
native_context_length: 32768
trust_remote_code: false
draft_model_config: aspec/config/domino_qwen3_8b_draft_config.json
dataset:
eval_interval: 100
eval_micro_batch_size: 1
train_data_path: ${oc.env:ASPEC_TRAIN_DATA}
eval_data_path: ${oc.env:ASPEC_EVAL_DATA}
chat_template: qwen3-instruct
prompt_key: conversations
min_loss_tokens: 32
shuffle_dataset: true
dataset_seed: 42
training:
attention_backend: flex_attention
draft_context_parallel_size: 1
micro_batch_size: 1
draft_accumulation_steps: 2
learning_rate: 6e-4
min_lr: 6e-5
weight_decay: 0.0
max_grad_norm: 1.0
max_seq_length: 4096
num_epochs: 3
warmup_ratio: 0.04
seed: 42
continual_training: false
deterministic_anchor_sampling: false
training_num_gpus_per_node: 2
training_num_nodes: 1
placement_strategy: inference_first
fsdp_strategy: REPLICATE
fsdp_reduce_dtype: bfloat16
compile_model: false
max_concurrent_batches: 1
streaming_dispatch_with_training: true
pipeline_dispatch_with_training: true
prefetch_depth: 1
save_interval: 100
save_per_epoch: false
save_final_checkpoint: true
max_checkpoints: 2
dflash_block_size: 16
dflash_packing: false
dflash_num_target_layers: 5
dflash_num_anchors: 128
dflash_loss_objective: decay
dflash_loss_decay_gamma: 7.0
dflash_ce_loss_alpha: 1.0
dflash_l1_loss_alpha: 0.0
domino_lambda_base: 0.3
domino_lambda_base_start: 1.0
domino_lambda_base_decay_ratio: 1.0
domino_eval_backbone: true
inference:
inference_engine_type: sgl
store_last_hidden_states: false
target_prefill_memory_optimized: false
inference_num_gpus: 1
inference_num_gpus_per_engine: 1
inference_num_gpus_per_node: 3
inference_fetch_batch: 8
inference_buffer_threshold: 2
max_sample_pool_size: 12
inference_batch_size: 2
sglang:
tp_size: 1
mem_fraction_static: 0.5
mooncake:
master_server_address: null
metadata_server: null
data_plane: store
protocol: tcp
enable_gpu_direct: false
allow_host_fallback: false
global_segment_size: 32GB
local_buffer_size: 8GB
host_buffer_size: 1GB
async_put_pool_size: 1
enable_hard_pin: true
max_seq_len: 4096
hidden_dim: 4096
logging:
report_to: none
debug:
enable_perf_metrics: true
output_dir: outputs/domino-qwen3-8b
cache_dir: cache/domino-qwen3-8b
2 changes: 1 addition & 1 deletion docs/support.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ and tests do not imply CPU training support.
| DSpark / DFlash | Training, exact recovery, evaluation and export; Dense and selected MoE/attention structures | Objectives, feature bindings and assets remain algorithm-specific |
| EAGLE3 | Recurrent TTT, full/pruned vocabulary, GQA/MHA or materialized MLA; FA4 packing/CP | No EAGLE3 MoE, anchored mode or whole-model compile |
| P-EAGLE core | Qwen3 full-vocabulary parallel model/loss, deterministic dense slots, eager/SDPA | Model/loss prerequisite only; no public trainer, COD sampler, resume, export or consumer validation; see [scope](user-guide/peagle.md) |
| Domino | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts; training, `lambda_base` curriculum, held-out corrected/backbone observations and exact recovery with FlexAttention, CP1, padded/non-packing | No packed/CP>1, MoE, sparse attention or MTP conditioning; native export and consumer acceptance pending; see [scope](user-guide/domino.md) |
| Domino | DFlash block backbone plus GRU causal correction head, mixed backbone/corrected CE, both documented label layouts; training, `lambda_base` curriculum, held-out corrected/backbone observations and exact recovery with FlexAttention, CP1, padded/non-packing | No packed/CP>1, MoE, sparse attention or MTP conditioning; native export, strict reload, proposal consumer and a bounded SpecForge comparison are validated; longer convergence/decoding acceptance pending; see [scope](user-guide/domino.md) |
| Medusa heads-only | Parallel full-vocabulary future-token heads on the frozen Target post-norm state; training, exact recovery and native export; SDPA CP1, MBS1, non-packing, REPLICATE, one node | No packing/CP, warmstart or candidate-tree acceptance yet; see [scope](user-guide/medusa.md) |
| EAGLE4 | Dense and selected upstream MoE; Draft/candidate stages; packing/CP and native assets | Candidate Top-K forward-KL is explicitly implemented from public semantics, not an unpublished official recipe; no serving integration |
| DFlash2 | Experimental full/causal/SWA training with Flex, cuDNN THD or FA4 packing/CP | Experimental objective; no claim to reproduce an official training recipe |
Expand Down
Loading