[Bugfix][Spec Decode] Take the DFlash draft's RoPE layout from its own config (#54373)

Signed-off-by: SubSir <tiancaizhangdaxian@sjtu.edu.cn>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Zihan Zhang
2026-08-31 18:13:20 -04:00
committed by GitHub
co-authored by Claude Opus 5
parent 24d42f3553
commit d61b6e1878
3 changed files with 5 additions and 46 deletions
+5 -25
View File
@@ -75,26 +75,6 @@ def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
)
def dflash_target_rope_is_neox_style(target_model: nn.Module) -> bool | None:
"""The target's RoPE layout, from its first attention layer.
A DFlash head must rotate Q/K the way the target it was distilled against
does, and a mismatch is silent — acceptance collapses but nothing errors and
the output stays correct. Draft checkpoints do not carry this, so take it
from the target. None if the target uses no RoPE.
"""
language_model = (
target_model.get_language_model()
if hasattr(target_model, "get_language_model")
else target_model
)
for module in language_model.modules():
style = getattr(module, "is_neox_style", None)
if isinstance(style, bool):
return style
return None
def _get_dflash_fc_input_size(vllm_config: VllmConfig) -> int:
spec_config = vllm_config.speculative_config
config = spec_config.draft_model_config.hf_config
@@ -316,11 +296,11 @@ class DFlashQwen3DecoderLayer(nn.Module):
# non-causal) from the draft config.
sliding_window, causal = _resolve_layer_attention(config, layer_idx)
# RoPE layout, copied off the target at load time by the draft loader
# (see `dflash_target_rope_is_neox_style`). Checkpoints do not carry it:
# a head distilled from an interleaved-RoPE target must rotate the way
# that target does, or every drafted Q/K is wrong and acceptance
# collapses with no error raised.
# RoPE layout. The rotation applies to the draft's own Q/K, so this is
# fixed by how the head was distilled, not by the target: a neox-trained
# head on an interleaved target still needs neox. A mismatch is silent --
# acceptance collapses and nothing errors -- so a checkpoint that was
# distilled the other way has to say so here.
is_neox_style = getattr(config, "is_neox_style", True)
self.self_attn = DFlashQwen3Attention(