[Bugfix][Spec Decode] Take the DFlash draft's RoPE layout from its own config (#54373)

Signed-off-by: SubSir <tiancaizhangdaxian@sjtu.edu.cn>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Zihan Zhang
2026-08-31 18:13:20 -04:00
committed by GitHub
co-authored by Claude Opus 5
parent 24d42f3553
commit d61b6e1878
3 changed files with 5 additions and 46 deletions
+5 -25
View File
@@ -75,26 +75,6 @@ def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
)
def dflash_target_rope_is_neox_style(target_model: nn.Module) -> bool | None:
"""The target's RoPE layout, from its first attention layer.
A DFlash head must rotate Q/K the way the target it was distilled against
does, and a mismatch is silent — acceptance collapses but nothing errors and
the output stays correct. Draft checkpoints do not carry this, so take it
from the target. None if the target uses no RoPE.
"""
language_model = (
target_model.get_language_model()
if hasattr(target_model, "get_language_model")
else target_model
)
for module in language_model.modules():
style = getattr(module, "is_neox_style", None)
if isinstance(style, bool):
return style
return None
def _get_dflash_fc_input_size(vllm_config: VllmConfig) -> int:
spec_config = vllm_config.speculative_config
config = spec_config.draft_model_config.hf_config
@@ -316,11 +296,11 @@ class DFlashQwen3DecoderLayer(nn.Module):
# non-causal) from the draft config.
sliding_window, causal = _resolve_layer_attention(config, layer_idx)
# RoPE layout, copied off the target at load time by the draft loader
# (see `dflash_target_rope_is_neox_style`). Checkpoints do not carry it:
# a head distilled from an interleaved-RoPE target must rotate the way
# that target does, or every drafted Q/K is wrong and acceptance
# collapses with no error raised.
# RoPE layout. The rotation applies to the draft's own Q/K, so this is
# fixed by how the head was distilled, not by the target: a neox-trained
# head on an interleaved target still needs neox. A mismatch is silent --
# acceptance collapses and nothing errors -- so a checkpoint that was
# distilled the other way has to say so here.
is_neox_style = getattr(config, "is_neox_style", True)
self.self_attn = DFlashQwen3Attention(
-15
View File
@@ -80,21 +80,6 @@ class DFlashProposer(SpecDecodeBaseProposer):
self.draft_model_config.hf_config
)
@override
def load_model(self, target_model: torch.nn.Module) -> None:
from vllm.model_executor.models.qwen3_dflash import (
dflash_target_rope_is_neox_style,
)
draft_model_config = self.speculative_config.draft_model_config
assert draft_model_config is not None
# The drafter must rotate Q/K the way its target does. Take that from the
# built target before super() constructs the draft.
is_neox_style = dflash_target_rope_is_neox_style(target_model)
if is_neox_style is not None:
draft_model_config.hf_config.is_neox_style = is_neox_style
super().load_model(target_model)
@override
def _create_draft_vllm_config(self) -> VllmConfig:
base = super()._create_draft_vllm_config()
@@ -15,17 +15,11 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
from vllm.compilation.backends import set_model_tag
from vllm.model_executor.models.qwen3_dflash import (
dflash_has_any_non_causal,
dflash_target_rope_is_neox_style,
)
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
draft_model_config = speculative_config.draft_model_config
# The drafter must rotate Q/K the way its target does. Take that from the
# built target before super() constructs the draft.
is_neox_style = dflash_target_rope_is_neox_style(target_model)
if is_neox_style is not None:
draft_model_config.hf_config.is_neox_style = is_neox_style
# Select an attention backend that supports the drafter's attention: mixing
# a non-causal layer onto a causal-only backend would fail.
draft_vllm_config = replace(