[Bugfix][Spec Decode] Take the DFlash draft's RoPE layout from its own config (#54373)
Signed-off-by: SubSir <tiancaizhangdaxian@sjtu.edu.cn> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
24d42f3553
commit
d61b6e1878
@@ -75,26 +75,6 @@ def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def dflash_target_rope_is_neox_style(target_model: nn.Module) -> bool | None:
|
||||
"""The target's RoPE layout, from its first attention layer.
|
||||
|
||||
A DFlash head must rotate Q/K the way the target it was distilled against
|
||||
does, and a mismatch is silent — acceptance collapses but nothing errors and
|
||||
the output stays correct. Draft checkpoints do not carry this, so take it
|
||||
from the target. None if the target uses no RoPE.
|
||||
"""
|
||||
language_model = (
|
||||
target_model.get_language_model()
|
||||
if hasattr(target_model, "get_language_model")
|
||||
else target_model
|
||||
)
|
||||
for module in language_model.modules():
|
||||
style = getattr(module, "is_neox_style", None)
|
||||
if isinstance(style, bool):
|
||||
return style
|
||||
return None
|
||||
|
||||
|
||||
def _get_dflash_fc_input_size(vllm_config: VllmConfig) -> int:
|
||||
spec_config = vllm_config.speculative_config
|
||||
config = spec_config.draft_model_config.hf_config
|
||||
@@ -316,11 +296,11 @@ class DFlashQwen3DecoderLayer(nn.Module):
|
||||
# non-causal) from the draft config.
|
||||
sliding_window, causal = _resolve_layer_attention(config, layer_idx)
|
||||
|
||||
# RoPE layout, copied off the target at load time by the draft loader
|
||||
# (see `dflash_target_rope_is_neox_style`). Checkpoints do not carry it:
|
||||
# a head distilled from an interleaved-RoPE target must rotate the way
|
||||
# that target does, or every drafted Q/K is wrong and acceptance
|
||||
# collapses with no error raised.
|
||||
# RoPE layout. The rotation applies to the draft's own Q/K, so this is
|
||||
# fixed by how the head was distilled, not by the target: a neox-trained
|
||||
# head on an interleaved target still needs neox. A mismatch is silent --
|
||||
# acceptance collapses and nothing errors -- so a checkpoint that was
|
||||
# distilled the other way has to say so here.
|
||||
is_neox_style = getattr(config, "is_neox_style", True)
|
||||
|
||||
self.self_attn = DFlashQwen3Attention(
|
||||
|
||||
@@ -80,21 +80,6 @@ class DFlashProposer(SpecDecodeBaseProposer):
|
||||
self.draft_model_config.hf_config
|
||||
)
|
||||
|
||||
@override
|
||||
def load_model(self, target_model: torch.nn.Module) -> None:
|
||||
from vllm.model_executor.models.qwen3_dflash import (
|
||||
dflash_target_rope_is_neox_style,
|
||||
)
|
||||
|
||||
draft_model_config = self.speculative_config.draft_model_config
|
||||
assert draft_model_config is not None
|
||||
# The drafter must rotate Q/K the way its target does. Take that from the
|
||||
# built target before super() constructs the draft.
|
||||
is_neox_style = dflash_target_rope_is_neox_style(target_model)
|
||||
if is_neox_style is not None:
|
||||
draft_model_config.hf_config.is_neox_style = is_neox_style
|
||||
super().load_model(target_model)
|
||||
|
||||
@override
|
||||
def _create_draft_vllm_config(self) -> VllmConfig:
|
||||
base = super()._create_draft_vllm_config()
|
||||
|
||||
@@ -15,17 +15,11 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
from vllm.model_executor.models.qwen3_dflash import (
|
||||
dflash_has_any_non_causal,
|
||||
dflash_target_rope_is_neox_style,
|
||||
)
|
||||
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
# The drafter must rotate Q/K the way its target does. Take that from the
|
||||
# built target before super() constructs the draft.
|
||||
is_neox_style = dflash_target_rope_is_neox_style(target_model)
|
||||
if is_neox_style is not None:
|
||||
draft_model_config.hf_config.is_neox_style = is_neox_style
|
||||
# Select an attention backend that supports the drafter's attention: mixing
|
||||
# a non-causal layer onto a causal-only backend would fail.
|
||||
draft_vllm_config = replace(
|
||||
|
||||
Reference in New Issue
Block a user