[Spec Decode] Support hybrid (SWA + full attention) DFlash drafters (#47914)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-07-08 11:12:45 -04:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 68b4a1d582
commit 0d12618e98
9 changed files with 79 additions and 30 deletions
+17 -9
View File
@@ -86,14 +86,15 @@ def _resolve_layer_attention(
if layer_types is not None:
num_sliding = sum(lt == SLIDING_ATTENTION for lt in layer_types)
any_sliding = num_sliding > 0
all_sliding = num_sliding == len(layer_types)
if any_sliding and not all_sliding:
# Mixed sliding/full attention needs per-layer causal metadata and
# multiple KV-cache groups, which DFlash does not yet support.
# Mixed sliding/full attention needs multiple KV groups (V2 runner only).
if (
0 < num_sliding < len(layer_types)
and not get_current_vllm_config().use_v2_model_runner
):
raise NotImplementedError(
"DFlash does not yet support mixed sliding/full attention via "
"layer_types; see "
"https://github.com/vllm-project/vllm/issues/40898."
"DFlash drafters with mixed sliding/full attention require "
"the V2 model runner; relaunch with "
"VLLM_USE_V2_MODEL_RUNNER=1."
)
default_causal = False
@@ -207,8 +208,7 @@ class DFlashQwen3Attention(nn.Module):
attn_type=attn_type,
sinks=self.attention_sink_bias,
)
# NOTE: `causal` is currently unused here, but will be needed in the future
# to support models with different causality per-layer.
self.causal = causal
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
@@ -709,6 +709,14 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
) -> torch.Tensor:
return self.model(input_ids, positions, inputs_embeds)
def get_draft_kv_cache_layer_names(self) -> list[str]:
return [layer.self_attn.attn.layer_name for layer in self.model.layers]
def get_draft_attn_causal(self) -> list[bool]:
"""Per-layer attention causality, aligned with
get_draft_kv_cache_layer_names."""
return [layer.self_attn.causal for layer in self.model.layers]
def compute_logits(
self,
hidden_states: torch.Tensor,