[Spec Decode] Support hybrid (SWA + full attention) DFlash drafters (#47914)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
68b4a1d582
commit
0d12618e98
@@ -86,14 +86,15 @@ def _resolve_layer_attention(
|
||||
if layer_types is not None:
|
||||
num_sliding = sum(lt == SLIDING_ATTENTION for lt in layer_types)
|
||||
any_sliding = num_sliding > 0
|
||||
all_sliding = num_sliding == len(layer_types)
|
||||
if any_sliding and not all_sliding:
|
||||
# Mixed sliding/full attention needs per-layer causal metadata and
|
||||
# multiple KV-cache groups, which DFlash does not yet support.
|
||||
# Mixed sliding/full attention needs multiple KV groups (V2 runner only).
|
||||
if (
|
||||
0 < num_sliding < len(layer_types)
|
||||
and not get_current_vllm_config().use_v2_model_runner
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"DFlash does not yet support mixed sliding/full attention via "
|
||||
"layer_types; see "
|
||||
"https://github.com/vllm-project/vllm/issues/40898."
|
||||
"DFlash drafters with mixed sliding/full attention require "
|
||||
"the V2 model runner; relaunch with "
|
||||
"VLLM_USE_V2_MODEL_RUNNER=1."
|
||||
)
|
||||
|
||||
default_causal = False
|
||||
@@ -207,8 +208,7 @@ class DFlashQwen3Attention(nn.Module):
|
||||
attn_type=attn_type,
|
||||
sinks=self.attention_sink_bias,
|
||||
)
|
||||
# NOTE: `causal` is currently unused here, but will be needed in the future
|
||||
# to support models with different causality per-layer.
|
||||
self.causal = causal
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
|
||||
@@ -709,6 +709,14 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
) -> torch.Tensor:
|
||||
return self.model(input_ids, positions, inputs_embeds)
|
||||
|
||||
def get_draft_kv_cache_layer_names(self) -> list[str]:
|
||||
return [layer.self_attn.attn.layer_name for layer in self.model.layers]
|
||||
|
||||
def get_draft_attn_causal(self) -> list[bool]:
|
||||
"""Per-layer attention causality, aligned with
|
||||
get_draft_kv_cache_layer_names."""
|
||||
return [layer.self_attn.causal for layer in self.model.layers]
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user