[Spec Decode] Support hybrid (SWA + full attention) DFlash drafters (#47914)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-07-08 11:12:45 -04:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 68b4a1d582
commit 0d12618e98
9 changed files with 79 additions and 30 deletions
+17
View File
@@ -542,6 +542,11 @@ class VllmConfig:
):
return True
# Mixed sliding/full DFlash drafts need multiple KV groups (V2 only);
# force V2 as for dspark, since a hybrid target otherwise defaults to V1.
if self._dflash_needs_multi_kv_group():
return True
if self.model_config is not None and self.model_config.is_diffusion:
return True
@@ -565,6 +570,18 @@ class VllmConfig:
return True
def _dflash_needs_multi_kv_group(self) -> bool:
"""Whether a DFlash draft mixes sliding-window and full attention."""
spec = self.speculative_config
if spec is None or spec.method != "dflash":
return False
draft_config = getattr(spec, "draft_model_config", None)
if draft_config is None:
return False
layer_types = getattr(draft_config.hf_config, "layer_types", None) or []
num_sliding = sum(lt == "sliding_attention" for lt in layer_types)
return 0 < num_sliding < len(layer_types)
def _is_default_v2_model_runner_model(self) -> bool:
model_config = self.model_config
if model_config is None:
+17 -9
View File
@@ -86,14 +86,15 @@ def _resolve_layer_attention(
if layer_types is not None:
num_sliding = sum(lt == SLIDING_ATTENTION for lt in layer_types)
any_sliding = num_sliding > 0
all_sliding = num_sliding == len(layer_types)
if any_sliding and not all_sliding:
# Mixed sliding/full attention needs per-layer causal metadata and
# multiple KV-cache groups, which DFlash does not yet support.
# Mixed sliding/full attention needs multiple KV groups (V2 runner only).
if (
0 < num_sliding < len(layer_types)
and not get_current_vllm_config().use_v2_model_runner
):
raise NotImplementedError(
"DFlash does not yet support mixed sliding/full attention via "
"layer_types; see "
"https://github.com/vllm-project/vllm/issues/40898."
"DFlash drafters with mixed sliding/full attention require "
"the V2 model runner; relaunch with "
"VLLM_USE_V2_MODEL_RUNNER=1."
)
default_causal = False
@@ -207,8 +208,7 @@ class DFlashQwen3Attention(nn.Module):
attn_type=attn_type,
sinks=self.attention_sink_bias,
)
# NOTE: `causal` is currently unused here, but will be needed in the future
# to support models with different causality per-layer.
self.causal = causal
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
@@ -709,6 +709,14 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
) -> torch.Tensor:
return self.model(input_ids, positions, inputs_embeds)
def get_draft_kv_cache_layer_names(self) -> list[str]:
return [layer.self_attn.attn.layer_name for layer in self.model.layers]
def get_draft_attn_causal(self) -> list[bool]:
"""Per-layer attention causality, aligned with
get_draft_kv_cache_layer_names."""
return [layer.self_attn.causal for layer in self.model.layers]
def compute_logits(
self,
hidden_states: torch.Tensor,
+5 -3
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Sequence
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass, replace
from math import prod
from typing import Any, cast
@@ -573,7 +573,7 @@ def build_attn_metadata(
mm_req_doc_ranges: dict[int, list[tuple[int, int]]] | None = None,
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
for_cudagraph_capture: bool = False,
causal: bool = True,
causal: bool | Mapping[int, bool] = True,
rswa_prefix_lens: torch.Tensor | None = None,
) -> dict[str, Any]:
seq_lens = seq_lens[:num_reqs]
@@ -587,6 +587,8 @@ def build_attn_metadata(
for i in range(num_kv_cache_groups):
block_table = block_tables[i]
slot_mapping = slot_mappings[i]
# Per-group causal for hybrid drafters (mixed SWA/full attention).
group_causal = causal if isinstance(causal, bool) else causal.get(i, True)
common_attn_metadata_extra_kwargs = (
model_specific_attn_metadata.get_extra_common_attn_kwargs(i, num_reqs)
@@ -604,7 +606,7 @@ def build_attn_metadata(
max_query_len=max_query_len,
block_table_tensor=block_table,
slot_mapping=slot_mapping,
causal=causal,
causal=group_causal,
dcp_local_seq_lens=dcp_local_seq_lens,
positions=positions,
mm_req_doc_ranges=mm_req_doc_ranges,
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from collections.abc import Callable, Mapping
import torch
@@ -29,7 +29,7 @@ def _prepare_dflash_inputs_to_capture(
kv_cache_config: KVCacheConfig,
max_model_len: int,
skip_attn: bool,
causal: bool,
causal: bool | Mapping[int, bool],
) -> AttentionState:
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
@@ -63,10 +63,6 @@ class DFlashCudaGraphManager(CudaGraphManager):
"""DFlash CudaGraphManager for the parallel-drafting query forward,
building its own attention metadata from scratch."""
def __init__(self, *args, causal: bool = False, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.causal = causal
def capture(
self,
forward_fn: Callable,
@@ -75,6 +71,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
max_model_len: int,
causal: bool | Mapping[int, bool],
progress_bar_desc: str = "Capturing CUDA graphs",
) -> None:
def create_forward_fn(
@@ -97,7 +94,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
kv_cache_config,
max_model_len,
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
causal=self.causal,
causal=causal,
)
attn_metadata, slot_mappings = attn_state
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from typing import Any
import torch
@@ -94,7 +95,6 @@ class DFlashSpeculator(DraftModelSpeculator):
self.device,
cudagraph_mode,
decode_query_len=self.num_query_per_req,
causal=self.dflash_causal,
)
def capture(self, attn_states: dict | None = None) -> None:
@@ -112,6 +112,7 @@ class DFlashSpeculator(DraftModelSpeculator):
self.attn_groups,
self.kv_cache_config,
self.max_model_len,
causal=self._group_causal,
progress_bar_desc=f"Capturing {self._speculator_name.lower()} CUDA graphs",
)
@@ -148,7 +149,10 @@ class DFlashSpeculator(DraftModelSpeculator):
# of the kv-cache group its cache belongs to. Models that share a single group
# leave this as None and share one context slot mapping.
self._layer_group_idx: list[int] | None = None
# Per-KV-group causal, falling back to the scalar dflash_causal.
self._group_causal: dict[int, bool] | bool = self.dflash_causal
if hasattr(self.model, "get_draft_kv_cache_layer_names"):
layer_names = self.model.get_draft_kv_cache_layer_names()
name_to_gid = {
ln: gid
for gid, group in enumerate(kv_cache_config.kv_cache_groups)
@@ -156,9 +160,15 @@ class DFlashSpeculator(DraftModelSpeculator):
}
gid_to_idx = {gid: i for i, gid in enumerate(self.draft_kv_cache_group_ids)}
self._layer_group_idx = [
gid_to_idx[name_to_gid[name]]
for name in self.model.get_draft_kv_cache_layer_names()
gid_to_idx[name_to_gid[name]] for name in layer_names
]
if hasattr(self.model, "get_draft_attn_causal"):
self._group_causal = {
name_to_gid[name]: layer_causal
for name, layer_causal in zip(
layer_names, self.model.get_draft_attn_causal()
)
}
@torch.inference_mode()
def _run_model(
@@ -226,7 +236,7 @@ class DFlashSpeculator(DraftModelSpeculator):
num_reqs_padded: int,
num_tokens_padded: int,
num_query_per_req: int | None = None,
causal: bool = False,
causal: bool | Mapping[int, bool] = False,
) -> dict[str, Any] | None:
if not self.draft_attn_layer_names:
return None
@@ -383,7 +393,7 @@ class DFlashSpeculator(DraftModelSpeculator):
num_reqs=num_reqs,
num_reqs_padded=num_reqs_padded,
num_tokens_padded=num_tokens_padded,
causal=self.dflash_causal,
causal=self._group_causal,
)
draft_slot_mappings_by_layer = build_slot_mappings_by_layer(
self.block_tables.slot_mappings[:, :num_tokens_padded],
@@ -5,7 +5,10 @@ import torch.nn as nn
from vllm.config import ModelConfig, VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.model_loader import get_model
from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
_should_share,
get_target_lm_head,
)
def get_dflash_causal(draft_model_config: ModelConfig) -> bool:
@@ -57,7 +60,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
target_lm_head = getattr(target_model, "lm_head", None)
target_lm_head = get_target_lm_head(target_model, target_language_model)
draft_lm_head = getattr(dflash_model, "lm_head", None)
if target_lm_head is not None and _should_share(
dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head
@@ -6,7 +6,10 @@ import torch.nn as nn
from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.model_loader import get_model
from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
_should_share,
get_target_lm_head,
)
def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
@@ -52,7 +55,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
target_lm_head = getattr(target_model, "lm_head", None)
target_lm_head = get_target_lm_head(target_model, target_language_model)
draft_lm_head = getattr(draft_model, "lm_head", None)
if target_lm_head is not None and _should_share(
draft_model, "has_own_lm_head", draft_lm_head, target_lm_head
@@ -25,6 +25,14 @@ def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
return torch.equal(w, target.weight)
def get_target_lm_head(target_model: nn.Module, target_language_model: nn.Module):
"""The target's lm_head — from get_language_model() for
*ForConditionalGeneration targets, else the top-level module."""
return getattr(target_language_model, "lm_head", None) or getattr(
target_model, "lm_head", None
)
def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
from vllm.compilation.backends import set_model_tag
@@ -64,7 +72,7 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
target_lm_head = getattr(target_model, "lm_head", None)
target_lm_head = get_target_lm_head(target_model, target_language_model)
draft_lm_head = getattr(eagle_model, "lm_head", None)
if target_lm_head is not None and _should_share(
eagle_model, "has_own_lm_head", draft_lm_head, target_lm_head
+2 -1
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import Any
import torch
@@ -202,7 +203,7 @@ class DraftModelSpeculator(BaseSpeculator):
num_reqs_padded: int,
num_tokens_padded: int,
num_query_per_req: int = 1,
causal: bool = True,
causal: bool | Mapping[int, bool] = True,
) -> dict[str, Any] | None:
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
# Clamp keeps the series non-decreasing past num_reqs, which some