From 0d12618e98ff2d21d36081e0e9b4eb23573b6d38 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Wed, 8 Jul 2026 11:12:45 -0400 Subject: [PATCH] [Spec Decode] Support hybrid (SWA + full attention) DFlash drafters (#47914) Signed-off-by: mgoin Co-authored-by: Claude Opus 4.8 (1M context) --- vllm/config/vllm.py | 17 ++++++++++++ vllm/model_executor/models/qwen3_dflash.py | 26 ++++++++++++------- vllm/v1/worker/gpu/attn_utils.py | 8 +++--- .../gpu/spec_decode/dflash/cudagraph.py | 11 +++----- .../gpu/spec_decode/dflash/speculator.py | 20 ++++++++++---- .../v1/worker/gpu/spec_decode/dflash/utils.py | 7 +++-- .../v1/worker/gpu/spec_decode/dspark/utils.py | 7 +++-- vllm/v1/worker/gpu/spec_decode/eagle/utils.py | 10 ++++++- vllm/v1/worker/gpu/spec_decode/speculator.py | 3 ++- 9 files changed, 79 insertions(+), 30 deletions(-) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index dadf7e35f1..062877b169 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -542,6 +542,11 @@ class VllmConfig: ): return True + # Mixed sliding/full DFlash drafts need multiple KV groups (V2 only); + # force V2 as for dspark, since a hybrid target otherwise defaults to V1. + if self._dflash_needs_multi_kv_group(): + return True + if self.model_config is not None and self.model_config.is_diffusion: return True @@ -565,6 +570,18 @@ class VllmConfig: return True + def _dflash_needs_multi_kv_group(self) -> bool: + """Whether a DFlash draft mixes sliding-window and full attention.""" + spec = self.speculative_config + if spec is None or spec.method != "dflash": + return False + draft_config = getattr(spec, "draft_model_config", None) + if draft_config is None: + return False + layer_types = getattr(draft_config.hf_config, "layer_types", None) or [] + num_sliding = sum(lt == "sliding_attention" for lt in layer_types) + return 0 < num_sliding < len(layer_types) + def _is_default_v2_model_runner_model(self) -> bool: model_config = self.model_config if model_config is None: diff --git a/vllm/model_executor/models/qwen3_dflash.py b/vllm/model_executor/models/qwen3_dflash.py index 6261fc50fa..12bc143072 100644 --- a/vllm/model_executor/models/qwen3_dflash.py +++ b/vllm/model_executor/models/qwen3_dflash.py @@ -86,14 +86,15 @@ def _resolve_layer_attention( if layer_types is not None: num_sliding = sum(lt == SLIDING_ATTENTION for lt in layer_types) any_sliding = num_sliding > 0 - all_sliding = num_sliding == len(layer_types) - if any_sliding and not all_sliding: - # Mixed sliding/full attention needs per-layer causal metadata and - # multiple KV-cache groups, which DFlash does not yet support. + # Mixed sliding/full attention needs multiple KV groups (V2 runner only). + if ( + 0 < num_sliding < len(layer_types) + and not get_current_vllm_config().use_v2_model_runner + ): raise NotImplementedError( - "DFlash does not yet support mixed sliding/full attention via " - "layer_types; see " - "https://github.com/vllm-project/vllm/issues/40898." + "DFlash drafters with mixed sliding/full attention require " + "the V2 model runner; relaunch with " + "VLLM_USE_V2_MODEL_RUNNER=1." ) default_causal = False @@ -207,8 +208,7 @@ class DFlashQwen3Attention(nn.Module): attn_type=attn_type, sinks=self.attention_sink_bias, ) - # NOTE: `causal` is currently unused here, but will be needed in the future - # to support models with different causality per-layer. + self.causal = causal self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) @@ -709,6 +709,14 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): ) -> torch.Tensor: return self.model(input_ids, positions, inputs_embeds) + def get_draft_kv_cache_layer_names(self) -> list[str]: + return [layer.self_attn.attn.layer_name for layer in self.model.layers] + + def get_draft_attn_causal(self) -> list[bool]: + """Per-layer attention causality, aligned with + get_draft_kv_cache_layer_names.""" + return [layer.self_attn.causal for layer in self.model.layers] + def compute_logits( self, hidden_states: torch.Tensor, diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index dfa2daacf9..228fd08a75 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Iterable, Sequence +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, replace from math import prod from typing import Any, cast @@ -573,7 +573,7 @@ def build_attn_metadata( mm_req_doc_ranges: dict[int, list[tuple[int, int]]] | None = None, model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None, for_cudagraph_capture: bool = False, - causal: bool = True, + causal: bool | Mapping[int, bool] = True, rswa_prefix_lens: torch.Tensor | None = None, ) -> dict[str, Any]: seq_lens = seq_lens[:num_reqs] @@ -587,6 +587,8 @@ def build_attn_metadata( for i in range(num_kv_cache_groups): block_table = block_tables[i] slot_mapping = slot_mappings[i] + # Per-group causal for hybrid drafters (mixed SWA/full attention). + group_causal = causal if isinstance(causal, bool) else causal.get(i, True) common_attn_metadata_extra_kwargs = ( model_specific_attn_metadata.get_extra_common_attn_kwargs(i, num_reqs) @@ -604,7 +606,7 @@ def build_attn_metadata( max_query_len=max_query_len, block_table_tensor=block_table, slot_mapping=slot_mapping, - causal=causal, + causal=group_causal, dcp_local_seq_lens=dcp_local_seq_lens, positions=positions, mm_req_doc_ranges=mm_req_doc_ranges, diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py b/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py index 3e4b2b7e7f..a4b4033caf 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/cudagraph.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Callable +from collections.abc import Callable, Mapping import torch @@ -29,7 +29,7 @@ def _prepare_dflash_inputs_to_capture( kv_cache_config: KVCacheConfig, max_model_len: int, skip_attn: bool, - causal: bool, + causal: bool | Mapping[int, bool], ) -> AttentionState: input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers) input_block_tables = block_tables.get_dummy_block_tables(num_reqs) @@ -63,10 +63,6 @@ class DFlashCudaGraphManager(CudaGraphManager): """DFlash CudaGraphManager for the parallel-drafting query forward, building its own attention metadata from scratch.""" - def __init__(self, *args, causal: bool = False, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.causal = causal - def capture( self, forward_fn: Callable, @@ -75,6 +71,7 @@ class DFlashCudaGraphManager(CudaGraphManager): attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, max_model_len: int, + causal: bool | Mapping[int, bool], progress_bar_desc: str = "Capturing CUDA graphs", ) -> None: def create_forward_fn( @@ -97,7 +94,7 @@ class DFlashCudaGraphManager(CudaGraphManager): kv_cache_config, max_model_len, skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE), - causal=self.causal, + causal=causal, ) attn_metadata, slot_mappings = attn_state diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index b2762244a4..f9ffd3135d 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Mapping from typing import Any import torch @@ -94,7 +95,6 @@ class DFlashSpeculator(DraftModelSpeculator): self.device, cudagraph_mode, decode_query_len=self.num_query_per_req, - causal=self.dflash_causal, ) def capture(self, attn_states: dict | None = None) -> None: @@ -112,6 +112,7 @@ class DFlashSpeculator(DraftModelSpeculator): self.attn_groups, self.kv_cache_config, self.max_model_len, + causal=self._group_causal, progress_bar_desc=f"Capturing {self._speculator_name.lower()} CUDA graphs", ) @@ -148,7 +149,10 @@ class DFlashSpeculator(DraftModelSpeculator): # of the kv-cache group its cache belongs to. Models that share a single group # leave this as None and share one context slot mapping. self._layer_group_idx: list[int] | None = None + # Per-KV-group causal, falling back to the scalar dflash_causal. + self._group_causal: dict[int, bool] | bool = self.dflash_causal if hasattr(self.model, "get_draft_kv_cache_layer_names"): + layer_names = self.model.get_draft_kv_cache_layer_names() name_to_gid = { ln: gid for gid, group in enumerate(kv_cache_config.kv_cache_groups) @@ -156,9 +160,15 @@ class DFlashSpeculator(DraftModelSpeculator): } gid_to_idx = {gid: i for i, gid in enumerate(self.draft_kv_cache_group_ids)} self._layer_group_idx = [ - gid_to_idx[name_to_gid[name]] - for name in self.model.get_draft_kv_cache_layer_names() + gid_to_idx[name_to_gid[name]] for name in layer_names ] + if hasattr(self.model, "get_draft_attn_causal"): + self._group_causal = { + name_to_gid[name]: layer_causal + for name, layer_causal in zip( + layer_names, self.model.get_draft_attn_causal() + ) + } @torch.inference_mode() def _run_model( @@ -226,7 +236,7 @@ class DFlashSpeculator(DraftModelSpeculator): num_reqs_padded: int, num_tokens_padded: int, num_query_per_req: int | None = None, - causal: bool = False, + causal: bool | Mapping[int, bool] = False, ) -> dict[str, Any] | None: if not self.draft_attn_layer_names: return None @@ -383,7 +393,7 @@ class DFlashSpeculator(DraftModelSpeculator): num_reqs=num_reqs, num_reqs_padded=num_reqs_padded, num_tokens_padded=num_tokens_padded, - causal=self.dflash_causal, + causal=self._group_causal, ) draft_slot_mappings_by_layer = build_slot_mappings_by_layer( self.block_tables.slot_mappings[:, :num_tokens_padded], diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/utils.py b/vllm/v1/worker/gpu/spec_decode/dflash/utils.py index c4f98e715b..37fe693bbb 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/utils.py @@ -5,7 +5,10 @@ import torch.nn as nn from vllm.config import ModelConfig, VllmConfig, replace from vllm.distributed.parallel_state import get_pp_group from vllm.model_executor.model_loader import get_model -from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share +from vllm.v1.worker.gpu.spec_decode.eagle.utils import ( + _should_share, + get_target_lm_head, +) def get_dflash_causal(draft_model_config: ModelConfig) -> bool: @@ -57,7 +60,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo del draft_inner.embed_tokens draft_inner.embed_tokens = target_embed - target_lm_head = getattr(target_model, "lm_head", None) + target_lm_head = get_target_lm_head(target_model, target_language_model) draft_lm_head = getattr(dflash_model, "lm_head", None) if target_lm_head is not None and _should_share( dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/utils.py b/vllm/v1/worker/gpu/spec_decode/dspark/utils.py index acc32dafcd..08ff30e5fd 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/utils.py @@ -6,7 +6,10 @@ import torch.nn as nn from vllm.config import VllmConfig, replace from vllm.distributed.parallel_state import get_pp_group from vllm.model_executor.model_loader import get_model -from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share +from vllm.v1.worker.gpu.spec_decode.eagle.utils import ( + _should_share, + get_target_lm_head, +) def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module: @@ -52,7 +55,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo del draft_inner.embed_tokens draft_inner.embed_tokens = target_embed - target_lm_head = getattr(target_model, "lm_head", None) + target_lm_head = get_target_lm_head(target_model, target_language_model) draft_lm_head = getattr(draft_model, "lm_head", None) if target_lm_head is not None and _should_share( draft_model, "has_own_lm_head", draft_lm_head, target_lm_head diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/utils.py b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py index 11961ceef4..bdd588e578 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py @@ -25,6 +25,14 @@ def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool: return torch.equal(w, target.weight) +def get_target_lm_head(target_model: nn.Module, target_language_model: nn.Module): + """The target's lm_head — from get_language_model() for + *ForConditionalGeneration targets, else the top-level module.""" + return getattr(target_language_model, "lm_head", None) or getattr( + target_model, "lm_head", None + ) + + def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module: from vllm.compilation.backends import set_model_tag @@ -64,7 +72,7 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod del draft_inner.embed_tokens draft_inner.embed_tokens = target_embed - target_lm_head = getattr(target_model, "lm_head", None) + target_lm_head = get_target_lm_head(target_model, target_language_model) draft_lm_head = getattr(eagle_model, "lm_head", None) if target_lm_head is not None and _should_share( eagle_model, "has_own_lm_head", draft_lm_head, target_lm_head diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index 9e36c7cc65..4b7d7fef41 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from abc import ABC, abstractmethod +from collections.abc import Mapping from typing import Any import torch @@ -202,7 +203,7 @@ class DraftModelSpeculator(BaseSpeculator): num_reqs_padded: int, num_tokens_padded: int, num_query_per_req: int = 1, - causal: bool = True, + causal: bool | Mapping[int, bool] = True, ) -> dict[str, Any] | None: # Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req. # Clamp keeps the series non-decreasing past num_reqs, which some