[Spec Decode] Support hybrid (SWA + full attention) DFlash drafters (#47914)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-07-08 11:12:45 -04:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 68b4a1d582
commit 0d12618e98
9 changed files with 79 additions and 30 deletions
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from collections.abc import Callable, Mapping
import torch
@@ -29,7 +29,7 @@ def _prepare_dflash_inputs_to_capture(
kv_cache_config: KVCacheConfig,
max_model_len: int,
skip_attn: bool,
causal: bool,
causal: bool | Mapping[int, bool],
) -> AttentionState:
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
@@ -63,10 +63,6 @@ class DFlashCudaGraphManager(CudaGraphManager):
"""DFlash CudaGraphManager for the parallel-drafting query forward,
building its own attention metadata from scratch."""
def __init__(self, *args, causal: bool = False, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.causal = causal
def capture(
self,
forward_fn: Callable,
@@ -75,6 +71,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
max_model_len: int,
causal: bool | Mapping[int, bool],
progress_bar_desc: str = "Capturing CUDA graphs",
) -> None:
def create_forward_fn(
@@ -97,7 +94,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
kv_cache_config,
max_model_len,
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
causal=self.causal,
causal=causal,
)
attn_metadata, slot_mappings = attn_state
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from typing import Any
import torch
@@ -94,7 +95,6 @@ class DFlashSpeculator(DraftModelSpeculator):
self.device,
cudagraph_mode,
decode_query_len=self.num_query_per_req,
causal=self.dflash_causal,
)
def capture(self, attn_states: dict | None = None) -> None:
@@ -112,6 +112,7 @@ class DFlashSpeculator(DraftModelSpeculator):
self.attn_groups,
self.kv_cache_config,
self.max_model_len,
causal=self._group_causal,
progress_bar_desc=f"Capturing {self._speculator_name.lower()} CUDA graphs",
)
@@ -148,7 +149,10 @@ class DFlashSpeculator(DraftModelSpeculator):
# of the kv-cache group its cache belongs to. Models that share a single group
# leave this as None and share one context slot mapping.
self._layer_group_idx: list[int] | None = None
# Per-KV-group causal, falling back to the scalar dflash_causal.
self._group_causal: dict[int, bool] | bool = self.dflash_causal
if hasattr(self.model, "get_draft_kv_cache_layer_names"):
layer_names = self.model.get_draft_kv_cache_layer_names()
name_to_gid = {
ln: gid
for gid, group in enumerate(kv_cache_config.kv_cache_groups)
@@ -156,9 +160,15 @@ class DFlashSpeculator(DraftModelSpeculator):
}
gid_to_idx = {gid: i for i, gid in enumerate(self.draft_kv_cache_group_ids)}
self._layer_group_idx = [
gid_to_idx[name_to_gid[name]]
for name in self.model.get_draft_kv_cache_layer_names()
gid_to_idx[name_to_gid[name]] for name in layer_names
]
if hasattr(self.model, "get_draft_attn_causal"):
self._group_causal = {
name_to_gid[name]: layer_causal
for name, layer_causal in zip(
layer_names, self.model.get_draft_attn_causal()
)
}
@torch.inference_mode()
def _run_model(
@@ -226,7 +236,7 @@ class DFlashSpeculator(DraftModelSpeculator):
num_reqs_padded: int,
num_tokens_padded: int,
num_query_per_req: int | None = None,
causal: bool = False,
causal: bool | Mapping[int, bool] = False,
) -> dict[str, Any] | None:
if not self.draft_attn_layer_names:
return None
@@ -383,7 +393,7 @@ class DFlashSpeculator(DraftModelSpeculator):
num_reqs=num_reqs,
num_reqs_padded=num_reqs_padded,
num_tokens_padded=num_tokens_padded,
causal=self.dflash_causal,
causal=self._group_causal,
)
draft_slot_mappings_by_layer = build_slot_mappings_by_layer(
self.block_tables.slot_mappings[:, :num_tokens_padded],
@@ -5,7 +5,10 @@ import torch.nn as nn
from vllm.config import ModelConfig, VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.model_loader import get_model
from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
_should_share,
get_target_lm_head,
)
def get_dflash_causal(draft_model_config: ModelConfig) -> bool:
@@ -57,7 +60,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
del draft_inner.embed_tokens
draft_inner.embed_tokens = target_embed
target_lm_head = getattr(target_model, "lm_head", None)
target_lm_head = get_target_lm_head(target_model, target_language_model)
draft_lm_head = getattr(dflash_model, "lm_head", None)
if target_lm_head is not None and _should_share(
dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head