[Spec Decode] Support hybrid (SWA + full attention) DFlash drafters (#47914)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
68b4a1d582
commit
0d12618e98
@@ -542,6 +542,11 @@ class VllmConfig:
|
||||
):
|
||||
return True
|
||||
|
||||
# Mixed sliding/full DFlash drafts need multiple KV groups (V2 only);
|
||||
# force V2 as for dspark, since a hybrid target otherwise defaults to V1.
|
||||
if self._dflash_needs_multi_kv_group():
|
||||
return True
|
||||
|
||||
if self.model_config is not None and self.model_config.is_diffusion:
|
||||
return True
|
||||
|
||||
@@ -565,6 +570,18 @@ class VllmConfig:
|
||||
|
||||
return True
|
||||
|
||||
def _dflash_needs_multi_kv_group(self) -> bool:
|
||||
"""Whether a DFlash draft mixes sliding-window and full attention."""
|
||||
spec = self.speculative_config
|
||||
if spec is None or spec.method != "dflash":
|
||||
return False
|
||||
draft_config = getattr(spec, "draft_model_config", None)
|
||||
if draft_config is None:
|
||||
return False
|
||||
layer_types = getattr(draft_config.hf_config, "layer_types", None) or []
|
||||
num_sliding = sum(lt == "sliding_attention" for lt in layer_types)
|
||||
return 0 < num_sliding < len(layer_types)
|
||||
|
||||
def _is_default_v2_model_runner_model(self) -> bool:
|
||||
model_config = self.model_config
|
||||
if model_config is None:
|
||||
|
||||
@@ -86,14 +86,15 @@ def _resolve_layer_attention(
|
||||
if layer_types is not None:
|
||||
num_sliding = sum(lt == SLIDING_ATTENTION for lt in layer_types)
|
||||
any_sliding = num_sliding > 0
|
||||
all_sliding = num_sliding == len(layer_types)
|
||||
if any_sliding and not all_sliding:
|
||||
# Mixed sliding/full attention needs per-layer causal metadata and
|
||||
# multiple KV-cache groups, which DFlash does not yet support.
|
||||
# Mixed sliding/full attention needs multiple KV groups (V2 runner only).
|
||||
if (
|
||||
0 < num_sliding < len(layer_types)
|
||||
and not get_current_vllm_config().use_v2_model_runner
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"DFlash does not yet support mixed sliding/full attention via "
|
||||
"layer_types; see "
|
||||
"https://github.com/vllm-project/vllm/issues/40898."
|
||||
"DFlash drafters with mixed sliding/full attention require "
|
||||
"the V2 model runner; relaunch with "
|
||||
"VLLM_USE_V2_MODEL_RUNNER=1."
|
||||
)
|
||||
|
||||
default_causal = False
|
||||
@@ -207,8 +208,7 @@ class DFlashQwen3Attention(nn.Module):
|
||||
attn_type=attn_type,
|
||||
sinks=self.attention_sink_bias,
|
||||
)
|
||||
# NOTE: `causal` is currently unused here, but will be needed in the future
|
||||
# to support models with different causality per-layer.
|
||||
self.causal = causal
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
|
||||
@@ -709,6 +709,14 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
) -> torch.Tensor:
|
||||
return self.model(input_ids, positions, inputs_embeds)
|
||||
|
||||
def get_draft_kv_cache_layer_names(self) -> list[str]:
|
||||
return [layer.self_attn.attn.layer_name for layer in self.model.layers]
|
||||
|
||||
def get_draft_attn_causal(self) -> list[bool]:
|
||||
"""Per-layer attention causality, aligned with
|
||||
get_draft_kv_cache_layer_names."""
|
||||
return [layer.self_attn.causal for layer in self.model.layers]
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable, Sequence
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from math import prod
|
||||
from typing import Any, cast
|
||||
@@ -573,7 +573,7 @@ def build_attn_metadata(
|
||||
mm_req_doc_ranges: dict[int, list[tuple[int, int]]] | None = None,
|
||||
model_specific_attn_metadata: ModelSpecificAttnMetadata | None = None,
|
||||
for_cudagraph_capture: bool = False,
|
||||
causal: bool = True,
|
||||
causal: bool | Mapping[int, bool] = True,
|
||||
rswa_prefix_lens: torch.Tensor | None = None,
|
||||
) -> dict[str, Any]:
|
||||
seq_lens = seq_lens[:num_reqs]
|
||||
@@ -587,6 +587,8 @@ def build_attn_metadata(
|
||||
for i in range(num_kv_cache_groups):
|
||||
block_table = block_tables[i]
|
||||
slot_mapping = slot_mappings[i]
|
||||
# Per-group causal for hybrid drafters (mixed SWA/full attention).
|
||||
group_causal = causal if isinstance(causal, bool) else causal.get(i, True)
|
||||
|
||||
common_attn_metadata_extra_kwargs = (
|
||||
model_specific_attn_metadata.get_extra_common_attn_kwargs(i, num_reqs)
|
||||
@@ -604,7 +606,7 @@ def build_attn_metadata(
|
||||
max_query_len=max_query_len,
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=slot_mapping,
|
||||
causal=causal,
|
||||
causal=group_causal,
|
||||
dcp_local_seq_lens=dcp_local_seq_lens,
|
||||
positions=positions,
|
||||
mm_req_doc_ranges=mm_req_doc_ranges,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping
|
||||
|
||||
import torch
|
||||
|
||||
@@ -29,7 +29,7 @@ def _prepare_dflash_inputs_to_capture(
|
||||
kv_cache_config: KVCacheConfig,
|
||||
max_model_len: int,
|
||||
skip_attn: bool,
|
||||
causal: bool,
|
||||
causal: bool | Mapping[int, bool],
|
||||
) -> AttentionState:
|
||||
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
|
||||
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
|
||||
@@ -63,10 +63,6 @@ class DFlashCudaGraphManager(CudaGraphManager):
|
||||
"""DFlash CudaGraphManager for the parallel-drafting query forward,
|
||||
building its own attention metadata from scratch."""
|
||||
|
||||
def __init__(self, *args, causal: bool = False, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.causal = causal
|
||||
|
||||
def capture(
|
||||
self,
|
||||
forward_fn: Callable,
|
||||
@@ -75,6 +71,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
max_model_len: int,
|
||||
causal: bool | Mapping[int, bool],
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> None:
|
||||
def create_forward_fn(
|
||||
@@ -97,7 +94,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
|
||||
kv_cache_config,
|
||||
max_model_len,
|
||||
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
|
||||
causal=self.causal,
|
||||
causal=causal,
|
||||
)
|
||||
attn_metadata, slot_mappings = attn_state
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -94,7 +95,6 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.device,
|
||||
cudagraph_mode,
|
||||
decode_query_len=self.num_query_per_req,
|
||||
causal=self.dflash_causal,
|
||||
)
|
||||
|
||||
def capture(self, attn_states: dict | None = None) -> None:
|
||||
@@ -112,6 +112,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
self.max_model_len,
|
||||
causal=self._group_causal,
|
||||
progress_bar_desc=f"Capturing {self._speculator_name.lower()} CUDA graphs",
|
||||
)
|
||||
|
||||
@@ -148,7 +149,10 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
# of the kv-cache group its cache belongs to. Models that share a single group
|
||||
# leave this as None and share one context slot mapping.
|
||||
self._layer_group_idx: list[int] | None = None
|
||||
# Per-KV-group causal, falling back to the scalar dflash_causal.
|
||||
self._group_causal: dict[int, bool] | bool = self.dflash_causal
|
||||
if hasattr(self.model, "get_draft_kv_cache_layer_names"):
|
||||
layer_names = self.model.get_draft_kv_cache_layer_names()
|
||||
name_to_gid = {
|
||||
ln: gid
|
||||
for gid, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
@@ -156,9 +160,15 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
}
|
||||
gid_to_idx = {gid: i for i, gid in enumerate(self.draft_kv_cache_group_ids)}
|
||||
self._layer_group_idx = [
|
||||
gid_to_idx[name_to_gid[name]]
|
||||
for name in self.model.get_draft_kv_cache_layer_names()
|
||||
gid_to_idx[name_to_gid[name]] for name in layer_names
|
||||
]
|
||||
if hasattr(self.model, "get_draft_attn_causal"):
|
||||
self._group_causal = {
|
||||
name_to_gid[name]: layer_causal
|
||||
for name, layer_causal in zip(
|
||||
layer_names, self.model.get_draft_attn_causal()
|
||||
)
|
||||
}
|
||||
|
||||
@torch.inference_mode()
|
||||
def _run_model(
|
||||
@@ -226,7 +236,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_reqs_padded: int,
|
||||
num_tokens_padded: int,
|
||||
num_query_per_req: int | None = None,
|
||||
causal: bool = False,
|
||||
causal: bool | Mapping[int, bool] = False,
|
||||
) -> dict[str, Any] | None:
|
||||
if not self.draft_attn_layer_names:
|
||||
return None
|
||||
@@ -383,7 +393,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
num_tokens_padded=num_tokens_padded,
|
||||
causal=self.dflash_causal,
|
||||
causal=self._group_causal,
|
||||
)
|
||||
draft_slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
self.block_tables.slot_mappings[:, :num_tokens_padded],
|
||||
|
||||
@@ -5,7 +5,10 @@ import torch.nn as nn
|
||||
from vllm.config import ModelConfig, VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
||||
_should_share,
|
||||
get_target_lm_head,
|
||||
)
|
||||
|
||||
|
||||
def get_dflash_causal(draft_model_config: ModelConfig) -> bool:
|
||||
@@ -57,7 +60,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
del draft_inner.embed_tokens
|
||||
draft_inner.embed_tokens = target_embed
|
||||
|
||||
target_lm_head = getattr(target_model, "lm_head", None)
|
||||
target_lm_head = get_target_lm_head(target_model, target_language_model)
|
||||
draft_lm_head = getattr(dflash_model, "lm_head", None)
|
||||
if target_lm_head is not None and _should_share(
|
||||
dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head
|
||||
|
||||
@@ -6,7 +6,10 @@ import torch.nn as nn
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
||||
_should_share,
|
||||
get_target_lm_head,
|
||||
)
|
||||
|
||||
|
||||
def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
|
||||
@@ -52,7 +55,7 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
del draft_inner.embed_tokens
|
||||
draft_inner.embed_tokens = target_embed
|
||||
|
||||
target_lm_head = getattr(target_model, "lm_head", None)
|
||||
target_lm_head = get_target_lm_head(target_model, target_language_model)
|
||||
draft_lm_head = getattr(draft_model, "lm_head", None)
|
||||
if target_lm_head is not None and _should_share(
|
||||
draft_model, "has_own_lm_head", draft_lm_head, target_lm_head
|
||||
|
||||
@@ -25,6 +25,14 @@ def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
|
||||
return torch.equal(w, target.weight)
|
||||
|
||||
|
||||
def get_target_lm_head(target_model: nn.Module, target_language_model: nn.Module):
|
||||
"""The target's lm_head — from get_language_model() for
|
||||
*ForConditionalGeneration targets, else the top-level module."""
|
||||
return getattr(target_language_model, "lm_head", None) or getattr(
|
||||
target_model, "lm_head", None
|
||||
)
|
||||
|
||||
|
||||
def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
|
||||
@@ -64,7 +72,7 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
|
||||
del draft_inner.embed_tokens
|
||||
draft_inner.embed_tokens = target_embed
|
||||
|
||||
target_lm_head = getattr(target_model, "lm_head", None)
|
||||
target_lm_head = get_target_lm_head(target_model, target_language_model)
|
||||
draft_lm_head = getattr(eagle_model, "lm_head", None)
|
||||
if target_lm_head is not None and _should_share(
|
||||
eagle_model, "has_own_lm_head", draft_lm_head, target_lm_head
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -202,7 +203,7 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
num_reqs_padded: int,
|
||||
num_tokens_padded: int,
|
||||
num_query_per_req: int = 1,
|
||||
causal: bool = True,
|
||||
causal: bool | Mapping[int, bool] = True,
|
||||
) -> dict[str, Any] | None:
|
||||
# Uniform query: query_start_loc[i] = min(i, num_reqs) * num_query_per_req.
|
||||
# Clamp keeps the series non-decreasing past num_reqs, which some
|
||||
|
||||
Reference in New Issue
Block a user