[Spec Decode] Support hybrid (SWA + full attention) DFlash drafters (#47914)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
68b4a1d582
commit
0d12618e98
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Mapping
|
||||
|
||||
import torch
|
||||
|
||||
@@ -29,7 +29,7 @@ def _prepare_dflash_inputs_to_capture(
|
||||
kv_cache_config: KVCacheConfig,
|
||||
max_model_len: int,
|
||||
skip_attn: bool,
|
||||
causal: bool,
|
||||
causal: bool | Mapping[int, bool],
|
||||
) -> AttentionState:
|
||||
input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers)
|
||||
input_block_tables = block_tables.get_dummy_block_tables(num_reqs)
|
||||
@@ -63,10 +63,6 @@ class DFlashCudaGraphManager(CudaGraphManager):
|
||||
"""DFlash CudaGraphManager for the parallel-drafting query forward,
|
||||
building its own attention metadata from scratch."""
|
||||
|
||||
def __init__(self, *args, causal: bool = False, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.causal = causal
|
||||
|
||||
def capture(
|
||||
self,
|
||||
forward_fn: Callable,
|
||||
@@ -75,6 +71,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
max_model_len: int,
|
||||
causal: bool | Mapping[int, bool],
|
||||
progress_bar_desc: str = "Capturing CUDA graphs",
|
||||
) -> None:
|
||||
def create_forward_fn(
|
||||
@@ -97,7 +94,7 @@ class DFlashCudaGraphManager(CudaGraphManager):
|
||||
kv_cache_config,
|
||||
max_model_len,
|
||||
skip_attn=(desc.cg_mode == CUDAGraphMode.PIECEWISE),
|
||||
causal=self.causal,
|
||||
causal=causal,
|
||||
)
|
||||
attn_metadata, slot_mappings = attn_state
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -94,7 +95,6 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.device,
|
||||
cudagraph_mode,
|
||||
decode_query_len=self.num_query_per_req,
|
||||
causal=self.dflash_causal,
|
||||
)
|
||||
|
||||
def capture(self, attn_states: dict | None = None) -> None:
|
||||
@@ -112,6 +112,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.attn_groups,
|
||||
self.kv_cache_config,
|
||||
self.max_model_len,
|
||||
causal=self._group_causal,
|
||||
progress_bar_desc=f"Capturing {self._speculator_name.lower()} CUDA graphs",
|
||||
)
|
||||
|
||||
@@ -148,7 +149,10 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
# of the kv-cache group its cache belongs to. Models that share a single group
|
||||
# leave this as None and share one context slot mapping.
|
||||
self._layer_group_idx: list[int] | None = None
|
||||
# Per-KV-group causal, falling back to the scalar dflash_causal.
|
||||
self._group_causal: dict[int, bool] | bool = self.dflash_causal
|
||||
if hasattr(self.model, "get_draft_kv_cache_layer_names"):
|
||||
layer_names = self.model.get_draft_kv_cache_layer_names()
|
||||
name_to_gid = {
|
||||
ln: gid
|
||||
for gid, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
@@ -156,9 +160,15 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
}
|
||||
gid_to_idx = {gid: i for i, gid in enumerate(self.draft_kv_cache_group_ids)}
|
||||
self._layer_group_idx = [
|
||||
gid_to_idx[name_to_gid[name]]
|
||||
for name in self.model.get_draft_kv_cache_layer_names()
|
||||
gid_to_idx[name_to_gid[name]] for name in layer_names
|
||||
]
|
||||
if hasattr(self.model, "get_draft_attn_causal"):
|
||||
self._group_causal = {
|
||||
name_to_gid[name]: layer_causal
|
||||
for name, layer_causal in zip(
|
||||
layer_names, self.model.get_draft_attn_causal()
|
||||
)
|
||||
}
|
||||
|
||||
@torch.inference_mode()
|
||||
def _run_model(
|
||||
@@ -226,7 +236,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_reqs_padded: int,
|
||||
num_tokens_padded: int,
|
||||
num_query_per_req: int | None = None,
|
||||
causal: bool = False,
|
||||
causal: bool | Mapping[int, bool] = False,
|
||||
) -> dict[str, Any] | None:
|
||||
if not self.draft_attn_layer_names:
|
||||
return None
|
||||
@@ -383,7 +393,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_padded=num_reqs_padded,
|
||||
num_tokens_padded=num_tokens_padded,
|
||||
causal=self.dflash_causal,
|
||||
causal=self._group_causal,
|
||||
)
|
||||
draft_slot_mappings_by_layer = build_slot_mappings_by_layer(
|
||||
self.block_tables.slot_mappings[:, :num_tokens_padded],
|
||||
|
||||
@@ -5,7 +5,10 @@ import torch.nn as nn
|
||||
from vllm.config import ModelConfig, VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import _should_share
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
||||
_should_share,
|
||||
get_target_lm_head,
|
||||
)
|
||||
|
||||
|
||||
def get_dflash_causal(draft_model_config: ModelConfig) -> bool:
|
||||
@@ -57,7 +60,7 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
del draft_inner.embed_tokens
|
||||
draft_inner.embed_tokens = target_embed
|
||||
|
||||
target_lm_head = getattr(target_model, "lm_head", None)
|
||||
target_lm_head = get_target_lm_head(target_model, target_language_model)
|
||||
draft_lm_head = getattr(dflash_model, "lm_head", None)
|
||||
if target_lm_head is not None and _should_share(
|
||||
dflash_model, "has_own_lm_head", draft_lm_head, target_lm_head
|
||||
|
||||
Reference in New Issue
Block a user