[Bugfix] Fix FlashInfer non-causal draft attention (DFlash/DSpark) on Blackwell (#48167)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
49e777cf08
commit
ecf4aa5ce2
@@ -6,11 +6,12 @@ from typing import Any
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
|
||||
@@ -19,10 +20,7 @@ from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.cudagraph import DFlashCudaGraphManager
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.utils import (
|
||||
get_dflash_causal,
|
||||
load_dflash_model,
|
||||
)
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.utils import load_dflash_model
|
||||
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.utils import get_parallel_drafting_token_id
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
@@ -50,7 +48,11 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.draft_model_config.hf_config
|
||||
)
|
||||
|
||||
self.dflash_causal = get_dflash_causal(self.draft_model_config)
|
||||
from vllm.model_executor.models.qwen3_dflash import dflash_has_any_non_causal
|
||||
|
||||
self.requires_non_causal = dflash_has_any_non_causal(
|
||||
self.draft_model_config.hf_config
|
||||
)
|
||||
|
||||
# Whether the anchor query position is itself a prediction. DFlash default uses
|
||||
# the anchor as the bonus token (only mask tokens predict); DSpark samples from
|
||||
@@ -84,9 +86,32 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
self.query_cudagraph_manager: DFlashCudaGraphManager | None = None
|
||||
self.draft_kv_cache_group_id: int = -1
|
||||
|
||||
@property
|
||||
def attn_vllm_config(self) -> VllmConfig:
|
||||
# The draft's attention differs from the target's in causality.
|
||||
return replace(
|
||||
self.vllm_config,
|
||||
attention_config=replace(
|
||||
self.vllm_config.attention_config,
|
||||
use_non_causal=self.requires_non_causal,
|
||||
),
|
||||
)
|
||||
|
||||
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
|
||||
# PIECEWISE cudagraphs are not supported for dflash
|
||||
if cudagraph_mode.decode_mode() == CUDAGraphMode.FULL:
|
||||
wants_full = cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
|
||||
supports_full = (
|
||||
self.attn_cg_support.min_cg_support.value
|
||||
>= AttentionCGSupport.UNIFORM_BATCH.value
|
||||
)
|
||||
if wants_full and not supports_full:
|
||||
logger.warning(
|
||||
"%s draft attention (%s) does not support full CUDA graphs; "
|
||||
"running the draft eagerly.",
|
||||
self._speculator_name,
|
||||
self.attn_cg_support.min_cg_attn_backend,
|
||||
)
|
||||
# PIECEWISE cudagraphs are not supported for dflash.
|
||||
if wants_full and supports_full:
|
||||
cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
|
||||
else:
|
||||
cudagraph_mode = CUDAGraphMode.NONE
|
||||
@@ -158,8 +183,8 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
# of the kv-cache group its cache belongs to. Models that share a single group
|
||||
# leave this as None and share one context slot mapping.
|
||||
self._layer_group_idx: list[int] | None = None
|
||||
# Per-KV-group causal, falling back to the scalar dflash_causal.
|
||||
self._group_causal: dict[int, bool] | bool = self.dflash_causal
|
||||
# Per-KV-group causal, falling back to whether the drafter is all-causal.
|
||||
self._group_causal: dict[int, bool] | bool = not self.requires_non_causal
|
||||
if hasattr(self.model, "get_draft_kv_cache_layer_names"):
|
||||
layer_names = self.model.get_draft_kv_cache_layer_names()
|
||||
name_to_gid = {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig, replace
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
||||
@@ -11,26 +11,20 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
|
||||
)
|
||||
|
||||
|
||||
def get_dflash_causal(draft_model_config: ModelConfig) -> bool:
|
||||
"""Whether the DFlash draft uses causal (vs non-causal) attention."""
|
||||
dflash_config = getattr(draft_model_config.hf_config, "dflash_config", None) or {}
|
||||
return dflash_config.get("causal", False)
|
||||
|
||||
|
||||
def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
from vllm.model_executor.models.qwen3_dflash import dflash_has_any_non_causal
|
||||
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
# Modify the attention config so that we select an attention backend that matches
|
||||
# the causal/non-causal mode of the dflash model.
|
||||
causal = get_dflash_causal(draft_model_config)
|
||||
# Select an attention backend that supports the drafter's attention: mixing
|
||||
# a non-causal layer onto a causal-only backend would fail.
|
||||
draft_vllm_config = replace(
|
||||
vllm_config,
|
||||
attention_config=replace(
|
||||
vllm_config.attention_config,
|
||||
use_non_causal=not causal,
|
||||
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
|
||||
backend=speculative_config.attention_backend,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -59,8 +59,6 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
self.max_num_tokens, draft_hidden, dtype=self.dtype, device=device
|
||||
)
|
||||
|
||||
self.dflash_causal = False
|
||||
|
||||
self._step_cols = torch.arange(
|
||||
self.num_speculative_steps, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
@@ -18,14 +18,13 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
from vllm.model_executor.models.qwen3_dflash import dflash_has_any_non_causal
|
||||
|
||||
# DSpark uses non-causal attention.
|
||||
causal = False
|
||||
draft_vllm_config = replace(
|
||||
vllm_config,
|
||||
attention_config=replace(
|
||||
vllm_config.attention_config,
|
||||
use_non_causal=not causal,
|
||||
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
|
||||
backend=speculative_config.attention_backend,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -175,6 +175,12 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
num_unpadded_tokens,
|
||||
)
|
||||
|
||||
@property
|
||||
def attn_vllm_config(self) -> VllmConfig:
|
||||
"""Config for the draft's attention metadata builders. Overridden by
|
||||
speculators whose attention mode differs from the target's."""
|
||||
return self.vllm_config
|
||||
|
||||
def set_attn(
|
||||
self,
|
||||
model_state: ModelState,
|
||||
@@ -185,9 +191,9 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
) -> None:
|
||||
self.model_state = model_state
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.attn_groups, _, _ = init_attn_backend(
|
||||
self.attn_groups, self.attn_cg_support, _ = init_attn_backend(
|
||||
kv_cache_config,
|
||||
self.vllm_config,
|
||||
self.attn_vllm_config,
|
||||
self.device,
|
||||
active_layer_names=self.draft_attn_layer_names,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user