[Bugfix] Fix FlashInfer non-causal draft attention (DFlash/DSpark) on Blackwell (#48167)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-07-15 12:44:01 -07:00
committed by GitHub
co-authored by Claude Fable 5
parent 49e777cf08
commit ecf4aa5ce2
12 changed files with 149 additions and 47 deletions
@@ -6,11 +6,12 @@ from typing import Any
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config import VllmConfig, replace
from vllm.config.compilation import CUDAGraphMode
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
@@ -19,10 +20,7 @@ from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.spec_decode.dflash.cudagraph import DFlashCudaGraphManager
from vllm.v1.worker.gpu.spec_decode.dflash.utils import (
get_dflash_causal,
load_dflash_model,
)
from vllm.v1.worker.gpu.spec_decode.dflash.utils import load_dflash_model
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.utils import get_parallel_drafting_token_id
from vllm.v1.worker.utils import AttentionGroup
@@ -50,7 +48,11 @@ class DFlashSpeculator(DraftModelSpeculator):
self.draft_model_config.hf_config
)
self.dflash_causal = get_dflash_causal(self.draft_model_config)
from vllm.model_executor.models.qwen3_dflash import dflash_has_any_non_causal
self.requires_non_causal = dflash_has_any_non_causal(
self.draft_model_config.hf_config
)
# Whether the anchor query position is itself a prediction. DFlash default uses
# the anchor as the bonus token (only mask tokens predict); DSpark samples from
@@ -84,9 +86,32 @@ class DFlashSpeculator(DraftModelSpeculator):
self.query_cudagraph_manager: DFlashCudaGraphManager | None = None
self.draft_kv_cache_group_id: int = -1
@property
def attn_vllm_config(self) -> VllmConfig:
# The draft's attention differs from the target's in causality.
return replace(
self.vllm_config,
attention_config=replace(
self.vllm_config.attention_config,
use_non_causal=self.requires_non_causal,
),
)
def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
# PIECEWISE cudagraphs are not supported for dflash
if cudagraph_mode.decode_mode() == CUDAGraphMode.FULL:
wants_full = cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
supports_full = (
self.attn_cg_support.min_cg_support.value
>= AttentionCGSupport.UNIFORM_BATCH.value
)
if wants_full and not supports_full:
logger.warning(
"%s draft attention (%s) does not support full CUDA graphs; "
"running the draft eagerly.",
self._speculator_name,
self.attn_cg_support.min_cg_attn_backend,
)
# PIECEWISE cudagraphs are not supported for dflash.
if wants_full and supports_full:
cudagraph_mode = CUDAGraphMode.FULL_DECODE_ONLY
else:
cudagraph_mode = CUDAGraphMode.NONE
@@ -158,8 +183,8 @@ class DFlashSpeculator(DraftModelSpeculator):
# of the kv-cache group its cache belongs to. Models that share a single group
# leave this as None and share one context slot mapping.
self._layer_group_idx: list[int] | None = None
# Per-KV-group causal, falling back to the scalar dflash_causal.
self._group_causal: dict[int, bool] | bool = self.dflash_causal
# Per-KV-group causal, falling back to whether the drafter is all-causal.
self._group_causal: dict[int, bool] | bool = not self.requires_non_causal
if hasattr(self.model, "get_draft_kv_cache_layer_names"):
layer_names = self.model.get_draft_kv_cache_layer_names()
name_to_gid = {
+5 -11
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch.nn as nn
from vllm.config import ModelConfig, VllmConfig, replace
from vllm.config import VllmConfig, replace
from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.model_loader import get_model
from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
@@ -11,26 +11,20 @@ from vllm.v1.worker.gpu.spec_decode.eagle.utils import (
)
def get_dflash_causal(draft_model_config: ModelConfig) -> bool:
"""Whether the DFlash draft uses causal (vs non-causal) attention."""
dflash_config = getattr(draft_model_config.hf_config, "dflash_config", None) or {}
return dflash_config.get("causal", False)
def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module:
from vllm.compilation.backends import set_model_tag
from vllm.model_executor.models.qwen3_dflash import dflash_has_any_non_causal
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
draft_model_config = speculative_config.draft_model_config
# Modify the attention config so that we select an attention backend that matches
# the causal/non-causal mode of the dflash model.
causal = get_dflash_causal(draft_model_config)
# Select an attention backend that supports the drafter's attention: mixing
# a non-causal layer onto a causal-only backend would fail.
draft_vllm_config = replace(
vllm_config,
attention_config=replace(
vllm_config.attention_config,
use_non_causal=not causal,
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
backend=speculative_config.attention_backend,
),
)
@@ -59,8 +59,6 @@ class DSparkSpeculator(DFlashSpeculator):
self.max_num_tokens, draft_hidden, dtype=self.dtype, device=device
)
self.dflash_causal = False
self._step_cols = torch.arange(
self.num_speculative_steps, dtype=torch.int32, device=device
)
@@ -18,14 +18,13 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
draft_model_config = speculative_config.draft_model_config
from vllm.compilation.backends import set_model_tag
from vllm.model_executor.models.qwen3_dflash import dflash_has_any_non_causal
# DSpark uses non-causal attention.
causal = False
draft_vllm_config = replace(
vllm_config,
attention_config=replace(
vllm_config.attention_config,
use_non_causal=not causal,
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
backend=speculative_config.attention_backend,
),
)
+8 -2
View File
@@ -175,6 +175,12 @@ class DraftModelSpeculator(BaseSpeculator):
num_unpadded_tokens,
)
@property
def attn_vllm_config(self) -> VllmConfig:
"""Config for the draft's attention metadata builders. Overridden by
speculators whose attention mode differs from the target's."""
return self.vllm_config
def set_attn(
self,
model_state: ModelState,
@@ -185,9 +191,9 @@ class DraftModelSpeculator(BaseSpeculator):
) -> None:
self.model_state = model_state
self.kv_cache_config = kv_cache_config
self.attn_groups, _, _ = init_attn_backend(
self.attn_groups, self.attn_cg_support, _ = init_attn_backend(
kv_cache_config,
self.vllm_config,
self.attn_vllm_config,
self.device,
active_layer_names=self.draft_attn_layer_names,
)