[Bugfix] Fix FlashInfer non-causal draft attention (DFlash/DSpark) on Blackwell (#48167)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
49e777cf08
commit
ecf4aa5ce2
@@ -59,8 +59,6 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
self.max_num_tokens, draft_hidden, dtype=self.dtype, device=device
|
||||
)
|
||||
|
||||
self.dflash_causal = False
|
||||
|
||||
self._step_cols = torch.arange(
|
||||
self.num_speculative_steps, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
@@ -18,14 +18,13 @@ def load_dspark_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
||||
draft_model_config = speculative_config.draft_model_config
|
||||
|
||||
from vllm.compilation.backends import set_model_tag
|
||||
from vllm.model_executor.models.qwen3_dflash import dflash_has_any_non_causal
|
||||
|
||||
# DSpark uses non-causal attention.
|
||||
causal = False
|
||||
draft_vllm_config = replace(
|
||||
vllm_config,
|
||||
attention_config=replace(
|
||||
vllm_config.attention_config,
|
||||
use_non_causal=not causal,
|
||||
use_non_causal=dflash_has_any_non_causal(draft_model_config.hf_config),
|
||||
backend=speculative_config.attention_backend,
|
||||
),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user