[Bugfix] Fix FlashInfer non-causal draft attention (DFlash/DSpark) on Blackwell (#48167)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
49e777cf08
commit
ecf4aa5ce2
@@ -472,9 +472,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
decode_query_len=self.decode_query_len,
|
||||
lora_capture_cases=self.lora_capture_cases,
|
||||
)
|
||||
if self.speculator is not None:
|
||||
self.speculator.init_cudagraph_manager(cudagraph_mode)
|
||||
|
||||
check_attention_cp_compatibility(self.vllm_config)
|
||||
if isinstance(self.speculator, DraftModelSpeculator):
|
||||
# HACK(woosuk)
|
||||
@@ -485,6 +482,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.input_buffers,
|
||||
self.attn_groups,
|
||||
)
|
||||
if self.speculator is not None:
|
||||
# After set_attn, so the speculator can size its cudagraph mode
|
||||
# to its own attention support.
|
||||
self.speculator.init_cudagraph_manager(cudagraph_mode)
|
||||
|
||||
self.kv_caches: list[torch.Tensor] = []
|
||||
kv_caches_dict = init_kv_cache(
|
||||
|
||||
Reference in New Issue
Block a user