[Model Runner V2][DFlash] Enable dflash attention backend selection (#46770)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -26,7 +26,9 @@ def load_dflash_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mo
|
|||||||
draft_vllm_config = replace(
|
draft_vllm_config = replace(
|
||||||
vllm_config,
|
vllm_config,
|
||||||
attention_config=replace(
|
attention_config=replace(
|
||||||
vllm_config.attention_config, use_non_causal=not causal
|
vllm_config.attention_config,
|
||||||
|
use_non_causal=not causal,
|
||||||
|
backend=speculative_config.attention_backend,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
with set_model_tag("dflash_head"):
|
with set_model_tag("dflash_head"):
|
||||||
|
|||||||
Reference in New Issue
Block a user