[CustomOp][MM] Extract MMEncoderAttention as CustomOp and replace the backend of QwenVisionAttention with it. (#30125)

Signed-off-by: shen-shanshan <467638484@qq.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
Shanshan Shen
2025-12-15 11:13:32 +08:00
committed by GitHub
co-authored by Isotr0py tjtanaa
parent 84e23d103d
commit 87b4d1557d
24 changed files with 1262 additions and 851 deletions
+8 -5
View File
@@ -88,14 +88,17 @@ def get_vit_attn_backend(
"""
Get the available attention backend for Vision Transformer.
"""
if attn_backend_override is not None:
return attn_backend_override
attn_backend = attn_backend_override
selected_backend = get_current_vllm_config().attention_config.backend
if selected_backend is not None:
return selected_backend
if attn_backend is None:
attn_backend = selected_backend
return current_platform.get_vit_attn_backend(head_size, dtype)
return current_platform.get_vit_attn_backend(
head_size,
dtype,
backend=attn_backend,
)
def should_torch_compile_mm_vit(vllm_config: VllmConfig) -> bool: