[CustomOp][MM] Extract MMEncoderAttention as CustomOp and replace the backend of QwenVisionAttention with it. (#30125)
Signed-off-by: shen-shanshan <467638484@qq.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
Isotr0py
tjtanaa
parent
84e23d103d
commit
87b4d1557d
@@ -7,7 +7,7 @@ import platform
|
||||
import random
|
||||
import sys
|
||||
from datetime import timedelta
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -222,12 +222,6 @@ class Platform:
|
||||
with contextlib.suppress(ImportError):
|
||||
import vllm._moe_C # noqa: F401
|
||||
|
||||
@classmethod
|
||||
def get_vit_attn_backend(
|
||||
cls, head_size: int, dtype: torch.dtype
|
||||
) -> "AttentionBackendEnum":
|
||||
return AttentionBackendEnum.TORCH_SDPA
|
||||
|
||||
@classmethod
|
||||
def get_attn_backend_cls(
|
||||
cls,
|
||||
@@ -245,6 +239,43 @@ class Platform:
|
||||
"""Get the attention backend class of a device."""
|
||||
return ""
|
||||
|
||||
@classmethod
|
||||
def get_supported_vit_attn_backends(cls) -> list["AttentionBackendEnum"]:
|
||||
return [
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def get_vit_attn_backend(
|
||||
cls,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
backend: Optional["AttentionBackendEnum"] = None,
|
||||
) -> "AttentionBackendEnum":
|
||||
"""
|
||||
Get the vision attention backend class of a device.
|
||||
|
||||
NOTE: ViT Attention should be checked and override in the platform-specific
|
||||
implementation. we should not override this in any other places, like
|
||||
the model_executor/models/<model_name>.py.
|
||||
|
||||
We check if the backend is None or not:
|
||||
1. If not, check if the backend is supported by the platform.
|
||||
2. If None, continue to the default selection logic.
|
||||
"""
|
||||
if backend is not None:
|
||||
assert backend in cls.get_supported_vit_attn_backends(), (
|
||||
f"Backend {backend} is not supported for vit attention"
|
||||
f"Supported backends are: {cls.get_supported_vit_attn_backends()}"
|
||||
)
|
||||
logger.info_once(f"Using backend {backend} for vit attention")
|
||||
return backend
|
||||
|
||||
logger.info_once(
|
||||
f"Using default backend {AttentionBackendEnum.TORCH_SDPA} for vit attention"
|
||||
)
|
||||
return AttentionBackendEnum.TORCH_SDPA
|
||||
|
||||
@classmethod
|
||||
def get_device_capability(
|
||||
cls,
|
||||
|
||||
Reference in New Issue
Block a user