[Platform] Support platform-specific MLA prefill backend selection (#56959)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
fd928e5f3c
commit
072d49bb6e
@@ -18,6 +18,7 @@ from vllm.utils.cpu_resource_utils import (
|
||||
get_visible_memory_node,
|
||||
)
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .interface import CpuArchEnum, Platform, PlatformEnum
|
||||
@@ -26,6 +27,10 @@ logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.attention.backends.mla.prefill.base import (
|
||||
MLAPrefillBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import MLAPrefillSelectorConfig
|
||||
from vllm.v1.attention.selector import AttentionSelectorConfig
|
||||
else:
|
||||
VllmConfig = None
|
||||
@@ -173,6 +178,21 @@ class CpuPlatform(Platform):
|
||||
logger.info("Cannot use %s backend on CPU.", selected_backend)
|
||||
return AttentionBackendEnum.CPU_ATTN.get_path()
|
||||
|
||||
@classmethod
|
||||
def get_mla_prefill_backend_cls(
|
||||
cls,
|
||||
mla_selector_config: "MLAPrefillSelectorConfig",
|
||||
) -> "type[MLAPrefillBackend]":
|
||||
"""Get the MLA prefill backend class of a device."""
|
||||
for backend_enum in (MLAPrefillBackendEnum.ZEN_CPU, MLAPrefillBackendEnum.CPU):
|
||||
try:
|
||||
cpu_backend_cls = backend_enum.get_class()
|
||||
except ImportError:
|
||||
continue
|
||||
if cpu_backend_cls.is_available():
|
||||
return cpu_backend_cls
|
||||
raise ValueError("No valid CPU MLA prefill backend found.")
|
||||
|
||||
@classmethod
|
||||
def get_device_total_memory(cls, device_id: int = 0) -> int:
|
||||
meminfo = get_memory_node_info(device_id)
|
||||
|
||||
@@ -26,6 +26,8 @@ if TYPE_CHECKING:
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import MLAPrefillSelectorConfig
|
||||
from vllm.v1.attention.selector import AttentionSelectorConfig
|
||||
else:
|
||||
FlexibleArgumentParser = object
|
||||
@@ -429,6 +431,88 @@ class Platform:
|
||||
)
|
||||
return AttentionBackendEnum.TORCH_SDPA
|
||||
|
||||
@classmethod
|
||||
def get_mla_prefill_backend_cls(
|
||||
cls,
|
||||
mla_selector_config: "MLAPrefillSelectorConfig",
|
||||
) -> "type[MLAPrefillBackend]":
|
||||
"""Get the MLA prefill backend class of a device.
|
||||
|
||||
Platforms that have a single, fixed MLA prefill backend can override
|
||||
this to return it directly. The default implementation selects the
|
||||
best backend from a device-capability-based priority list.
|
||||
|
||||
Returns:
|
||||
The selected backend class.
|
||||
|
||||
Raises:
|
||||
ValueError: If no MLA prefill backend is valid for the
|
||||
configuration.
|
||||
|
||||
"""
|
||||
from vllm.v1.attention.backends.mla.prefill.base import MLADimensions
|
||||
from vllm.v1.attention.backends.mla.prefill.registry import (
|
||||
MLAPrefillBackendEnum,
|
||||
)
|
||||
|
||||
device_capability = cls.get_device_capability()
|
||||
if device_capability is None:
|
||||
return MLAPrefillBackendEnum.FLASH_ATTN.get_class()
|
||||
|
||||
if device_capability.major == 10: # Blackwell
|
||||
if mla_selector_config.mla_dimensions == MLADimensions(
|
||||
qk_nope_head_dim=192,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=256,
|
||||
):
|
||||
priorities = [
|
||||
MLAPrefillBackendEnum.TRTLLM_RAGGED,
|
||||
MLAPrefillBackendEnum.FLASH_ATTN,
|
||||
MLAPrefillBackendEnum.FLASHINFER,
|
||||
MLAPrefillBackendEnum.TOKENSPEED_MLA,
|
||||
]
|
||||
else:
|
||||
priorities = [
|
||||
MLAPrefillBackendEnum.FLASH_ATTN,
|
||||
MLAPrefillBackendEnum.TRTLLM_RAGGED,
|
||||
MLAPrefillBackendEnum.FLASHINFER,
|
||||
MLAPrefillBackendEnum.TOKENSPEED_MLA,
|
||||
]
|
||||
else: # Hopper (SM90) and older
|
||||
priorities = [MLAPrefillBackendEnum.FLASH_ATTN]
|
||||
|
||||
all_invalid_reasons: dict[str, list[str]] = {}
|
||||
|
||||
for backend_enum in priorities:
|
||||
try:
|
||||
backend_cls = backend_enum.get_class()
|
||||
invalid_reasons = backend_cls.validate_configuration(
|
||||
device_capability, mla_selector_config
|
||||
)
|
||||
except ImportError:
|
||||
invalid_reasons = ["ImportError"]
|
||||
if not invalid_reasons:
|
||||
return backend_cls
|
||||
all_invalid_reasons[backend_enum.name] = invalid_reasons
|
||||
|
||||
reasons_str = (
|
||||
"{"
|
||||
+ ", ".join(
|
||||
f"{name}: [{', '.join(reasons)}]"
|
||||
for name, reasons in all_invalid_reasons.items()
|
||||
)
|
||||
+ "}"
|
||||
)
|
||||
logger.debug_once(
|
||||
"Some MLA prefill backends are not valid with %s. Reasons: %s.",
|
||||
repr(mla_selector_config),
|
||||
reasons_str,
|
||||
)
|
||||
raise ValueError(
|
||||
f"No valid MLA prefill backend found with {mla_selector_config!r}. "
|
||||
f"Reasons: {reasons_str}."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_device_capability(
|
||||
cls,
|
||||
|
||||
@@ -24,6 +24,10 @@ if TYPE_CHECKING:
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import (
|
||||
MLAPrefillSelectorConfig,
|
||||
)
|
||||
from vllm.v1.attention.selector import AttentionSelectorConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -904,6 +908,57 @@ class RocmPlatform(Platform):
|
||||
logger.info_once("Using Torch SDPA backend for ViT model.")
|
||||
return AttentionBackendEnum.TORCH_SDPA
|
||||
|
||||
@classmethod
|
||||
def get_mla_prefill_backend_cls(
|
||||
cls,
|
||||
mla_selector_config: "MLAPrefillSelectorConfig",
|
||||
) -> "type[MLAPrefillBackend]":
|
||||
"""On ROCm, prefer the AITER FlashAttention backend, falling back to
|
||||
FlashAttention.
|
||||
|
||||
Raises:
|
||||
ValueError: If neither backend is valid for the configuration.
|
||||
|
||||
"""
|
||||
from vllm.v1.attention.backends.mla.prefill.registry import (
|
||||
MLAPrefillBackendEnum,
|
||||
)
|
||||
|
||||
device_capability = cls.get_device_capability()
|
||||
all_invalid_reasons: dict[str, list[str]] = {}
|
||||
for backend_enum in (
|
||||
MLAPrefillBackendEnum.ROCM_AITER_FA,
|
||||
MLAPrefillBackendEnum.FLASH_ATTN,
|
||||
):
|
||||
try:
|
||||
backend_cls = backend_enum.get_class()
|
||||
invalid_reasons = backend_cls.validate_configuration(
|
||||
device_capability, mla_selector_config
|
||||
)
|
||||
except ImportError:
|
||||
invalid_reasons = ["ImportError"]
|
||||
if not invalid_reasons:
|
||||
return backend_cls
|
||||
all_invalid_reasons[backend_enum.name] = invalid_reasons
|
||||
|
||||
reasons_str = (
|
||||
"{"
|
||||
+ ", ".join(
|
||||
f"{name}: [{', '.join(reasons)}]"
|
||||
for name, reasons in all_invalid_reasons.items()
|
||||
)
|
||||
+ "}"
|
||||
)
|
||||
logger.debug_once(
|
||||
"Some MLA prefill backends are not valid with %s. Reasons: %s.",
|
||||
repr(mla_selector_config),
|
||||
reasons_str,
|
||||
)
|
||||
raise ValueError(
|
||||
f"No valid MLA prefill backend found with {mla_selector_config!r}. "
|
||||
f"Reasons: {reasons_str}."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def set_device(cls, device: torch.device) -> None:
|
||||
"""Set the device for the current platform."""
|
||||
|
||||
Reference in New Issue
Block a user