[MLA][Attention] Add OOT MLA prefill backend registration mechanism (#43325)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
@@ -0,0 +1,137 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for MLA prefill backend registry."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
|
||||
from vllm.v1.attention.backends.mla.prefill.registry import (
|
||||
MLAPrefillBackendEnum,
|
||||
register_mla_prefill_backend,
|
||||
)
|
||||
|
||||
|
||||
class CustomMLAPrefillBackend(MLAPrefillBackend):
|
||||
"""Mock custom MLA prefill backend for testing."""
|
||||
|
||||
supported_dtypes = [torch.bfloat16, torch.float16]
|
||||
requires_r1_mla_dimensions = False
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "CUSTOM"
|
||||
|
||||
def run_prefill_new_tokens(self, q, k, v, return_softmax_lse):
|
||||
raise NotImplementedError
|
||||
|
||||
def run_prefill_context_chunk(self, chunk_idx, q, k, v):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cleanup_overrides():
|
||||
"""Clear any overrides after each test."""
|
||||
yield
|
||||
for member in MLAPrefillBackendEnum:
|
||||
member.clear_override()
|
||||
|
||||
|
||||
def test_custom_is_not_alias_of_any_backend():
|
||||
all_backends = list(MLAPrefillBackendEnum)
|
||||
|
||||
aliases = []
|
||||
for backend in all_backends:
|
||||
if backend.name != "CUSTOM" and backend is MLAPrefillBackendEnum.CUSTOM:
|
||||
aliases.append(backend.name)
|
||||
|
||||
assert len(aliases) == 0, (
|
||||
f"BUG! CUSTOM is an alias of: {', '.join(aliases)}!\n"
|
||||
f"CUSTOM.value = {repr(MLAPrefillBackendEnum.CUSTOM.value)}\n"
|
||||
f"All MLA prefill backend values:\n"
|
||||
+ "\n".join(f" {b.name}: {repr(b.value)}" for b in all_backends)
|
||||
)
|
||||
|
||||
assert MLAPrefillBackendEnum.CUSTOM.name == "CUSTOM"
|
||||
|
||||
|
||||
def test_custom_unregistered_raises():
|
||||
with pytest.raises(ValueError, match="must be registered before use"):
|
||||
MLAPrefillBackendEnum.CUSTOM.get_path()
|
||||
|
||||
|
||||
def test_register_custom_backend_with_class_path():
|
||||
register_mla_prefill_backend(
|
||||
backend=MLAPrefillBackendEnum.CUSTOM,
|
||||
class_path=(
|
||||
"tests.v1.attention.test_mla_prefill_registry.CustomMLAPrefillBackend"
|
||||
),
|
||||
)
|
||||
|
||||
assert MLAPrefillBackendEnum.CUSTOM.is_overridden()
|
||||
|
||||
class_path = MLAPrefillBackendEnum.CUSTOM.get_path()
|
||||
assert class_path == (
|
||||
"tests.v1.attention.test_mla_prefill_registry.CustomMLAPrefillBackend"
|
||||
)
|
||||
|
||||
backend_cls = MLAPrefillBackendEnum.CUSTOM.get_class()
|
||||
assert backend_cls.get_name() == "CUSTOM"
|
||||
|
||||
|
||||
def test_register_custom_backend_as_decorator():
|
||||
@register_mla_prefill_backend(MLAPrefillBackendEnum.CUSTOM)
|
||||
class DecoratedPrefillBackend(MLAPrefillBackend):
|
||||
supported_dtypes = [torch.bfloat16]
|
||||
requires_r1_mla_dimensions = False
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "DECORATED"
|
||||
|
||||
def run_prefill_new_tokens(self, q, k, v, return_softmax_lse):
|
||||
raise NotImplementedError
|
||||
|
||||
def run_prefill_context_chunk(self, chunk_idx, q, k, v):
|
||||
raise NotImplementedError
|
||||
|
||||
assert MLAPrefillBackendEnum.CUSTOM.is_overridden()
|
||||
assert "DecoratedPrefillBackend" in MLAPrefillBackendEnum.CUSTOM.get_path()
|
||||
|
||||
|
||||
def test_override_existing_backend():
|
||||
original_path = MLAPrefillBackendEnum.FLASH_ATTN.get_path()
|
||||
|
||||
register_mla_prefill_backend(
|
||||
backend=MLAPrefillBackendEnum.FLASH_ATTN,
|
||||
class_path=(
|
||||
"tests.v1.attention.test_mla_prefill_registry.CustomMLAPrefillBackend"
|
||||
),
|
||||
)
|
||||
|
||||
assert MLAPrefillBackendEnum.FLASH_ATTN.is_overridden()
|
||||
assert MLAPrefillBackendEnum.FLASH_ATTN.get_path() != original_path
|
||||
|
||||
backend_cls = MLAPrefillBackendEnum.FLASH_ATTN.get_class()
|
||||
assert backend_cls.get_name() == "CUSTOM"
|
||||
|
||||
|
||||
def test_clear_override():
|
||||
original_path = MLAPrefillBackendEnum.FLASH_ATTN.get_path()
|
||||
|
||||
register_mla_prefill_backend(
|
||||
backend=MLAPrefillBackendEnum.FLASH_ATTN,
|
||||
class_path=(
|
||||
"tests.v1.attention.test_mla_prefill_registry.CustomMLAPrefillBackend"
|
||||
),
|
||||
)
|
||||
assert MLAPrefillBackendEnum.FLASH_ATTN.is_overridden()
|
||||
|
||||
MLAPrefillBackendEnum.FLASH_ATTN.clear_override()
|
||||
assert not MLAPrefillBackendEnum.FLASH_ATTN.is_overridden()
|
||||
assert MLAPrefillBackendEnum.FLASH_ATTN.get_path() == original_path
|
||||
|
||||
|
||||
def test_unknown_backend_name_raises():
|
||||
with pytest.raises(ValueError, match="Unknown MLA prefill backend"):
|
||||
MLAPrefillBackendEnum["NONEXISTENT"]
|
||||
@@ -1,11 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
|
||||
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
|
||||
from vllm.v1.attention.backends.mla.prefill.registry import (
|
||||
MLAPrefillBackendEnum,
|
||||
register_mla_prefill_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import get_mla_prefill_backend
|
||||
|
||||
__all__ = [
|
||||
"MLAPrefillBackend",
|
||||
"MLAPrefillBackendEnum",
|
||||
"get_mla_prefill_backend",
|
||||
"register_mla_prefill_backend",
|
||||
]
|
||||
|
||||
@@ -3,9 +3,10 @@
|
||||
"""Registry for MLA prefill backends.
|
||||
|
||||
This module provides an enumeration of all available MLA prefill backends
|
||||
and utilities for loading them.
|
||||
and utilities for loading and registering them.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from enum import Enum, EnumMeta
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -47,11 +48,91 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta):
|
||||
"vllm.v1.attention.backends.mla.prefill.tokenspeed_mla."
|
||||
"TokenspeedMLAPrefillBackend"
|
||||
)
|
||||
# Placeholder for third-party/custom backends - must be registered before use
|
||||
# set to None to avoid alias with other backend, whose value is an empty string
|
||||
CUSTOM = None
|
||||
|
||||
def get_path(self) -> str:
|
||||
"""Get the fully qualified class path for this backend."""
|
||||
return self.value
|
||||
"""Get the class path for this backend (respects overrides).
|
||||
|
||||
Returns:
|
||||
The fully qualified class path string
|
||||
|
||||
Raises:
|
||||
ValueError: If Backend.CUSTOM is used without being registered
|
||||
"""
|
||||
path = _MLA_PREFILL_OVERRIDES.get(self, self.value)
|
||||
if not path:
|
||||
raise ValueError(
|
||||
f"MLA prefill backend {self.name} must be registered before "
|
||||
f"use. Use register_mla_prefill_backend("
|
||||
f"MLAPrefillBackendEnum.{self.name}, "
|
||||
f"'your.module.YourClass')"
|
||||
)
|
||||
return path
|
||||
|
||||
def get_class(self) -> "type[MLAPrefillBackend]":
|
||||
"""Lazy load and return the backend class."""
|
||||
"""Get the backend class (respects overrides).
|
||||
|
||||
Returns:
|
||||
The backend class
|
||||
|
||||
Raises:
|
||||
ImportError: If the backend class cannot be imported
|
||||
ValueError: If CUSTOM is used without being registered
|
||||
"""
|
||||
return resolve_obj_by_qualname(self.get_path())
|
||||
|
||||
def is_overridden(self) -> bool:
|
||||
"""Check if this backend has been overridden."""
|
||||
return self in _MLA_PREFILL_OVERRIDES
|
||||
|
||||
def clear_override(self) -> None:
|
||||
"""Clear any override for this backend, reverting to the default."""
|
||||
_MLA_PREFILL_OVERRIDES.pop(self, None)
|
||||
|
||||
|
||||
_MLA_PREFILL_OVERRIDES: dict[MLAPrefillBackendEnum, str] = {}
|
||||
|
||||
|
||||
def register_mla_prefill_backend(
|
||||
backend: MLAPrefillBackendEnum,
|
||||
class_path: str | None = None,
|
||||
) -> Callable[[type], type]:
|
||||
"""Register or override an MLA prefill backend implementation.
|
||||
|
||||
Args:
|
||||
backend: The MLAPrefillBackendEnum member to register.
|
||||
class_path: Optional class path. If not provided and used as
|
||||
decorator, will be auto-generated from the class.
|
||||
|
||||
Returns:
|
||||
Decorator function if class_path is None, otherwise a no-op.
|
||||
|
||||
Examples:
|
||||
# Override an existing MLA prefill backend
|
||||
@register_mla_prefill_backend(MLAPrefillBackendEnum.FLASH_ATTN)
|
||||
class MyCustomFlashAttn(MLAPrefillBackend):
|
||||
...
|
||||
|
||||
# Register a custom third-party MLA prefill backend
|
||||
@register_mla_prefill_backend(MLAPrefillBackendEnum.CUSTOM)
|
||||
class MyCustomPrefillBackend(MLAPrefillBackend):
|
||||
...
|
||||
|
||||
# Direct registration
|
||||
register_mla_prefill_backend(
|
||||
MLAPrefillBackendEnum.CUSTOM,
|
||||
"my.module.MyCustomPrefillBackend"
|
||||
)
|
||||
"""
|
||||
|
||||
def decorator(cls: type) -> type:
|
||||
_MLA_PREFILL_OVERRIDES[backend] = f"{cls.__module__}.{cls.__qualname__}"
|
||||
return cls
|
||||
|
||||
if class_path is not None:
|
||||
_MLA_PREFILL_OVERRIDES[backend] = class_path
|
||||
return lambda x: x
|
||||
|
||||
return decorator
|
||||
|
||||
Reference in New Issue
Block a user