[Attention] Implement universal BACKEND_MAP (#25900)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
@@ -34,7 +34,7 @@ DEVICE_MLA_BACKENDS = {
|
||||
|
||||
DEVICE_REGULAR_ATTN_BACKENDS = {
|
||||
"cuda": ["XFORMERS", "FLASHINFER", "FLASH_ATTN"],
|
||||
"hip": ["ROCM_FLASH"],
|
||||
"hip": ["ROCM_ATTN"],
|
||||
"cpu": ["TORCH_SDPA"],
|
||||
}
|
||||
|
||||
@@ -122,7 +122,7 @@ def test_env(
|
||||
backend = get_attn_backend(
|
||||
16, torch.float16, None, block_size, use_mla=use_mla
|
||||
)
|
||||
expected = "TRITON_ATTN"
|
||||
expected = "ROCM_ATTN"
|
||||
assert backend.get_name() == expected
|
||||
|
||||
elif device == "cuda":
|
||||
|
||||
@@ -18,7 +18,7 @@ def clear_cache():
|
||||
@pytest.mark.skip(reason="Skipped for now. Should be revisited.")
|
||||
def test_selector(monkeypatch: pytest.MonkeyPatch):
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv(STR_BACKEND_ENV_VAR, "ROCM_FLASH")
|
||||
m.setenv(STR_BACKEND_ENV_VAR, "ROCM_ATTN")
|
||||
|
||||
# Set the current platform to ROCm using monkeypatch
|
||||
monkeypatch.setattr("vllm.attention.selector.current_platform", RocmPlatform())
|
||||
|
||||
@@ -14,7 +14,7 @@ from tests.v1.attention.utils import (
|
||||
create_common_attn_metadata,
|
||||
create_standard_kv_cache_spec,
|
||||
create_vllm_config,
|
||||
get_attention_backend,
|
||||
try_get_attention_backend,
|
||||
)
|
||||
from vllm.attention.backends.registry import _Backend
|
||||
from vllm.config import ModelConfig
|
||||
@@ -214,7 +214,7 @@ def run_attention_backend(
|
||||
actual_backend = _Backend.FLEX_ATTENTION
|
||||
use_direct_block_mask = False
|
||||
|
||||
builder_cls, impl_cls = get_attention_backend(actual_backend)
|
||||
builder_cls, impl_cls = try_get_attention_backend(actual_backend)
|
||||
|
||||
# Mock flashinfer's get_per_layer_parameters if needed
|
||||
if actual_backend == _Backend.FLASHINFER:
|
||||
|
||||
@@ -12,7 +12,7 @@ from tests.v1.attention.utils import (
|
||||
create_common_attn_metadata,
|
||||
create_standard_kv_cache_spec,
|
||||
create_vllm_config,
|
||||
get_attention_backend,
|
||||
try_get_attention_backend,
|
||||
)
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.attention.backends.registry import _Backend
|
||||
@@ -239,7 +239,7 @@ def run_attention_backend(
|
||||
) -> torch.Tensor:
|
||||
"""Run attention computation using the specified backend's AttentionImpl."""
|
||||
|
||||
builder_cls, impl_cls = get_attention_backend(backend)
|
||||
builder_cls, impl_cls = try_get_attention_backend(backend)
|
||||
|
||||
# Build metadata
|
||||
builder = builder_cls(kv_cache_spec, layer_names, vllm_config, device)
|
||||
@@ -400,7 +400,7 @@ def test_backend_correctness(dist_init, batch_spec_name: str, model: str):
|
||||
# Determine if this is decode or prefill
|
||||
is_decode = []
|
||||
for i, backend in enumerate(BACKENDS_TO_TEST):
|
||||
builder_cls, _ = get_attention_backend(backend)
|
||||
builder_cls, _ = try_get_attention_backend(backend)
|
||||
is_decode.append(q_len <= builder_cls.reorder_batch_threshold)
|
||||
|
||||
# Split q into nope and rope components
|
||||
|
||||
+14
-38
@@ -8,7 +8,8 @@ from typing import Optional, Union
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.attention.backends.registry import _Backend
|
||||
from vllm.attention.backends.abstract import AttentionImpl
|
||||
from vllm.attention.backends.registry import _Backend, backend_to_class_str
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
@@ -20,9 +21,11 @@ from vllm.config import (
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.model import ModelDType
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import resolve_obj_by_qualname
|
||||
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
|
||||
@@ -117,44 +120,17 @@ def create_common_attn_metadata(
|
||||
)
|
||||
|
||||
|
||||
def get_attention_backend(backend_name: _Backend):
|
||||
"""Set up attention backend classes for testing.
|
||||
|
||||
Args:
|
||||
backend_name: Name of the backend ("flash_attn", "flashinfer", etc.)
|
||||
vllm_config: VllmConfig instance
|
||||
|
||||
Returns:
|
||||
Tuple of (backend_builder_class, backend_impl_class)
|
||||
"""
|
||||
backend_map = {
|
||||
_Backend.FLASH_ATTN: (
|
||||
"vllm.v1.attention.backends.flash_attn.FlashAttentionBackend"
|
||||
if current_platform.is_cuda()
|
||||
else "vllm.v1.attention.backends.rocm_aiter_fa.AiterFlashAttentionBackend"
|
||||
),
|
||||
_Backend.FLASHINFER: "vllm.v1.attention.backends.flashinfer.FlashInferBackend",
|
||||
_Backend.FLEX_ATTENTION: "vllm.v1.attention.backends.flex_attention.FlexAttentionBackend", # noqa: E501
|
||||
_Backend.TRITON_ATTN: "vllm.v1.attention.backends.triton_attn.TritonAttentionBackend", # noqa: E501
|
||||
_Backend.TREE_ATTN: "vllm.v1.attention.backends.tree_attn.TreeAttentionBackend",
|
||||
_Backend.XFORMERS: "vllm.v1.attention.backends.xformers.XFormersAttentionBackend", # noqa: E501
|
||||
_Backend.CUTLASS_MLA: "vllm.v1.attention.backends.mla.cutlass_mla.CutlassMLABackend", # noqa: E501
|
||||
_Backend.FLASHMLA: "vllm.v1.attention.backends.mla.flashmla.FlashMLABackend",
|
||||
_Backend.FLASH_ATTN_MLA: "vllm.v1.attention.backends.mla.flashattn_mla.FlashAttnMLABackend", # noqa: E501
|
||||
_Backend.FLASHINFER_MLA: "vllm.v1.attention.backends.mla.flashinfer_mla.FlashInferMLABackend", # noqa: E501
|
||||
_Backend.TRITON_MLA: "vllm.v1.attention.backends.mla.triton_mla.TritonMLABackend", # noqa: E501
|
||||
}
|
||||
|
||||
if backend_name not in backend_map:
|
||||
raise ValueError(f"Unknown backend: {backend_name}")
|
||||
|
||||
backend_class_name = backend_map[backend_name]
|
||||
|
||||
def try_get_attention_backend(
|
||||
backend: _Backend,
|
||||
) -> tuple[type[AttentionMetadataBuilder], type[AttentionImpl]]:
|
||||
"""Try to get the attention backend class, skipping test if not found."""
|
||||
backend_class_str = backend_to_class_str(backend)
|
||||
try:
|
||||
backend_class = resolve_obj_by_qualname(backend_class_name)
|
||||
backend_class = resolve_obj_by_qualname(backend_class_str)
|
||||
return backend_class.get_builder_cls(), backend_class.get_impl_cls()
|
||||
except ImportError as e:
|
||||
pytest.skip(f"{backend_name} not available: {e}")
|
||||
pytest.skip(f"{backend_class_str} not available: {e}")
|
||||
raise AssertionError("unreachable") from None
|
||||
|
||||
|
||||
def create_standard_kv_cache_spec(vllm_config: VllmConfig) -> FullAttentionSpec:
|
||||
|
||||
@@ -12,7 +12,7 @@ from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_standard_kv_cache_spec,
|
||||
get_attention_backend,
|
||||
try_get_attention_backend,
|
||||
)
|
||||
from vllm.attention.backends.registry import _Backend
|
||||
from vllm.config import (
|
||||
@@ -535,11 +535,11 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch):
|
||||
sampling_metadata = mock.MagicMock()
|
||||
|
||||
if attn_backend == "FLASH_ATTN":
|
||||
attn_metadata_builder_cls, _ = get_attention_backend(_Backend.FLASH_ATTN)
|
||||
attn_metadata_builder_cls, _ = try_get_attention_backend(_Backend.FLASH_ATTN)
|
||||
elif attn_backend == "TRITON_ATTN":
|
||||
attn_metadata_builder_cls, _ = get_attention_backend(_Backend.TRITON_ATTN)
|
||||
attn_metadata_builder_cls, _ = try_get_attention_backend(_Backend.TRITON_ATTN)
|
||||
elif attn_backend == "TREE_ATTN":
|
||||
attn_metadata_builder_cls, _ = get_attention_backend(_Backend.TREE_ATTN)
|
||||
attn_metadata_builder_cls, _ = try_get_attention_backend(_Backend.TREE_ATTN)
|
||||
else:
|
||||
raise ValueError(f"Unsupported attention backend: {attn_backend}")
|
||||
|
||||
@@ -674,7 +674,7 @@ def test_propose_tree(spec_token_tree):
|
||||
proposer.attn_layer_names = ["layer.0"]
|
||||
|
||||
# Get the tree attention metadata builder.
|
||||
attn_metadata_builder_cls, _ = get_attention_backend(_Backend.TREE_ATTN)
|
||||
attn_metadata_builder_cls, _ = try_get_attention_backend(_Backend.TREE_ATTN)
|
||||
attn_metadata_builder = attn_metadata_builder_cls(
|
||||
kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config),
|
||||
layer_names=proposer.attn_layer_names,
|
||||
|
||||
@@ -10,7 +10,7 @@ from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_standard_kv_cache_spec,
|
||||
get_attention_backend,
|
||||
try_get_attention_backend,
|
||||
)
|
||||
from vllm.attention.backends.registry import _Backend
|
||||
from vllm.config import (
|
||||
@@ -177,7 +177,7 @@ def test_mtp_propose(num_speculative_tokens, monkeypatch):
|
||||
sampling_metadata = mock.MagicMock()
|
||||
|
||||
# Setup attention metadata
|
||||
attn_metadata_builder_cls, _ = get_attention_backend(_Backend.FLASH_ATTN)
|
||||
attn_metadata_builder_cls, _ = try_get_attention_backend(_Backend.FLASH_ATTN)
|
||||
|
||||
attn_metadata_builder = attn_metadata_builder_cls(
|
||||
kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config),
|
||||
|
||||
@@ -9,7 +9,7 @@ import torch
|
||||
from tests.v1.attention.utils import (
|
||||
create_standard_kv_cache_spec,
|
||||
create_vllm_config,
|
||||
get_attention_backend,
|
||||
try_get_attention_backend,
|
||||
)
|
||||
from vllm.attention.backends.registry import _Backend
|
||||
from vllm.config import ParallelConfig, SpeculativeConfig
|
||||
@@ -63,7 +63,7 @@ def forward_attention(
|
||||
|
||||
# Build common metadata.
|
||||
model_name = "meta-llama/Meta-Llama-3-8B"
|
||||
builder_cls, impl_cls = get_attention_backend(backend)
|
||||
builder_cls, impl_cls = try_get_attention_backend(backend)
|
||||
vllm_config = create_vllm_config(model_name=model_name, max_model_len=max(seq_lens))
|
||||
if spec_token_tree is not None:
|
||||
# Create speculative config if token tree is specified.
|
||||
|
||||
Reference in New Issue
Block a user