[Attention] Abstract the MLA prefill backends and eliminate cuDNN (#32623)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-05-01 13:36:20 -04:00
committed by GitHub
co-authored by Michael Goin Lucas Wilkinson mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 51295793a2
commit f3fef12350
16 changed files with 1629 additions and 708 deletions
-3
View File
@@ -333,8 +333,6 @@ def test_attention_config():
"true",
"--attention-config.flash_attn_max_num_splits_for_cuda_graph",
"16",
"--attention-config.use_cudnn_prefill",
"true",
"--attention-config.use_trtllm_ragged_deepseek_prefill",
"true",
"--attention-config.use_trtllm_attention",
@@ -352,7 +350,6 @@ def test_attention_config():
assert engine_args.attention_config.flash_attn_version == 3
assert engine_args.attention_config.use_prefill_decode_attention is True
assert engine_args.attention_config.flash_attn_max_num_splits_for_cuda_graph == 16
assert engine_args.attention_config.use_cudnn_prefill is True
assert engine_args.attention_config.use_trtllm_ragged_deepseek_prefill is True
assert engine_args.attention_config.use_trtllm_attention is True
assert engine_args.attention_config.disable_flashinfer_prefill is True
+1
View File
@@ -672,6 +672,7 @@ def run_attention_backend(
def test_backend_correctness(
default_vllm_config,
dist_init,
workspace_init,
batch_spec_name: str,
model: str,
tensor_parallel_size: int,
@@ -0,0 +1,304 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for MLA prefill backend selector."""
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.config import AttentionConfig, ModelConfig, VllmConfig
from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
from vllm.v1.attention.backends.mla.prefill.selector import (
MLAPrefillSelectorConfig,
_auto_select_mla_prefill_backend,
get_mla_prefill_backend,
is_deepseek_r1_mla_compatible,
)
@pytest.fixture(autouse=True)
def clear_cache():
"""Clear lru cache to ensure each test case runs without caching."""
_auto_select_mla_prefill_backend.cache_clear()
def _make_mock_model_config(
qk_nope_head_dim: int = 128,
qk_rope_head_dim: int = 64,
v_head_dim: int = 128,
dtype: torch.dtype = torch.bfloat16,
) -> ModelConfig:
mock_config = MagicMock(spec=ModelConfig)
mock_config.dtype = dtype
mock_config.hf_text_config = MagicMock()
mock_config.hf_text_config.qk_nope_head_dim = qk_nope_head_dim
mock_config.hf_text_config.qk_rope_head_dim = qk_rope_head_dim
mock_config.hf_text_config.v_head_dim = v_head_dim
return mock_config
def _make_vllm_config(
model_config: ModelConfig | None = None,
mla_prefill_backend: MLAPrefillBackendEnum | None = None,
) -> VllmConfig:
if model_config is None:
model_config = _make_mock_model_config()
attention_config = AttentionConfig(mla_prefill_backend=mla_prefill_backend)
mock_vllm_config = MagicMock(spec=VllmConfig)
mock_vllm_config.model_config = model_config
mock_vllm_config.attention_config = attention_config
return mock_vllm_config
class TestGetMLAPrefillBackend:
"""Tests for get_mla_prefill_backend (public API)."""
def test_no_device_capability_returns_flash_attn(self):
vllm_config = _make_vllm_config()
with patch("vllm.platforms.current_platform") as mock_platform:
mock_platform.get_device_capability.return_value = None
backend = get_mla_prefill_backend(vllm_config)
assert backend.get_name() == "FLASH_ATTN"
def test_explicit_flash_attn_selection(self):
try:
flash_attn_cls = MLAPrefillBackendEnum.FLASH_ATTN.get_class()
except ImportError:
pytest.skip("FLASH_ATTN backend not available")
return
vllm_config = _make_vllm_config(
mla_prefill_backend=MLAPrefillBackendEnum.FLASH_ATTN,
)
with patch("vllm.platforms.current_platform") as mock_platform:
mock_platform.get_device_capability.return_value = DeviceCapability(
major=9, minor=0
)
with patch.object(
flash_attn_cls,
"validate_configuration",
return_value=[],
):
backend = get_mla_prefill_backend(vllm_config)
assert backend.get_name() == "FLASH_ATTN"
def test_explicit_backend_invalid_raises_error(self):
vllm_config = _make_vllm_config(
mla_prefill_backend=MLAPrefillBackendEnum.FLASHINFER,
)
with patch("vllm.platforms.current_platform") as mock_platform:
mock_platform.get_device_capability.return_value = DeviceCapability(
major=9, minor=0
)
with pytest.raises(ValueError, match="is not valid"):
get_mla_prefill_backend(vllm_config)
def test_explicit_backend_import_error_raises(self):
vllm_config = _make_vllm_config(
mla_prefill_backend=MLAPrefillBackendEnum.TRTLLM_RAGGED,
)
with patch("vllm.platforms.current_platform") as mock_platform:
mock_platform.get_device_capability.return_value = DeviceCapability(
major=10, minor=0
)
with (
patch.object(
MLAPrefillBackendEnum.TRTLLM_RAGGED,
"get_class",
side_effect=ImportError("trtllm not installed"),
),
pytest.raises(ValueError, match="is not valid"),
):
get_mla_prefill_backend(vllm_config)
def test_auto_selection_on_hopper(self):
try:
flash_attn_cls = MLAPrefillBackendEnum.FLASH_ATTN.get_class()
except ImportError:
pytest.skip("FLASH_ATTN backend not available")
return
vllm_config = _make_vllm_config()
with patch("vllm.platforms.current_platform") as mock_platform:
mock_platform.get_device_capability.return_value = DeviceCapability(
major=9, minor=0
)
with patch.object(
flash_attn_cls,
"validate_configuration",
return_value=[],
):
backend = get_mla_prefill_backend(vllm_config)
assert backend.get_name() == "FLASH_ATTN"
class TestAutoSelectMLAPrefillBackend:
"""Tests for fallback and error paths in auto-selection."""
def test_blackwell_falls_back_to_trtllm(self):
vllm_config = _make_vllm_config()
capability = DeviceCapability(major=10, minor=0)
selector_config = MLAPrefillSelectorConfig(
dtype=torch.bfloat16,
is_r1_compatible=is_deepseek_r1_mla_compatible(vllm_config),
)
try:
trtllm_cls = MLAPrefillBackendEnum.TRTLLM_RAGGED.get_class()
except ImportError:
pytest.skip("TRTLLM_RAGGED backend not available")
return
with (
patch.object(
MLAPrefillBackendEnum.FLASH_ATTN,
"get_class",
side_effect=ImportError("FLASH_ATTN not available"),
),
patch.object(trtllm_cls, "validate_configuration", return_value=[]),
):
backend = _auto_select_mla_prefill_backend(
capability,
selector_config,
)
assert backend.get_name() == "TRTLLM_RAGGED"
def test_all_fail_raises_error(self):
vllm_config = _make_vllm_config()
capability = DeviceCapability(major=10, minor=0)
selector_config = MLAPrefillSelectorConfig(
dtype=torch.bfloat16,
is_r1_compatible=is_deepseek_r1_mla_compatible(vllm_config),
)
def mock_get_class(backend_enum): # noqa: ARG001
cls = MagicMock()
cls.validate_configuration.return_value = ["not available"]
return cls
with patch.object(MLAPrefillBackendEnum, "get_class", mock_get_class):
_auto_select_mla_prefill_backend.cache_clear()
with pytest.raises(ValueError, match="No valid MLA"):
_auto_select_mla_prefill_backend(
capability,
selector_config,
)
class TestBackendValidation:
"""Tests for backend validation logic."""
def test_r1_dimension_requirement(self):
try:
from vllm.v1.attention.backends.mla.prefill.flashinfer import (
FlashInferPrefillBackend,
)
except ImportError:
pytest.skip("FlashInfer prefill backend not available")
return
assert FlashInferPrefillBackend.requires_r1_mla_dimensions is True
vllm_config = _make_vllm_config(
model_config=_make_mock_model_config(
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
)
)
capability = DeviceCapability(major=10, minor=0)
selector_config = MLAPrefillSelectorConfig(
dtype=torch.bfloat16,
is_r1_compatible=is_deepseek_r1_mla_compatible(vllm_config),
)
with patch.object(FlashInferPrefillBackend, "is_available", return_value=True):
invalid_reasons = FlashInferPrefillBackend.validate_configuration(
capability,
selector_config,
)
assert len(invalid_reasons) == 0
vllm_config_invalid = _make_vllm_config(
model_config=_make_mock_model_config(
qk_nope_head_dim=64,
qk_rope_head_dim=64,
v_head_dim=128,
)
)
selector_config_invalid = MLAPrefillSelectorConfig(
dtype=torch.bfloat16,
is_r1_compatible=is_deepseek_r1_mla_compatible(vllm_config_invalid),
)
with patch.object(FlashInferPrefillBackend, "is_available", return_value=True):
invalid_reasons = FlashInferPrefillBackend.validate_configuration(
capability,
selector_config_invalid,
)
assert len(invalid_reasons) == 1
assert "DeepSeek R1 MLA dimensions" in invalid_reasons[0]
class TestMLAPrefillBackendParsing:
"""Tests for string-based mla_prefill_backend parsing from CLI args."""
def test_valid_string_parses_to_enum(self):
config = AttentionConfig(
mla_prefill_backend="FLASH_ATTN", # type: ignore[arg-type]
)
assert config.mla_prefill_backend == MLAPrefillBackendEnum.FLASH_ATTN
def test_invalid_string_raises_error(self):
with pytest.raises(ValueError, match="Unknown MLA prefill backend"):
AttentionConfig(
mla_prefill_backend="NONEXISTENT", # type: ignore[arg-type]
)
class TestDeprecatedFlagMigration:
"""Tests for _migrate_deprecated_mla_prefill_flags in AttentionConfig."""
def test_no_deprecated_flags_leaves_backend_none(self):
config = AttentionConfig()
assert config.mla_prefill_backend is None
def test_use_trtllm_ragged_migrates_to_trtllm_ragged(self):
config = AttentionConfig(use_trtllm_ragged_deepseek_prefill=True)
assert config.mla_prefill_backend == MLAPrefillBackendEnum.TRTLLM_RAGGED
def test_disable_flashinfer_prefill_migrates_to_flash_attn(self):
config = AttentionConfig(disable_flashinfer_prefill=True)
assert config.mla_prefill_backend == MLAPrefillBackendEnum.FLASH_ATTN
def test_explicit_backend_ignores_deprecated_flags(self):
config = AttentionConfig(
mla_prefill_backend=MLAPrefillBackendEnum.FLASH_ATTN,
use_cudnn_prefill=True,
)
assert config.mla_prefill_backend == MLAPrefillBackendEnum.FLASH_ATTN
def test_cudnn_raises_error(self):
match = "cuDNN MLA prefill backend has been removed"
with pytest.raises(ValueError, match=match):
AttentionConfig(use_cudnn_prefill=True)
def test_trtllm_takes_priority_over_disable_flashinfer(self):
config = AttentionConfig(
use_trtllm_ragged_deepseek_prefill=True,
disable_flashinfer_prefill=True,
)
assert config.mla_prefill_backend == MLAPrefillBackendEnum.TRTLLM_RAGGED