[Kernel] Support GLM-5 dimensions for TRT-LLM ragged MLA prefill (#43525)
Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
parent
b9684d99e9
commit
9d4dc4ca2f
@@ -765,7 +765,8 @@ def test_backend_correctness(
|
||||
if not backends_to_test:
|
||||
pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}")
|
||||
|
||||
# Skip prefill backends that can't satisfy capability/deps/R1 constraints.
|
||||
# Skip prefill backends that can't satisfy capability/deps/dimension constraints.
|
||||
from vllm.v1.attention.backends.mla.prefill.base import MLADimensions
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import (
|
||||
MLAPrefillSelectorConfig,
|
||||
)
|
||||
@@ -773,7 +774,14 @@ def test_backend_correctness(
|
||||
try:
|
||||
prefill_invalid_reasons = prefill_backend.get_class().validate_configuration(
|
||||
current_platform.get_device_capability(),
|
||||
MLAPrefillSelectorConfig(dtype=torch.bfloat16, is_r1_compatible=True),
|
||||
MLAPrefillSelectorConfig(
|
||||
dtype=torch.bfloat16,
|
||||
mla_dimensions=MLADimensions(
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
),
|
||||
),
|
||||
)
|
||||
except ImportError:
|
||||
prefill_invalid_reasons = ["ImportError"]
|
||||
|
||||
@@ -16,7 +16,6 @@ class CustomMLAPrefillBackend(MLAPrefillBackend):
|
||||
"""Mock custom MLA prefill backend for testing."""
|
||||
|
||||
supported_dtypes = [torch.bfloat16, torch.float16]
|
||||
requires_r1_mla_dimensions = False
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
@@ -83,7 +82,6 @@ def test_register_custom_backend_as_decorator():
|
||||
@register_mla_prefill_backend(MLAPrefillBackendEnum.CUSTOM)
|
||||
class DecoratedPrefillBackend(MLAPrefillBackend):
|
||||
supported_dtypes = [torch.bfloat16]
|
||||
requires_r1_mla_dimensions = False
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
|
||||
@@ -9,12 +9,12 @@ import torch
|
||||
|
||||
from vllm.config import AttentionConfig, ModelConfig, VllmConfig
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends.mla.prefill.base import MLADimensions
|
||||
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import (
|
||||
MLAPrefillSelectorConfig,
|
||||
_auto_select_mla_prefill_backend,
|
||||
get_mla_prefill_backend,
|
||||
is_deepseek_r1_mla_compatible,
|
||||
)
|
||||
|
||||
|
||||
@@ -149,11 +149,14 @@ class TestAutoSelectMLAPrefillBackend:
|
||||
"""Tests for fallback and error paths in auto-selection."""
|
||||
|
||||
def test_blackwell_falls_back_to_trtllm(self):
|
||||
vllm_config = _make_vllm_config()
|
||||
capability = DeviceCapability(major=10, minor=0)
|
||||
selector_config = MLAPrefillSelectorConfig(
|
||||
dtype=torch.bfloat16,
|
||||
is_r1_compatible=is_deepseek_r1_mla_compatible(vllm_config),
|
||||
mla_dimensions=MLADimensions(
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
),
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -177,11 +180,14 @@ class TestAutoSelectMLAPrefillBackend:
|
||||
assert backend.get_name() == "TRTLLM_RAGGED"
|
||||
|
||||
def test_all_fail_raises_error(self):
|
||||
vllm_config = _make_vllm_config()
|
||||
capability = DeviceCapability(major=10, minor=0)
|
||||
selector_config = MLAPrefillSelectorConfig(
|
||||
dtype=torch.bfloat16,
|
||||
is_r1_compatible=is_deepseek_r1_mla_compatible(vllm_config),
|
||||
mla_dimensions=MLADimensions(
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
),
|
||||
)
|
||||
|
||||
def mock_get_class(backend_enum): # noqa: ARG001
|
||||
@@ -201,28 +207,26 @@ class TestAutoSelectMLAPrefillBackend:
|
||||
class TestBackendValidation:
|
||||
"""Tests for backend validation logic."""
|
||||
|
||||
def test_r1_dimension_requirement(self):
|
||||
def test_backend_supported_dimension_validation(self):
|
||||
try:
|
||||
from vllm.v1.attention.backends.mla.prefill.flashinfer import (
|
||||
FlashInferPrefillBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill.trtllm_ragged import (
|
||||
TrtllmRaggedPrefillBackend,
|
||||
)
|
||||
except ImportError:
|
||||
pytest.skip("FlashInfer prefill backend not available")
|
||||
pytest.skip("MLA prefill backend not available")
|
||||
return
|
||||
|
||||
assert FlashInferPrefillBackend.requires_r1_mla_dimensions is True
|
||||
|
||||
vllm_config = _make_vllm_config(
|
||||
model_config=_make_mock_model_config(
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
)
|
||||
)
|
||||
capability = DeviceCapability(major=10, minor=0)
|
||||
selector_config = MLAPrefillSelectorConfig(
|
||||
dtype=torch.bfloat16,
|
||||
is_r1_compatible=is_deepseek_r1_mla_compatible(vllm_config),
|
||||
mla_dimensions=MLADimensions(
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
),
|
||||
)
|
||||
|
||||
with patch.object(FlashInferPrefillBackend, "is_available", return_value=True):
|
||||
@@ -232,16 +236,13 @@ class TestBackendValidation:
|
||||
)
|
||||
assert len(invalid_reasons) == 0
|
||||
|
||||
vllm_config_invalid = _make_vllm_config(
|
||||
model_config=_make_mock_model_config(
|
||||
selector_config_invalid = MLAPrefillSelectorConfig(
|
||||
dtype=torch.bfloat16,
|
||||
mla_dimensions=MLADimensions(
|
||||
qk_nope_head_dim=64,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
)
|
||||
)
|
||||
selector_config_invalid = MLAPrefillSelectorConfig(
|
||||
dtype=torch.bfloat16,
|
||||
is_r1_compatible=is_deepseek_r1_mla_compatible(vllm_config_invalid),
|
||||
),
|
||||
)
|
||||
|
||||
with patch.object(FlashInferPrefillBackend, "is_available", return_value=True):
|
||||
@@ -250,7 +251,25 @@ class TestBackendValidation:
|
||||
selector_config_invalid,
|
||||
)
|
||||
assert len(invalid_reasons) == 1
|
||||
assert "DeepSeek R1 MLA dimensions" in invalid_reasons[0]
|
||||
assert "supported MLA dimensions" in invalid_reasons[0]
|
||||
|
||||
selector_config_glm5 = MLAPrefillSelectorConfig(
|
||||
dtype=torch.bfloat16,
|
||||
mla_dimensions=MLADimensions(
|
||||
qk_nope_head_dim=192,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=256,
|
||||
),
|
||||
)
|
||||
|
||||
with patch.object(
|
||||
TrtllmRaggedPrefillBackend, "is_available", return_value=True
|
||||
):
|
||||
invalid_reasons = TrtllmRaggedPrefillBackend.validate_configuration(
|
||||
capability,
|
||||
selector_config_glm5,
|
||||
)
|
||||
assert invalid_reasons == []
|
||||
|
||||
|
||||
class TestMLAPrefillBackendParsing:
|
||||
|
||||
Reference in New Issue
Block a user