[Kernel] Support GLM-5 dimensions for TRT-LLM ragged MLA prefill (#43525)

Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Mohammad Miadh Angkad
2026-06-16 20:49:47 +00:00
committed by GitHub
co-authored by Matthew Bonanni
parent b9684d99e9
commit 9d4dc4ca2f
10 changed files with 200 additions and 77 deletions
+3 -3
View File
@@ -214,9 +214,9 @@ hardware and configuration.
| Backend | Description | Dtypes | Compute Cap. | Notes | | Backend | Description | Dtypes | Compute Cap. | Notes |
| ------- | ----------- | ------ | ------------ | ----- | | ------- | ----------- | ------ | ------------ | ----- |
| `FLASH_ATTN`‡ | FlashAttention varlen (FA2/FA3/FA4) | fp16, bf16 | Any | FA4 on SM100+, FA3 on SM90, FA2 otherwise | | `FLASH_ATTN`‡ | FlashAttention varlen (FA2/FA3/FA4) | fp16, bf16 | Any | FA4 on SM100+, FA3 on SM90, FA2 otherwise |
| `TRTLLM_RAGGED` | TensorRT-LLM ragged attention | fp16, bf16 | 10.x | DeepSeek R1 dims only | | `TRTLLM_RAGGED` | TensorRT-LLM ragged attention | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) or (qk_nope_head_dim=192, qk_rope_head_dim=64, v_head_dim=256) only |
| `FLASHINFER` | FlashInfer CUTLASS backend | fp16, bf16 | 10.x | DeepSeek R1 dims only | | `FLASHINFER` | FlashInfer CUTLASS backend | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only |
| `TOKENSPEED_MLA` | | fp16, bf16 | 10.x | DeepSeek R1 dims only | | `TOKENSPEED_MLA` | | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only |
> **‡** Automatic selection tries FlashAttention first. On Blackwell > **‡** Automatic selection tries FlashAttention first. On Blackwell
> (SM100), the fallback order is TRT-LLM Ragged, FlashInfer, then > (SM100), the fallback order is TRT-LLM Ragged, FlashInfer, then
+10 -2
View File
@@ -765,7 +765,8 @@ def test_backend_correctness(
if not backends_to_test: if not backends_to_test:
pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}") pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}")
# Skip prefill backends that can't satisfy capability/deps/R1 constraints. # Skip prefill backends that can't satisfy capability/deps/dimension constraints.
from vllm.v1.attention.backends.mla.prefill.base import MLADimensions
from vllm.v1.attention.backends.mla.prefill.selector import ( from vllm.v1.attention.backends.mla.prefill.selector import (
MLAPrefillSelectorConfig, MLAPrefillSelectorConfig,
) )
@@ -773,7 +774,14 @@ def test_backend_correctness(
try: try:
prefill_invalid_reasons = prefill_backend.get_class().validate_configuration( prefill_invalid_reasons = prefill_backend.get_class().validate_configuration(
current_platform.get_device_capability(), current_platform.get_device_capability(),
MLAPrefillSelectorConfig(dtype=torch.bfloat16, is_r1_compatible=True), MLAPrefillSelectorConfig(
dtype=torch.bfloat16,
mla_dimensions=MLADimensions(
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
),
),
) )
except ImportError: except ImportError:
prefill_invalid_reasons = ["ImportError"] prefill_invalid_reasons = ["ImportError"]
@@ -16,7 +16,6 @@ class CustomMLAPrefillBackend(MLAPrefillBackend):
"""Mock custom MLA prefill backend for testing.""" """Mock custom MLA prefill backend for testing."""
supported_dtypes = [torch.bfloat16, torch.float16] supported_dtypes = [torch.bfloat16, torch.float16]
requires_r1_mla_dimensions = False
@staticmethod @staticmethod
def get_name() -> str: def get_name() -> str:
@@ -83,7 +82,6 @@ def test_register_custom_backend_as_decorator():
@register_mla_prefill_backend(MLAPrefillBackendEnum.CUSTOM) @register_mla_prefill_backend(MLAPrefillBackendEnum.CUSTOM)
class DecoratedPrefillBackend(MLAPrefillBackend): class DecoratedPrefillBackend(MLAPrefillBackend):
supported_dtypes = [torch.bfloat16] supported_dtypes = [torch.bfloat16]
requires_r1_mla_dimensions = False
@staticmethod @staticmethod
def get_name() -> str: def get_name() -> str:
+44 -25
View File
@@ -9,12 +9,12 @@ import torch
from vllm.config import AttentionConfig, ModelConfig, VllmConfig from vllm.config import AttentionConfig, ModelConfig, VllmConfig
from vllm.platforms.interface import DeviceCapability from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backends.mla.prefill.base import MLADimensions
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
from vllm.v1.attention.backends.mla.prefill.selector import ( from vllm.v1.attention.backends.mla.prefill.selector import (
MLAPrefillSelectorConfig, MLAPrefillSelectorConfig,
_auto_select_mla_prefill_backend, _auto_select_mla_prefill_backend,
get_mla_prefill_backend, get_mla_prefill_backend,
is_deepseek_r1_mla_compatible,
) )
@@ -149,11 +149,14 @@ class TestAutoSelectMLAPrefillBackend:
"""Tests for fallback and error paths in auto-selection.""" """Tests for fallback and error paths in auto-selection."""
def test_blackwell_falls_back_to_trtllm(self): def test_blackwell_falls_back_to_trtllm(self):
vllm_config = _make_vllm_config()
capability = DeviceCapability(major=10, minor=0) capability = DeviceCapability(major=10, minor=0)
selector_config = MLAPrefillSelectorConfig( selector_config = MLAPrefillSelectorConfig(
dtype=torch.bfloat16, dtype=torch.bfloat16,
is_r1_compatible=is_deepseek_r1_mla_compatible(vllm_config), mla_dimensions=MLADimensions(
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
),
) )
try: try:
@@ -177,11 +180,14 @@ class TestAutoSelectMLAPrefillBackend:
assert backend.get_name() == "TRTLLM_RAGGED" assert backend.get_name() == "TRTLLM_RAGGED"
def test_all_fail_raises_error(self): def test_all_fail_raises_error(self):
vllm_config = _make_vllm_config()
capability = DeviceCapability(major=10, minor=0) capability = DeviceCapability(major=10, minor=0)
selector_config = MLAPrefillSelectorConfig( selector_config = MLAPrefillSelectorConfig(
dtype=torch.bfloat16, dtype=torch.bfloat16,
is_r1_compatible=is_deepseek_r1_mla_compatible(vllm_config), mla_dimensions=MLADimensions(
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
),
) )
def mock_get_class(backend_enum): # noqa: ARG001 def mock_get_class(backend_enum): # noqa: ARG001
@@ -201,28 +207,26 @@ class TestAutoSelectMLAPrefillBackend:
class TestBackendValidation: class TestBackendValidation:
"""Tests for backend validation logic.""" """Tests for backend validation logic."""
def test_r1_dimension_requirement(self): def test_backend_supported_dimension_validation(self):
try: try:
from vllm.v1.attention.backends.mla.prefill.flashinfer import ( from vllm.v1.attention.backends.mla.prefill.flashinfer import (
FlashInferPrefillBackend, FlashInferPrefillBackend,
) )
from vllm.v1.attention.backends.mla.prefill.trtllm_ragged import (
TrtllmRaggedPrefillBackend,
)
except ImportError: except ImportError:
pytest.skip("FlashInfer prefill backend not available") pytest.skip("MLA prefill backend not available")
return return
assert FlashInferPrefillBackend.requires_r1_mla_dimensions is True
vllm_config = _make_vllm_config(
model_config=_make_mock_model_config(
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
)
)
capability = DeviceCapability(major=10, minor=0) capability = DeviceCapability(major=10, minor=0)
selector_config = MLAPrefillSelectorConfig( selector_config = MLAPrefillSelectorConfig(
dtype=torch.bfloat16, dtype=torch.bfloat16,
is_r1_compatible=is_deepseek_r1_mla_compatible(vllm_config), mla_dimensions=MLADimensions(
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
),
) )
with patch.object(FlashInferPrefillBackend, "is_available", return_value=True): with patch.object(FlashInferPrefillBackend, "is_available", return_value=True):
@@ -232,16 +236,13 @@ class TestBackendValidation:
) )
assert len(invalid_reasons) == 0 assert len(invalid_reasons) == 0
vllm_config_invalid = _make_vllm_config( selector_config_invalid = MLAPrefillSelectorConfig(
model_config=_make_mock_model_config( dtype=torch.bfloat16,
mla_dimensions=MLADimensions(
qk_nope_head_dim=64, qk_nope_head_dim=64,
qk_rope_head_dim=64, qk_rope_head_dim=64,
v_head_dim=128, v_head_dim=128,
) ),
)
selector_config_invalid = MLAPrefillSelectorConfig(
dtype=torch.bfloat16,
is_r1_compatible=is_deepseek_r1_mla_compatible(vllm_config_invalid),
) )
with patch.object(FlashInferPrefillBackend, "is_available", return_value=True): with patch.object(FlashInferPrefillBackend, "is_available", return_value=True):
@@ -250,7 +251,25 @@ class TestBackendValidation:
selector_config_invalid, selector_config_invalid,
) )
assert len(invalid_reasons) == 1 assert len(invalid_reasons) == 1
assert "DeepSeek R1 MLA dimensions" in invalid_reasons[0] assert "supported MLA dimensions" in invalid_reasons[0]
selector_config_glm5 = MLAPrefillSelectorConfig(
dtype=torch.bfloat16,
mla_dimensions=MLADimensions(
qk_nope_head_dim=192,
qk_rope_head_dim=64,
v_head_dim=256,
),
)
with patch.object(
TrtllmRaggedPrefillBackend, "is_available", return_value=True
):
invalid_reasons = TrtllmRaggedPrefillBackend.validate_configuration(
capability,
selector_config_glm5,
)
assert invalid_reasons == []
class TestMLAPrefillBackendParsing: class TestMLAPrefillBackendParsing:
@@ -384,6 +384,49 @@ def parse_mla_prefill_priorities() -> dict[str, list[str]]:
return priorities return priorities
def parse_mla_dimensions_call(node: ast.AST) -> str | None:
"""Parse an MLADimensions(...) call into a compact display string."""
if not isinstance(node, ast.Call):
return None
func = node.func
if not isinstance(func, ast.Name) or func.id != "MLADimensions":
return None
dimensions: dict[str, int] = {}
for keyword in node.keywords:
if (
keyword.arg is not None
and isinstance(keyword.value, ast.Constant)
and isinstance(keyword.value.value, int)
):
dimensions[keyword.arg] = keyword.value.value
qk_nope_head_dim = dimensions.get("qk_nope_head_dim")
qk_rope_head_dim = dimensions.get("qk_rope_head_dim")
v_head_dim = dimensions.get("v_head_dim")
if qk_nope_head_dim is None or qk_rope_head_dim is None or v_head_dim is None:
return None
return (
f"(qk_nope_head_dim={qk_nope_head_dim}, "
f"qk_rope_head_dim={qk_rope_head_dim}, v_head_dim={v_head_dim})"
)
def parse_supported_mla_dimensions(node: ast.AST | None) -> list[str]:
"""Parse a supported_mla_dimensions class variable."""
if not isinstance(node, ast.List):
return []
supported_dimensions = []
for element in node.elts:
dimensions = parse_mla_dimensions_call(element)
if dimensions is not None:
supported_dimensions.append(dimensions)
return supported_dimensions
def parse_mla_prefill_backend_file(class_path: str) -> dict[str, Any] | None: def parse_mla_prefill_backend_file(class_path: str) -> dict[str, Any] | None:
"""Parse a single MLA prefill backend file to extract its properties. """Parse a single MLA prefill backend file to extract its properties.
@@ -409,20 +452,20 @@ def parse_mla_prefill_backend_file(class_path: str) -> dict[str, Any] | None:
info: dict[str, Any] = { info: dict[str, Any] = {
"compute_capability": "Any", "compute_capability": "Any",
"requires_r1_dims": False, "supported_mla_dimensions": [],
"dtypes": "fp16, bf16", # Default from base class "dtypes": "fp16, bf16", # Default from base class
} }
# Parse class variables # Parse class variables
for item in class_node.body: for item in class_node.body:
if isinstance(item, ast.Assign): if (
for target in item.targets: isinstance(item, ast.AnnAssign)
if ( and isinstance(item.target, ast.Name)
isinstance(target, ast.Name) and item.target.id == "supported_mla_dimensions"
and target.id == "requires_r1_mla_dimensions" ):
and isinstance(item.value, ast.Constant) info["supported_mla_dimensions"] = parse_supported_mla_dimensions(
): item.value
info["requires_r1_dims"] = item.value.value )
# Parse supported_dtypes class variable # Parse supported_dtypes class variable
if ( if (
@@ -515,8 +558,9 @@ def parse_mla_prefill_backends() -> list[dict[str, Any]]:
marker = "‡" marker = "‡"
notes = "" notes = ""
if backend_info.get("requires_r1_dims"): supported_mla_dimensions = backend_info.get("supported_mla_dimensions", [])
notes = "DeepSeek R1 dims only" if supported_mla_dimensions:
notes = " or ".join(supported_mla_dimensions) + " only"
elif backend_name == "FLASH_ATTN": elif backend_name == "FLASH_ATTN":
notes = "FA4 on SM100+, FA3 on SM90, FA2 otherwise" notes = "FA4 on SM100+, FA3 on SM90, FA2 otherwise"
+23 -4
View File
@@ -3,6 +3,7 @@
"""Abstract base class for MLA prefill backends.""" """Abstract base class for MLA prefill backends."""
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar from typing import TYPE_CHECKING, ClassVar
import torch import torch
@@ -19,6 +20,20 @@ if TYPE_CHECKING:
) )
@dataclass(frozen=True, kw_only=True)
class MLADimensions:
qk_nope_head_dim: int
qk_rope_head_dim: int
v_head_dim: int
def __str__(self) -> str:
return (
f"(qk_nope_head_dim={self.qk_nope_head_dim}, "
f"qk_rope_head_dim={self.qk_rope_head_dim}, "
f"v_head_dim={self.v_head_dim})"
)
class MLAPrefillBackend(ABC): class MLAPrefillBackend(ABC):
"""Abstract base class for MLA prefill backends.""" """Abstract base class for MLA prefill backends."""
@@ -26,7 +41,7 @@ class MLAPrefillBackend(ABC):
torch.float16, torch.float16,
torch.bfloat16, torch.bfloat16,
] ]
requires_r1_mla_dimensions: ClassVar[bool] = False supported_mla_dimensions: ClassVar[list[MLADimensions]] = []
@staticmethod @staticmethod
@abstractmethod @abstractmethod
@@ -71,10 +86,14 @@ class MLAPrefillBackend(ABC):
if not cls.is_available(): if not cls.is_available():
invalid_reasons.append("required dependencies not available") invalid_reasons.append("required dependencies not available")
if cls.requires_r1_mla_dimensions and not selector_config.is_r1_compatible: if (
cls.supported_mla_dimensions
and selector_config.mla_dimensions not in cls.supported_mla_dimensions
):
supported = ", ".join(str(dims) for dims in cls.supported_mla_dimensions)
invalid_reasons.append( invalid_reasons.append(
"model does not have DeepSeek R1 MLA dimensions " "Model does not have supported MLA dimensions "
"(qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128)" f"(got {selector_config.mla_dimensions}; supported: {supported})"
) )
return invalid_reasons return invalid_reasons
@@ -2,12 +2,15 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FlashInfer backend for MLA prefill.""" """FlashInfer backend for MLA prefill."""
from typing import TYPE_CHECKING from typing import TYPE_CHECKING, ClassVar
import torch import torch
import vllm.envs as envs import vllm.envs as envs
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend from vllm.v1.attention.backends.mla.prefill.base import (
MLADimensions,
MLAPrefillBackend,
)
from vllm.v1.attention.backends.utils import ( from vllm.v1.attention.backends.utils import (
PerLayerParameters, PerLayerParameters,
get_per_layer_parameters, get_per_layer_parameters,
@@ -33,7 +36,13 @@ _DEFAULT_NUM_CHUNKS = 32
class FlashInferPrefillBackend(MLAPrefillBackend): class FlashInferPrefillBackend(MLAPrefillBackend):
"""FlashInfer backend for MLA prefill.""" """FlashInfer backend for MLA prefill."""
requires_r1_mla_dimensions = True supported_mla_dimensions: ClassVar[list[MLADimensions]] = [
MLADimensions(
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
),
]
@staticmethod @staticmethod
def get_name() -> str: def get_name() -> str:
@@ -13,6 +13,7 @@ import torch
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.platforms.interface import DeviceCapability from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backends.mla.prefill.base import MLADimensions
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -31,24 +32,17 @@ class MLAPrefillSelectorConfig(NamedTuple):
""" """
dtype: torch.dtype dtype: torch.dtype
is_r1_compatible: bool mla_dimensions: MLADimensions = MLADimensions(
qk_nope_head_dim=0,
qk_rope_head_dim=0,
v_head_dim=0,
)
def __repr__(self):
def is_deepseek_r1_mla_compatible(vllm_config: "VllmConfig") -> bool: return (
"""Check if model has DeepSeek R1 compatible MLA dimensions. f"MLAPrefillSelectorConfig(dtype={self.dtype}, "
f"mla_dimensions={self.mla_dimensions})"
DeepSeek R1 MLA dimensions are: )
- qk_nope_head_dim = 128
- qk_rope_head_dim = 64
- v_head_dim = 128
"""
if vllm_config.model_config is None:
return False
hf_text_config = vllm_config.model_config.hf_text_config
qk_nope_head_dim = getattr(hf_text_config, "qk_nope_head_dim", 1)
qk_rope_head_dim = getattr(hf_text_config, "qk_rope_head_dim", 1)
v_head_dim = getattr(hf_text_config, "v_head_dim", 1)
return qk_nope_head_dim == 128 and qk_rope_head_dim == 64 and v_head_dim == 128
def _get_mla_prefill_backend_priorities( def _get_mla_prefill_backend_priorities(
@@ -101,10 +95,19 @@ def get_mla_prefill_backend(
attention_config = vllm_config.attention_config attention_config = vllm_config.attention_config
selector_config = MLAPrefillSelectorConfig( model_config = vllm_config.model_config
dtype=vllm_config.model_config.dtype, if model_config is None:
is_r1_compatible=is_deepseek_r1_mla_compatible(vllm_config), selector_config = MLAPrefillSelectorConfig(dtype=torch.get_default_dtype())
) else:
hf_text_config = model_config.hf_text_config
selector_config = MLAPrefillSelectorConfig(
dtype=model_config.dtype,
mla_dimensions=MLADimensions(
qk_nope_head_dim=getattr(hf_text_config, "qk_nope_head_dim", 0),
qk_rope_head_dim=getattr(hf_text_config, "qk_rope_head_dim", 0),
v_head_dim=getattr(hf_text_config, "v_head_dim", 0),
),
)
if attention_config.mla_prefill_backend is not None: if attention_config.mla_prefill_backend is not None:
selected_backend = attention_config.mla_prefill_backend selected_backend = attention_config.mla_prefill_backend
@@ -2,11 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TokenSpeed CuTe DSL backend for MLA prefill.""" """TokenSpeed CuTe DSL backend for MLA prefill."""
from typing import TYPE_CHECKING from typing import TYPE_CHECKING, ClassVar
import torch import torch
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend from vllm.v1.attention.backends.mla.prefill.base import (
MLADimensions,
MLAPrefillBackend,
)
if TYPE_CHECKING: if TYPE_CHECKING:
from vllm.config import VllmConfig from vllm.config import VllmConfig
@@ -19,7 +22,13 @@ if TYPE_CHECKING:
class TokenspeedMLAPrefillBackend(MLAPrefillBackend): class TokenspeedMLAPrefillBackend(MLAPrefillBackend):
"""TokenSpeed CuTe DSL backend for MLA prefill.""" """TokenSpeed CuTe DSL backend for MLA prefill."""
requires_r1_mla_dimensions = True supported_mla_dimensions: ClassVar[list[MLADimensions]] = [
MLADimensions(
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
),
]
@staticmethod @staticmethod
def get_name() -> str: def get_name() -> str:
@@ -2,12 +2,15 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TRT-LLM Ragged backend for MLA prefill.""" """TRT-LLM Ragged backend for MLA prefill."""
from typing import TYPE_CHECKING from typing import TYPE_CHECKING, ClassVar
import torch import torch
import vllm.envs as envs import vllm.envs as envs
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend from vllm.v1.attention.backends.mla.prefill.base import (
MLADimensions,
MLAPrefillBackend,
)
from vllm.v1.worker.workspace import current_workspace_manager from vllm.v1.worker.workspace import current_workspace_manager
if TYPE_CHECKING: if TYPE_CHECKING:
@@ -21,7 +24,18 @@ if TYPE_CHECKING:
class TrtllmRaggedPrefillBackend(MLAPrefillBackend): class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
"""TRT-LLM Ragged backend for MLA prefill.""" """TRT-LLM Ragged backend for MLA prefill."""
requires_r1_mla_dimensions = True supported_mla_dimensions: ClassVar[list[MLADimensions]] = [
MLADimensions(
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
),
MLADimensions(
qk_nope_head_dim=192,
qk_rope_head_dim=64,
v_head_dim=256,
),
]
@staticmethod @staticmethod
def get_name() -> str: def get_name() -> str: