From 01b9b5af67b77715fd1c33f19598f213ae4fec6a Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Tue, 5 May 2026 19:22:41 -0400 Subject: [PATCH] [Attention] Minor refactor: layer takes ownership of the MLA prefill backend (#41744) Signed-off-by: Matthew Bonanni --- tests/v1/attention/test_mla_backends.py | 18 ++++- .../v1/attention/test_mla_prefill_selector.py | 61 --------------- .../layers/attention/mla_attention.py | 76 ++++++------------- .../v1/attention/backends/mla/prefill/base.py | 4 - .../backends/mla/prefill/flash_attn.py | 4 - .../backends/mla/prefill/flashinfer.py | 57 ++++++++------ .../backends/mla/prefill/trtllm_ragged.py | 4 - 7 files changed, 74 insertions(+), 150 deletions(-) diff --git a/tests/v1/attention/test_mla_backends.py b/tests/v1/attention/test_mla_backends.py index 8ab47b6189..3807ee69ec 100644 --- a/tests/v1/attention/test_mla_backends.py +++ b/tests/v1/attention/test_mla_backends.py @@ -22,7 +22,6 @@ from vllm.config.vllm import set_current_vllm_config from vllm.model_executor.layers.attention.mla_attention import ( QueryLenSupport, _DecodeConcatQuantFP8, - get_mla_prefill_scale, ) from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape @@ -31,6 +30,7 @@ from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla +from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported from vllm.v1.kv_cache_interface import MLAAttentionSpec @@ -622,6 +622,19 @@ def run_attention_backend( k_scale=k_scale, ) + # Attach prefill backend (normally created by MLAAttention.__init__) + prefill_scale = (qk_nope_head_dim + qk_rope_head_dim) ** -0.5 + prefill_backend_cls = get_mla_prefill_backend(vllm_config) + mock_layer.prefill_backend = prefill_backend_cls( + num_heads=num_heads, + scale=prefill_scale, + kv_lora_rank=kv_lora_rank, + qk_nope_head_dim=qk_nope_head_dim, + qk_rope_head_dim=qk_rope_head_dim, + v_head_dim=v_head_dim, + vllm_config=vllm_config, + ) + # Populate static_forward_context with mock attention layers for layer_name in layer_names: vllm_config.compilation_config.static_forward_context[layer_name] = ( @@ -787,7 +800,8 @@ def test_backend_correctness( f"MLA dimensions don't match: {total_head_size} != {head_size}" ) decode_scale = 1.0 / (total_head_size**0.5) - prefill_scale = get_mla_prefill_scale(vllm_config.model_config) + qk_head_dim = qk_nope_head_dim + qk_rope_head_dim + prefill_scale = qk_head_dim**-0.5 # 2. Generate data and compute SDPA reference output for MLA all_q_vllm, all_kv_c_vllm, all_k_pe_vllm = [], [], [] diff --git a/tests/v1/attention/test_mla_prefill_selector.py b/tests/v1/attention/test_mla_prefill_selector.py index 873cfb1870..068eb43faf 100644 --- a/tests/v1/attention/test_mla_prefill_selector.py +++ b/tests/v1/attention/test_mla_prefill_selector.py @@ -2,17 +2,12 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Tests for MLA prefill backend selector.""" -from types import SimpleNamespace from unittest.mock import MagicMock, patch import pytest import torch from vllm.config import AttentionConfig, ModelConfig, VllmConfig -from vllm.model_executor.layers.attention.mla_attention import get_mla_prefill_scale -from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import ( - yarn_get_mscale, -) from vllm.platforms.interface import DeviceCapability from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum from vllm.v1.attention.backends.mla.prefill.selector import ( @@ -58,62 +53,6 @@ def _make_vllm_config( return mock_vllm_config -class TestMLAPrefillScale: - """Tests for the MLA prefill softmax scale.""" - - def test_uses_qk_head_dim_for_deepseek_v2_style_mla(self): - model_config = SimpleNamespace( - hf_text_config=SimpleNamespace( - q_lora_rank=None, - kv_lora_rank=512, - qk_nope_head_dim=128, - qk_rope_head_dim=64, - v_head_dim=128, - rope_parameters={"rope_type": "default"}, - ) - ) - - assert get_mla_prefill_scale(model_config) == pytest.approx(192**-0.5) - - def test_applies_deepseek_yarn_mscale(self): - model_config = SimpleNamespace( - hf_text_config=SimpleNamespace( - q_lora_rank=None, - kv_lora_rank=512, - qk_nope_head_dim=128, - qk_rope_head_dim=64, - v_head_dim=128, - rope_parameters={ - "rope_type": "yarn", - "factor": 40, - "mscale_all_dim": 0.707, - }, - ) - ) - - mscale = yarn_get_mscale(40, 0.707) - assert get_mla_prefill_scale(model_config) == pytest.approx( - 192**-0.5 * mscale * mscale - ) - - def test_deepseek_v4_style_mla_does_not_apply_yarn_mscale(self): - model_config = SimpleNamespace( - hf_text_config=SimpleNamespace( - compress_ratios=[4], - q_lora_rank=1536, - head_dim=128, - qk_rope_head_dim=64, - rope_parameters={ - "rope_type": "yarn", - "factor": 40, - "mscale_all_dim": 0.707, - }, - ) - ) - - assert get_mla_prefill_scale(model_config) == pytest.approx(128**-0.5) - - class TestGetMLAPrefillBackend: """Tests for get_mla_prefill_backend (public API).""" diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 20981f60cd..9cf7ebf5cd 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -238,9 +238,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8StaticTensorSym, kNvfp4Dynamic, ) -from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import ( - yarn_get_mscale, -) from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer from vllm.utils.math_utils import cdiv, round_down @@ -262,7 +259,10 @@ from vllm.v1.attention.backend import ( MLAAttentionImpl, SparseMLAAttentionImpl, ) -from vllm.v1.attention.backends.mla.prefill import MLAPrefillBackend +from vllm.v1.attention.backends.mla.prefill import ( + MLAPrefillBackend, + get_mla_prefill_backend, +) from vllm.v1.attention.backends.utils import ( get_dcp_local_seq_lens, split_decodes_and_prefills, @@ -454,20 +454,32 @@ class MLAAttention(nn.Module, AttentionLayerBase): self.q_pad_num_heads = getattr(self.impl, "q_pad_num_heads", None) self.use_direct_call = not current_platform.opaque_attention_op() - compilation_config = get_current_vllm_config().compilation_config + vllm_config = get_current_vllm_config() + compilation_config = vllm_config.compilation_config if prefix in compilation_config.static_forward_context: raise ValueError(f"Duplicate layer name: {prefix}") compilation_config.static_forward_context[prefix] = self + prefill_backend_cls = get_mla_prefill_backend(vllm_config) + self.prefill_backend = prefill_backend_cls( + num_heads=self.num_heads, + scale=self.scale, + kv_lora_rank=self.kv_lora_rank, + qk_nope_head_dim=self.qk_nope_head_dim, + qk_rope_head_dim=self.qk_rope_head_dim, + v_head_dim=self.v_head_dim, + vllm_config=vllm_config, + ) + self.kv_cache = torch.tensor([]) self.use_sparse = use_sparse - vllm_config = get_current_vllm_config_or_none() + _vllm_config = get_current_vllm_config_or_none() self.dcp_a2a = ( - vllm_config is not None - and vllm_config.parallel_config.decode_context_parallel_size > 1 - and vllm_config.parallel_config.dcp_comm_backend == "a2a" + _vllm_config is not None + and _vllm_config.parallel_config.decode_context_parallel_size > 1 + and _vllm_config.parallel_config.dcp_comm_backend == "a2a" ) # Initialize q/k/v range constants. @@ -1330,35 +1342,6 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims: ) -def get_mla_prefill_scale(model_config: ModelConfig) -> float: - hf_text_config = model_config.hf_text_config - mla_dims = get_mla_dims(model_config) - qk_head_dim = mla_dims.qk_nope_head_dim + mla_dims.qk_rope_head_dim - scale = qk_head_dim**-0.5 - - # Deepseek V4 disables YaRN mscale for attention; Deepseek V2/V3 applies - # the same mscale correction when constructing the MLA attention module. - if hasattr(hf_text_config, "compress_ratios"): - return scale - - rope_parameters = getattr(hf_text_config, "rope_parameters", None) - if rope_parameters is None: - rope_parameters = getattr(hf_text_config, "rope_scaling", None) - - if rope_parameters is None: - return scale - - rope_type = rope_parameters.get("rope_type", rope_parameters.get("type")) - apply_yarn_scaling = rope_parameters.get("apply_yarn_scaling", True) - if rope_type != "default" and apply_yarn_scaling: - mscale_all_dim = rope_parameters.get("mscale_all_dim", False) - scaling_factor = rope_parameters["factor"] - mscale = yarn_get_mscale(float(scaling_factor), float(mscale_all_dim)) - scale *= mscale * mscale - - return scale - - @functools.cache def backend_supports_prefill_query_quantization() -> bool: """Check if the selected MLA prefill backend supports query quantization. @@ -1554,20 +1537,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): device=device, ) - from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend - - prefill_backend_cls = get_mla_prefill_backend(vllm_config) - self._prefill_backend = prefill_backend_cls( - num_heads=self.num_heads, - scale=get_mla_prefill_scale(self.model_config), - kv_lora_rank=self.mla_dims.kv_lora_rank, - qk_nope_head_dim=self.mla_dims.qk_nope_head_dim, - qk_rope_head_dim=self.mla_dims.qk_rope_head_dim, - v_head_dim=self.mla_dims.v_head_dim, - vllm_config=vllm_config, - device=device, - layer_names=layer_names, - ) + self._prefill_backend = self.compilation_config.static_forward_context[ + layer_names[0] + ].prefill_backend supports_spec_decode = self.query_len_support != QueryLenSupport.SINGLE_ONLY self._init_reorder_batch_threshold( diff --git a/vllm/v1/attention/backends/mla/prefill/base.py b/vllm/v1/attention/backends/mla/prefill/base.py index 9c850a0b1d..91d668826f 100644 --- a/vllm/v1/attention/backends/mla/prefill/base.py +++ b/vllm/v1/attention/backends/mla/prefill/base.py @@ -81,8 +81,6 @@ class MLAPrefillBackend(ABC): qk_rope_head_dim: int, v_head_dim: int, vllm_config: "VllmConfig", - device: torch.device, - layer_names: list[str] | None = None, ) -> None: self.num_heads = num_heads self.scale = scale @@ -91,8 +89,6 @@ class MLAPrefillBackend(ABC): self.qk_rope_head_dim = qk_rope_head_dim self.v_head_dim = v_head_dim self.vllm_config = vllm_config - self.device = device - self.layer_names = layer_names def prepare_metadata( # noqa: B027 self, diff --git a/vllm/v1/attention/backends/mla/prefill/flash_attn.py b/vllm/v1/attention/backends/mla/prefill/flash_attn.py index 42d77b12d0..029bd8ec95 100644 --- a/vllm/v1/attention/backends/mla/prefill/flash_attn.py +++ b/vllm/v1/attention/backends/mla/prefill/flash_attn.py @@ -44,8 +44,6 @@ class FlashAttnPrefillBackend(MLAPrefillBackend): qk_rope_head_dim: int, v_head_dim: int, vllm_config: "VllmConfig", - device: torch.device, - layer_names: list[str] | None = None, ) -> None: super().__init__( num_heads=num_heads, @@ -55,8 +53,6 @@ class FlashAttnPrefillBackend(MLAPrefillBackend): qk_rope_head_dim=qk_rope_head_dim, v_head_dim=v_head_dim, vllm_config=vllm_config, - device=device, - layer_names=layer_names, ) # Handle the differences between the flash_attn_varlen from diff --git a/vllm/v1/attention/backends/mla/prefill/flashinfer.py b/vllm/v1/attention/backends/mla/prefill/flashinfer.py index a1107df304..77199fb523 100644 --- a/vllm/v1/attention/backends/mla/prefill/flashinfer.py +++ b/vllm/v1/attention/backends/mla/prefill/flashinfer.py @@ -9,6 +9,7 @@ import torch import vllm.envs as envs from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend from vllm.v1.attention.backends.utils import ( + PerLayerParameters, get_per_layer_parameters, infer_global_hyperparameters, ) @@ -62,8 +63,6 @@ class FlashInferPrefillBackend(MLAPrefillBackend): qk_rope_head_dim: int, v_head_dim: int, vllm_config: "VllmConfig", - device: torch.device, - layer_names: list[str] | None = None, ) -> None: super().__init__( num_heads=num_heads, @@ -73,25 +72,11 @@ class FlashInferPrefillBackend(MLAPrefillBackend): qk_rope_head_dim=qk_rope_head_dim, v_head_dim=v_head_dim, vllm_config=vllm_config, - device=device, - layer_names=layer_names, ) self._prefill_main: BatchPrefillWithRaggedKVCacheWrapper | None = None self._prefill_chunks: list[BatchPrefillWithRaggedKVCacheWrapper] = [] - if layer_names is None: - raise ValueError( - "FlashInferPrefillBackend requires layer_names to " - "initialize global hyperparameters." - ) - - from vllm.model_executor.layers.attention.mla_attention import ( - MLACommonImpl, - ) - - self._global_hyperparameters = infer_global_hyperparameters( - get_per_layer_parameters(vllm_config, layer_names, MLACommonImpl) # type: ignore[type-abstract] - ) + self._global_hyperparameters: PerLayerParameters | None = None def _ensure_chunks( self, @@ -106,10 +91,36 @@ class FlashInferPrefillBackend(MLAPrefillBackend): ) ) + def _resolve_global_hyperparameters(self) -> PerLayerParameters: + if self._global_hyperparameters is not None: + return self._global_hyperparameters + + from vllm.model_executor.layers.attention.mla_attention import ( + MLAAttention, + MLACommonImpl, + ) + + forward_context = self.vllm_config.compilation_config.static_forward_context + layer_names = [ + name + for name, layer in forward_context.items() + if isinstance(layer, MLAAttention) + ] + + self._global_hyperparameters = infer_global_hyperparameters( + get_per_layer_parameters( + self.vllm_config, + layer_names, + MLACommonImpl, # type: ignore[type-abstract] + ) + ) + return self._global_hyperparameters + def prepare_metadata( self, prefill_metadata: "MLACommonPrefillMetadata", ) -> None: + global_hyperparameters = self._resolve_global_hyperparameters() qo_indptr = prefill_metadata.query_start_loc has_context = prefill_metadata.chunked_context is not None (workspace_buffer,) = current_workspace_manager().get_simultaneous( @@ -144,9 +155,9 @@ class FlashInferPrefillBackend(MLAPrefillBackend): head_dim_qk=head_dim_qk, head_dim_vo=head_dim_vo, causal=True, - sm_scale=self._global_hyperparameters.sm_scale, - window_left=self._global_hyperparameters.window_left, - logits_soft_cap=self._global_hyperparameters.logits_soft_cap, + sm_scale=global_hyperparameters.sm_scale, + window_left=global_hyperparameters.window_left, + logits_soft_cap=global_hyperparameters.logits_soft_cap, q_data_type=prefill_metadata.q_data_type, o_data_type=prefill_metadata.output_dtype, ) @@ -165,9 +176,9 @@ class FlashInferPrefillBackend(MLAPrefillBackend): head_dim_qk=head_dim_qk, head_dim_vo=head_dim_vo, causal=False, - sm_scale=self._global_hyperparameters.sm_scale, - window_left=self._global_hyperparameters.window_left, - logits_soft_cap=self._global_hyperparameters.logits_soft_cap, + sm_scale=global_hyperparameters.sm_scale, + window_left=global_hyperparameters.window_left, + logits_soft_cap=global_hyperparameters.logits_soft_cap, q_data_type=prefill_metadata.q_data_type, o_data_type=prefill_metadata.output_dtype, ) diff --git a/vllm/v1/attention/backends/mla/prefill/trtllm_ragged.py b/vllm/v1/attention/backends/mla/prefill/trtllm_ragged.py index 2ffe244ebe..7462dc39e7 100644 --- a/vllm/v1/attention/backends/mla/prefill/trtllm_ragged.py +++ b/vllm/v1/attention/backends/mla/prefill/trtllm_ragged.py @@ -51,8 +51,6 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend): qk_rope_head_dim: int, v_head_dim: int, vllm_config: "VllmConfig", - device: torch.device, - layer_names: list[str] | None = None, ) -> None: super().__init__( num_heads=num_heads, @@ -62,8 +60,6 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend): qk_rope_head_dim=qk_rope_head_dim, v_head_dim=v_head_dim, vllm_config=vllm_config, - device=device, - layer_names=layer_names, ) def _get_workspace_buffer(self) -> torch.Tensor: