[Attention] Minor refactor: layer takes ownership of the MLA prefill backend (#41744)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Matthew Bonanni
2026-05-05 23:22:41 +00:00
committed by GitHub
parent 8c57b6e7bc
commit 01b9b5af67
7 changed files with 74 additions and 150 deletions
+16 -2
View File
@@ -22,7 +22,6 @@ from vllm.config.vllm import set_current_vllm_config
from vllm.model_executor.layers.attention.mla_attention import (
QueryLenSupport,
_DecodeConcatQuantFP8,
get_mla_prefill_scale,
)
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
@@ -31,6 +30,7 @@ from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported
from vllm.v1.kv_cache_interface import MLAAttentionSpec
@@ -622,6 +622,19 @@ def run_attention_backend(
k_scale=k_scale,
)
# Attach prefill backend (normally created by MLAAttention.__init__)
prefill_scale = (qk_nope_head_dim + qk_rope_head_dim) ** -0.5
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
mock_layer.prefill_backend = prefill_backend_cls(
num_heads=num_heads,
scale=prefill_scale,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
vllm_config=vllm_config,
)
# Populate static_forward_context with mock attention layers
for layer_name in layer_names:
vllm_config.compilation_config.static_forward_context[layer_name] = (
@@ -787,7 +800,8 @@ def test_backend_correctness(
f"MLA dimensions don't match: {total_head_size} != {head_size}"
)
decode_scale = 1.0 / (total_head_size**0.5)
prefill_scale = get_mla_prefill_scale(vllm_config.model_config)
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
prefill_scale = qk_head_dim**-0.5
# 2. Generate data and compute SDPA reference output for MLA
all_q_vllm, all_kv_c_vllm, all_k_pe_vllm = [], [], []
@@ -2,17 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for MLA prefill backend selector."""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.config import AttentionConfig, ModelConfig, VllmConfig
from vllm.model_executor.layers.attention.mla_attention import get_mla_prefill_scale
from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import (
yarn_get_mscale,
)
from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
from vllm.v1.attention.backends.mla.prefill.selector import (
@@ -58,62 +53,6 @@ def _make_vllm_config(
return mock_vllm_config
class TestMLAPrefillScale:
"""Tests for the MLA prefill softmax scale."""
def test_uses_qk_head_dim_for_deepseek_v2_style_mla(self):
model_config = SimpleNamespace(
hf_text_config=SimpleNamespace(
q_lora_rank=None,
kv_lora_rank=512,
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
rope_parameters={"rope_type": "default"},
)
)
assert get_mla_prefill_scale(model_config) == pytest.approx(192**-0.5)
def test_applies_deepseek_yarn_mscale(self):
model_config = SimpleNamespace(
hf_text_config=SimpleNamespace(
q_lora_rank=None,
kv_lora_rank=512,
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
rope_parameters={
"rope_type": "yarn",
"factor": 40,
"mscale_all_dim": 0.707,
},
)
)
mscale = yarn_get_mscale(40, 0.707)
assert get_mla_prefill_scale(model_config) == pytest.approx(
192**-0.5 * mscale * mscale
)
def test_deepseek_v4_style_mla_does_not_apply_yarn_mscale(self):
model_config = SimpleNamespace(
hf_text_config=SimpleNamespace(
compress_ratios=[4],
q_lora_rank=1536,
head_dim=128,
qk_rope_head_dim=64,
rope_parameters={
"rope_type": "yarn",
"factor": 40,
"mscale_all_dim": 0.707,
},
)
)
assert get_mla_prefill_scale(model_config) == pytest.approx(128**-0.5)
class TestGetMLAPrefillBackend:
"""Tests for get_mla_prefill_backend (public API)."""
@@ -238,9 +238,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import (
yarn_get_mscale,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.math_utils import cdiv, round_down
@@ -262,7 +259,10 @@ from vllm.v1.attention.backend import (
MLAAttentionImpl,
SparseMLAAttentionImpl,
)
from vllm.v1.attention.backends.mla.prefill import MLAPrefillBackend
from vllm.v1.attention.backends.mla.prefill import (
MLAPrefillBackend,
get_mla_prefill_backend,
)
from vllm.v1.attention.backends.utils import (
get_dcp_local_seq_lens,
split_decodes_and_prefills,
@@ -454,20 +454,32 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self.q_pad_num_heads = getattr(self.impl, "q_pad_num_heads", None)
self.use_direct_call = not current_platform.opaque_attention_op()
compilation_config = get_current_vllm_config().compilation_config
vllm_config = get_current_vllm_config()
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
self.prefill_backend = prefill_backend_cls(
num_heads=self.num_heads,
scale=self.scale,
kv_lora_rank=self.kv_lora_rank,
qk_nope_head_dim=self.qk_nope_head_dim,
qk_rope_head_dim=self.qk_rope_head_dim,
v_head_dim=self.v_head_dim,
vllm_config=vllm_config,
)
self.kv_cache = torch.tensor([])
self.use_sparse = use_sparse
vllm_config = get_current_vllm_config_or_none()
_vllm_config = get_current_vllm_config_or_none()
self.dcp_a2a = (
vllm_config is not None
and vllm_config.parallel_config.decode_context_parallel_size > 1
and vllm_config.parallel_config.dcp_comm_backend == "a2a"
_vllm_config is not None
and _vllm_config.parallel_config.decode_context_parallel_size > 1
and _vllm_config.parallel_config.dcp_comm_backend == "a2a"
)
# Initialize q/k/v range constants.
@@ -1330,35 +1342,6 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims:
)
def get_mla_prefill_scale(model_config: ModelConfig) -> float:
hf_text_config = model_config.hf_text_config
mla_dims = get_mla_dims(model_config)
qk_head_dim = mla_dims.qk_nope_head_dim + mla_dims.qk_rope_head_dim
scale = qk_head_dim**-0.5
# Deepseek V4 disables YaRN mscale for attention; Deepseek V2/V3 applies
# the same mscale correction when constructing the MLA attention module.
if hasattr(hf_text_config, "compress_ratios"):
return scale
rope_parameters = getattr(hf_text_config, "rope_parameters", None)
if rope_parameters is None:
rope_parameters = getattr(hf_text_config, "rope_scaling", None)
if rope_parameters is None:
return scale
rope_type = rope_parameters.get("rope_type", rope_parameters.get("type"))
apply_yarn_scaling = rope_parameters.get("apply_yarn_scaling", True)
if rope_type != "default" and apply_yarn_scaling:
mscale_all_dim = rope_parameters.get("mscale_all_dim", False)
scaling_factor = rope_parameters["factor"]
mscale = yarn_get_mscale(float(scaling_factor), float(mscale_all_dim))
scale *= mscale * mscale
return scale
@functools.cache
def backend_supports_prefill_query_quantization() -> bool:
"""Check if the selected MLA prefill backend supports query quantization.
@@ -1554,20 +1537,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
device=device,
)
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
self._prefill_backend = prefill_backend_cls(
num_heads=self.num_heads,
scale=get_mla_prefill_scale(self.model_config),
kv_lora_rank=self.mla_dims.kv_lora_rank,
qk_nope_head_dim=self.mla_dims.qk_nope_head_dim,
qk_rope_head_dim=self.mla_dims.qk_rope_head_dim,
v_head_dim=self.mla_dims.v_head_dim,
vllm_config=vllm_config,
device=device,
layer_names=layer_names,
)
self._prefill_backend = self.compilation_config.static_forward_context[
layer_names[0]
].prefill_backend
supports_spec_decode = self.query_len_support != QueryLenSupport.SINGLE_ONLY
self._init_reorder_batch_threshold(
@@ -81,8 +81,6 @@ class MLAPrefillBackend(ABC):
qk_rope_head_dim: int,
v_head_dim: int,
vllm_config: "VllmConfig",
device: torch.device,
layer_names: list[str] | None = None,
) -> None:
self.num_heads = num_heads
self.scale = scale
@@ -91,8 +89,6 @@ class MLAPrefillBackend(ABC):
self.qk_rope_head_dim = qk_rope_head_dim
self.v_head_dim = v_head_dim
self.vllm_config = vllm_config
self.device = device
self.layer_names = layer_names
def prepare_metadata( # noqa: B027
self,
@@ -44,8 +44,6 @@ class FlashAttnPrefillBackend(MLAPrefillBackend):
qk_rope_head_dim: int,
v_head_dim: int,
vllm_config: "VllmConfig",
device: torch.device,
layer_names: list[str] | None = None,
) -> None:
super().__init__(
num_heads=num_heads,
@@ -55,8 +53,6 @@ class FlashAttnPrefillBackend(MLAPrefillBackend):
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
vllm_config=vllm_config,
device=device,
layer_names=layer_names,
)
# Handle the differences between the flash_attn_varlen from
@@ -9,6 +9,7 @@ import torch
import vllm.envs as envs
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
from vllm.v1.attention.backends.utils import (
PerLayerParameters,
get_per_layer_parameters,
infer_global_hyperparameters,
)
@@ -62,8 +63,6 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
qk_rope_head_dim: int,
v_head_dim: int,
vllm_config: "VllmConfig",
device: torch.device,
layer_names: list[str] | None = None,
) -> None:
super().__init__(
num_heads=num_heads,
@@ -73,25 +72,11 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
vllm_config=vllm_config,
device=device,
layer_names=layer_names,
)
self._prefill_main: BatchPrefillWithRaggedKVCacheWrapper | None = None
self._prefill_chunks: list[BatchPrefillWithRaggedKVCacheWrapper] = []
if layer_names is None:
raise ValueError(
"FlashInferPrefillBackend requires layer_names to "
"initialize global hyperparameters."
)
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonImpl,
)
self._global_hyperparameters = infer_global_hyperparameters(
get_per_layer_parameters(vllm_config, layer_names, MLACommonImpl) # type: ignore[type-abstract]
)
self._global_hyperparameters: PerLayerParameters | None = None
def _ensure_chunks(
self,
@@ -106,10 +91,36 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
)
)
def _resolve_global_hyperparameters(self) -> PerLayerParameters:
if self._global_hyperparameters is not None:
return self._global_hyperparameters
from vllm.model_executor.layers.attention.mla_attention import (
MLAAttention,
MLACommonImpl,
)
forward_context = self.vllm_config.compilation_config.static_forward_context
layer_names = [
name
for name, layer in forward_context.items()
if isinstance(layer, MLAAttention)
]
self._global_hyperparameters = infer_global_hyperparameters(
get_per_layer_parameters(
self.vllm_config,
layer_names,
MLACommonImpl, # type: ignore[type-abstract]
)
)
return self._global_hyperparameters
def prepare_metadata(
self,
prefill_metadata: "MLACommonPrefillMetadata",
) -> None:
global_hyperparameters = self._resolve_global_hyperparameters()
qo_indptr = prefill_metadata.query_start_loc
has_context = prefill_metadata.chunked_context is not None
(workspace_buffer,) = current_workspace_manager().get_simultaneous(
@@ -144,9 +155,9 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
head_dim_qk=head_dim_qk,
head_dim_vo=head_dim_vo,
causal=True,
sm_scale=self._global_hyperparameters.sm_scale,
window_left=self._global_hyperparameters.window_left,
logits_soft_cap=self._global_hyperparameters.logits_soft_cap,
sm_scale=global_hyperparameters.sm_scale,
window_left=global_hyperparameters.window_left,
logits_soft_cap=global_hyperparameters.logits_soft_cap,
q_data_type=prefill_metadata.q_data_type,
o_data_type=prefill_metadata.output_dtype,
)
@@ -165,9 +176,9 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
head_dim_qk=head_dim_qk,
head_dim_vo=head_dim_vo,
causal=False,
sm_scale=self._global_hyperparameters.sm_scale,
window_left=self._global_hyperparameters.window_left,
logits_soft_cap=self._global_hyperparameters.logits_soft_cap,
sm_scale=global_hyperparameters.sm_scale,
window_left=global_hyperparameters.window_left,
logits_soft_cap=global_hyperparameters.logits_soft_cap,
q_data_type=prefill_metadata.q_data_type,
o_data_type=prefill_metadata.output_dtype,
)
@@ -51,8 +51,6 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
qk_rope_head_dim: int,
v_head_dim: int,
vllm_config: "VllmConfig",
device: torch.device,
layer_names: list[str] | None = None,
) -> None:
super().__init__(
num_heads=num_heads,
@@ -62,8 +60,6 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
vllm_config=vllm_config,
device=device,
layer_names=layer_names,
)
def _get_workspace_buffer(self) -> torch.Tensor: