[Attention] Minor refactor: layer takes ownership of the MLA prefill backend (#41744)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
@@ -22,7 +22,6 @@ from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
QueryLenSupport,
|
||||
_DecodeConcatQuantFP8,
|
||||
get_mla_prefill_scale,
|
||||
)
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
@@ -31,6 +30,7 @@ from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
|
||||
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
@@ -622,6 +622,19 @@ def run_attention_backend(
|
||||
k_scale=k_scale,
|
||||
)
|
||||
|
||||
# Attach prefill backend (normally created by MLAAttention.__init__)
|
||||
prefill_scale = (qk_nope_head_dim + qk_rope_head_dim) ** -0.5
|
||||
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
|
||||
mock_layer.prefill_backend = prefill_backend_cls(
|
||||
num_heads=num_heads,
|
||||
scale=prefill_scale,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
# Populate static_forward_context with mock attention layers
|
||||
for layer_name in layer_names:
|
||||
vllm_config.compilation_config.static_forward_context[layer_name] = (
|
||||
@@ -787,7 +800,8 @@ def test_backend_correctness(
|
||||
f"MLA dimensions don't match: {total_head_size} != {head_size}"
|
||||
)
|
||||
decode_scale = 1.0 / (total_head_size**0.5)
|
||||
prefill_scale = get_mla_prefill_scale(vllm_config.model_config)
|
||||
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
prefill_scale = qk_head_dim**-0.5
|
||||
|
||||
# 2. Generate data and compute SDPA reference output for MLA
|
||||
all_q_vllm, all_kv_c_vllm, all_k_pe_vllm = [], [], []
|
||||
|
||||
@@ -2,17 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for MLA prefill backend selector."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import AttentionConfig, ModelConfig, VllmConfig
|
||||
from vllm.model_executor.layers.attention.mla_attention import get_mla_prefill_scale
|
||||
from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import (
|
||||
yarn_get_mscale,
|
||||
)
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import (
|
||||
@@ -58,62 +53,6 @@ def _make_vllm_config(
|
||||
return mock_vllm_config
|
||||
|
||||
|
||||
class TestMLAPrefillScale:
|
||||
"""Tests for the MLA prefill softmax scale."""
|
||||
|
||||
def test_uses_qk_head_dim_for_deepseek_v2_style_mla(self):
|
||||
model_config = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
rope_parameters={"rope_type": "default"},
|
||||
)
|
||||
)
|
||||
|
||||
assert get_mla_prefill_scale(model_config) == pytest.approx(192**-0.5)
|
||||
|
||||
def test_applies_deepseek_yarn_mscale(self):
|
||||
model_config = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
rope_parameters={
|
||||
"rope_type": "yarn",
|
||||
"factor": 40,
|
||||
"mscale_all_dim": 0.707,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
mscale = yarn_get_mscale(40, 0.707)
|
||||
assert get_mla_prefill_scale(model_config) == pytest.approx(
|
||||
192**-0.5 * mscale * mscale
|
||||
)
|
||||
|
||||
def test_deepseek_v4_style_mla_does_not_apply_yarn_mscale(self):
|
||||
model_config = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(
|
||||
compress_ratios=[4],
|
||||
q_lora_rank=1536,
|
||||
head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
rope_parameters={
|
||||
"rope_type": "yarn",
|
||||
"factor": 40,
|
||||
"mscale_all_dim": 0.707,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert get_mla_prefill_scale(model_config) == pytest.approx(128**-0.5)
|
||||
|
||||
|
||||
class TestGetMLAPrefillBackend:
|
||||
"""Tests for get_mla_prefill_backend (public API)."""
|
||||
|
||||
|
||||
@@ -238,9 +238,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import (
|
||||
yarn_get_mscale,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.math_utils import cdiv, round_down
|
||||
@@ -262,7 +259,10 @@ from vllm.v1.attention.backend import (
|
||||
MLAAttentionImpl,
|
||||
SparseMLAAttentionImpl,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill import MLAPrefillBackend
|
||||
from vllm.v1.attention.backends.mla.prefill import (
|
||||
MLAPrefillBackend,
|
||||
get_mla_prefill_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
get_dcp_local_seq_lens,
|
||||
split_decodes_and_prefills,
|
||||
@@ -454,20 +454,32 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self.q_pad_num_heads = getattr(self.impl, "q_pad_num_heads", None)
|
||||
self.use_direct_call = not current_platform.opaque_attention_op()
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
vllm_config = get_current_vllm_config()
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
|
||||
self.prefill_backend = prefill_backend_cls(
|
||||
num_heads=self.num_heads,
|
||||
scale=self.scale,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
self.kv_cache = torch.tensor([])
|
||||
|
||||
self.use_sparse = use_sparse
|
||||
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
_vllm_config = get_current_vllm_config_or_none()
|
||||
self.dcp_a2a = (
|
||||
vllm_config is not None
|
||||
and vllm_config.parallel_config.decode_context_parallel_size > 1
|
||||
and vllm_config.parallel_config.dcp_comm_backend == "a2a"
|
||||
_vllm_config is not None
|
||||
and _vllm_config.parallel_config.decode_context_parallel_size > 1
|
||||
and _vllm_config.parallel_config.dcp_comm_backend == "a2a"
|
||||
)
|
||||
|
||||
# Initialize q/k/v range constants.
|
||||
@@ -1330,35 +1342,6 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims:
|
||||
)
|
||||
|
||||
|
||||
def get_mla_prefill_scale(model_config: ModelConfig) -> float:
|
||||
hf_text_config = model_config.hf_text_config
|
||||
mla_dims = get_mla_dims(model_config)
|
||||
qk_head_dim = mla_dims.qk_nope_head_dim + mla_dims.qk_rope_head_dim
|
||||
scale = qk_head_dim**-0.5
|
||||
|
||||
# Deepseek V4 disables YaRN mscale for attention; Deepseek V2/V3 applies
|
||||
# the same mscale correction when constructing the MLA attention module.
|
||||
if hasattr(hf_text_config, "compress_ratios"):
|
||||
return scale
|
||||
|
||||
rope_parameters = getattr(hf_text_config, "rope_parameters", None)
|
||||
if rope_parameters is None:
|
||||
rope_parameters = getattr(hf_text_config, "rope_scaling", None)
|
||||
|
||||
if rope_parameters is None:
|
||||
return scale
|
||||
|
||||
rope_type = rope_parameters.get("rope_type", rope_parameters.get("type"))
|
||||
apply_yarn_scaling = rope_parameters.get("apply_yarn_scaling", True)
|
||||
if rope_type != "default" and apply_yarn_scaling:
|
||||
mscale_all_dim = rope_parameters.get("mscale_all_dim", False)
|
||||
scaling_factor = rope_parameters["factor"]
|
||||
mscale = yarn_get_mscale(float(scaling_factor), float(mscale_all_dim))
|
||||
scale *= mscale * mscale
|
||||
|
||||
return scale
|
||||
|
||||
|
||||
@functools.cache
|
||||
def backend_supports_prefill_query_quantization() -> bool:
|
||||
"""Check if the selected MLA prefill backend supports query quantization.
|
||||
@@ -1554,20 +1537,9 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
device=device,
|
||||
)
|
||||
|
||||
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
|
||||
|
||||
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
|
||||
self._prefill_backend = prefill_backend_cls(
|
||||
num_heads=self.num_heads,
|
||||
scale=get_mla_prefill_scale(self.model_config),
|
||||
kv_lora_rank=self.mla_dims.kv_lora_rank,
|
||||
qk_nope_head_dim=self.mla_dims.qk_nope_head_dim,
|
||||
qk_rope_head_dim=self.mla_dims.qk_rope_head_dim,
|
||||
v_head_dim=self.mla_dims.v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
layer_names=layer_names,
|
||||
)
|
||||
self._prefill_backend = self.compilation_config.static_forward_context[
|
||||
layer_names[0]
|
||||
].prefill_backend
|
||||
|
||||
supports_spec_decode = self.query_len_support != QueryLenSupport.SINGLE_ONLY
|
||||
self._init_reorder_batch_threshold(
|
||||
|
||||
@@ -81,8 +81,6 @@ class MLAPrefillBackend(ABC):
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
layer_names: list[str] | None = None,
|
||||
) -> None:
|
||||
self.num_heads = num_heads
|
||||
self.scale = scale
|
||||
@@ -91,8 +89,6 @@ class MLAPrefillBackend(ABC):
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
self.layer_names = layer_names
|
||||
|
||||
def prepare_metadata( # noqa: B027
|
||||
self,
|
||||
|
||||
@@ -44,8 +44,6 @@ class FlashAttnPrefillBackend(MLAPrefillBackend):
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
layer_names: list[str] | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_heads=num_heads,
|
||||
@@ -55,8 +53,6 @@ class FlashAttnPrefillBackend(MLAPrefillBackend):
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
layer_names=layer_names,
|
||||
)
|
||||
|
||||
# Handle the differences between the flash_attn_varlen from
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
import vllm.envs as envs
|
||||
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
PerLayerParameters,
|
||||
get_per_layer_parameters,
|
||||
infer_global_hyperparameters,
|
||||
)
|
||||
@@ -62,8 +63,6 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
layer_names: list[str] | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_heads=num_heads,
|
||||
@@ -73,25 +72,11 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
layer_names=layer_names,
|
||||
)
|
||||
|
||||
self._prefill_main: BatchPrefillWithRaggedKVCacheWrapper | None = None
|
||||
self._prefill_chunks: list[BatchPrefillWithRaggedKVCacheWrapper] = []
|
||||
if layer_names is None:
|
||||
raise ValueError(
|
||||
"FlashInferPrefillBackend requires layer_names to "
|
||||
"initialize global hyperparameters."
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonImpl,
|
||||
)
|
||||
|
||||
self._global_hyperparameters = infer_global_hyperparameters(
|
||||
get_per_layer_parameters(vllm_config, layer_names, MLACommonImpl) # type: ignore[type-abstract]
|
||||
)
|
||||
self._global_hyperparameters: PerLayerParameters | None = None
|
||||
|
||||
def _ensure_chunks(
|
||||
self,
|
||||
@@ -106,10 +91,36 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
|
||||
)
|
||||
)
|
||||
|
||||
def _resolve_global_hyperparameters(self) -> PerLayerParameters:
|
||||
if self._global_hyperparameters is not None:
|
||||
return self._global_hyperparameters
|
||||
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLAAttention,
|
||||
MLACommonImpl,
|
||||
)
|
||||
|
||||
forward_context = self.vllm_config.compilation_config.static_forward_context
|
||||
layer_names = [
|
||||
name
|
||||
for name, layer in forward_context.items()
|
||||
if isinstance(layer, MLAAttention)
|
||||
]
|
||||
|
||||
self._global_hyperparameters = infer_global_hyperparameters(
|
||||
get_per_layer_parameters(
|
||||
self.vllm_config,
|
||||
layer_names,
|
||||
MLACommonImpl, # type: ignore[type-abstract]
|
||||
)
|
||||
)
|
||||
return self._global_hyperparameters
|
||||
|
||||
def prepare_metadata(
|
||||
self,
|
||||
prefill_metadata: "MLACommonPrefillMetadata",
|
||||
) -> None:
|
||||
global_hyperparameters = self._resolve_global_hyperparameters()
|
||||
qo_indptr = prefill_metadata.query_start_loc
|
||||
has_context = prefill_metadata.chunked_context is not None
|
||||
(workspace_buffer,) = current_workspace_manager().get_simultaneous(
|
||||
@@ -144,9 +155,9 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
|
||||
head_dim_qk=head_dim_qk,
|
||||
head_dim_vo=head_dim_vo,
|
||||
causal=True,
|
||||
sm_scale=self._global_hyperparameters.sm_scale,
|
||||
window_left=self._global_hyperparameters.window_left,
|
||||
logits_soft_cap=self._global_hyperparameters.logits_soft_cap,
|
||||
sm_scale=global_hyperparameters.sm_scale,
|
||||
window_left=global_hyperparameters.window_left,
|
||||
logits_soft_cap=global_hyperparameters.logits_soft_cap,
|
||||
q_data_type=prefill_metadata.q_data_type,
|
||||
o_data_type=prefill_metadata.output_dtype,
|
||||
)
|
||||
@@ -165,9 +176,9 @@ class FlashInferPrefillBackend(MLAPrefillBackend):
|
||||
head_dim_qk=head_dim_qk,
|
||||
head_dim_vo=head_dim_vo,
|
||||
causal=False,
|
||||
sm_scale=self._global_hyperparameters.sm_scale,
|
||||
window_left=self._global_hyperparameters.window_left,
|
||||
logits_soft_cap=self._global_hyperparameters.logits_soft_cap,
|
||||
sm_scale=global_hyperparameters.sm_scale,
|
||||
window_left=global_hyperparameters.window_left,
|
||||
logits_soft_cap=global_hyperparameters.logits_soft_cap,
|
||||
q_data_type=prefill_metadata.q_data_type,
|
||||
o_data_type=prefill_metadata.output_dtype,
|
||||
)
|
||||
|
||||
@@ -51,8 +51,6 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
vllm_config: "VllmConfig",
|
||||
device: torch.device,
|
||||
layer_names: list[str] | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_heads=num_heads,
|
||||
@@ -62,8 +60,6 @@ class TrtllmRaggedPrefillBackend(MLAPrefillBackend):
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
layer_names=layer_names,
|
||||
)
|
||||
|
||||
def _get_workspace_buffer(self) -> torch.Tensor:
|
||||
|
||||
Reference in New Issue
Block a user