[Attention] Minor refactor: layer takes ownership of the MLA prefill backend (#41744)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
@@ -22,7 +22,6 @@ from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
QueryLenSupport,
|
||||
_DecodeConcatQuantFP8,
|
||||
get_mla_prefill_scale,
|
||||
)
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
@@ -31,6 +30,7 @@ from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
|
||||
from vllm.v1.attention.backends.mla.prefill import get_mla_prefill_backend
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
@@ -622,6 +622,19 @@ def run_attention_backend(
|
||||
k_scale=k_scale,
|
||||
)
|
||||
|
||||
# Attach prefill backend (normally created by MLAAttention.__init__)
|
||||
prefill_scale = (qk_nope_head_dim + qk_rope_head_dim) ** -0.5
|
||||
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
|
||||
mock_layer.prefill_backend = prefill_backend_cls(
|
||||
num_heads=num_heads,
|
||||
scale=prefill_scale,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
# Populate static_forward_context with mock attention layers
|
||||
for layer_name in layer_names:
|
||||
vllm_config.compilation_config.static_forward_context[layer_name] = (
|
||||
@@ -787,7 +800,8 @@ def test_backend_correctness(
|
||||
f"MLA dimensions don't match: {total_head_size} != {head_size}"
|
||||
)
|
||||
decode_scale = 1.0 / (total_head_size**0.5)
|
||||
prefill_scale = get_mla_prefill_scale(vllm_config.model_config)
|
||||
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
prefill_scale = qk_head_dim**-0.5
|
||||
|
||||
# 2. Generate data and compute SDPA reference output for MLA
|
||||
all_q_vllm, all_kv_c_vllm, all_k_pe_vllm = [], [], []
|
||||
|
||||
@@ -2,17 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for MLA prefill backend selector."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import AttentionConfig, ModelConfig, VllmConfig
|
||||
from vllm.model_executor.layers.attention.mla_attention import get_mla_prefill_scale
|
||||
from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import (
|
||||
yarn_get_mscale,
|
||||
)
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import (
|
||||
@@ -58,62 +53,6 @@ def _make_vllm_config(
|
||||
return mock_vllm_config
|
||||
|
||||
|
||||
class TestMLAPrefillScale:
|
||||
"""Tests for the MLA prefill softmax scale."""
|
||||
|
||||
def test_uses_qk_head_dim_for_deepseek_v2_style_mla(self):
|
||||
model_config = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
rope_parameters={"rope_type": "default"},
|
||||
)
|
||||
)
|
||||
|
||||
assert get_mla_prefill_scale(model_config) == pytest.approx(192**-0.5)
|
||||
|
||||
def test_applies_deepseek_yarn_mscale(self):
|
||||
model_config = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
rope_parameters={
|
||||
"rope_type": "yarn",
|
||||
"factor": 40,
|
||||
"mscale_all_dim": 0.707,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
mscale = yarn_get_mscale(40, 0.707)
|
||||
assert get_mla_prefill_scale(model_config) == pytest.approx(
|
||||
192**-0.5 * mscale * mscale
|
||||
)
|
||||
|
||||
def test_deepseek_v4_style_mla_does_not_apply_yarn_mscale(self):
|
||||
model_config = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(
|
||||
compress_ratios=[4],
|
||||
q_lora_rank=1536,
|
||||
head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
rope_parameters={
|
||||
"rope_type": "yarn",
|
||||
"factor": 40,
|
||||
"mscale_all_dim": 0.707,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert get_mla_prefill_scale(model_config) == pytest.approx(128**-0.5)
|
||||
|
||||
|
||||
class TestGetMLAPrefillBackend:
|
||||
"""Tests for get_mla_prefill_backend (public API)."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user