[ROCm][CI] Fix MLA prefill scale for DeepSeek GSM8K (#41569)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-05-04 16:32:55 -07:00
committed by GitHub
parent be5983b874
commit 1cb0838721
3 changed files with 99 additions and 4 deletions
+5 -3
View File
@@ -22,6 +22,7 @@ from vllm.config.vllm import set_current_vllm_config
from vllm.model_executor.layers.attention.mla_attention import (
QueryLenSupport,
_DecodeConcatQuantFP8,
get_mla_prefill_scale,
)
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
@@ -785,7 +786,8 @@ def test_backend_correctness(
assert kv_lora_rank + qk_rope_head_dim == head_size, (
f"MLA dimensions don't match: {total_head_size} != {head_size}"
)
scale = 1.0 / (total_head_size**0.5)
decode_scale = 1.0 / (total_head_size**0.5)
prefill_scale = get_mla_prefill_scale(vllm_config.model_config)
# 2. Generate data and compute SDPA reference output for MLA
all_q_vllm, all_kv_c_vllm, all_k_pe_vllm = [], [], []
@@ -902,7 +904,7 @@ def test_backend_correctness(
v_sdpa_in = v_mqa.unsqueeze(0).transpose(1, 2)
sdpa_out_i_decode = torch.nn.functional.scaled_dot_product_attention(
q_sdpa_in, k_sdpa_in, v_sdpa_in, attn_mask=attn_mask, scale=scale
q_sdpa_in, k_sdpa_in, v_sdpa_in, attn_mask=attn_mask, scale=decode_scale
)
sdpa_out_i_decode = sdpa_out_i_decode.transpose(1, 2).squeeze(
0
@@ -938,7 +940,7 @@ def test_backend_correctness(
# Single attention call with custom mask
sdpa_out_i_prefill = torch.nn.functional.scaled_dot_product_attention(
q_sdpa_in, k_sdpa_in, v_sdpa_in, attn_mask=attn_mask, scale=scale
q_sdpa_in, k_sdpa_in, v_sdpa_in, attn_mask=attn_mask, scale=prefill_scale
)
sdpa_out_i_prefill = sdpa_out_i_prefill.transpose(1, 2).squeeze(0)
sdpa_out_i_prefill = sdpa_out_i_prefill.flatten(start_dim=-2)
@@ -2,12 +2,17 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for MLA prefill backend selector."""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.config import AttentionConfig, ModelConfig, VllmConfig
from vllm.model_executor.layers.attention.mla_attention import get_mla_prefill_scale
from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import (
yarn_get_mscale,
)
from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
from vllm.v1.attention.backends.mla.prefill.selector import (
@@ -53,6 +58,62 @@ def _make_vllm_config(
return mock_vllm_config
class TestMLAPrefillScale:
"""Tests for the MLA prefill softmax scale."""
def test_uses_qk_head_dim_for_deepseek_v2_style_mla(self):
model_config = SimpleNamespace(
hf_text_config=SimpleNamespace(
q_lora_rank=None,
kv_lora_rank=512,
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
rope_parameters={"rope_type": "default"},
)
)
assert get_mla_prefill_scale(model_config) == pytest.approx(192**-0.5)
def test_applies_deepseek_yarn_mscale(self):
model_config = SimpleNamespace(
hf_text_config=SimpleNamespace(
q_lora_rank=None,
kv_lora_rank=512,
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
rope_parameters={
"rope_type": "yarn",
"factor": 40,
"mscale_all_dim": 0.707,
},
)
)
mscale = yarn_get_mscale(40, 0.707)
assert get_mla_prefill_scale(model_config) == pytest.approx(
192**-0.5 * mscale * mscale
)
def test_deepseek_v4_style_mla_does_not_apply_yarn_mscale(self):
model_config = SimpleNamespace(
hf_text_config=SimpleNamespace(
compress_ratios=[4],
q_lora_rank=1536,
head_dim=128,
qk_rope_head_dim=64,
rope_parameters={
"rope_type": "yarn",
"factor": 40,
"mscale_all_dim": 0.707,
},
)
)
assert get_mla_prefill_scale(model_config) == pytest.approx(128**-0.5)
class TestGetMLAPrefillBackend:
"""Tests for get_mla_prefill_backend (public API)."""
@@ -238,6 +238,9 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import (
yarn_get_mscale,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.math_utils import cdiv, round_down
@@ -1327,6 +1330,35 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims:
)
def get_mla_prefill_scale(model_config: ModelConfig) -> float:
hf_text_config = model_config.hf_text_config
mla_dims = get_mla_dims(model_config)
qk_head_dim = mla_dims.qk_nope_head_dim + mla_dims.qk_rope_head_dim
scale = qk_head_dim**-0.5
# Deepseek V4 disables YaRN mscale for attention; Deepseek V2/V3 applies
# the same mscale correction when constructing the MLA attention module.
if hasattr(hf_text_config, "compress_ratios"):
return scale
rope_parameters = getattr(hf_text_config, "rope_parameters", None)
if rope_parameters is None:
rope_parameters = getattr(hf_text_config, "rope_scaling", None)
if rope_parameters is None:
return scale
rope_type = rope_parameters.get("rope_type", rope_parameters.get("type"))
apply_yarn_scaling = rope_parameters.get("apply_yarn_scaling", True)
if rope_type != "default" and apply_yarn_scaling:
mscale_all_dim = rope_parameters.get("mscale_all_dim", False)
scaling_factor = rope_parameters["factor"]
mscale = yarn_get_mscale(float(scaling_factor), float(mscale_all_dim))
scale *= mscale * mscale
return scale
@functools.cache
def backend_supports_prefill_query_quantization() -> bool:
"""Check if the selected MLA prefill backend supports query quantization.
@@ -1527,7 +1559,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
self._prefill_backend = prefill_backend_cls(
num_heads=self.num_heads,
scale=self.model_config.get_head_size() ** -0.5,
scale=get_mla_prefill_scale(self.model_config),
kv_lora_rank=self.mla_dims.kv_lora_rank,
qk_nope_head_dim=self.mla_dims.qk_nope_head_dim,
qk_rope_head_dim=self.mla_dims.qk_rope_head_dim,