From 1cb08387214fba82c2705429100c5942fab65f05 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Mon, 4 May 2026 18:32:55 -0500 Subject: [PATCH] [ROCm][CI] Fix MLA prefill scale for DeepSeek GSM8K (#41569) Signed-off-by: Andreas Karatzas --- tests/v1/attention/test_mla_backends.py | 8 ++- .../v1/attention/test_mla_prefill_selector.py | 61 +++++++++++++++++++ .../layers/attention/mla_attention.py | 34 ++++++++++- 3 files changed, 99 insertions(+), 4 deletions(-) diff --git a/tests/v1/attention/test_mla_backends.py b/tests/v1/attention/test_mla_backends.py index f91ea85779..8ab47b6189 100644 --- a/tests/v1/attention/test_mla_backends.py +++ b/tests/v1/attention/test_mla_backends.py @@ -22,6 +22,7 @@ from vllm.config.vllm import set_current_vllm_config from vllm.model_executor.layers.attention.mla_attention import ( QueryLenSupport, _DecodeConcatQuantFP8, + get_mla_prefill_scale, ) from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape @@ -785,7 +786,8 @@ def test_backend_correctness( assert kv_lora_rank + qk_rope_head_dim == head_size, ( f"MLA dimensions don't match: {total_head_size} != {head_size}" ) - scale = 1.0 / (total_head_size**0.5) + decode_scale = 1.0 / (total_head_size**0.5) + prefill_scale = get_mla_prefill_scale(vllm_config.model_config) # 2. Generate data and compute SDPA reference output for MLA all_q_vllm, all_kv_c_vllm, all_k_pe_vllm = [], [], [] @@ -902,7 +904,7 @@ def test_backend_correctness( v_sdpa_in = v_mqa.unsqueeze(0).transpose(1, 2) sdpa_out_i_decode = torch.nn.functional.scaled_dot_product_attention( - q_sdpa_in, k_sdpa_in, v_sdpa_in, attn_mask=attn_mask, scale=scale + q_sdpa_in, k_sdpa_in, v_sdpa_in, attn_mask=attn_mask, scale=decode_scale ) sdpa_out_i_decode = sdpa_out_i_decode.transpose(1, 2).squeeze( 0 @@ -938,7 +940,7 @@ def test_backend_correctness( # Single attention call with custom mask sdpa_out_i_prefill = torch.nn.functional.scaled_dot_product_attention( - q_sdpa_in, k_sdpa_in, v_sdpa_in, attn_mask=attn_mask, scale=scale + q_sdpa_in, k_sdpa_in, v_sdpa_in, attn_mask=attn_mask, scale=prefill_scale ) sdpa_out_i_prefill = sdpa_out_i_prefill.transpose(1, 2).squeeze(0) sdpa_out_i_prefill = sdpa_out_i_prefill.flatten(start_dim=-2) diff --git a/tests/v1/attention/test_mla_prefill_selector.py b/tests/v1/attention/test_mla_prefill_selector.py index 068eb43faf..873cfb1870 100644 --- a/tests/v1/attention/test_mla_prefill_selector.py +++ b/tests/v1/attention/test_mla_prefill_selector.py @@ -2,12 +2,17 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Tests for MLA prefill backend selector.""" +from types import SimpleNamespace from unittest.mock import MagicMock, patch import pytest import torch from vllm.config import AttentionConfig, ModelConfig, VllmConfig +from vllm.model_executor.layers.attention.mla_attention import get_mla_prefill_scale +from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import ( + yarn_get_mscale, +) from vllm.platforms.interface import DeviceCapability from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum from vllm.v1.attention.backends.mla.prefill.selector import ( @@ -53,6 +58,62 @@ def _make_vllm_config( return mock_vllm_config +class TestMLAPrefillScale: + """Tests for the MLA prefill softmax scale.""" + + def test_uses_qk_head_dim_for_deepseek_v2_style_mla(self): + model_config = SimpleNamespace( + hf_text_config=SimpleNamespace( + q_lora_rank=None, + kv_lora_rank=512, + qk_nope_head_dim=128, + qk_rope_head_dim=64, + v_head_dim=128, + rope_parameters={"rope_type": "default"}, + ) + ) + + assert get_mla_prefill_scale(model_config) == pytest.approx(192**-0.5) + + def test_applies_deepseek_yarn_mscale(self): + model_config = SimpleNamespace( + hf_text_config=SimpleNamespace( + q_lora_rank=None, + kv_lora_rank=512, + qk_nope_head_dim=128, + qk_rope_head_dim=64, + v_head_dim=128, + rope_parameters={ + "rope_type": "yarn", + "factor": 40, + "mscale_all_dim": 0.707, + }, + ) + ) + + mscale = yarn_get_mscale(40, 0.707) + assert get_mla_prefill_scale(model_config) == pytest.approx( + 192**-0.5 * mscale * mscale + ) + + def test_deepseek_v4_style_mla_does_not_apply_yarn_mscale(self): + model_config = SimpleNamespace( + hf_text_config=SimpleNamespace( + compress_ratios=[4], + q_lora_rank=1536, + head_dim=128, + qk_rope_head_dim=64, + rope_parameters={ + "rope_type": "yarn", + "factor": 40, + "mscale_all_dim": 0.707, + }, + ) + ) + + assert get_mla_prefill_scale(model_config) == pytest.approx(128**-0.5) + + class TestGetMLAPrefillBackend: """Tests for get_mla_prefill_backend (public API).""" diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 82eecc8cd4..20981f60cd 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -238,6 +238,9 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8StaticTensorSym, kNvfp4Dynamic, ) +from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import ( + yarn_get_mscale, +) from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer from vllm.utils.math_utils import cdiv, round_down @@ -1327,6 +1330,35 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims: ) +def get_mla_prefill_scale(model_config: ModelConfig) -> float: + hf_text_config = model_config.hf_text_config + mla_dims = get_mla_dims(model_config) + qk_head_dim = mla_dims.qk_nope_head_dim + mla_dims.qk_rope_head_dim + scale = qk_head_dim**-0.5 + + # Deepseek V4 disables YaRN mscale for attention; Deepseek V2/V3 applies + # the same mscale correction when constructing the MLA attention module. + if hasattr(hf_text_config, "compress_ratios"): + return scale + + rope_parameters = getattr(hf_text_config, "rope_parameters", None) + if rope_parameters is None: + rope_parameters = getattr(hf_text_config, "rope_scaling", None) + + if rope_parameters is None: + return scale + + rope_type = rope_parameters.get("rope_type", rope_parameters.get("type")) + apply_yarn_scaling = rope_parameters.get("apply_yarn_scaling", True) + if rope_type != "default" and apply_yarn_scaling: + mscale_all_dim = rope_parameters.get("mscale_all_dim", False) + scaling_factor = rope_parameters["factor"] + mscale = yarn_get_mscale(float(scaling_factor), float(mscale_all_dim)) + scale *= mscale * mscale + + return scale + + @functools.cache def backend_supports_prefill_query_quantization() -> bool: """Check if the selected MLA prefill backend supports query quantization. @@ -1527,7 +1559,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): prefill_backend_cls = get_mla_prefill_backend(vllm_config) self._prefill_backend = prefill_backend_cls( num_heads=self.num_heads, - scale=self.model_config.get_head_size() ** -0.5, + scale=get_mla_prefill_scale(self.model_config), kv_lora_rank=self.mla_dims.kv_lora_rank, qk_nope_head_dim=self.mla_dims.qk_nope_head_dim, qk_rope_head_dim=self.mla_dims.qk_rope_head_dim,