[ROCm][CI] Fix MLA prefill scale for DeepSeek GSM8K (#41569)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -22,6 +22,7 @@ from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
QueryLenSupport,
|
||||
_DecodeConcatQuantFP8,
|
||||
get_mla_prefill_scale,
|
||||
)
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
@@ -785,7 +786,8 @@ def test_backend_correctness(
|
||||
assert kv_lora_rank + qk_rope_head_dim == head_size, (
|
||||
f"MLA dimensions don't match: {total_head_size} != {head_size}"
|
||||
)
|
||||
scale = 1.0 / (total_head_size**0.5)
|
||||
decode_scale = 1.0 / (total_head_size**0.5)
|
||||
prefill_scale = get_mla_prefill_scale(vllm_config.model_config)
|
||||
|
||||
# 2. Generate data and compute SDPA reference output for MLA
|
||||
all_q_vllm, all_kv_c_vllm, all_k_pe_vllm = [], [], []
|
||||
@@ -902,7 +904,7 @@ def test_backend_correctness(
|
||||
v_sdpa_in = v_mqa.unsqueeze(0).transpose(1, 2)
|
||||
|
||||
sdpa_out_i_decode = torch.nn.functional.scaled_dot_product_attention(
|
||||
q_sdpa_in, k_sdpa_in, v_sdpa_in, attn_mask=attn_mask, scale=scale
|
||||
q_sdpa_in, k_sdpa_in, v_sdpa_in, attn_mask=attn_mask, scale=decode_scale
|
||||
)
|
||||
sdpa_out_i_decode = sdpa_out_i_decode.transpose(1, 2).squeeze(
|
||||
0
|
||||
@@ -938,7 +940,7 @@ def test_backend_correctness(
|
||||
|
||||
# Single attention call with custom mask
|
||||
sdpa_out_i_prefill = torch.nn.functional.scaled_dot_product_attention(
|
||||
q_sdpa_in, k_sdpa_in, v_sdpa_in, attn_mask=attn_mask, scale=scale
|
||||
q_sdpa_in, k_sdpa_in, v_sdpa_in, attn_mask=attn_mask, scale=prefill_scale
|
||||
)
|
||||
sdpa_out_i_prefill = sdpa_out_i_prefill.transpose(1, 2).squeeze(0)
|
||||
sdpa_out_i_prefill = sdpa_out_i_prefill.flatten(start_dim=-2)
|
||||
|
||||
@@ -2,12 +2,17 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for MLA prefill backend selector."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import AttentionConfig, ModelConfig, VllmConfig
|
||||
from vllm.model_executor.layers.attention.mla_attention import get_mla_prefill_scale
|
||||
from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import (
|
||||
yarn_get_mscale,
|
||||
)
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
|
||||
from vllm.v1.attention.backends.mla.prefill.selector import (
|
||||
@@ -53,6 +58,62 @@ def _make_vllm_config(
|
||||
return mock_vllm_config
|
||||
|
||||
|
||||
class TestMLAPrefillScale:
|
||||
"""Tests for the MLA prefill softmax scale."""
|
||||
|
||||
def test_uses_qk_head_dim_for_deepseek_v2_style_mla(self):
|
||||
model_config = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
rope_parameters={"rope_type": "default"},
|
||||
)
|
||||
)
|
||||
|
||||
assert get_mla_prefill_scale(model_config) == pytest.approx(192**-0.5)
|
||||
|
||||
def test_applies_deepseek_yarn_mscale(self):
|
||||
model_config = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
rope_parameters={
|
||||
"rope_type": "yarn",
|
||||
"factor": 40,
|
||||
"mscale_all_dim": 0.707,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
mscale = yarn_get_mscale(40, 0.707)
|
||||
assert get_mla_prefill_scale(model_config) == pytest.approx(
|
||||
192**-0.5 * mscale * mscale
|
||||
)
|
||||
|
||||
def test_deepseek_v4_style_mla_does_not_apply_yarn_mscale(self):
|
||||
model_config = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(
|
||||
compress_ratios=[4],
|
||||
q_lora_rank=1536,
|
||||
head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
rope_parameters={
|
||||
"rope_type": "yarn",
|
||||
"factor": 40,
|
||||
"mscale_all_dim": 0.707,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert get_mla_prefill_scale(model_config) == pytest.approx(128**-0.5)
|
||||
|
||||
|
||||
class TestGetMLAPrefillBackend:
|
||||
"""Tests for get_mla_prefill_backend (public API)."""
|
||||
|
||||
|
||||
@@ -238,6 +238,9 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding.deepseek_scaling_rope import (
|
||||
yarn_get_mscale,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.utils.math_utils import cdiv, round_down
|
||||
@@ -1327,6 +1330,35 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims:
|
||||
)
|
||||
|
||||
|
||||
def get_mla_prefill_scale(model_config: ModelConfig) -> float:
|
||||
hf_text_config = model_config.hf_text_config
|
||||
mla_dims = get_mla_dims(model_config)
|
||||
qk_head_dim = mla_dims.qk_nope_head_dim + mla_dims.qk_rope_head_dim
|
||||
scale = qk_head_dim**-0.5
|
||||
|
||||
# Deepseek V4 disables YaRN mscale for attention; Deepseek V2/V3 applies
|
||||
# the same mscale correction when constructing the MLA attention module.
|
||||
if hasattr(hf_text_config, "compress_ratios"):
|
||||
return scale
|
||||
|
||||
rope_parameters = getattr(hf_text_config, "rope_parameters", None)
|
||||
if rope_parameters is None:
|
||||
rope_parameters = getattr(hf_text_config, "rope_scaling", None)
|
||||
|
||||
if rope_parameters is None:
|
||||
return scale
|
||||
|
||||
rope_type = rope_parameters.get("rope_type", rope_parameters.get("type"))
|
||||
apply_yarn_scaling = rope_parameters.get("apply_yarn_scaling", True)
|
||||
if rope_type != "default" and apply_yarn_scaling:
|
||||
mscale_all_dim = rope_parameters.get("mscale_all_dim", False)
|
||||
scaling_factor = rope_parameters["factor"]
|
||||
mscale = yarn_get_mscale(float(scaling_factor), float(mscale_all_dim))
|
||||
scale *= mscale * mscale
|
||||
|
||||
return scale
|
||||
|
||||
|
||||
@functools.cache
|
||||
def backend_supports_prefill_query_quantization() -> bool:
|
||||
"""Check if the selected MLA prefill backend supports query quantization.
|
||||
@@ -1527,7 +1559,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]):
|
||||
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
|
||||
self._prefill_backend = prefill_backend_cls(
|
||||
num_heads=self.num_heads,
|
||||
scale=self.model_config.get_head_size() ** -0.5,
|
||||
scale=get_mla_prefill_scale(self.model_config),
|
||||
kv_lora_rank=self.mla_dims.kv_lora_rank,
|
||||
qk_nope_head_dim=self.mla_dims.qk_nope_head_dim,
|
||||
qk_rope_head_dim=self.mla_dims.qk_rope_head_dim,
|
||||
|
||||
Reference in New Issue
Block a user