[Bugfix] Fix KV scales inconsistency in fp8 MLA & FlashInfer kv_cache_dtype "auto" leading to gibberish (#37054)

Signed-off-by: Andy Lo <andy@mistral.ai>
This commit is contained in:
Andy Lo
2026-03-18 23:07:29 +00:00
committed by GitHub
parent 04244fd0e1
commit 577df69b26
8 changed files with 71 additions and 43 deletions
+31 -28
View File
@@ -266,22 +266,6 @@ def create_and_prepopulate_kv_cache(
return kv_cache
class MockAttentionLayer:
"""A mock attention layer for testing."""
def __init__(self, device: torch.device):
self._q_scale = torch.tensor(1.0, device=device)
self._k_scale = torch.tensor(1.0, device=device)
self._v_scale = torch.tensor(1.0, device=device)
self._prob_scale = torch.tensor(1.0, device=device)
self._q_scale_float = 1.0
self._k_scale_float = 1.0
self._v_scale_float = 1.0
def forward(self, *_args, **_kwargs):
raise NotImplementedError
class MockSparseMLAAttentionLayer:
"""A mock sparse MLA attention layer for testing.
@@ -304,6 +288,8 @@ class MockSparseMLAAttentionLayer:
device: torch.device,
W_UK: torch.Tensor,
W_UV: torch.Tensor,
q_scale: float,
k_scale: float,
):
self.impl = impl
self.num_heads = num_heads
@@ -319,13 +305,13 @@ class MockSparseMLAAttentionLayer:
self.W_UV = W_UV.transpose(0, 1)
# Scale attributes needed by attention backends
self._q_scale = torch.tensor(1.0, device=device)
self._k_scale = torch.tensor(1.0, device=device)
self._v_scale = torch.tensor(1.0, device=device)
self._q_scale = torch.tensor(q_scale, device=device)
self._k_scale = torch.tensor(k_scale, device=device)
self._v_scale = torch.tensor(float("nan"), device=device)
self._prob_scale = torch.tensor(1.0, device=device)
self._q_scale_float = 1.0
self._k_scale_float = 1.0
self._v_scale_float = 1.0
self._q_scale_float = q_scale
self._k_scale_float = k_scale
self._v_scale_float = float("nan")
self._decode_concat_quant_fp8_op = _DecodeConcatQuantFP8(
static=True,
@@ -420,6 +406,8 @@ class MockMLAAttentionLayer(AttentionLayerBase):
kv_lora_rank: int,
device: torch.device,
kv_b_proj,
q_scale: float,
k_scale: float,
):
self.impl = impl
self.num_heads = num_heads
@@ -443,13 +431,13 @@ class MockMLAAttentionLayer(AttentionLayerBase):
self.W_UK_T = W_UK.permute(1, 2, 0)
# Scale attributes needed by attention backends
self._q_scale = torch.tensor(1.0, device=device)
self._k_scale = torch.tensor(1.0, device=device)
self._v_scale = torch.tensor(1.0, device=device)
self._q_scale = torch.tensor(q_scale, device=device)
self._k_scale = torch.tensor(k_scale, device=device)
self._v_scale = torch.tensor(float("nan"), device=device)
self._prob_scale = torch.tensor(1.0, device=device)
self._q_scale_float = 1.0
self._k_scale_float = 1.0
self._v_scale_float = 1.0
self._q_scale_float = q_scale
self._k_scale_float = k_scale
self._v_scale_float = float("nan")
self._decode_concat_quant_fp8_op = _DecodeConcatQuantFP8(
static=True,
@@ -568,6 +556,8 @@ def run_attention_backend(
qk_rope_head_dim: int,
v_head_dim: int,
mock_kv_b_proj,
q_scale: float,
k_scale: float,
kv_cache_dtype: str = "auto",
) -> torch.Tensor:
"""Run attention computation using the specified backend's AttentionImpl."""
@@ -625,6 +615,8 @@ def run_attention_backend(
kv_lora_rank=kv_lora_rank,
device=device,
kv_b_proj=mock_kv_b_proj,
q_scale=q_scale,
k_scale=k_scale,
)
# Populate static_forward_context with mock attention layers
@@ -674,6 +666,7 @@ def run_attention_backend(
@pytest.mark.parametrize("model", ["deepseek-ai/DeepSeek-R1"])
@pytest.mark.parametrize("tensor_parallel_size", [1, 4, 8, 16])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
def test_backend_correctness(
default_vllm_config,
dist_init,
@@ -681,6 +674,8 @@ def test_backend_correctness(
model: str,
tensor_parallel_size: int,
kv_cache_dtype: str,
q_scale: float,
k_scale: float,
):
"""
Test that all backends produce similar outputs to a reference implementation
@@ -709,6 +704,11 @@ def test_backend_correctness(
for b in BACKENDS_TO_TEST
if kv_cache_dtype in b.get_class().supported_kv_cache_dtypes
]
if (
q_scale != 1.0 or k_scale != 1.0
) and AttentionBackendEnum.CUTLASS_MLA in backends_to_test:
# CUTLASS_MLA does not support non-1 Q/K scales
backends_to_test.remove(AttentionBackendEnum.CUTLASS_MLA)
if not backends_to_test:
pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}")
@@ -1029,6 +1029,7 @@ def test_backend_correctness(
common_attn_metadata=common_attn_metadata,
randomize_blocks=True,
kv_cache_dtype=kv_cache_dtype,
scale=k_scale,
)
kv_cache_per_block_size[block_size] = kv_cache
@@ -1072,6 +1073,8 @@ def test_backend_correctness(
qk_rope_head_dim,
v_head_dim,
mock_kv_b_proj,
q_scale=q_scale,
k_scale=k_scale,
kv_cache_dtype=kv_cache_dtype,
)