[Attention][MLA] Support fp8_ds_mla KV cache for NoPE-512 models on SM90 (#59246)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: Leoyzen <leoyzen@gmail.com>
This commit is contained in:
Nicolò Lucchesi
2026-10-05 13:12:00 +00:00
committed by GitHub
co-authored by Leoyzen
parent d0d6e5f3a2
commit ff53f32409
5 changed files with 82 additions and 44 deletions
+20 -6
View File
@@ -295,7 +295,8 @@ def _quantize_dequantize_fp8_ds_mla(
rope_dim = k_pe.shape[-1]
num_tokens = kv_c.shape[0]
num_blocks = max(1, math.ceil(num_tokens / block_size))
entry_size = kv_lora_rank + 4 * 4 + 2 * rope_dim
# The 64-dim RoPE slot is always there; NoPE rows zero-fill it.
entry_size = kv_lora_rank + 4 * 4 + 2 * 64
tmp_cache = torch.zeros(
num_blocks, block_size, entry_size, dtype=torch.uint8, device=kv_c.device
@@ -413,6 +414,13 @@ def _quantize_dequantize_nvfp4_ds_mla(
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
@pytest.mark.parametrize("block_size", [32, 64])
@pytest.mark.parametrize(("q_scale", "k_scale"), [(1.0, 1.0), (2.0, 3.0)])
@pytest.mark.parametrize(
("total_num_heads", "qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim"),
[
pytest.param(128, 128, 64, 128, id="deepseek"),
pytest.param(64, 256, 0, 256, id="glm53_flash_nope"),
],
)
def test_sparse_backend_decode_correctness(
default_vllm_config,
dist_init,
@@ -424,8 +432,18 @@ def test_sparse_backend_decode_correctness(
workspace_init,
q_scale: float,
k_scale: float,
total_num_heads: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
monkeypatch,
):
if qk_rope_head_dim == 0 and (
backend_cls != FlashMLASparseBackend
or kv_cache_dtype not in ("auto", "fp8_ds_mla")
or torch.cuda.get_device_capability()[0] != 9
):
pytest.skip("NoPE sparse MLA is served by FlashMLA on SM90 only")
if (
batch_name == "large_q_pure_prefill"
and backend_cls == FlashMLASparseBackend
@@ -484,15 +502,10 @@ def test_sparse_backend_decode_correctness(
device = torch.device(DEVICE_TYPE)
dtype = torch.bfloat16
# Model hyper-parameters (kept intentionally small for the unit test)
total_num_heads = 128
# Compute per-rank heads for simulated TP
num_heads = max(1, total_num_heads // tensor_parallel_size)
kv_lora_rank = 512
qk_nope_head_dim = 128
qk_rope_head_dim = 64
v_head_dim = 128
head_size = kv_lora_rank + qk_rope_head_dim
topk_tokens = 128
@@ -3809,6 +3822,7 @@ def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: b
topk_indices_buffer=topk_indices,
num_heads=2,
kv_lora_rank=1,
q_head_size=3,
index_group=None,
index_group_index=0,
dcp_world_size=1,