From f0553889c04ae5bf3df48245302ba5ac19fe3432 Mon Sep 17 00:00:00 2001 From: Nick Iusiumbeli Date: Mon, 27 Jul 2026 04:06:07 +0300 Subject: [PATCH] [Bugfix] Prevent NaN poisoning in xpu_mla_sparse for fully-masked index chunks (#48366) Signed-off-by: Nick Iusiumbeli Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Kunshang Ji --- .../kernels/attention/test_xpu_mla_sparse.py | 52 +++++++++++++++++++ vllm/v1/attention/ops/xpu_mla_sparse.py | 7 ++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/tests/kernels/attention/test_xpu_mla_sparse.py b/tests/kernels/attention/test_xpu_mla_sparse.py index 419644923e..965c6f735f 100644 --- a/tests/kernels/attention/test_xpu_mla_sparse.py +++ b/tests/kernels/attention/test_xpu_mla_sparse.py @@ -116,3 +116,55 @@ def test_bf16_triton_sparse_mla(device_str, dtype): assert torch.allclose(out, ref_out, atol=1e-2, rtol=1e-2) assert torch.allclose(max_logits, ref_max_logits, atol=1e-3, rtol=1e-3) assert torch.allclose(lse, ref_lse, atol=1e-3, rtol=1e-3) + + +@pytest.mark.parametrize("device_str", ["xpu"]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.skipif( + not torch.xpu.is_available(), + reason="XPU is required", +) +def test_bf16_triton_sparse_mla_masked_chunks(device_str, dtype): + """Rows whose leading BLOCK_N index entries are all masked must not NaN. + + Regression test: with an -inf running max, a fully-masked leading chunk + produced re_scale = exp2(-inf - -inf) = NaN, permanently poisoning the + accumulator even though valid keys followed in later chunks. + """ + device = torch.device(device_str) + s_q = 3 + s_kv = 256 + h_q = 64 + h_kv = 1 + d_qk = 576 + d_v = 512 + topk = 128 # 8 chunks of BLOCK_N=16 + + torch.random.manual_seed(1234) + + q = torch.randn((s_q, h_q, d_qk), dtype=dtype, device=device) + kv = torch.randn((s_kv, h_kv, d_qk), dtype=dtype, device=device) + indices = torch.full((s_q, h_kv, topk), -1, dtype=torch.int32, device=device) + # row 0: valid keys only in chunks 1-2 -> leading AND trailing masked chunks + indices[0, 0, 16:48] = torch.arange(32, dtype=torch.int32, device=device) + # row 1: fully valid + indices[1, 0, :] = torch.arange(topk, dtype=torch.int32, device=device) + # row 2: no valid key at all + + sm_scale = d_qk**-0.5 + + out, max_logits, lse = triton_bf16_mla_sparse_interface( + q, kv, indices, sm_scale, d_v + ) + assert out.isfinite().all() + + ref_out, _, ref_max_logits, ref_lse = reference_mla_sparse_prefill( + q, kv, indices, sm_scale, d_v + ) + assert torch.allclose(out[:2], ref_out[:2], atol=1e-2, rtol=1e-2) + assert torch.allclose(max_logits[:2], ref_max_logits[:2], atol=1e-3, rtol=1e-3) + assert torch.allclose(lse[:2], ref_lse[:2], atol=1e-3, rtol=1e-3) + # A row with no valid key yields zeros (the reference's convention); its + # lse/max_logits are large-negative finite rather than the reference's + # +inf/-inf placeholders, so only the output is compared here. + assert torch.allclose(out[2], torch.zeros_like(out[2])) diff --git a/vllm/v1/attention/ops/xpu_mla_sparse.py b/vllm/v1/attention/ops/xpu_mla_sparse.py index e73e5a2b28..9c1a92e10f 100644 --- a/vllm/v1/attention/ops/xpu_mla_sparse.py +++ b/vllm/v1/attention/ops/xpu_mla_sparse.py @@ -73,7 +73,10 @@ def _bf16_mla_sparse_kernel( q_buffer + off_qpe, mask=(mask_h[:, None]) & (mask_dpe[None, :]), other=0.0 ) - e_max = tl.zeros([BLOCK_H], dtype=tl.float32) - float("inf") + # Use a finite sentinel rather than -inf: if every key in a chunk is + # masked (e.g. leading -1 padding in `indices`) an -inf running max gives + # re_scale = exp2(-inf - -inf) = NaN, permanently poisoning acc / e_sum. + e_max = tl.zeros([BLOCK_H], dtype=tl.float32) - 1.0e30 e_sum = tl.zeros([BLOCK_H], dtype=tl.float32) acc = tl.zeros([BLOCK_H, BLOCK_DV], dtype=tl.float32) @@ -122,7 +125,7 @@ def _bf16_mla_sparse_kernel( # apply scaling qk *= sm_scale - qk = tl.where((mask_h[:, None]) & (mask_kv[None, :]), qk, -float("inf")) + qk = tl.where((mask_h[:, None]) & (mask_kv[None, :]), qk, -1.0e30) # load v mask_v_d = offs_dv < dim_v