[Bugfix] Prevent NaN poisoning in xpu_mla_sparse for fully-masked index chunks (#48366)
Signed-off-by: Nick Iusiumbeli <nickuspro@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
Kunshang Ji
parent
fdaa0d9e59
commit
f0553889c0
@@ -116,3 +116,55 @@ def test_bf16_triton_sparse_mla(device_str, dtype):
|
||||
assert torch.allclose(out, ref_out, atol=1e-2, rtol=1e-2)
|
||||
assert torch.allclose(max_logits, ref_max_logits, atol=1e-3, rtol=1e-3)
|
||||
assert torch.allclose(lse, ref_lse, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device_str", ["xpu"])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.skipif(
|
||||
not torch.xpu.is_available(),
|
||||
reason="XPU is required",
|
||||
)
|
||||
def test_bf16_triton_sparse_mla_masked_chunks(device_str, dtype):
|
||||
"""Rows whose leading BLOCK_N index entries are all masked must not NaN.
|
||||
|
||||
Regression test: with an -inf running max, a fully-masked leading chunk
|
||||
produced re_scale = exp2(-inf - -inf) = NaN, permanently poisoning the
|
||||
accumulator even though valid keys followed in later chunks.
|
||||
"""
|
||||
device = torch.device(device_str)
|
||||
s_q = 3
|
||||
s_kv = 256
|
||||
h_q = 64
|
||||
h_kv = 1
|
||||
d_qk = 576
|
||||
d_v = 512
|
||||
topk = 128 # 8 chunks of BLOCK_N=16
|
||||
|
||||
torch.random.manual_seed(1234)
|
||||
|
||||
q = torch.randn((s_q, h_q, d_qk), dtype=dtype, device=device)
|
||||
kv = torch.randn((s_kv, h_kv, d_qk), dtype=dtype, device=device)
|
||||
indices = torch.full((s_q, h_kv, topk), -1, dtype=torch.int32, device=device)
|
||||
# row 0: valid keys only in chunks 1-2 -> leading AND trailing masked chunks
|
||||
indices[0, 0, 16:48] = torch.arange(32, dtype=torch.int32, device=device)
|
||||
# row 1: fully valid
|
||||
indices[1, 0, :] = torch.arange(topk, dtype=torch.int32, device=device)
|
||||
# row 2: no valid key at all
|
||||
|
||||
sm_scale = d_qk**-0.5
|
||||
|
||||
out, max_logits, lse = triton_bf16_mla_sparse_interface(
|
||||
q, kv, indices, sm_scale, d_v
|
||||
)
|
||||
assert out.isfinite().all()
|
||||
|
||||
ref_out, _, ref_max_logits, ref_lse = reference_mla_sparse_prefill(
|
||||
q, kv, indices, sm_scale, d_v
|
||||
)
|
||||
assert torch.allclose(out[:2], ref_out[:2], atol=1e-2, rtol=1e-2)
|
||||
assert torch.allclose(max_logits[:2], ref_max_logits[:2], atol=1e-3, rtol=1e-3)
|
||||
assert torch.allclose(lse[:2], ref_lse[:2], atol=1e-3, rtol=1e-3)
|
||||
# A row with no valid key yields zeros (the reference's convention); its
|
||||
# lse/max_logits are large-negative finite rather than the reference's
|
||||
# +inf/-inf placeholders, so only the output is compared here.
|
||||
assert torch.allclose(out[2], torch.zeros_like(out[2]))
|
||||
|
||||
@@ -73,7 +73,10 @@ def _bf16_mla_sparse_kernel(
|
||||
q_buffer + off_qpe, mask=(mask_h[:, None]) & (mask_dpe[None, :]), other=0.0
|
||||
)
|
||||
|
||||
e_max = tl.zeros([BLOCK_H], dtype=tl.float32) - float("inf")
|
||||
# Use a finite sentinel rather than -inf: if every key in a chunk is
|
||||
# masked (e.g. leading -1 padding in `indices`) an -inf running max gives
|
||||
# re_scale = exp2(-inf - -inf) = NaN, permanently poisoning acc / e_sum.
|
||||
e_max = tl.zeros([BLOCK_H], dtype=tl.float32) - 1.0e30
|
||||
e_sum = tl.zeros([BLOCK_H], dtype=tl.float32)
|
||||
acc = tl.zeros([BLOCK_H, BLOCK_DV], dtype=tl.float32)
|
||||
|
||||
@@ -122,7 +125,7 @@ def _bf16_mla_sparse_kernel(
|
||||
|
||||
# apply scaling
|
||||
qk *= sm_scale
|
||||
qk = tl.where((mask_h[:, None]) & (mask_kv[None, :]), qk, -float("inf"))
|
||||
qk = tl.where((mask_h[:, None]) & (mask_kv[None, :]), qk, -1.0e30)
|
||||
|
||||
# load v
|
||||
mask_v_d = offs_dv < dim_v
|
||||
|
||||
Reference in New Issue
Block a user