[Bugfix][Attention] Preserve small FP8 softmax weights in Triton attention (#60156)

Signed-off-by: Andy Lo <andy@mistral.ai>
This commit is contained in:
Andy Lo
2026-10-08 12:35:22 +02:00
committed by GitHub
parent e161a0798c
commit ba77c4c130
2 changed files with 56 additions and 1 deletions
@@ -181,6 +181,53 @@ def ref_paged_attn(
return torch.cat(outputs, dim=0)
@torch.inference_mode()
def test_fp8_softmax_preserves_small_probabilities() -> None:
"""Keep exp(-8) contributions that underflow when cast directly to E4M3."""
device = torch.device(DEVICE_TYPE)
num_tokens = block_size = 32
head_size = 128
query = torch.zeros(1, 1, head_size, dtype=FP8_DTYPE, device=device)
query[..., 0] = 1
key_cache = torch.zeros(1, block_size, 1, head_size, dtype=FP8_DTYPE, device=device)
key_cache[:, 1:, :, 0] = -8
value_cache = torch.ones_like(key_cache)
value_cache[:, 0] = 0
output = torch.empty(1, 1, head_size, dtype=torch.bfloat16, device=device)
cu_seqlens_q = torch.tensor([0, 1], dtype=torch.int32, device=device)
seqused_k = torch.tensor([num_tokens], dtype=torch.int32, device=device)
block_table = torch.tensor([[0]], dtype=torch.int32, device=device)
scale = torch.ones(1, dtype=torch.float32, device=device)
unified_attention(
q=query,
k=key_cache,
v=value_cache,
out=output,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=1,
seqused_k=seqused_k,
max_seqlen_k=num_tokens,
softmax_scale=1.0,
causal=True,
window_size=(-1, -1),
block_table=block_table,
softcap=0,
q_descale=scale,
k_descale=scale,
v_descale=scale,
kv_quant_mode=KVQuantMode.FP8_PER_TENSOR,
)
scores = torch.einsum("qhd,thd->qht", query.float(), key_cache[0].float())
probabilities = torch.softmax(scores, dim=-1)
expected = torch.einsum("qht,thd->qhd", probabilities, value_cache[0].float())
torch.testing.assert_close(output.float(), expected, atol=5e-5, rtol=1e-2)
def ref_paged_clamped_mm_attn(
query: torch.Tensor,
key_cache: torch.Tensor,
@@ -586,7 +586,15 @@ def kernel_unified_attention(
P_v = (P * v_token_head_scales[None, :]).to(V.dtype)
acc += tl.dot(P_v, V)
else:
acc += tl.dot(P.to(V.dtype), V)
# Keep softmax weights representable when the P @ V dot uses FP8
# tensor cores. Undo the power-of-two scale after the dot.
if V.dtype == tl.float8e4nv:
p_scale = 256.0
elif V.dtype.is_fp8():
p_scale = 128.0
else:
p_scale = 1.0
acc += tl.dot((P * p_scale).to(V.dtype), V) / p_scale
# ---- Epilogue ---------------------------------------------------------
if IS_3D: