[Bugfix][Attention] Preserve small FP8 softmax weights in Triton attention (#60156)
Signed-off-by: Andy Lo <andy@mistral.ai>
This commit is contained in:
@@ -181,6 +181,53 @@ def ref_paged_attn(
|
||||
return torch.cat(outputs, dim=0)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_fp8_softmax_preserves_small_probabilities() -> None:
|
||||
"""Keep exp(-8) contributions that underflow when cast directly to E4M3."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_tokens = block_size = 32
|
||||
head_size = 128
|
||||
|
||||
query = torch.zeros(1, 1, head_size, dtype=FP8_DTYPE, device=device)
|
||||
query[..., 0] = 1
|
||||
key_cache = torch.zeros(1, block_size, 1, head_size, dtype=FP8_DTYPE, device=device)
|
||||
key_cache[:, 1:, :, 0] = -8
|
||||
value_cache = torch.ones_like(key_cache)
|
||||
value_cache[:, 0] = 0
|
||||
output = torch.empty(1, 1, head_size, dtype=torch.bfloat16, device=device)
|
||||
|
||||
cu_seqlens_q = torch.tensor([0, 1], dtype=torch.int32, device=device)
|
||||
seqused_k = torch.tensor([num_tokens], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor([[0]], dtype=torch.int32, device=device)
|
||||
scale = torch.ones(1, dtype=torch.float32, device=device)
|
||||
|
||||
unified_attention(
|
||||
q=query,
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
out=output,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=1,
|
||||
seqused_k=seqused_k,
|
||||
max_seqlen_k=num_tokens,
|
||||
softmax_scale=1.0,
|
||||
causal=True,
|
||||
window_size=(-1, -1),
|
||||
block_table=block_table,
|
||||
softcap=0,
|
||||
q_descale=scale,
|
||||
k_descale=scale,
|
||||
v_descale=scale,
|
||||
kv_quant_mode=KVQuantMode.FP8_PER_TENSOR,
|
||||
)
|
||||
|
||||
scores = torch.einsum("qhd,thd->qht", query.float(), key_cache[0].float())
|
||||
probabilities = torch.softmax(scores, dim=-1)
|
||||
expected = torch.einsum("qht,thd->qhd", probabilities, value_cache[0].float())
|
||||
|
||||
torch.testing.assert_close(output.float(), expected, atol=5e-5, rtol=1e-2)
|
||||
|
||||
|
||||
def ref_paged_clamped_mm_attn(
|
||||
query: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
|
||||
@@ -586,7 +586,15 @@ def kernel_unified_attention(
|
||||
P_v = (P * v_token_head_scales[None, :]).to(V.dtype)
|
||||
acc += tl.dot(P_v, V)
|
||||
else:
|
||||
acc += tl.dot(P.to(V.dtype), V)
|
||||
# Keep softmax weights representable when the P @ V dot uses FP8
|
||||
# tensor cores. Undo the power-of-two scale after the dot.
|
||||
if V.dtype == tl.float8e4nv:
|
||||
p_scale = 256.0
|
||||
elif V.dtype.is_fp8():
|
||||
p_scale = 128.0
|
||||
else:
|
||||
p_scale = 1.0
|
||||
acc += tl.dot((P * p_scale).to(V.dtype), V) / p_scale
|
||||
|
||||
# ---- Epilogue ---------------------------------------------------------
|
||||
if IS_3D:
|
||||
|
||||
Reference in New Issue
Block a user