diff --git a/tests/kernels/attention/test_triton_unified_attention.py b/tests/kernels/attention/test_triton_unified_attention.py index e30d9225ea..d936e37704 100644 --- a/tests/kernels/attention/test_triton_unified_attention.py +++ b/tests/kernels/attention/test_triton_unified_attention.py @@ -181,6 +181,53 @@ def ref_paged_attn( return torch.cat(outputs, dim=0) +@torch.inference_mode() +def test_fp8_softmax_preserves_small_probabilities() -> None: + """Keep exp(-8) contributions that underflow when cast directly to E4M3.""" + device = torch.device(DEVICE_TYPE) + num_tokens = block_size = 32 + head_size = 128 + + query = torch.zeros(1, 1, head_size, dtype=FP8_DTYPE, device=device) + query[..., 0] = 1 + key_cache = torch.zeros(1, block_size, 1, head_size, dtype=FP8_DTYPE, device=device) + key_cache[:, 1:, :, 0] = -8 + value_cache = torch.ones_like(key_cache) + value_cache[:, 0] = 0 + output = torch.empty(1, 1, head_size, dtype=torch.bfloat16, device=device) + + cu_seqlens_q = torch.tensor([0, 1], dtype=torch.int32, device=device) + seqused_k = torch.tensor([num_tokens], dtype=torch.int32, device=device) + block_table = torch.tensor([[0]], dtype=torch.int32, device=device) + scale = torch.ones(1, dtype=torch.float32, device=device) + + unified_attention( + q=query, + k=key_cache, + v=value_cache, + out=output, + cu_seqlens_q=cu_seqlens_q, + max_seqlen_q=1, + seqused_k=seqused_k, + max_seqlen_k=num_tokens, + softmax_scale=1.0, + causal=True, + window_size=(-1, -1), + block_table=block_table, + softcap=0, + q_descale=scale, + k_descale=scale, + v_descale=scale, + kv_quant_mode=KVQuantMode.FP8_PER_TENSOR, + ) + + scores = torch.einsum("qhd,thd->qht", query.float(), key_cache[0].float()) + probabilities = torch.softmax(scores, dim=-1) + expected = torch.einsum("qht,thd->qhd", probabilities, value_cache[0].float()) + + torch.testing.assert_close(output.float(), expected, atol=5e-5, rtol=1e-2) + + def ref_paged_clamped_mm_attn( query: torch.Tensor, key_cache: torch.Tensor, diff --git a/vllm/v1/attention/ops/triton_unified_attention.py b/vllm/v1/attention/ops/triton_unified_attention.py index 4d73e49e91..63a9cdc7f9 100644 --- a/vllm/v1/attention/ops/triton_unified_attention.py +++ b/vllm/v1/attention/ops/triton_unified_attention.py @@ -586,7 +586,15 @@ def kernel_unified_attention( P_v = (P * v_token_head_scales[None, :]).to(V.dtype) acc += tl.dot(P_v, V) else: - acc += tl.dot(P.to(V.dtype), V) + # Keep softmax weights representable when the P @ V dot uses FP8 + # tensor cores. Undo the power-of-two scale after the dot. + if V.dtype == tl.float8e4nv: + p_scale = 256.0 + elif V.dtype.is_fp8(): + p_scale = 128.0 + else: + p_scale = 1.0 + acc += tl.dot((P * p_scale).to(V.dtype), V) / p_scale # ---- Epilogue --------------------------------------------------------- if IS_3D: