Fix ROCm AITER FP8 KV test tolerances. (#53590)
Signed-off-by: Aarushi Jain <aarushjain29@users.noreply.github.com> Co-authored-by: Aarushi Jain <aarushjain29@users.noreply.github.com>
This commit is contained in:
co-authored by
Aarushi Jain
parent
42d76ee35a
commit
1454b71727
@@ -805,14 +805,14 @@ def test_aiter_mha_decode_single_token(dtype):
|
||||
def test_aiter_mha_varlen_fp8_kv(dtype):
|
||||
"""AITER flash attention with FP8 KV cache matches reference on BF16-cast KV.
|
||||
|
||||
cp_mha_gather_cache is called with dequant=True to dequantize FP8 to dtype
|
||||
before passing to flash_attn_varlen_func. We compare to ref_paged_attn on
|
||||
the dtype-cast KV cache with FP8 quantization tolerance.
|
||||
|
||||
Exercises: VLLM_ROCM_USE_AITER, VLLM_ROCM_USE_AITER_MHA, FP8 KV path.
|
||||
The reference dequantizes the same FP8 KV tensors the kernel reads, so
|
||||
e4m3 rounding cancels on both sides; the residual is bf16/fp16 kernel
|
||||
rounding (~few millidecimals), not fp8-scale quantization error.
|
||||
"""
|
||||
atol = 0.15
|
||||
rtol = 0.05
|
||||
if dtype == torch.bfloat16:
|
||||
atol, rtol = 6e-3, 1e-2
|
||||
else:
|
||||
atol, rtol = 7e-4, 1e-3
|
||||
_assert_aiter_supported()
|
||||
if not current_platform.supports_fp8():
|
||||
pytest.skip("FP8 not supported on this hardware")
|
||||
@@ -900,7 +900,8 @@ def test_aiter_mha_varlen_fp8_kv(dtype):
|
||||
scale=scale,
|
||||
)
|
||||
|
||||
# FP8 quantization + dequantization introduces noise
|
||||
# FP8 quantization + dequantization cancels on the reference side; compare
|
||||
# kernel rounding at the working precision.
|
||||
_print_close_stats(
|
||||
f"varlen_fp8_kv dtype={dtype} query_len={query_len} kv_len={kv_len}",
|
||||
output,
|
||||
|
||||
@@ -49,14 +49,20 @@ PREFILL_SEQ_LENS = [
|
||||
[(256, 1024), (128, 2048)],
|
||||
]
|
||||
|
||||
Fp8Variant = Literal["fp8_kv", "fp8_query", "fp8_query_kv"]
|
||||
|
||||
DEFAULT_ATOL, DEFAULT_RTOL = 1.5e-2, 1e-2
|
||||
FP8_ATOL, FP8_RTOL = 1.5e-1, 1.5e-1
|
||||
# fp8_kv: reference dequantizes the same KV, so error is bf16-scale (~2e-3).
|
||||
# fp8_query / fp8_query_kv: query and/or KV e4m3 rounding dominates (~0.1-0.4).
|
||||
FP8_TOL_BY_VARIANT: dict[Fp8Variant, tuple[float, float]] = {
|
||||
"fp8_kv": (2.5e-3, 1e-2),
|
||||
"fp8_query": (1.5e-1, 1.5e-1),
|
||||
"fp8_query_kv": (1.5e-1, 1.5e-1),
|
||||
}
|
||||
# Non-unity scale so q_descale handling is exercised explicitly.
|
||||
Q_SCALE = 0.75
|
||||
K_SCALE, V_SCALE = 0.5, 0.25
|
||||
|
||||
Fp8Variant = Literal["fp8_kv", "fp8_query", "fp8_query_kv"]
|
||||
|
||||
FP8_VARIANTS = [
|
||||
pytest.param("fp8_kv", id="fp8_kv"),
|
||||
pytest.param("fp8_query", id="fp8_query"),
|
||||
@@ -336,4 +342,5 @@ def test_aiter_unified_attn_fp8(
|
||||
block_size=block_size,
|
||||
variant=variant,
|
||||
)
|
||||
_assert_matches_reference(case, atol=FP8_ATOL, rtol=FP8_RTOL)
|
||||
atol, rtol = FP8_TOL_BY_VARIANT[variant]
|
||||
_assert_matches_reference(case, atol=atol, rtol=rtol)
|
||||
|
||||
Reference in New Issue
Block a user