[ROCm][CI] Relax fused layernorm quant test tolerances for one-ULP outliers (#46658)

Signed-off-by: Divakar Verma <divakar.verma@amd.com>
This commit is contained in:
Divakar Verma
2026-06-27 10:44:29 +08:00
committed by GitHub
parent 2e058851d3
commit 17a71d8702
4 changed files with 90 additions and 55 deletions
@@ -8,7 +8,7 @@ import pytest
import torch
import vllm._custom_ops as ops
from tests.kernels.utils import opcheck
from tests.kernels.utils import fp8_ulp_distance, opcheck
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
@@ -250,30 +250,54 @@ def test_rms_norm(
assert ref_out.dtype == quant_dtype
assert ops_out.dtype == quant_dtype
# Per-block bf16 scales: allow a small relative tolerance for a few groups
# whose abs-max flips by one ULP between the fused and reference paths. The
# per-token and fp32 paths stay strict.
relax_block_rocm = (
group_size is not None
and dtype == torch.bfloat16
and current_platform.is_rocm()
)
def scales_close(rtol: float, atol: float) -> bool:
if torch.allclose(ref_scales, ops_scales, rtol=rtol, atol=atol):
return True
return relax_block_rocm and torch.allclose(
ref_scales, ops_scales, rtol=1e-2, atol=atol
)
if quant_dtype == torch.int8:
assert torch.allclose(ref_scales, ops_scales, atol=1e-6)
assert scales_close(rtol=1e-5, atol=1e-6)
# big atol to account for round-off errors.
assert torch.allclose(ref_out, ops_out, atol=1)
else:
assert torch.allclose(ref_scales, ops_scales)
assert scales_close(rtol=1e-5, atol=1e-8)
a = ref_out.to(dtype=torch.float32)
b = ops_out.to(dtype=torch.float32)
ok = torch.allclose(a, b, atol=1e-6)
if not ok:
# fallback: compare dequantized values with relaxed tolerance
if group_size is None:
a_deq = a * ref_scales.view(-1, 1)
b_deq = b * ops_scales.view(-1, 1)
if relax_block_rocm:
# ULP-flipped group scale can cross an E4M3 tie; tolerate a
# bounded count of isolated fp8 outliers.
ulp = fp8_ulp_distance(ref_out, ops_out)
max_outliers = ulp.numel() // 100_000 + 8
ok = int((ulp > 0).sum().item()) <= max_outliers
else:
a_deq = a * ref_scales.repeat_interleave(group_size[1], dim=1)
b_deq = b * ops_scales.repeat_interleave(group_size[1], dim=1)
# NOTE: It is possible that some future test cases trigger this
# max diff due to precision issues. If such an error is
# encountered, it's recommended to inspect the differences between
# all corresponding elements from each tensor (e.g. by looping over
# them) and checking how many the max diff error shows up on (just
# a few bad elements should still be considered acceptable).
ok = torch.allclose(a_deq, b_deq, rtol=5e-2, atol=5e-2)
# CUDA (& non-bf16): compare dequantized values with relaxed tolerance.
if group_size is None:
a_deq = a * ref_scales.view(-1, 1)
b_deq = b * ops_scales.view(-1, 1)
else:
a_deq = a * ref_scales.repeat_interleave(group_size[1], dim=1)
b_deq = b * ops_scales.repeat_interleave(group_size[1], dim=1)
# NOTE: It is possible that some future test cases trigger this
# max diff due to precision issues. If such an error is
# encountered, it's recommended to inspect the differences between
# all corresponding elements from each tensor (e.g. by looping over
# them) and checking how many the max diff error shows up on (just
# a few bad elements should still be considered acceptable).
ok = torch.allclose(a_deq, b_deq, rtol=5e-2, atol=5e-2)
assert ok
if add_residual:
assert torch.allclose(ref_residual, ops_residual)
+17 -7
View File
@@ -5,7 +5,7 @@ import pytest
import torch
from tests.kernels.quant_utils import FP8_DTYPE
from tests.kernels.utils import opcheck
from tests.kernels.utils import fp8_ulp_distance, opcheck
from vllm import ir
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
from vllm.platforms import current_platform
@@ -204,12 +204,22 @@ def test_fused_rms_norm_quant(
(out_quant_fused, x, weight, quant_scale_t, 1e-6),
)
torch.testing.assert_close(
out_quant.to(dtype=torch.float32),
out_quant_fused.to(dtype=torch.float32),
atol=1e-3,
rtol=1e-3,
)
if current_platform.is_rocm():
# Fused and unfused FP8 paths can land on opposite sides of an E4M3 tie;
# tolerate a tiny number of isolated fp8 outliers on ROCm.
ulp = fp8_ulp_distance(out_quant, out_quant_fused)
max_outliers = ulp.numel() // 100_000 + 8
num_outliers = int((ulp > 0).sum().item())
assert num_outliers <= max_outliers, (
f"FP8 quant mismatch: {num_outliers} fp8 outliers (allowed {max_outliers})"
)
else:
torch.testing.assert_close(
out_quant.to(dtype=torch.float32),
out_quant_fused.to(dtype=torch.float32),
atol=1e-3,
rtol=1e-3,
)
@torch.inference_mode()