[ROCm][CI] Add ROCm score absolute tolerance floor (#41341)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-04-30 18:59:35 -07:00
committed by GitHub
parent 4d5c89295b
commit 1adaa5056b
@@ -38,6 +38,15 @@ BACKEND_TOL: dict[str, float] = {
"FLEX_ATTENTION": 0.045, # gfx950:~3.25%, gfx942:~1.10%
}
# ROCm 7.2/gfx950 shows small absolute drift on the low text-vs-text
# probability even though larger scores remain well inside the relative
# tolerance. Keep the relative tolerances tight and add only a small floor.
BACKEND_ABS_TOL: dict[str, float] = {
"default": 0.0,
"ROCM_AITER_FA": 0.005,
"FLEX_ATTENTION": 0.006,
}
# ROCm: disable skinny GEMM to avoid non-deterministic results from
# atomic reductions in wvSplitKrc kernel.
# See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975
@@ -57,18 +66,23 @@ def get_tol(backend: str) -> float:
return BACKEND_TOL.get(backend, BACKEND_TOL["default"])
def get_abs_tol(backend: str) -> float:
return BACKEND_ABS_TOL.get(backend, BACKEND_ABS_TOL["default"])
def assert_score(actual: float, expected: float, backend: str, label: str):
tol = get_tol(backend)
abs_tol = get_abs_tol(backend)
diff = abs(actual - expected)
rel_diff = diff / abs(expected) if expected != 0 else diff
print(
f"[{backend}] {label}: actual={actual:.6f} expected={expected:.6f} "
f"diff={diff:.6f} rel_diff={rel_diff:.4f} tol={tol}"
f"diff={diff:.6f} rel_diff={rel_diff:.4f} tol={tol} abs_tol={abs_tol}"
)
assert actual == pytest.approx(expected, rel=tol), (
assert actual == pytest.approx(expected, rel=tol, abs=abs_tol), (
f"[{backend}] {label}: score mismatch — "
f"actual={actual:.6f}, expected={expected:.6f}, "
f"rel_diff={rel_diff:.4f}, tol={tol}"
f"rel_diff={rel_diff:.4f}, tol={tol}, abs_tol={abs_tol}"
)