[ROCm][CI] Add ROCm score absolute tolerance floor (#41341)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -38,6 +38,15 @@ BACKEND_TOL: dict[str, float] = {
|
||||
"FLEX_ATTENTION": 0.045, # gfx950:~3.25%, gfx942:~1.10%
|
||||
}
|
||||
|
||||
# ROCm 7.2/gfx950 shows small absolute drift on the low text-vs-text
|
||||
# probability even though larger scores remain well inside the relative
|
||||
# tolerance. Keep the relative tolerances tight and add only a small floor.
|
||||
BACKEND_ABS_TOL: dict[str, float] = {
|
||||
"default": 0.0,
|
||||
"ROCM_AITER_FA": 0.005,
|
||||
"FLEX_ATTENTION": 0.006,
|
||||
}
|
||||
|
||||
# ROCm: disable skinny GEMM to avoid non-deterministic results from
|
||||
# atomic reductions in wvSplitKrc kernel.
|
||||
# See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975
|
||||
@@ -57,18 +66,23 @@ def get_tol(backend: str) -> float:
|
||||
return BACKEND_TOL.get(backend, BACKEND_TOL["default"])
|
||||
|
||||
|
||||
def get_abs_tol(backend: str) -> float:
|
||||
return BACKEND_ABS_TOL.get(backend, BACKEND_ABS_TOL["default"])
|
||||
|
||||
|
||||
def assert_score(actual: float, expected: float, backend: str, label: str):
|
||||
tol = get_tol(backend)
|
||||
abs_tol = get_abs_tol(backend)
|
||||
diff = abs(actual - expected)
|
||||
rel_diff = diff / abs(expected) if expected != 0 else diff
|
||||
print(
|
||||
f"[{backend}] {label}: actual={actual:.6f} expected={expected:.6f} "
|
||||
f"diff={diff:.6f} rel_diff={rel_diff:.4f} tol={tol}"
|
||||
f"diff={diff:.6f} rel_diff={rel_diff:.4f} tol={tol} abs_tol={abs_tol}"
|
||||
)
|
||||
assert actual == pytest.approx(expected, rel=tol), (
|
||||
assert actual == pytest.approx(expected, rel=tol, abs=abs_tol), (
|
||||
f"[{backend}] {label}: score mismatch — "
|
||||
f"actual={actual:.6f}, expected={expected:.6f}, "
|
||||
f"rel_diff={rel_diff:.4f}, tol={tol}"
|
||||
f"rel_diff={rel_diff:.4f}, tol={tol}, abs_tol={abs_tol}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user