From 1adaa5056b0e1b7ce9918d30baa5c7b8b7d86e0d Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Thu, 30 Apr 2026 20:59:35 -0500 Subject: [PATCH] [ROCm][CI] Add ROCm score absolute tolerance floor (#41341) Signed-off-by: Andreas Karatzas --- .../test_cross_encoder_online_vision.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py b/tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py index d773802a9c..a815e6e509 100644 --- a/tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py +++ b/tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py @@ -38,6 +38,15 @@ BACKEND_TOL: dict[str, float] = { "FLEX_ATTENTION": 0.045, # gfx950:~3.25%, gfx942:~1.10% } +# ROCm 7.2/gfx950 shows small absolute drift on the low text-vs-text +# probability even though larger scores remain well inside the relative +# tolerance. Keep the relative tolerances tight and add only a small floor. +BACKEND_ABS_TOL: dict[str, float] = { + "default": 0.0, + "ROCM_AITER_FA": 0.005, + "FLEX_ATTENTION": 0.006, +} + # ROCm: disable skinny GEMM to avoid non-deterministic results from # atomic reductions in wvSplitKrc kernel. # See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975 @@ -57,18 +66,23 @@ def get_tol(backend: str) -> float: return BACKEND_TOL.get(backend, BACKEND_TOL["default"]) +def get_abs_tol(backend: str) -> float: + return BACKEND_ABS_TOL.get(backend, BACKEND_ABS_TOL["default"]) + + def assert_score(actual: float, expected: float, backend: str, label: str): tol = get_tol(backend) + abs_tol = get_abs_tol(backend) diff = abs(actual - expected) rel_diff = diff / abs(expected) if expected != 0 else diff print( f"[{backend}] {label}: actual={actual:.6f} expected={expected:.6f} " - f"diff={diff:.6f} rel_diff={rel_diff:.4f} tol={tol}" + f"diff={diff:.6f} rel_diff={rel_diff:.4f} tol={tol} abs_tol={abs_tol}" ) - assert actual == pytest.approx(expected, rel=tol), ( + assert actual == pytest.approx(expected, rel=tol, abs=abs_tol), ( f"[{backend}] {label}: score mismatch — " f"actual={actual:.6f}, expected={expected:.6f}, " - f"rel_diff={rel_diff:.4f}, tol={tol}" + f"rel_diff={rel_diff:.4f}, tol={tol}, abs_tol={abs_tol}" )