From 6c300ddb9e9d9582324032ee68525aae60c9800c Mon Sep 17 00:00:00 2001 From: stefankoncarevic Date: Mon, 28 Sep 2026 19:29:59 +0200 Subject: [PATCH] [CI][Bugfix] Relax packed_qk_rope_ correctness test to one ULP (#59008) Signed-off-by: Stefan Koncarevic Co-authored-by: Andreas Karatzas --- tests/kernels/core/test_apply_rotary_emb.py | 23 +++++++++++++++---- .../layers/rotary_embedding/packed_qk_rope.py | 6 +++-- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/kernels/core/test_apply_rotary_emb.py b/tests/kernels/core/test_apply_rotary_emb.py index 7d63c3499c..ddb717f394 100644 --- a/tests/kernels/core/test_apply_rotary_emb.py +++ b/tests/kernels/core/test_apply_rotary_emb.py @@ -233,9 +233,15 @@ def test_apply_rotary_emb_xpu_matches_native( def test_packed_qk_rope_correctness( num_tokens: int, dtype: torch.dtype, default_vllm_config ): - """packed_qk_rope_ must be bitwise identical to the per-tensor - ApplyRotaryEmb path with enable_fp32_compute=True, and must leave the V - slice untouched.""" + """packed_qk_rope_ must match the per-tensor ApplyRotaryEmb path with + enable_fp32_compute=True to within one ULP, and must leave the V slice + untouched. + + Both sides compute in fp32 on identical inputs. The only gap is which + product the backend keeps exact inside the o1 multiply-add, and that + follows the register layout, so it is decided by codegen rather than by + the source and holds neither across backends nor across Triton versions. + """ from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import ( packed_qk_rope_, @@ -265,6 +271,13 @@ def test_packed_qk_rope_correctness( xk_ref = op(xk, cos, sin) packed_qk_rope_(xqkv, freqs_cis) - torch.testing.assert_close(xqkv[:, 0], xq_ref, atol=0, rtol=0) - torch.testing.assert_close(xqkv[:, 1], xk_ref, atol=0, rtol=0) + + # One ULP at the largest magnitude present, not relative: o1 can cancel, + # so the error scales with the inputs to the sum, not with the result. + eps = torch.finfo(dtype).eps + q_atol = eps * xq_ref.abs().max().item() + k_atol = eps * xk_ref.abs().max().item() + torch.testing.assert_close(xqkv[:, 0], xq_ref, atol=q_atol, rtol=0) + torch.testing.assert_close(xqkv[:, 1], xk_ref, atol=k_atol, rtol=0) + # V is never read or written by the kernel, so it stays exact. torch.testing.assert_close(xqkv[:, 2], xv, atol=0, rtol=0) diff --git a/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py b/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py index e7d916e48f..71f73473c6 100644 --- a/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py +++ b/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py @@ -46,8 +46,10 @@ def packed_qk_rope_(xqkv: torch.Tensor, freqs_cis: torch.Tensor) -> None: freqs_cis: contiguous (seqlen, headdim // 2) complex64 rotary freqs, read through its interleaved re/im fp32 view. - Bitwise identical to ``ApplyRotaryEmb(enable_fp32_compute=True)`` applied - to Q and K separately, but in one kernel with ~5x less memory traffic. + Matches ``ApplyRotaryEmb(enable_fp32_compute=True)`` applied to Q and K + separately to within one ULP, but in one kernel with ~5x less memory + traffic. Both compute in fp32; the gap is which product the compiler + keeps exact inside the fused multiply-add, which differs per backend. Requires triton: callers must check HAS_TRITON and use the unfused path otherwise. Contract violations raise.