From b74ac0bc61d8b8f74848664b1a3b4e878f047861 Mon Sep 17 00:00:00 2001 From: Roy Wang Date: Fri, 28 Aug 2026 10:08:27 +0800 Subject: [PATCH] [Bugfix] Allocate packed outputs in fused_q_kv_rmsnorm so q_b_proj keeps its low-latency GEMM path at decode (#53109) Signed-off-by: yasong.wang Co-authored-by: Claude Fable 5 --- tests/kernels/core/test_fused_q_kv_rmsnorm.py | 31 +++++++++++++++++++ vllm/models/common/ops/fused_qk_rmsnorm.py | 11 +++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/tests/kernels/core/test_fused_q_kv_rmsnorm.py b/tests/kernels/core/test_fused_q_kv_rmsnorm.py index e5f2c878ff..c10590085e 100644 --- a/tests/kernels/core/test_fused_q_kv_rmsnorm.py +++ b/tests/kernels/core/test_fused_q_kv_rmsnorm.py @@ -51,6 +51,37 @@ def test_fused_q_kv_rmsnorm_correctness(num_tokens: int, dtype: torch.dtype): torch.testing.assert_close(kv_out, kv_ref, **tol) +@pytest.mark.parametrize("num_tokens", [1, 16]) +def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int): + """Regression guard: the outputs must be packed row-major even when the + inputs are column slices of a wider fused-projection buffer. + + empty_like preserves the strides of size-1 dims, so with num_tokens == 1 + a [1, q_size] slice of a [1, q_size + kv_size] buffer used to produce a + qr_out with row stride q_size + kv_size. Downstream dispatchers that + require packed row-major inputs then reject the tensor and silently fall + back to a slower GEMM path on every decode step.""" + device = "cuda" + dtype = torch.bfloat16 + q_size, kv_size = 192, 576 + fused = torch.randn(num_tokens, q_size + kv_size, dtype=dtype, device=device) + qr, kv = fused.split([q_size, kv_size], dim=-1) + qw = torch.randn(q_size, dtype=dtype, device=device) + kvw = torch.randn(kv_size, dtype=dtype, device=device) + eps = 1e-6 + + qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, eps) + + assert qr_out.stride() == (q_size, 1) + assert kv_out.stride() == (kv_size, 1) + + qr_ref = _ref_rmsnorm(qr, qw, eps) + kv_ref = _ref_rmsnorm(kv, kvw, eps) + tol = dict(rtol=1e-2, atol=1e-2) + torch.testing.assert_close(qr_out, qr_ref, **tol) + torch.testing.assert_close(kv_out, kv_ref, **tol) + + @pytest.mark.parametrize("num_tokens", [65535, 65536, 131072]) def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int): """Regression guard: grid used to be (2, num_tokens), hitting CUDA's diff --git a/vllm/models/common/ops/fused_qk_rmsnorm.py b/vllm/models/common/ops/fused_qk_rmsnorm.py index ac7d0dfd4a..77c2be6319 100644 --- a/vllm/models/common/ops/fused_qk_rmsnorm.py +++ b/vllm/models/common/ops/fused_qk_rmsnorm.py @@ -77,8 +77,15 @@ def fused_q_kv_rmsnorm( q_size = qr.shape[1] kv_size = kv.shape[1] num_tokens = qr.shape[0] - qr_out = torch.empty_like(qr) - kv_out = torch.empty_like(kv) + # Allocate packed outputs rather than empty_like: qr/kv are typically + # column slices of a fused qkv_a projection, and for num_tokens == 1 + # empty_like preserves the parent's row stride (torch keeps strides of + # size-1 dims), e.g. (2112, 1) for a [1, 1536] q slice. Downstream + # dispatchers that require packed row-major inputs (e.g. the Kimi-K3 + # low-latency GEMM's _is_packed_row_major check) then reject the tensor + # and silently fall back to cuBLAS on every decode step. + qr_out = torch.empty(qr.shape, dtype=qr.dtype, device=qr.device) + kv_out = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device) if num_tokens == 0: return qr_out, kv_out