[Bugfix] Allocate packed outputs in fused_q_kv_rmsnorm so q_b_proj keeps its low-latency GEMM path at decode (#53109)

Signed-off-by: yasong.wang <yasong.wang@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Roy Wang
2026-08-28 02:08:27 +00:00
committed by GitHub
co-authored by Claude Fable 5
parent f3b91f1af2
commit b74ac0bc61
2 changed files with 40 additions and 2 deletions
@@ -51,6 +51,37 @@ def test_fused_q_kv_rmsnorm_correctness(num_tokens: int, dtype: torch.dtype):
torch.testing.assert_close(kv_out, kv_ref, **tol)
@pytest.mark.parametrize("num_tokens", [1, 16])
def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int):
"""Regression guard: the outputs must be packed row-major even when the
inputs are column slices of a wider fused-projection buffer.
empty_like preserves the strides of size-1 dims, so with num_tokens == 1
a [1, q_size] slice of a [1, q_size + kv_size] buffer used to produce a
qr_out with row stride q_size + kv_size. Downstream dispatchers that
require packed row-major inputs then reject the tensor and silently fall
back to a slower GEMM path on every decode step."""
device = "cuda"
dtype = torch.bfloat16
q_size, kv_size = 192, 576
fused = torch.randn(num_tokens, q_size + kv_size, dtype=dtype, device=device)
qr, kv = fused.split([q_size, kv_size], dim=-1)
qw = torch.randn(q_size, dtype=dtype, device=device)
kvw = torch.randn(kv_size, dtype=dtype, device=device)
eps = 1e-6
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, eps)
assert qr_out.stride() == (q_size, 1)
assert kv_out.stride() == (kv_size, 1)
qr_ref = _ref_rmsnorm(qr, qw, eps)
kv_ref = _ref_rmsnorm(kv, kvw, eps)
tol = dict(rtol=1e-2, atol=1e-2)
torch.testing.assert_close(qr_out, qr_ref, **tol)
torch.testing.assert_close(kv_out, kv_ref, **tol)
@pytest.mark.parametrize("num_tokens", [65535, 65536, 131072])
def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int):
"""Regression guard: grid used to be (2, num_tokens), hitting CUDA's
+9 -2
View File
@@ -77,8 +77,15 @@ def fused_q_kv_rmsnorm(
q_size = qr.shape[1]
kv_size = kv.shape[1]
num_tokens = qr.shape[0]
qr_out = torch.empty_like(qr)
kv_out = torch.empty_like(kv)
# Allocate packed outputs rather than empty_like: qr/kv are typically
# column slices of a fused qkv_a projection, and for num_tokens == 1
# empty_like preserves the parent's row stride (torch keeps strides of
# size-1 dims), e.g. (2112, 1) for a [1, 1536] q slice. Downstream
# dispatchers that require packed row-major inputs (e.g. the Kimi-K3
# low-latency GEMM's _is_packed_row_major check) then reject the tensor
# and silently fall back to cuBLAS on every decode step.
qr_out = torch.empty(qr.shape, dtype=qr.dtype, device=qr.device)
kv_out = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device)
if num_tokens == 0:
return qr_out, kv_out