DSv4 fused Q-norm kernel grid refactor (#42353)

This commit is contained in:
gnovack
2026-05-22 15:21:33 -07:00
committed by GitHub
parent 47d4407d7c
commit f743254143
2 changed files with 327 additions and 213 deletions
@@ -67,29 +67,26 @@ def apply_rope_gptj_last_k(
head_dim = x.shape[-1]
nope_dim = head_dim - rope_dim
# Gather cos/sin for each token position: [num_tokens, rope_dim]
cs = cos_sin_cache[positions].to(torch.float32) # [N, rope_dim]
cos = cs[..., :half] # [N, half]
sin = cs[..., half:] # [N, half]
cs = cos_sin_cache[positions].to(torch.float32)
cos = cs[..., :half]
sin = cs[..., half:]
# Reshape leading dims so we can broadcast: x shape [..., head_dim].
# Bring token dim to front; assume x is [num_tokens, ..., head_dim].
# We rely on positions being per-token and all other dims sharing the same pos.
rope = x[..., nope_dim:].float() # [..., rope_dim]
# Make rope pairs: reshape last dim to [half, 2]
rope = x[..., nope_dim:].float()
shape = rope.shape
rope = rope.reshape(*shape[:-1], half, 2)
even = rope[..., 0] # [..., half]
even = rope[..., 0]
odd = rope[..., 1]
# Broadcast cos/sin over any heads dim in between. cos/sin are [N, half].
# Add singleton dims for intermediate axes.
for _ in range(rope.ndim - 3):
cos = cos.unsqueeze(1)
sin = sin.unsqueeze(1)
new_even = even * cos - odd * sin
new_odd = even * sin + odd * cos
# Use addcmul (compiles to FMA on CUDA) for the 2x2 rotation. nvcc lowers
# the kernel's `e*c - o*s` to fma(e, c, -o*s); matching that here keeps
# near-cancellation pairs on the same bf16 grid as the kernel output and
# avoids spurious 1-ULP boundary flips at high num_tokens.
new_even = torch.addcmul(-odd * sin, even, cos)
new_odd = torch.addcmul(odd * cos, even, sin)
rope_rotated = torch.stack((new_even, new_odd), dim=-1).reshape(shape)
out = x.clone().float()
@@ -99,11 +96,15 @@ def apply_rope_gptj_last_k(
def rmsnorm_no_weight(x: torch.Tensor, eps: float) -> torch.Tensor:
"""RMSNorm with no learnable weight, matching
`RMSNorm(head_dim, has_weight=False)`."""
orig_dtype = x.dtype
`RMSNorm(head_dim, has_weight=False)`.
Returns fp32 so callers can chain RoPE without an intermediate bf16 round
(the kernel keeps the whole RMSNorm→RoPE pipeline in fp32 and rounds once
at the final store).
"""
xf = x.float()
variance = xf.pow(2).mean(dim=-1, keepdim=True)
return (xf * torch.rsqrt(variance + eps)).to(orig_dtype)
return xf * torch.rsqrt(variance + eps)
# ── Dispatch to the CUDA op (skip test cleanly if it isn't built in) ─────────
@@ -128,7 +129,7 @@ def _call_fused(q, kv, k_cache, slot_mapping, positions, cos_sin_cache, eps, bs)
# ── Test 1: Q path numerical parity ──────────────────────────────────────────
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64])
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64, 2048])
@pytest.mark.parametrize("n_heads", [8, 64])
def test_q_path_matches_reference(num_tokens: int, n_heads: int):
torch.manual_seed(0)
@@ -142,8 +143,10 @@ def test_q_path_matches_reference(num_tokens: int, n_heads: int):
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
# Reference: RMSNorm (no weight) per head, then GPT-J RoPE on last 64.
# Keep the chain in fp32 (rmsnorm_no_weight returns fp32) and round to
# bf16 once at the end, matching the kernel.
q_ref = rmsnorm_no_weight(q, eps)
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache)
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache).to(dtype)
# Fused call with dummy KV tensors (KV branch will write slot_mapping=-1 → noop).
num_blocks = 2
@@ -173,7 +176,7 @@ def _ue8m0_per_block_scales(kv_roped_nope_f32: torch.Tensor, qblock: int):
return torch.pow(2.0, exponent) # [n_tok, n_blocks]
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64])
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64, 2048])
@pytest.mark.parametrize("block_size", [16, 64])
def test_kv_path_matches_reference(num_tokens: int, block_size: int):
torch.manual_seed(1)
@@ -261,7 +264,7 @@ def test_kv_path_matches_reference(num_tokens: int, block_size: int):
# ── Test 2b: DP padding (slot_mapping shorter than q/kv) ─────────────────────
@pytest.mark.parametrize("num_tokens", [4, 17])
@pytest.mark.parametrize("num_tokens", [4, 17, 2048])
@pytest.mark.parametrize("pad", [1, 5])
@pytest.mark.parametrize("block_size", [16, 64])
def test_kv_path_with_dp_padding(num_tokens: int, pad: int, block_size: int):
@@ -312,7 +315,7 @@ def test_kv_path_with_dp_padding(num_tokens: int, pad: int, block_size: int):
# ── Test 3: combined single-call Q + KV parity ───────────────────────────────
@pytest.mark.parametrize("num_tokens", [1, 4, 17])
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 2048])
@pytest.mark.parametrize("n_heads", [8, 64])
@pytest.mark.parametrize("block_size", [16, 64])
def test_combined_q_and_kv(num_tokens: int, n_heads: int, block_size: int):
@@ -332,7 +335,7 @@ def test_combined_q_and_kv(num_tokens: int, n_heads: int, block_size: int):
# Reference.
q_ref = rmsnorm_no_weight(q, eps)
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache)
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache).to(dtype)
kv_ref = apply_rope_gptj_last_k(kv, positions, cos_sin_cache)
k_cache_ref = torch.zeros(
num_blocks, block_size * HEAD_BYTES, dtype=torch.uint8, device=device