DSv4 fused Q-norm kernel grid refactor (#42353)
This commit is contained in:
@@ -67,29 +67,26 @@ def apply_rope_gptj_last_k(
|
||||
head_dim = x.shape[-1]
|
||||
nope_dim = head_dim - rope_dim
|
||||
|
||||
# Gather cos/sin for each token position: [num_tokens, rope_dim]
|
||||
cs = cos_sin_cache[positions].to(torch.float32) # [N, rope_dim]
|
||||
cos = cs[..., :half] # [N, half]
|
||||
sin = cs[..., half:] # [N, half]
|
||||
cs = cos_sin_cache[positions].to(torch.float32)
|
||||
cos = cs[..., :half]
|
||||
sin = cs[..., half:]
|
||||
|
||||
# Reshape leading dims so we can broadcast: x shape [..., head_dim].
|
||||
# Bring token dim to front; assume x is [num_tokens, ..., head_dim].
|
||||
# We rely on positions being per-token and all other dims sharing the same pos.
|
||||
rope = x[..., nope_dim:].float() # [..., rope_dim]
|
||||
# Make rope pairs: reshape last dim to [half, 2]
|
||||
rope = x[..., nope_dim:].float()
|
||||
shape = rope.shape
|
||||
rope = rope.reshape(*shape[:-1], half, 2)
|
||||
even = rope[..., 0] # [..., half]
|
||||
even = rope[..., 0]
|
||||
odd = rope[..., 1]
|
||||
|
||||
# Broadcast cos/sin over any heads dim in between. cos/sin are [N, half].
|
||||
# Add singleton dims for intermediate axes.
|
||||
for _ in range(rope.ndim - 3):
|
||||
cos = cos.unsqueeze(1)
|
||||
sin = sin.unsqueeze(1)
|
||||
|
||||
new_even = even * cos - odd * sin
|
||||
new_odd = even * sin + odd * cos
|
||||
# Use addcmul (compiles to FMA on CUDA) for the 2x2 rotation. nvcc lowers
|
||||
# the kernel's `e*c - o*s` to fma(e, c, -o*s); matching that here keeps
|
||||
# near-cancellation pairs on the same bf16 grid as the kernel output and
|
||||
# avoids spurious 1-ULP boundary flips at high num_tokens.
|
||||
new_even = torch.addcmul(-odd * sin, even, cos)
|
||||
new_odd = torch.addcmul(odd * cos, even, sin)
|
||||
rope_rotated = torch.stack((new_even, new_odd), dim=-1).reshape(shape)
|
||||
|
||||
out = x.clone().float()
|
||||
@@ -99,11 +96,15 @@ def apply_rope_gptj_last_k(
|
||||
|
||||
def rmsnorm_no_weight(x: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
"""RMSNorm with no learnable weight, matching
|
||||
`RMSNorm(head_dim, has_weight=False)`."""
|
||||
orig_dtype = x.dtype
|
||||
`RMSNorm(head_dim, has_weight=False)`.
|
||||
|
||||
Returns fp32 so callers can chain RoPE without an intermediate bf16 round
|
||||
(the kernel keeps the whole RMSNorm→RoPE pipeline in fp32 and rounds once
|
||||
at the final store).
|
||||
"""
|
||||
xf = x.float()
|
||||
variance = xf.pow(2).mean(dim=-1, keepdim=True)
|
||||
return (xf * torch.rsqrt(variance + eps)).to(orig_dtype)
|
||||
return xf * torch.rsqrt(variance + eps)
|
||||
|
||||
|
||||
# ── Dispatch to the CUDA op (skip test cleanly if it isn't built in) ─────────
|
||||
@@ -128,7 +129,7 @@ def _call_fused(q, kv, k_cache, slot_mapping, positions, cos_sin_cache, eps, bs)
|
||||
# ── Test 1: Q path numerical parity ──────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64, 2048])
|
||||
@pytest.mark.parametrize("n_heads", [8, 64])
|
||||
def test_q_path_matches_reference(num_tokens: int, n_heads: int):
|
||||
torch.manual_seed(0)
|
||||
@@ -142,8 +143,10 @@ def test_q_path_matches_reference(num_tokens: int, n_heads: int):
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
|
||||
|
||||
# Reference: RMSNorm (no weight) per head, then GPT-J RoPE on last 64.
|
||||
# Keep the chain in fp32 (rmsnorm_no_weight returns fp32) and round to
|
||||
# bf16 once at the end, matching the kernel.
|
||||
q_ref = rmsnorm_no_weight(q, eps)
|
||||
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache)
|
||||
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache).to(dtype)
|
||||
|
||||
# Fused call with dummy KV tensors (KV branch will write slot_mapping=-1 → noop).
|
||||
num_blocks = 2
|
||||
@@ -173,7 +176,7 @@ def _ue8m0_per_block_scales(kv_roped_nope_f32: torch.Tensor, qblock: int):
|
||||
return torch.pow(2.0, exponent) # [n_tok, n_blocks]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64, 2048])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_kv_path_matches_reference(num_tokens: int, block_size: int):
|
||||
torch.manual_seed(1)
|
||||
@@ -261,7 +264,7 @@ def test_kv_path_matches_reference(num_tokens: int, block_size: int):
|
||||
# ── Test 2b: DP padding (slot_mapping shorter than q/kv) ─────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [4, 17])
|
||||
@pytest.mark.parametrize("num_tokens", [4, 17, 2048])
|
||||
@pytest.mark.parametrize("pad", [1, 5])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_kv_path_with_dp_padding(num_tokens: int, pad: int, block_size: int):
|
||||
@@ -312,7 +315,7 @@ def test_kv_path_with_dp_padding(num_tokens: int, pad: int, block_size: int):
|
||||
# ── Test 3: combined single-call Q + KV parity ───────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 2048])
|
||||
@pytest.mark.parametrize("n_heads", [8, 64])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_combined_q_and_kv(num_tokens: int, n_heads: int, block_size: int):
|
||||
@@ -332,7 +335,7 @@ def test_combined_q_and_kv(num_tokens: int, n_heads: int, block_size: int):
|
||||
|
||||
# Reference.
|
||||
q_ref = rmsnorm_no_weight(q, eps)
|
||||
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache)
|
||||
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache).to(dtype)
|
||||
kv_ref = apply_rope_gptj_last_k(kv, positions, cos_sin_cache)
|
||||
k_cache_ref = torch.zeros(
|
||||
num_blocks, block_size * HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
|
||||
Reference in New Issue
Block a user