[Kernel] Optional Q-norm in fused DSv4 MLA epilogue; group_size=32 for packed FP8 quant (#56215)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-10 00:35:59 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 9e25706560
commit be1cb9834b
6 changed files with 267 additions and 148 deletions
@@ -80,6 +80,11 @@ def test_per_token_group_quant_fp8(
# Larger shapes with padding
(127, 7168, 128),
(253, 640, 128),
(1, 32, 32),
(3, 64, 32),
(7, 160, 32),
(32, 512, 32),
(127, 4096, 32),
],
)
@pytest.mark.parametrize("poisoned_scales", [False, True])
@@ -4,11 +4,11 @@
Standalone unit test for the horizontally-fused DeepseekV4-MLA kernel:
fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert
- Q side: per-head RMSNorm (no weight) + GPT-J RoPE on last 64 dims
- Q side: optional per-head RMSNorm + GPT-J RoPE on last 64 dims
- KV side: GPT-J RoPE on last 64 + UE8M0 FP8 quant + paged cache insert
We compare against:
- PyTorch reference for RMSNorm + GPT-J RoPE on Q
- PyTorch references for RoPE with and without RMSNorm on Q
- Existing Triton `quantize_and_insert_k_cache` + round-trip via
`dequantize_and_gather_k_cache` for KV
@@ -146,9 +146,18 @@ pytestmark = pytest.mark.skipif(
def _call_fused(
q_in, q_head_padded, kv, k_cache, slot_mapping, positions, cos_sin_cache, eps, bs
q_in,
q_head_padded,
kv,
k_cache,
slot_mapping,
positions,
cos_sin_cache,
eps,
bs,
apply_q_norm=None,
):
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
args = (
q_in,
kv,
k_cache,
@@ -159,6 +168,11 @@ def _call_fused(
eps,
bs,
)
if apply_q_norm is None:
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(*args)
return torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
*args, apply_q_norm
)
def _as_stored_fp8(t: torch.Tensor) -> torch.Tensor:
@@ -269,6 +283,42 @@ def test_q_path_matches_reference(num_tokens: int, n_heads: int, padded_heads: i
)
@pytest.mark.parametrize("num_tokens", [4, 2048])
def test_q_path_without_qnorm_matches_rope_only_reference(num_tokens: int):
"""Disabling Q norm must apply RoPE directly to the projected query."""
torch.manual_seed(7)
device = "cuda"
dtype = torch.bfloat16
eps = 1e-6
n_heads = 8
padded_heads = 16
block_size = 16
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(4096, ROPE_DIM, torch.float32, device)
q_ref = apply_rope_gptj_last_k(q, positions, cos_sin_cache)
kv = torch.zeros(num_tokens, HEAD_DIM, dtype=dtype, device=device)
k_cache = torch.zeros(2, block_size * HEAD_BYTES, dtype=torch.uint8, device=device)
slot_mapping = torch.full((num_tokens,), -1, dtype=torch.int64, device=device)
q_out = _call_fused(
q,
padded_heads,
kv,
k_cache,
slot_mapping,
positions,
cos_sin_cache,
eps,
block_size,
apply_q_norm=False,
)
torch.testing.assert_close(q_out[:, :n_heads], q_ref, rtol=0, atol=0)
assert q_out[:, n_heads:].count_nonzero().item() == 0
# ── Test 2: KV path round-trip byte/value parity ─────────────────────────────
@@ -382,7 +432,7 @@ def test_kv_path_matches_reference(num_tokens: int, block_size: int):
@pytest.mark.parametrize("block_size", [16, 64])
def test_kv_path_with_dp_padding(num_tokens: int, pad: int, block_size: int):
"""slot_mapping.size(0) < q.size(0): the kernel must skip padded
tokens in the KV branch while still running Q-norm+RoPE on all rows."""
tokens in the KV branch while still processing Q on all rows."""
torch.manual_seed(3)
device = "cuda"
dtype = torch.bfloat16
@@ -517,8 +567,9 @@ def _call_full_cache_fp8_fused(
q_fp8_scale_inv,
eps,
bs,
apply_q_norm=None,
):
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
args = (
q,
kv,
q_fp8,
@@ -531,6 +582,11 @@ def _call_full_cache_fp8_fused(
eps,
bs,
)
op = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert
if apply_q_norm is None:
op(*args)
else:
op(*args, apply_q_norm)
def _call_full_cache_bf16_fused(
@@ -542,8 +598,9 @@ def _call_full_cache_bf16_fused(
cos_sin_cache,
eps,
bs,
apply_q_norm=None,
):
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
args = (
q,
kv,
k_cache,
@@ -553,6 +610,11 @@ def _call_full_cache_bf16_fused(
eps,
bs,
)
op = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert
if apply_q_norm is None:
op(*args)
else:
op(*args, apply_q_norm)
def _fp8_full_cache_reference(
@@ -567,8 +629,9 @@ def _fp8_full_cache_reference(
block_size,
fp8_scale,
q_fp8_scale_inv,
apply_q_norm,
):
q_ref = rmsnorm_no_weight(q, eps)
q_ref = rmsnorm_no_weight(q, eps) if apply_q_norm else q.float()
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache)
q_fp8.copy_(
torch.clamp(q_ref.float() * q_fp8_scale_inv, -FP8_MAX, FP8_MAX).to(
@@ -595,9 +658,10 @@ def _bf16_full_cache_reference(
cos_sin_cache,
eps,
block_size,
apply_q_norm,
):
q_ref = rmsnorm_no_weight(q, eps)
# Kernel keeps RMSNorm+RoPE in fp32 and rounds to bf16 once at the store.
q_ref = rmsnorm_no_weight(q, eps) if apply_q_norm else q.float()
# Kernel keeps optional RMSNorm+RoPE in fp32 and rounds once at the store.
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache).to(q.dtype)
kv_ref = apply_rope_gptj_last_k(kv, positions, cos_sin_cache)
@@ -616,10 +680,19 @@ def _bf16_full_cache_reference(
@pytest.mark.parametrize("num_tokens", [4, 17])
@pytest.mark.parametrize("n_heads", [8, 17])
@pytest.mark.parametrize("positions_dtype", [torch.int32, torch.int64])
@pytest.mark.parametrize(
"apply_q_norm",
[
pytest.param(None, id="qnorm_default"),
pytest.param(True, id="qnorm_explicit"),
pytest.param(False, id="rope_only"),
],
)
def test_full_cache_per_tensor_fp8_matches_reference(
num_tokens: int,
n_heads: int,
positions_dtype: torch.dtype,
apply_q_norm: bool | None,
):
torch.manual_seed(4)
device = "cuda"
@@ -662,6 +735,7 @@ def test_full_cache_per_tensor_fp8_matches_reference(
block_size,
fp8_scale,
q_fp8_scale_inv,
apply_q_norm is not False,
)
_call_full_cache_fp8_fused(
q.clone(),
@@ -675,11 +749,12 @@ def test_full_cache_per_tensor_fp8_matches_reference(
q_fp8_scale_inv,
eps,
block_size,
apply_q_norm,
)
# Q is RMSNorm(no-weight)+RoPE in fp32 before fp8 quant; the RMSNorm
# reduction and RoPE rotation can land the kernel and the torch reference on
# opposite sides of an fp8 round-to-nearest tie, so allow <=1 fp8 ULP.
# Q uses optional RMSNorm followed by RoPE in fp32 before fp8 quant. The
# kernel and torch reference can land on opposite sides of an fp8
# round-to-nearest tie, so allow <=1 fp8 ULP.
q_fused = _as_stored_fp8(q_fp8_fused)
q_max_ulp = int(fp8_ulp_distance(q_fused, q_fp8_ref).max().item())
assert q_max_ulp <= 1, f"Q fp8 differs by {q_max_ulp} ULP (>1)"
@@ -709,10 +784,19 @@ def test_full_cache_per_tensor_fp8_matches_reference(
@pytest.mark.parametrize("num_tokens", [4, 17])
@pytest.mark.parametrize("n_heads", [8, 17])
@pytest.mark.parametrize("positions_dtype", [torch.int32, torch.int64])
@pytest.mark.parametrize(
"apply_q_norm",
[
pytest.param(None, id="qnorm_default"),
pytest.param(True, id="qnorm_explicit"),
pytest.param(False, id="rope_only"),
],
)
def test_full_cache_bf16_matches_reference(
num_tokens: int,
n_heads: int,
positions_dtype: torch.dtype,
apply_q_norm: bool | None,
):
torch.manual_seed(5)
device = "cuda"
@@ -743,6 +827,7 @@ def test_full_cache_bf16_matches_reference(
cos_sin_cache,
eps,
block_size,
apply_q_norm is not False,
)
_call_full_cache_bf16_fused(
q_fused,
@@ -753,6 +838,7 @@ def test_full_cache_bf16_matches_reference(
cos_sin_cache,
eps,
block_size,
apply_q_norm,
)
torch.testing.assert_close(q_fused, q_ref, rtol=1e-2, atol=1e-2)