Harden DeepSeek V3.2 fused kernel grids (#52381)

Signed-off-by: yimdev <5779256+yimdev@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
yimdev
2026-08-18 01:43:12 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent e8ad2855e7
commit eab1cff5b0
2 changed files with 79 additions and 6 deletions
@@ -362,6 +362,46 @@ def test_fused_norm_rope_ds_mla(num_tokens: int):
assert (topk == 7).all(), "topk buffer should be untouched (no indexer)"
def test_fused_norm_rope_supports_large_token_count():
"""Keep the token count off CUDA grid-y at its 65,536-block boundary."""
num_tokens = 65536
dev = "cuda"
dtype = torch.bfloat16
positions = torch.zeros(num_tokens, device=dev, dtype=torch.int64)
q_c = torch.ones((num_tokens, 1), device=dev, dtype=dtype)
kv_c = torch.ones((num_tokens, 1), device=dev, dtype=dtype)
k_pe = torch.ones((num_tokens, 2), device=dev, dtype=dtype)
norm_w = torch.ones(1, device=dev, dtype=dtype)
cos_sin = torch.tensor([[1.0, 0.0]], device=dev, dtype=torch.float32)
topk = torch.empty((num_tokens, 1), device=dev, dtype=torch.int32)
slot_mapping = torch.arange(num_tokens, device=dev, dtype=torch.int64)
mla_cache = torch.empty((1, num_tokens, 3), device=dev, dtype=dtype)
q_out = K.fused_norm_rope(
positions,
q_c,
norm_w,
EPS,
kv_c,
norm_w,
EPS,
k_pe,
cos_sin,
None,
None,
None,
EPS,
None,
topk,
slot_mapping=slot_mapping,
mla_kv_cache=mla_cache,
has_indexer=False,
)
rows = torch.tensor([0, num_tokens - 1], device=dev)
assert_bf16(q_out[rows], rms_norm(q_c[rows], norm_w), "large-token q norm")
# ── fused_q ──────────────────────────────────────────────────────────────────
@@ -539,6 +579,39 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool):
torch.testing.assert_close(iw_out, iw_ref, rtol=1e-3, atol=1e-3)
def test_fused_q_triton_supports_large_token_count():
"""Keep the token count off CUDA grid-y in the Triton fallback.
The minimal dimensions also bypass CuTeDSL on SM100.
"""
num_tokens = 65536
dev = "cuda"
dtype = torch.bfloat16
positions = torch.zeros(num_tokens, device=dev, dtype=torch.int64)
q_pe = torch.ones((num_tokens, 1, 2), device=dev, dtype=dtype)
ql_nope = torch.ones((num_tokens, 1, 1), device=dev, dtype=dtype)
cos_sin = torch.tensor([[1.0, 0.0]], device=dev, dtype=torch.float32)
q_scale = torch.ones(1, device=dev, dtype=torch.float32)
_, _, mqa_q = K.fused_q(
positions,
q_pe,
cos_sin,
None,
None,
ql_nope,
q_scale,
None,
0.0,
0.0,
has_indexer=False,
)
rows = torch.tensor([0, num_tokens - 1], device=dev)
ref = torch.cat([ql_nope, q_pe], dim=-1).to(FP8)
assert_fp8(mqa_q[rows], ref[rows], "large-token fused Q")
# ── fused_eh_norm (MTP) ──────────────────────────────────────────────────────