[Perf][DSv4.1] Exchange packed FP8 Engram rows across DP and feed wkv MXFP8 (#59923)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Juntian Liu
2026-10-09 15:30:18 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 82e89e9944
commit efaca9547d
6 changed files with 505 additions and 31 deletions
@@ -21,6 +21,18 @@ def _norm_row(x, weight, row, stride, eps, SIZE: tl.constexpr, BLOCK: tl.constex
return values * rrms * w
@triton.jit
def mxfp8_scale_offset(row, group, GROUPS: tl.constexpr):
"""F8_128x4: [row/128, group/4, row%32, row%128/32, group%4]."""
offset = (row // 128 * triton.cdiv(GROUPS, 4) + group // 4) * 512
return offset + row % 32 * 16 + row % 128 // 32 * 4 + group % 4
def mxfp8_scale_bytes(tokens: int, width: int) -> int:
"""Bytes of F8_128x4 scales for `tokens` rows of `width` MXFP8 values."""
return triton.cdiv(tokens, 128) * 128 * triton.cdiv(width // 32, 4) * 4
@triton.jit(do_not_specialize=["num_tokens"])
def _q_kv_norm_quant_kernel(
q,
@@ -68,14 +80,7 @@ def _q_kv_norm_quant_kernel(
sf = tl.full((BLOCK // 32,), 0, tl.uint32)
padded_groups: tl.constexpr = triton.cdiv(Q_SIZE // 32, 4) * 4
sf = tl.where(groups < Q_SIZE // 32, sf, 0)
# F8_128x4: [row/128, group/4, row%32, row%128/32, group%4].
offsets = (
row // 128 * (128 * padded_groups)
+ groups // 4 * 512
+ row % 32 * 16
+ row % 128 // 32 * 4
+ groups % 4
)
offsets = mxfp8_scale_offset(row, groups, Q_SIZE // 32)
tl.store(scales + offsets, sf, groups < padded_groups)
elif row < num_tokens:
y = _norm_row(kv, kvw, row, kv_stride, eps, KV_SIZE, BLOCK)
@@ -99,9 +104,8 @@ def fused_q_kv_rmsnorm_quant(
qo = torch.empty(qr.shape, dtype=torch.float8_e4m3fn, device=qr.device)
kvo = torch.empty(kv.shape, dtype=kv.dtype, device=kv.device)
padded_tokens = triton.cdiv(tokens, 128) * 128
padded_groups = triton.cdiv(q_size // 32, 4) * 4
scales = torch.empty(
padded_tokens * padded_groups, dtype=torch.uint8, device=qr.device
mxfp8_scale_bytes(tokens, q_size), dtype=torch.uint8, device=qr.device
)
if tokens:
block = triton.next_power_of_2(max(q_size, kv_size))