[Perf][MoE] Skip padded work in block-FP8 DeepGEMM experts (#59128)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Tyler Michael Smith
2026-10-06 18:49:57 -05:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b6d8e8afd9
commit 5ab4e4445b
5 changed files with 431 additions and 76 deletions
@@ -119,6 +119,7 @@ def _fwd_kernel_ep_scatter_1(
BLOCK_E: tl.constexpr,
BLOCK_EXPERT_NUM: tl.constexpr,
ALIGN_M: tl.constexpr,
USE_PSUM_LAYOUT: tl.constexpr,
):
cur_expert = tl.program_id(0)
@@ -140,20 +141,22 @@ def _fwd_kernel_ep_scatter_1(
tl.store(expert_start_loc + cur_expert, cur_expert_start)
cur_expert_token_num = tl.load(num_recv_tokens_per_expert + cur_expert)
m_indices_start_ptr = m_indices + cur_expert_start
off_expert = tl.arange(0, BLOCK_E)
if USE_PSUM_LAYOUT:
# Logical ends include preceding experts' alignment gaps.
tl.store(m_indices + cur_expert, cur_expert_start + cur_expert_token_num)
else:
m_indices_start_ptr = m_indices + cur_expert_start
off_expert = tl.arange(0, BLOCK_E)
# any rows in the per-expert aligned region that do not correspond to
# real tokens are left untouched here and should remain initialized to
# -1 so DeepGEMM can skip them
for start_m in tl.range(0, cur_expert_token_num, BLOCK_E):
offs = start_m + off_expert
mask = offs < cur_expert_token_num
tl.store(
m_indices_start_ptr + offs,
cur_expert,
mask=mask,
)
# Unwritten padding retains the -1 initialization.
for start_m in tl.range(0, cur_expert_token_num, BLOCK_E):
offs = start_m + off_expert
mask = offs < cur_expert_token_num
tl.store(
m_indices_start_ptr + offs,
cur_expert,
mask=mask,
)
@triton.jit
@@ -285,6 +288,7 @@ def ep_scatter(
align_m: int = 128,
block_size: int = 128,
pack_ue8m0: bool = False,
use_psum_layout: bool = False,
):
# BLOCK_E is the m_indices fill-loop tile (masked), independent of align_m.
BLOCK_E = 128
@@ -295,7 +299,10 @@ def ep_scatter(
# grid = (triton.cdiv(hidden_size, BLOCK_D), num_experts)
grid = num_experts
assert m_indices.shape[0] % align_m == 0
if use_psum_layout:
assert m_indices.shape == (num_experts,)
else:
assert m_indices.shape[0] % align_m == 0
assert expert_start_loc.shape[0] == num_experts
# pack_ue8m0: scatter packs 4 UE8M0 bytes per int32; else copies scales as-is.
@@ -311,6 +318,7 @@ def ep_scatter(
BLOCK_E=BLOCK_E,
BLOCK_EXPERT_NUM=triton.next_power_of_2(num_experts),
ALIGN_M=align_m,
USE_PSUM_LAYOUT=use_psum_layout,
)
grid = min(recv_topk.shape[0], 1024 * 8)
@@ -465,6 +473,7 @@ def deepgemm_moe_permute(
expert_tokens_meta: mk.ExpertTokensMetadata | None,
aq_out: torch.Tensor | None = None,
block_size: int | None = None,
use_psum_layout: bool = False,
):
assert aq.ndim == 2
assert topk_ids.dtype.is_signed, "The kernel uses -1 to represent invalid topk_ids"
@@ -509,17 +518,15 @@ def deepgemm_moe_permute(
else:
aq_scale_out = torch.zeros((M_sum, sf_k), device=device, dtype=torch.float32)
# DeepGEMM uses negative values in m_indices (here expert_ids) to mark
# completely invalid / padded blocks that should be skipped. We always
# initialize expert_ids to -1 so any row that is not explicitly written
# by the scatter kernel will be treated as invalid and skipped by
# DeepGEMM's scheduler.
expert_ids = torch.full(
(M_sum,),
fill_value=-1,
device=device,
dtype=torch.int32,
)
if use_psum_layout:
grouped_layout = torch.empty(
(local_num_experts,), device=device, dtype=torch.int32
)
else:
# Negative expert IDs mark padding for the contiguous scheduler.
grouped_layout = torch.full(
(M_sum,), fill_value=-1, device=device, dtype=torch.int32
)
inv_perm = torch.empty(topk_ids.shape, device=device, dtype=torch.int32)
expert_num_tokens = None
@@ -541,14 +548,15 @@ def deepgemm_moe_permute(
expert_map=expert_map,
output_tensor=aq_out,
output_tensor_scale=aq_scale_out,
m_indices=expert_ids,
m_indices=grouped_layout,
output_index=inv_perm,
align_m=align_used,
block_size=block_k,
pack_ue8m0=pack_ue8m0,
use_psum_layout=use_psum_layout,
)
return aq_out, aq_scale_out, expert_ids, inv_perm, align_used
return aq_out, aq_scale_out, grouped_layout, inv_perm, align_used
def deepgemm_unpermute_and_reduce(
@@ -244,7 +244,12 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
return (workspace1, workspace2, output)
def _act_mul_quant(
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
self,
input: torch.Tensor,
output: torch.Tensor,
activation: MoEActivation,
expert_ends: torch.Tensor | None = None,
expert_alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
assert self.block_shape is not None
block_k = self.block_shape[1]
@@ -271,6 +276,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
clamp_limit=self.gemm1_clamp_limit,
alpha=self.gemm1_alpha,
beta=self.gemm1_beta,
expert_ends=expert_ends,
expert_alignment=expert_alignment,
)
act_out = torch.empty(
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
@@ -294,6 +301,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
group_size=block_k,
alpha=self.gemm1_alpha,
beta=self.gemm1_beta,
expert_ends=expert_ends,
expert_alignment=expert_alignment,
)
# 3. fallback path for non-SiLU activations in non‑UE8M0 cases.
@@ -349,7 +358,20 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
a1q_perm = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
)
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
# Both GEMMs and quantization must agree on the live expert ranges.
use_psum_layout = (
not self.mxfp8
and (
current_platform.is_device_capability_family(90)
or (
current_platform.is_device_capability_family(100)
and DeepGemmQuantScaleFMT.from_oracle()
== DeepGemmQuantScaleFMT.UE8M0
)
)
and activation in (MoEActivation.SILU, MoEActivation.SWIGLUOAI_UNINTERLEAVE)
)
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
aq=a1q,
aq_scale=a1q_scale,
topk_ids=topk_ids,
@@ -360,17 +382,21 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
# MXFP8 uses a 32-element activation-scale group (block_shape[1]);
# FP8-block keeps the default (128) alignment.
block_size=self.block_shape[1] if self.mxfp8 else None,
use_psum_layout=use_psum_layout,
)
assert a1q.size(0) == M_sum
# MXFP8 (1x32) drives the fp8_fp4-aliased grouped GEMM with recipe
# (1, 32); the FP8 block path keeps the default (128) recipe.
gemm_kwargs = (
gemm_kwargs: dict = (
{"recipe_a": (1, self.block_shape[1]), "recipe_b": (1, self.block_shape[1])}
if self.mxfp8
else {}
)
if use_psum_layout:
gemm_kwargs["use_psum_layout"] = True
# Cap DG's BLOCK_M heuristic at the workspace's per-expert alignment;
# otherwise the scheduler can pick the wrong expert id from m_indices
# under cudagraph replay.
@@ -380,7 +406,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
(a1q, a1q_scale),
(w1, self.w1_scale),
mm1_out,
expert_ids,
grouped_layout,
**gemm_kwargs,
)
@@ -389,7 +415,11 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
)
a2q, a2q_scale = self._act_mul_quant(
input=mm1_out.view(-1, N), output=quant_out, activation=activation
input=mm1_out.view(-1, N),
output=quant_out,
activation=activation,
expert_ends=grouped_layout if use_psum_layout else None,
expert_alignment=align_used if use_psum_layout else 0,
)
mm2_out = _resize_cache(workspace2, (M_sum, K))
@@ -397,7 +427,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
(a2q, a2q_scale),
(w2, self.w2_scale),
mm2_out,
expert_ids,
grouped_layout,
**gemm_kwargs,
)
@@ -142,6 +142,7 @@ def _silu_mul_quant_fp8_packed_kernel(
input_ptr,
output_q_ptr,
output_scale_ptr,
expert_ends_ptr,
M,
input_stride_m,
output_q_stride_m,
@@ -158,6 +159,7 @@ def _silu_mul_quant_fp8_packed_kernel(
PACKS_PER_CTA: tl.constexpr,
BLOCK_M: tl.constexpr,
HAS_CLAMP: tl.constexpr,
EXPERT_ALIGNMENT: tl.constexpr,
):
GROUPS_PER_PACK: tl.constexpr = 4
hidden_size: tl.constexpr = N // 2
@@ -166,6 +168,13 @@ def _silu_mul_quant_fp8_packed_kernel(
row_start = tl.program_id(1).to(tl.int64) * BLOCK_M
row_step = tl.num_programs(1).to(tl.int64) * BLOCK_M
if EXPERT_ALIGNMENT:
expert = tl.program_id(2)
previous_end = tl.load(expert_ends_ptr + expert - 1, expert > 0, other=0)
expert_start = tl.cdiv(previous_end, EXPERT_ALIGNMENT) * EXPERT_ALIGNMENT
row_start += expert_start.to(tl.int64)
M = tl.load(expert_ends_ptr + expert)
groups_per_cta: tl.constexpr = PACKS_PER_CTA * GROUPS_PER_PACK
elems_per_cta: tl.constexpr = groups_per_cta * GROUP_SIZE
col_start = pack_tile * elems_per_cta
@@ -254,7 +263,19 @@ def silu_mul_quant_fp8_packed_triton(
clamp_limit: float | None = None,
alpha: float = 1.0,
beta: float = 0.0,
*,
expert_ends: torch.Tensor | None = None,
expert_alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fuse activation and FP8 quantization, optionally skipping expert padding.
``expert_ends`` uses DeepGEMM's prefix-sum layout: each entry is the
exclusive end of an expert's live rows, including earlier alignment gaps.
An expert starts at the preceding end rounded up to ``expert_alignment``.
Padding outputs are left unwritten and must not be consumed as live rows.
Without ``expert_ends``, the original dense launch configuration is used
and the metadata loads are compiled out.
"""
assert input.dim() == 2
assert input.is_contiguous()
@@ -293,13 +314,29 @@ def silu_mul_quant_fp8_packed_triton(
grid_n = triton.cdiv(packs_per_row, packs_per_cta)
grid_m = min(triton.cdiv(M, BM), 4096)
grid = (grid_n, grid_m)
grid: tuple[int, ...] = (grid_n, grid_m)
if expert_ends is not None:
assert expert_ends.ndim == 1 and expert_ends.numel() > 0
assert expert_ends.dtype == torch.int32
assert expert_ends.device == input.device and expert_ends.is_contiguous()
assert expert_alignment > 0
num_experts = expert_ends.numel()
# Bound empty CTAs while retaining enough parallelism for full experts.
max_ctas = 4096 if group_size < 128 else 8192
grid_m = min(
triton.cdiv(M, BM * num_experts),
max(1, triton.cdiv(max_ctas, grid_n * num_experts)),
)
grid = (grid_n, grid_m, num_experts)
else:
assert expert_alignment == 0
has_clamp = clamp_limit is not None
_silu_mul_quant_fp8_packed_kernel[grid](
input,
output_q,
output_scale_packed,
expert_ends,
M,
input.stride(0),
output_q.stride(0),
@@ -316,6 +353,7 @@ def silu_mul_quant_fp8_packed_triton(
PACKS_PER_CTA=packs_per_cta,
BLOCK_M=BM,
HAS_CLAMP=has_clamp,
EXPERT_ALIGNMENT=expert_alignment,
num_warps=num_warps,
num_stages=num_stages,
)
@@ -328,6 +366,7 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
y_ptr, # [M, N]
y_q_ptr, # [M, N // 2]
y_s_ptr, # [M, (N // 2) // GROUP_SIZE]
expert_ends_ptr,
M, # num tokens
N, # intermediate size
# Stride
@@ -345,8 +384,8 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
GROUP_SIZE: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_N: tl.constexpr,
EXPERT_ALIGNMENT: tl.constexpr,
):
# TODO(varun) : Add expert_ids so we may early-exit no-op thread blocks.
"""Each thread block (BLOCK_N) computes [BLOCK_M, GROUP_SIZE] act-mul outputs. Then
the thread block quantizes the [BLOCK_M, GROUP_SIZE] block of values and fills
the outputs tensors at the right positions.
@@ -357,56 +396,77 @@ def _silu_mul_per_token_group_quant_fp8_colmajor(
m_offset = pid_m.to(tl.int64) * BLOCK_M
n_offset = pid_n.to(tl.int64) * BLOCK_N
if m_offset >= M:
return
row_step = tl.num_programs(0).to(tl.int64) * BLOCK_M
num_iterations = 1
if EXPERT_ALIGNMENT:
expert = tl.program_id(2)
previous_end = tl.load(expert_ends_ptr + expert - 1, expert > 0, other=0)
expert_start = tl.cdiv(previous_end, EXPERT_ALIGNMENT) * EXPERT_ALIGNMENT
m_offset += expert_start.to(tl.int64)
M = tl.load(expert_ends_ptr + expert)
num_iterations = tl.cdiv(tl.maximum(M - m_offset, 0), row_step)
offs_n = tl.arange(0, BLOCK_N).to(tl.int64)
offs_m = tl.arange(0, BLOCK_M).to(tl.int64)
base_y_ptr = y_ptr + m_offset * N + n_offset
for _ in range(num_iterations):
base_y_ptr = y_ptr + m_offset * N + n_offset
act_in_ptrs = base_y_ptr + offs_m[:, None] * N + offs_n[None, :]
act_in_ptrs = base_y_ptr + offs_m[:, None] * N + offs_n[None, :]
act_in = tl.load(act_in_ptrs)
mul_in = tl.load(act_in_ptrs + N_2)
if EXPERT_ALIGNMENT:
live = m_offset + offs_m < M
act_in = tl.load(act_in_ptrs, live[:, None], other=0)
mul_in = tl.load(act_in_ptrs + N_2, live[:, None], other=0)
else:
act_in = tl.load(act_in_ptrs)
mul_in = tl.load(act_in_ptrs + N_2)
# silu & mul — match C++ silu_and_mul: clamp in fp32 then store back to the
# input dtype, run silu in fp32 then narrow, and do the mul at input
# precision so HAS_CLAMP True/False share the same multiplication path.
if HAS_CLAMP:
act_in = tl.minimum(act_in.to(tl.float32), clamp_limit).to(
y_ptr.dtype.element_ty
)
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
y_ptr.dtype.element_ty
)
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
# Keep glu/up at input precision (narrow before the mul) so the alpha=1,
# beta=0 defaults match the C++ silu_and_mul path bit-for-bit.
act_in = act_in.to(tl.float32)
glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty)
up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty)
y = (glu * up).to(tl.float32)
# silu & mul — match C++ silu_and_mul: clamp in fp32 then store back to the
# input dtype, run silu in fp32 then narrow, and do the mul at input
# precision so HAS_CLAMP True/False share the same multiplication path.
if HAS_CLAMP:
act_in = tl.minimum(act_in.to(tl.float32), clamp_limit).to(
y_ptr.dtype.element_ty
)
mul_in = tl.clamp(mul_in.to(tl.float32), -clamp_limit, clamp_limit).to(
y_ptr.dtype.element_ty
)
# Unified gated activation: silu == swigluoai with alpha=1, beta=0.
# glu = gate * sigmoid(alpha * gate); y = (up + beta) * glu
# Keep glu/up at input precision (narrow before the mul) so the alpha=1,
# beta=0 defaults match the C++ silu_and_mul path bit-for-bit.
act_in = act_in.to(tl.float32)
glu = (act_in / (1.0 + tl.exp(-act_in * alpha))).to(y_ptr.dtype.element_ty)
up = (mul_in.to(tl.float32) + beta).to(y_ptr.dtype.element_ty)
y = (glu * up).to(tl.float32)
# quant
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
scale_raw = _absmax * (1.0 / fp8_max)
y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw
y_s = tl.reshape(y_s, (BLOCK_M, 1))
y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
# quant
_absmax = tl.maximum(tl.max(tl.abs(y), axis=1), eps)
scale_raw = _absmax * (1.0 / fp8_max)
y_s = tl.math.exp2(tl.ceil(tl.log2(scale_raw))) if use_ue8m0 else scale_raw
y_s = tl.reshape(y_s, (BLOCK_M, 1))
y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty)
# store y_q
base_y_q_ptr = y_q_ptr + m_offset * N_2 + n_offset
y_q_ptrs = base_y_q_ptr + offs_m[:, None] * N_2 + offs_n[None, :]
tl.store(y_q_ptrs, y_q)
# store y_q
base_y_q_ptr = y_q_ptr + m_offset * N_2 + n_offset
y_q_ptrs = base_y_q_ptr + offs_m[:, None] * N_2 + offs_n[None, :]
if EXPERT_ALIGNMENT:
tl.store(y_q_ptrs, y_q, live[:, None])
else:
tl.store(y_q_ptrs, y_q)
# store y_s
group_id = n_offset // GROUP_SIZE
base_y_s_ptr = y_s_ptr + group_id * y_s_col_stride + m_offset
y_s_ptrs = base_y_s_ptr + offs_m
y_s = tl.reshape(y_s, (BLOCK_M,))
tl.store(y_s_ptrs, y_s)
# store y_s
group_id = n_offset // GROUP_SIZE
base_y_s_ptr = y_s_ptr + group_id * y_s_col_stride + m_offset
y_s_ptrs = base_y_s_ptr + offs_m
y_s = tl.reshape(y_s, (BLOCK_M,))
if EXPERT_ALIGNMENT:
tl.store(y_s_ptrs, y_s, live)
else:
tl.store(y_s_ptrs, y_s)
m_offset += row_step
def silu_mul_per_token_group_quant_fp8_colmajor(
@@ -418,9 +478,15 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
group_size: int = 128,
alpha: float = 1.0,
beta: float = 0.0,
*,
expert_ends: torch.Tensor | None = None,
expert_alignment: int = 0,
):
"""Gated activation + block-fp8 quant. ``alpha``/``beta`` select the gate
(silu: alpha=1, beta=0; swigluoai: alpha, beta from config).
Optional expert endpoints follow DeepGEMM's prefix-sum layout. Padding
values and scales are left unwritten and must not be consumed as live rows.
"""
GROUP_SIZE = group_size
assert input.ndim == 2
@@ -458,13 +524,27 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
# Force even division so we can avoid edgecases within the kernel.
assert M % BLOCK_M == 0
assert N_2 % BLOCK_N == 0
grid = (M // BLOCK_M, N_2 // BLOCK_N)
grid: tuple[int, ...] = (M // BLOCK_M, N_2 // BLOCK_N)
if expert_ends is not None:
assert expert_ends.ndim == 1 and expert_ends.numel() > 0
assert expert_ends.dtype == torch.int32
assert expert_ends.device == input.device and expert_ends.is_contiguous()
assert expert_alignment > 0 and expert_alignment % BLOCK_M == 0
num_experts = expert_ends.numel()
grid_m = min(
triton.cdiv(M, BLOCK_M * num_experts),
max(1, triton.cdiv(8192, grid[1] * num_experts)),
)
grid = (grid_m, grid[1], num_experts)
else:
assert expert_alignment == 0
has_clamp = clamp_limit is not None
_silu_mul_per_token_group_quant_fp8_colmajor[grid](
input,
output,
output_scales,
expert_ends,
M,
N,
output_scales.stride(-1),
@@ -479,6 +559,7 @@ def silu_mul_per_token_group_quant_fp8_colmajor(
GROUP_SIZE,
BLOCK_M,
BLOCK_N,
expert_alignment,
)
return output, output_scales