[Perf][MoE] Skip padded work in block-FP8 DeepGEMM experts (#59128)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Tyler Michael Smith
2026-10-06 18:49:57 -05:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b6d8e8afd9
commit 5ab4e4445b
5 changed files with 431 additions and 76 deletions
@@ -244,7 +244,12 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
return (workspace1, workspace2, output)
def _act_mul_quant(
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
self,
input: torch.Tensor,
output: torch.Tensor,
activation: MoEActivation,
expert_ends: torch.Tensor | None = None,
expert_alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
assert self.block_shape is not None
block_k = self.block_shape[1]
@@ -271,6 +276,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
clamp_limit=self.gemm1_clamp_limit,
alpha=self.gemm1_alpha,
beta=self.gemm1_beta,
expert_ends=expert_ends,
expert_alignment=expert_alignment,
)
act_out = torch.empty(
(M_sum, activation_out_dim), dtype=input.dtype, device=input.device
@@ -294,6 +301,8 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
group_size=block_k,
alpha=self.gemm1_alpha,
beta=self.gemm1_beta,
expert_ends=expert_ends,
expert_alignment=expert_alignment,
)
# 3. fallback path for non-SiLU activations in non‑UE8M0 cases.
@@ -349,7 +358,20 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
a1q_perm = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
)
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
# Both GEMMs and quantization must agree on the live expert ranges.
use_psum_layout = (
not self.mxfp8
and (
current_platform.is_device_capability_family(90)
or (
current_platform.is_device_capability_family(100)
and DeepGemmQuantScaleFMT.from_oracle()
== DeepGemmQuantScaleFMT.UE8M0
)
)
and activation in (MoEActivation.SILU, MoEActivation.SWIGLUOAI_UNINTERLEAVE)
)
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
aq=a1q,
aq_scale=a1q_scale,
topk_ids=topk_ids,
@@ -360,17 +382,21 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
# MXFP8 uses a 32-element activation-scale group (block_shape[1]);
# FP8-block keeps the default (128) alignment.
block_size=self.block_shape[1] if self.mxfp8 else None,
use_psum_layout=use_psum_layout,
)
assert a1q.size(0) == M_sum
# MXFP8 (1x32) drives the fp8_fp4-aliased grouped GEMM with recipe
# (1, 32); the FP8 block path keeps the default (128) recipe.
gemm_kwargs = (
gemm_kwargs: dict = (
{"recipe_a": (1, self.block_shape[1]), "recipe_b": (1, self.block_shape[1])}
if self.mxfp8
else {}
)
if use_psum_layout:
gemm_kwargs["use_psum_layout"] = True
# Cap DG's BLOCK_M heuristic at the workspace's per-expert alignment;
# otherwise the scheduler can pick the wrong expert id from m_indices
# under cudagraph replay.
@@ -380,7 +406,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
(a1q, a1q_scale),
(w1, self.w1_scale),
mm1_out,
expert_ids,
grouped_layout,
**gemm_kwargs,
)
@@ -389,7 +415,11 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
)
a2q, a2q_scale = self._act_mul_quant(
input=mm1_out.view(-1, N), output=quant_out, activation=activation
input=mm1_out.view(-1, N),
output=quant_out,
activation=activation,
expert_ends=grouped_layout if use_psum_layout else None,
expert_alignment=align_used if use_psum_layout else 0,
)
mm2_out = _resize_cache(workspace2, (M_sum, K))
@@ -397,7 +427,7 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
(a2q, a2q_scale),
(w2, self.w2_scale),
mm2_out,
expert_ids,
grouped_layout,
**gemm_kwargs,
)