[Perf][MoE] Skip padded work in SM100 FP4 DeepGEMM experts (#59044)

Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
Tyler Michael Smith
2026-10-16 16:20:52 -04:00
committed by GitHub
co-authored by Codex Robert Shaw
parent cc2e4fecf2
commit 68ebea789a
2 changed files with 27 additions and 8 deletions
@@ -116,7 +116,7 @@ def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
@pytest.mark.parametrize("scale_format", ["packed", "float32", "ceil_ue8m0"])
@pytest.mark.parametrize("hidden_size", [256, 2048])
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA graph test")
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm graph test")
def test_silu_quant_skips_expert_padding_on_graph_replay(
counts: list[int],
group_size: int,
@@ -135,7 +135,10 @@ def test_silu_quant_skips_expert_padding_on_graph_replay(
x *= 16
clean_x = x.clone()
ends = torch.empty(len(counts), device=DEVICE, dtype=torch.int32)
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=FLOAT8_DTYPE)
output_dtype = (
FLOAT8_DTYPE if scale_format == "packed" else current_platform.fp8_dtype()
)
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=output_dtype)
def set_counts(values):
positions = []
@@ -187,6 +190,7 @@ def test_silu_quant_skips_expert_padding_on_graph_replay(
output.view(torch.uint8).fill_(91)
graph.replay()
reference_q, reference_s = quantize(x)
assert actual_q.dtype == reference_q.dtype
torch.testing.assert_close(
actual_q.view(torch.uint8)[live],
reference_q.view(torch.uint8)[live],
@@ -545,7 +545,12 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
return (workspace1, workspace2, output)
def _act_mul_quant(
self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation
self,
input: torch.Tensor,
output: torch.Tensor,
activation: MoEActivation,
expert_ends: torch.Tensor | None = None,
expert_alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor]:
block_k = self._ACT_BLOCK_K
scale_fmt = DeepGemmQuantScaleFMT.from_oracle()
@@ -561,6 +566,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
output_q=output,
group_size=block_k,
clamp_limit=self.gemm1_clamp_limit,
expert_ends=expert_ends,
expert_alignment=expert_alignment,
)
use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
return silu_mul_per_token_group_quant_fp8_colmajor(
@@ -634,7 +641,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
a1q_perm = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K)
)
a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute(
use_psum_layout = current_platform.is_device_capability_family(100)
a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute(
aq=a1q,
aq_scale=a1q_scale,
topk_ids=topk_ids,
@@ -642,6 +650,7 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
expert_map=expert_map,
expert_tokens_meta=expert_tokens_meta,
aq_out=a1q_perm,
use_psum_layout=use_psum_layout,
)
assert a1q.size(0) == M_sum
@@ -655,18 +664,23 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
(a1q, a1q_scale),
(w1.view(torch.int8), self.w1_scale),
mm1_out,
expert_ids,
grouped_layout,
use_psum_layout=use_psum_layout,
recipe_a=(1, self._ACT_BLOCK_K),
recipe_b=(1, self._WEIGHT_BLOCK_K),
)
# SwiGLU activation + FP8 requant
# SiLU+mul and FP8 requant use the same live expert ranges as GEMM.
activation_out_dim = self.adjust_N_for_activation(N, activation)
quant_out = _resize_cache(
workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim)
)
a2q, a2q_scale = self._act_mul_quant(
input=mm1_out.view(-1, N), output=quant_out, activation=activation
input=mm1_out.view(-1, N),
output=quant_out,
activation=activation,
expert_ends=grouped_layout if use_psum_layout else None,
expert_alignment=align_used if use_psum_layout else 0,
)
# FC2: FP8 activations x FP4 weights
@@ -675,7 +689,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
(a2q, a2q_scale),
(w2.view(torch.int8), self.w2_scale),
mm2_out,
expert_ids,
grouped_layout,
use_psum_layout=use_psum_layout,
recipe_a=(1, self._ACT_BLOCK_K),
recipe_b=(1, self._WEIGHT_BLOCK_K),
)