From 68ebea789a4652a6b16665d50d79805866b933ec Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 16 Oct 2026 16:20:52 -0400 Subject: [PATCH] [Perf][MoE] Skip padded work in SM100 FP4 DeepGEMM experts (#59044) Signed-off-by: Tyler Michael Smith Co-authored-by: Codex Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> --- ..._mul_per_token_group_quant_fp8_colmajor.py | 8 ++++-- .../layers/fused_moe/experts/deep_gemm_moe.py | 27 ++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py b/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py index b7dcb956c8..771c1f2440 100644 --- a/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py +++ b/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py @@ -116,7 +116,7 @@ def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int): @pytest.mark.parametrize("scale_format", ["packed", "float32", "ceil_ue8m0"]) @pytest.mark.parametrize("hidden_size", [256, 2048]) @pytest.mark.parametrize("clamp_limit", [None, 10.0]) -@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA graph test") +@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm graph test") def test_silu_quant_skips_expert_padding_on_graph_replay( counts: list[int], group_size: int, @@ -135,7 +135,10 @@ def test_silu_quant_skips_expert_padding_on_graph_replay( x *= 16 clean_x = x.clone() ends = torch.empty(len(counts), device=DEVICE, dtype=torch.int32) - output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=FLOAT8_DTYPE) + output_dtype = ( + FLOAT8_DTYPE if scale_format == "packed" else current_platform.fp8_dtype() + ) + output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=output_dtype) def set_counts(values): positions = [] @@ -187,6 +190,7 @@ def test_silu_quant_skips_expert_padding_on_graph_replay( output.view(torch.uint8).fill_(91) graph.replay() reference_q, reference_s = quantize(x) + assert actual_q.dtype == reference_q.dtype torch.testing.assert_close( actual_q.view(torch.uint8)[live], reference_q.view(torch.uint8)[live], diff --git a/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py b/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py index 0eef7a8335..cfa380f9b4 100644 --- a/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/deep_gemm_moe.py @@ -545,7 +545,12 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular): return (workspace1, workspace2, output) def _act_mul_quant( - self, input: torch.Tensor, output: torch.Tensor, activation: MoEActivation + self, + input: torch.Tensor, + output: torch.Tensor, + activation: MoEActivation, + expert_ends: torch.Tensor | None = None, + expert_alignment: int = 0, ) -> tuple[torch.Tensor, torch.Tensor]: block_k = self._ACT_BLOCK_K scale_fmt = DeepGemmQuantScaleFMT.from_oracle() @@ -561,6 +566,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular): output_q=output, group_size=block_k, clamp_limit=self.gemm1_clamp_limit, + expert_ends=expert_ends, + expert_alignment=expert_alignment, ) use_ue8m0 = scale_fmt == DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0 return silu_mul_per_token_group_quant_fp8_colmajor( @@ -634,7 +641,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular): a1q_perm = _resize_cache( workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, K) ) - a1q, a1q_scale, expert_ids, inv_perm, align_used = deepgemm_moe_permute( + use_psum_layout = current_platform.is_device_capability_family(100) + a1q, a1q_scale, grouped_layout, inv_perm, align_used = deepgemm_moe_permute( aq=a1q, aq_scale=a1q_scale, topk_ids=topk_ids, @@ -642,6 +650,7 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular): expert_map=expert_map, expert_tokens_meta=expert_tokens_meta, aq_out=a1q_perm, + use_psum_layout=use_psum_layout, ) assert a1q.size(0) == M_sum @@ -655,18 +664,23 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular): (a1q, a1q_scale), (w1.view(torch.int8), self.w1_scale), mm1_out, - expert_ids, + grouped_layout, + use_psum_layout=use_psum_layout, recipe_a=(1, self._ACT_BLOCK_K), recipe_b=(1, self._WEIGHT_BLOCK_K), ) - # SwiGLU activation + FP8 requant + # SiLU+mul and FP8 requant use the same live expert ranges as GEMM. activation_out_dim = self.adjust_N_for_activation(N, activation) quant_out = _resize_cache( workspace13.view(dtype=torch.float8_e4m3fn), (M_sum, activation_out_dim) ) a2q, a2q_scale = self._act_mul_quant( - input=mm1_out.view(-1, N), output=quant_out, activation=activation + input=mm1_out.view(-1, N), + output=quant_out, + activation=activation, + expert_ends=grouped_layout if use_psum_layout else None, + expert_alignment=align_used if use_psum_layout else 0, ) # FC2: FP8 activations x FP4 weights @@ -675,7 +689,8 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular): (a2q, a2q_scale), (w2.view(torch.int8), self.w2_scale), mm2_out, - expert_ids, + grouped_layout, + use_psum_layout=use_psum_layout, recipe_a=(1, self._ACT_BLOCK_K), recipe_b=(1, self._WEIGHT_BLOCK_K), )