From 9ed81581a896a5bf0448b26d43f52486a435f799 Mon Sep 17 00:00:00 2001 From: Shuolei Wang <46160365+ShuoleiWang@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:03:21 +0800 Subject: [PATCH] =?UTF-8?q?[Perf][MoE]=20Skip=20top-k=20slots=20routed=20t?= =?UTF-8?q?o=20non-local=20experts=20in=20TritonExp=E2=80=A6=20(#58051)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Shuolei Wang Signed-off-by: Shuolei Wang <948904026@qq.com> --- tests/kernels/moe/test_block_fp8.py | 62 ++++++++++++++++++- .../layers/fused_moe/experts/triton_moe.py | 37 ++++++++++- 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/tests/kernels/moe/test_block_fp8.py b/tests/kernels/moe/test_block_fp8.py index 63f64c735a..68f6b3b0b3 100644 --- a/tests/kernels/moe/test_block_fp8.py +++ b/tests/kernels/moe/test_block_fp8.py @@ -107,7 +107,16 @@ SEEDS = [0] def torch_w8a8_block_fp8_moe( - a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, block_shape, silu_fp32=False + a, + w1, + w2, + w1_s, + w2_s, + topk_weight, + topk_ids, + block_shape, + silu_fp32=False, + expert_map=None, ): """Fused MoE with block-wise fp8 quantization using native torch. @@ -123,6 +132,9 @@ def torch_w8a8_block_fp8_moe( topk_weight = topk_weight.view(-1) topk_ids = topk_ids.view(-1) + if expert_map is not None: + # w1/w2 hold this rank's experts; slots mapped to -1 contribute nothing. + topk_ids = expert_map[topk_ids] _, block_k = block_shape[0], block_shape[1] # Quantize with the production per-token-group fp8 kernel (same HIP/CUDA op the @@ -301,6 +313,54 @@ def test_w8a8_block_fp8_clamp_skips_fused_silu_quant( ) +@pytest.mark.parametrize(("M", "N", "K"), [(83, 512, 512), (2048, 4608, 512)]) +@pytest.mark.parametrize("topk", [2, 6]) +@pytest.mark.parametrize("ep_size", [2, 4]) +@torch.inference_mode() +def test_w8a8_block_fp8_fused_moe_expert_map(M, N, K, topk, ep_size, workspace_init): + """One EP rank: local expert shard plus an expert_map with -1 entries.""" + E, block_size, dtype = 16, [128, 128], torch.bfloat16 + torch.manual_seed(0) + + a = torch.randn((M, K), dtype=dtype) / 10 + score = torch.randn((M, E), dtype=dtype) + w1, w2, quant_config = make_test_quant_config( + E, N, K, dtype, quant_dtype=torch.float8_e4m3fn, block_shape=block_size + ) + + local_e = E // ep_size + e_ids = torch.randperm(E, dtype=torch.int32)[:local_e] + e_map = torch.full((E,), -1, dtype=torch.int32) + e_map[e_ids] = torch.arange(local_e, dtype=torch.int32) + w1, w2 = w1[e_ids].contiguous(), w2[e_ids].contiguous() + w1_s = quant_config.w1_scale[e_ids].contiguous() + w2_s = quant_config.w2_scale[e_ids].contiguous() + quant_config = fp8_w8a8_moe_quant_config( + w1_scale=w1_s, w2_scale=w2_s, block_shape=block_size + ) + + m_fused_moe = modular_triton_fused_moe(make_dummy_moe_config(), quant_config) + topk_weights, topk_ids, _ = fused_topk(a, score.float(), topk, False) + + with set_current_vllm_config(vllm_config): + ref_out = torch_w8a8_block_fp8_moe( + a, w1, w2, w1_s, w2_s, topk_weights, topk_ids, block_size, expert_map=e_map + ) + m_out = m_fused_moe.apply( + a, + w1, + w2, + topk_weights, + topk_ids, + activation=MoEActivation.SILU, + apply_router_weight_on_input=False, + expert_map=e_map, + global_num_experts=E, + ) + + torch.testing.assert_close(m_out, ref_out, atol=0.035, rtol=0.035) + + @pytest.mark.parametrize(("M", "N", "K"), MNK_FACTORS_DG) @pytest.mark.parametrize("E", E) @pytest.mark.parametrize("topk", TOP_KS) diff --git a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py index 1e21a4fe2c..ee3b9b3f4b 100644 --- a/vllm/model_executor/layers/fused_moe/experts/triton_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/triton_moe.py @@ -337,6 +337,25 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): ) intermediate_cache3 = _resize_cache(workspace2, (num_tokens, top_k_num, K)) + # Under EP, drop the top-k slots routed to remote experts at alignment + # time instead of launching `off_experts == -1` GEMM blocks that only + # write zeros; the pad-aware `moe_sum` below skips the same slots. + # Their rows in the workspace caches are then never written, so every + # consumer of those caches must be row-local: a dynamic per-tensor + # scale for the second GEMM's activation would take its amax over the + # untouched rows, and LoRA aligns the full `topk_ids` itself. + a2_scale_is_global = ( + self.quant_dtype is not None + and a2_scale is None + and self.block_shape is None + and (not self.per_act_token_quant or self.quantization_emulation) + ) + skip_invalid = ( + expert_map is not None + and self._lora_context is None + and not a2_scale_is_global + ) + # Include fused shared-expert rows while preserving EP remapping. num_align_experts = w1.shape[0] if expert_map is None else global_num_experts sorted_token_ids, expert_ids, num_tokens_post_padded = ( @@ -350,6 +369,7 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): use_int8_w8a16=self.quant_config.use_int8_w8a16, use_int4_w4a16=self.quant_config.use_int4_w4a16, block_shape=self.block_shape, + ignore_invalid_experts=skip_invalid, ) ) @@ -585,10 +605,21 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): ) # separate function is required for MoE + LoRA - self.moe_sum(intermediate_cache3, output) + self.moe_sum(intermediate_cache3, output, topk_ids, expert_map) - def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None: - ops.moe_sum(input, output) + def moe_sum( + self, + input: torch.Tensor, + output: torch.Tensor, + topk_ids: torch.Tensor | None = None, + expert_map: torch.Tensor | None = None, + ) -> None: + if expert_map is not None: + # Skip the slots whose expert is not on this rank: the rows the + # alignment dropped, or the zeros the `-1` blocks wrote. + ops.moe_sum(input, output, topk_ids, expert_map) + else: + ops.moe_sum(input, output) class TritonWNA16Experts(TritonExperts):