diff --git a/tests/kernels/moe/test_trtllm_bf16_moe.py b/tests/kernels/moe/test_trtllm_bf16_moe.py new file mode 100644 index 0000000000..22dff237cb --- /dev/null +++ b/tests/kernels/moe/test_trtllm_bf16_moe.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Tests for the FlashInfer TRTLLM BF16 MoE backend +(`TrtLlmBf16ExpertsModular`). + +This mirrors the TRTLLM NvFP4 modular test shape: construct the modular +expert wrapper directly, pass production-format BlockMajorK weights, and +compare against a torch MoE reference using the original BF16 weights. +""" + +import pytest +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from tests.kernels.utils import torch_moe +from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.all2all_utils import ( + maybe_make_prepare_finalize, +) +from vllm.model_executor.layers.fused_moe.config import ( + FUSED_MOE_UNQUANTIZED_CONFIG, + FusedMoEConfig, + FusedMoEParallelConfig, + RoutingMethodType, +) +from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import ( + TrtLlmBf16ExpertsModular, +) +from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + convert_moe_weights_to_flashinfer_trtllm_block_layout, +) +from vllm.platforms import current_platform +from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe +from vllm.utils.math_utils import next_power_of_2 +from vllm.utils.torch_utils import set_random_seed + +if pytest and ( + not has_flashinfer_trtllm_fused_moe() + or not current_platform.has_device_capability(100) +): + pytest.skip( + "Requires flashinfer TRTLLM fused MoE BF16 backend (SM100)", + allow_module_level=True, + ) + +# (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim). +# Covers larger NvFP4-like shapes while keeping BF16's FlashInfer TRTLLM +# intermediate-size multiple-of-128 requirement. +MNK_FACTORS = [ + (2, 1024, 1024), + (64, 2048, 1536), + (64, 1024, 4096), +] + + +@pytest.mark.parametrize("m,n,k", MNK_FACTORS) +@pytest.mark.parametrize("e", [128]) +@pytest.mark.parametrize("topk", [8]) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@torch.inference_mode() +def test_trtllm_bf16_moe_modular_no_graph( + m: int, + n: int, + k: int, + e: int, + topk: int, + dtype: torch.dtype, + workspace_init, +): + set_random_seed(7) + with set_current_vllm_config( + VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1)) + ): + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 + w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10 + w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10 + score = torch.randn((m, e), device="cuda", dtype=dtype) + scores = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weights, topk_ids = torch.topk(scores, topk) + topk_weights = topk_weights.contiguous() + topk_ids = topk_ids.to(torch.int32).contiguous() + + moe_config = FusedMoEConfig( + num_experts=e, + experts_per_token=topk, + hidden_dim=k, + intermediate_size=n, + num_local_experts=e, + num_logical_experts=e, + activation=MoEActivation.SILU, + device="cuda", + moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), + in_dtype=dtype, + routing_method=RoutingMethodType.TopK, + max_num_tokens=next_power_of_2(m), + ) + + trtllm_w1, trtllm_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout( + {}, + w1, + w2, + ) + + trtllm_experts = mk.FusedMoEKernel( + maybe_make_prepare_finalize( + moe=moe_config, + quant_config=FUSED_MOE_UNQUANTIZED_CONFIG, + allow_new_interface=True, + use_monolithic=False, + ), + TrtLlmBf16ExpertsModular( + moe_config=moe_config, + quant_config=FUSED_MOE_UNQUANTIZED_CONFIG, + ), + ) + + trtllm_output = trtllm_experts.apply( + hidden_states=a, + w1=trtllm_w1, + w2=trtllm_w2, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=MoEActivation.SILU, + global_num_experts=e, + expert_map=None, + apply_router_weight_on_input=False, + ) + + torch_output = torch_moe( + a, + w1, + w2, + score, + topk, + activation=MoEActivation.SILU, + ) + + torch.testing.assert_close( + torch_output, + trtllm_output, + atol=1e-1, + rtol=2e-1, + ) diff --git a/tests/kernels/moe/test_unquantized_backend_selection.py b/tests/kernels/moe/test_unquantized_backend_selection.py index 9e1afbbdff..cdff3667ad 100644 --- a/tests/kernels/moe/test_unquantized_backend_selection.py +++ b/tests/kernels/moe/test_unquantized_backend_selection.py @@ -5,6 +5,7 @@ from unittest.mock import patch import pytest from tests.kernels.moe.utils import make_dummy_moe_config +from vllm.model_executor.layers.fused_moe.config import RoutingMethodType from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( UnquantizedMoeBackend, select_unquantized_moe_backend, @@ -117,13 +118,15 @@ def test_select_rocm_aiter_backend(mock_aiter_enabled, mock_has_flashinfer): @patch( - "vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16Experts.is_supported_config", + "vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config", return_value=(True, None), ) @pytest.mark.skipif( not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms." ) -def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm): +def test_select_cuda_flashinfer_trtllm_backend( + mock_is_supported_trtllm_monolithic, +): """Test CUDA backend selection when FlashInfer TRTLLM is available and enabled.""" with ( patch.object(current_platform, "is_cuda", return_value=True), @@ -146,6 +149,162 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm): assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM assert experts_cls is not None + assert experts_cls.__name__ == "TrtLlmBf16ExpertsMonolithic" + + +@patch( + "vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config", + return_value=(False, "monolithic unsupported"), +) +@patch( + "vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsModular.is_supported_config", + return_value=(True, None), +) +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms." +) +def test_select_cuda_flashinfer_trtllm_modular_backend( + mock_is_supported_trtllm_modular, + mock_is_supported_trtllm_monolithic, +): + """Test CUDA backend selection falls back to FlashInfer TRTLLM modular.""" + with ( + patch.object(current_platform, "is_cuda", return_value=True), + patch.object(current_platform, "is_rocm", return_value=False), + patch.object(current_platform, "is_cpu", return_value=False), + patch.object(current_platform, "is_xpu", return_value=False), + patch.object(current_platform, "is_tpu", return_value=False), + patch.object(current_platform, "is_out_of_tree", return_value=False), + patch.object(current_platform, "has_device_capability", return_value=True), + ): + moe_config = make_dummy_moe_config() + moe_config.moe_backend = "flashinfer_trtllm" + moe_config.moe_parallel_config.use_ep = True + moe_config.moe_parallel_config.use_dp = False + + selected_backend, experts_cls = select_unquantized_moe_backend( + moe_config=moe_config + ) + + assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM + assert experts_cls is not None + assert experts_cls.__name__ == "TrtLlmBf16ExpertsModular" + + +@patch( + "vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device", + return_value=True, +) +@pytest.mark.parametrize( + "all2all_backend", + [ + "mori_high_throughput", + "mori_low_latency", + "flashinfer_nvlink_two_sided", + "flashinfer_nvlink_one_sided", + ], +) +def test_select_cuda_flashinfer_trtllm_modular_for_standard_all2all( + mock_supports_current_device, + all2all_backend, +): + """Test non-AG/RS standard-format all2all backends select modular BF16.""" + with ( + patch.object(current_platform, "is_cuda", return_value=True), + patch.object(current_platform, "is_rocm", return_value=False), + patch.object(current_platform, "is_cpu", return_value=False), + patch.object(current_platform, "is_xpu", return_value=False), + patch.object(current_platform, "is_tpu", return_value=False), + patch.object(current_platform, "is_out_of_tree", return_value=False), + patch.object( + current_platform, "is_device_capability_family", return_value=False + ), + ): + moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2) + moe_config.moe_backend = "flashinfer_trtllm" + moe_config.moe_parallel_config.use_ep = True + moe_config.moe_parallel_config.dp_size = 2 + moe_config.moe_parallel_config.ep_size = 2 + moe_config.moe_parallel_config.all2all_backend = all2all_backend + + selected_backend, experts_cls = select_unquantized_moe_backend( + moe_config=moe_config + ) + + assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM + assert experts_cls is not None + assert experts_cls.__name__ == "TrtLlmBf16ExpertsModular" + + +@patch( + "vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device", + return_value=True, +) +def test_select_cuda_deepep_ht_falls_back_from_trtllm( + mock_supports_current_device, +): + """Test DeepEP HT avoids the unsupported BF16 TRTLLM modular path.""" + with ( + patch.object(current_platform, "is_cuda", return_value=True), + patch.object(current_platform, "is_rocm", return_value=False), + patch.object(current_platform, "is_cpu", return_value=False), + patch.object(current_platform, "is_xpu", return_value=False), + patch.object(current_platform, "is_tpu", return_value=False), + patch.object(current_platform, "is_out_of_tree", return_value=False), + patch.object( + current_platform, "is_device_capability_family", return_value=False + ), + ): + moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2) + moe_config.moe_backend = "auto" + moe_config.moe_parallel_config.use_ep = True + moe_config.moe_parallel_config.dp_size = 2 + moe_config.moe_parallel_config.ep_size = 2 + moe_config.moe_parallel_config.all2all_backend = "deepep_high_throughput" + + selected_backend, experts_cls = select_unquantized_moe_backend( + moe_config=moe_config + ) + + assert selected_backend == UnquantizedMoeBackend.TRITON + assert experts_cls is not None + assert experts_cls.__name__ == "TritonExperts" + + +@patch( + "vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device", + return_value=True, +) +def test_select_cuda_flashinfer_trtllm_ag_rs_uses_monolithic( + mock_supports_current_device, +): + """Test AG/RS stays on BF16 TRTLLM monolithic when TRTLLM is supported.""" + with ( + patch.object(current_platform, "is_cuda", return_value=True), + patch.object(current_platform, "is_rocm", return_value=False), + patch.object(current_platform, "is_cpu", return_value=False), + patch.object(current_platform, "is_xpu", return_value=False), + patch.object(current_platform, "is_tpu", return_value=False), + patch.object(current_platform, "is_out_of_tree", return_value=False), + patch.object( + current_platform, "is_device_capability_family", return_value=False + ), + ): + moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2) + moe_config.moe_backend = "flashinfer_trtllm" + moe_config.routing_method = RoutingMethodType.Renormalize + moe_config.moe_parallel_config.use_ep = True + moe_config.moe_parallel_config.dp_size = 2 + moe_config.moe_parallel_config.ep_size = 2 + moe_config.moe_parallel_config.all2all_backend = "allgather_reducescatter" + + selected_backend, experts_cls = select_unquantized_moe_backend( + moe_config=moe_config + ) + + assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM + assert experts_cls is not None + assert experts_cls.__name__ == "TrtLlmBf16ExpertsMonolithic" @patch( @@ -153,7 +312,11 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm): return_value=True, ) @patch( - "vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16Experts.is_supported_config", + "vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config", + return_value=(False, None), +) +@patch( + "vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsModular.is_supported_config", return_value=(False, None), ) @patch( @@ -164,9 +327,10 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm): not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms." ) def test_select_cuda_flashinfer_cutlass_backend( - mock_has_flashinfer, - mock_is_supported_trtllm, mock_is_supported_cutlass, + mock_is_supported_trtllm_modular, + mock_is_supported_trtllm_monolithic, + mock_has_flashinfer, ): """Test CUDA backend selection when FlashInfer TRTLLM is not available and FlashInfer CUTLASS is available.""" diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py index bd1b9ecaa8..143df9e81c 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py @@ -11,7 +11,13 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEQuantConfig, RoutingMethodType, ) -from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket +from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( + TopKWeightAndReduceNoOP, +) +from vllm.model_executor.layers.fused_moe.utils import ( + fi_moe_largest_bucket, + trtllm_moe_pack_topk_ids_weights, +) from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( activation_to_flashinfer_int, ) @@ -22,9 +28,10 @@ from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe -class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic): +class TrtLlmBf16ExpertsBase: """ - BF16 unquantized TRTLLM-Gen MoE kernels. Supports monolithic interface. + BF16 unquantized TRTLLM-Gen MoE kernels. Shared base for modular and + monolithic interfaces. """ def __init__( @@ -32,7 +39,6 @@ class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic): moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig, ): - super().__init__(moe_config, quant_config) self.routing_method_type = moe_config.routing_method self.topk = moe_config.experts_per_token self.intermediate_size_per_partition = ( @@ -42,6 +48,9 @@ class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic): self.local_num_experts = moe_config.num_local_experts self.ep_rank = moe_config.moe_parallel_config.ep_rank + self.moe_config = moe_config + self.quant_config = quant_config + @staticmethod def activation_format() -> mk.FusedMoEActivationFormat: return mk.FusedMoEActivationFormat.Standard @@ -74,6 +83,134 @@ class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic): """Supports SiLU (gated) and RELU^2 (non-gated) activations.""" return activation in [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL] + @staticmethod + def _supports_router_logits_dtype( + router_logits_dtype: torch.dtype | None, + routing_method: RoutingMethodType, + ) -> bool: + return True + + @property + def expects_unquantized_inputs(self) -> bool: + return True + + +class TrtLlmBf16ExpertsModular(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsModular): + """ + BF16 unquantized TRTLLM-Gen MoE kernels. Supports modular interface. + """ + + @staticmethod + def _supports_parallel_config( + moe_parallel_config: FusedMoEParallelConfig, + ) -> bool: + return ( + moe_parallel_config.use_all2all_kernels + and not moe_parallel_config.use_ag_rs_all2all_kernels + and not moe_parallel_config.use_deepep_ht_kernels + and not moe_parallel_config.enable_eplb + ) + + def moe_problem_size( + self, + a1: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_ids: torch.Tensor, + ) -> tuple[int, int, int, int, int]: + """Override to handle 4D BlockMajorK weights (E, K/bk, Mn, bk).""" + if w1.dim() == 4: + E = w1.shape[0] + N = w1.shape[2] + K = a1.size(-1) + M = a1.size(0) if a1.dim() == 2 else a1.size(1) + topk = topk_ids.size(1) + return E, M, N, K, topk + return super().moe_problem_size(a1, w1, w2, topk_ids) + + def workspace_shapes( + self, + M: int, + N: int, + K: int, + topk: int, + global_num_experts: int, + local_num_experts: int, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + activation: MoEActivation, + ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + # The workspaces for this implementation are managed by FlashInfer. + workspace1 = (0,) + workspace2 = (0,) + output = (M, K) + + return workspace1, workspace2, output + + def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: + return TopKWeightAndReduceNoOP() + + def apply( + self, + output: torch.Tensor, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, + workspace2: torch.Tensor, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + apply_router_weight_on_input: bool, + ): + import flashinfer + from flashinfer.fused_moe import WeightLayout + + # Pack topk ids and weights into format expected by the TRTLLM kernel. + packed_topk_ids = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights) + + result = flashinfer.fused_moe.trtllm_bf16_routed_moe( + topk_ids=packed_topk_ids, + hidden_states=hidden_states, + gemm1_weights=w1, + gemm2_weights=w2, + num_experts=global_num_experts, + top_k=self.topk, + n_group=None, + topk_group=None, + intermediate_size=self.intermediate_size_per_partition, + local_expert_offset=self.ep_rank * self.local_num_experts, + local_num_experts=self.local_num_experts, + routed_scaling_factor=None, + routing_method_type=1, # not used + use_shuffled_weight=True, + weight_layout=WeightLayout.BlockMajorK, + do_finalize=True, + activation_type=activation_to_flashinfer_int(activation), + ) + # FlashInfer's BF16 routed wrapper does not expose an output= argument. + output.copy_(result[0] if isinstance(result, list) else result) + + +class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonolithic): + """ + BF16 unquantized TRTLLM-Gen MoE kernels. Supports monolithic interface. + """ + + @staticmethod + def _supports_parallel_config( + moe_parallel_config: FusedMoEParallelConfig, + ) -> bool: + """Monolithic kernel supports no-all2all and AG/RS paths.""" + return ( + not moe_parallel_config.use_all2all_kernels + or moe_parallel_config.use_ag_rs_all2all_kernels + ) and not moe_parallel_config.enable_eplb + @staticmethod def _supports_routing_method( routing_method: RoutingMethodType, @@ -89,27 +226,6 @@ class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic): RoutingMethodType.Sigmoid, ] - @staticmethod - def _supports_parallel_config( - moe_parallel_config: FusedMoEParallelConfig, - ) -> bool: - """Monolithic kernel so only use with naive DP/EP and TP.""" - return ( - not moe_parallel_config.use_all2all_kernels - or moe_parallel_config.use_ag_rs_all2all_kernels - ) and not moe_parallel_config.enable_eplb - - @staticmethod - def _supports_router_logits_dtype( - router_logits_dtype: torch.dtype | None, - routing_method: RoutingMethodType, - ) -> bool: - return True - - @property - def expects_unquantized_inputs(self) -> bool: - return True - def apply( self, hidden_states: torch.Tensor, diff --git a/vllm/model_executor/layers/fused_moe/oracle/base.py b/vllm/model_executor/layers/fused_moe/oracle/base.py index 6f7b8a9855..06be6becf8 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/base.py +++ b/vllm/model_executor/layers/fused_moe/oracle/base.py @@ -63,9 +63,11 @@ class MoEKernelOracle(ABC, Generic[BackendT]): this `moe_config`.""" @abstractmethod - def backend_to_kernel_cls(self, backend: BackendT) -> type[mk.FusedMoEExperts]: + def backend_to_kernel_cls( + self, backend: BackendT + ) -> list[type[mk.FusedMoEExperts]]: """Map a backend enum value to its concrete `FusedMoEExperts` - subclass.""" + subclasses, in selection priority order.""" @abstractmethod def map_backend(self, runner_backend: MoEBackend) -> BackendT: diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index 639fce5234..8857488e4c 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -101,46 +101,47 @@ def _get_priority_backends(moe_config: FusedMoEConfig) -> list[UnquantizedMoeBac def backend_to_kernel_cls( backend: UnquantizedMoeBackend, -) -> type[mk.FusedMoEExperts]: +) -> list[type[mk.FusedMoEExperts]]: if backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM: from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import ( - TrtLlmBf16Experts, + TrtLlmBf16ExpertsModular, + TrtLlmBf16ExpertsMonolithic, ) - return TrtLlmBf16Experts + return [TrtLlmBf16ExpertsMonolithic, TrtLlmBf16ExpertsModular] elif backend == UnquantizedMoeBackend.FLASHINFER_CUTLASS: from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( # noqa: E501 FlashInferExperts, ) - return FlashInferExperts + return [FlashInferExperts] elif backend == UnquantizedMoeBackend.AITER: from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( AiterExperts, ) - return AiterExperts + return [AiterExperts] elif backend == UnquantizedMoeBackend.TRITON: from vllm.model_executor.layers.fused_moe.experts.triton_moe import ( TritonExperts, ) - return TritonExperts + return [TritonExperts] elif backend == UnquantizedMoeBackend.BATCHED_TRITON: from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import ( BatchedTritonExperts, ) - return BatchedTritonExperts + return [BatchedTritonExperts] elif backend == UnquantizedMoeBackend.XPU: from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExperts - return XPUExperts + return [XPUExperts] else: raise ValueError(f"Unknown unquantized MoE backend: {backend.value}") @@ -183,7 +184,7 @@ def select_unquantized_moe_backend( if moe_config.is_lora_enabled: return UnquantizedMoeBackend.TRITON, backend_to_kernel_cls( UnquantizedMoeBackend.TRITON - ) + )[0] # NOTE: the kernels are selected in the following order. AVAILABLE_BACKENDS = _get_priority_backends(moe_config) @@ -222,13 +223,14 @@ def select_unquantized_moe_backend( config: FusedMoEConfig, activation_format: mk.FusedMoEActivationFormat, ) -> tuple[UnquantizedMoeBackend, type[mk.FusedMoEExperts] | None]: - k_cls = backend_to_kernel_cls(backend) - supported, reason = k_cls.is_supported_config( - k_cls, config, None, None, activation_format - ) - if supported: - logger.info_once(_make_log_backend(backend)) - return backend, k_cls + reason = None + for k_cls in backend_to_kernel_cls(backend): + supported, reason = k_cls.is_supported_config( + k_cls, config, None, None, activation_format + ) + if supported: + logger.info_once(_make_log_backend(backend)) + return backend, k_cls raise ValueError(_make_log_unsupported(backend, reason)) runner_backend = moe_config.moe_backend @@ -254,15 +256,15 @@ def select_unquantized_moe_backend( return _return_or_raise(backend, moe_config, activation_format) for backend in AVAILABLE_BACKENDS: - k_cls = backend_to_kernel_cls(backend) - supported, reason = k_cls.is_supported_config( - k_cls, moe_config, None, None, activation_format - ) - if supported: - logger.info_once(_make_log_backend(backend)) - return backend, k_cls + for k_cls in backend_to_kernel_cls(backend): + supported, reason = k_cls.is_supported_config( + k_cls, moe_config, None, None, activation_format + ) + if supported: + logger.info_once(_make_log_backend(backend)) + return backend, k_cls - logger.debug_once(_make_log_unsupported(backend, reason)) + logger.debug_once(_make_log_unsupported(backend, reason)) raise NotImplementedError( "No Unquantized MoE backend supports the deployment configuration." @@ -332,6 +334,7 @@ def make_unquantized_moe_kernel( assert prepare_finalize is not None logger.info_once("Using %s", prepare_finalize.__class__.__name__) + logger.info_once("Using %s MoE backend", experts_cls.__name__) # Create Experts if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: @@ -383,7 +386,7 @@ class UnquantizedMoEKernelOracle(MoEKernelOracle[UnquantizedMoeBackend]): def backend_to_kernel_cls( self, backend: UnquantizedMoeBackend - ) -> type[mk.FusedMoEExperts]: + ) -> list[type[mk.FusedMoEExperts]]: return backend_to_kernel_cls(backend) def map_backend(self, runner_backend: MoEBackend) -> UnquantizedMoeBackend: