[Perf] Integrate TRTLLM BF16 MoE Modular Kernel (#45182)

Signed-off-by: Kaihang Jiang <kaihangj@login-lyris02.lyris.clusters.nvidia.com>
This commit is contained in:
Kaihang Jiang
2026-07-09 01:36:14 +04:00
committed by GitHub
parent a5d19cbb95
commit 089e412878
5 changed files with 487 additions and 57 deletions
@@ -11,7 +11,13 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import (
fi_moe_largest_bucket,
trtllm_moe_pack_topk_ids_weights,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
activation_to_flashinfer_int,
)
@@ -22,9 +28,10 @@ from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic):
class TrtLlmBf16ExpertsBase:
"""
BF16 unquantized TRTLLM-Gen MoE kernels. Supports monolithic interface.
BF16 unquantized TRTLLM-Gen MoE kernels. Shared base for modular and
monolithic interfaces.
"""
def __init__(
@@ -32,7 +39,6 @@ class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic):
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
self.routing_method_type = moe_config.routing_method
self.topk = moe_config.experts_per_token
self.intermediate_size_per_partition = (
@@ -42,6 +48,9 @@ class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic):
self.local_num_experts = moe_config.num_local_experts
self.ep_rank = moe_config.moe_parallel_config.ep_rank
self.moe_config = moe_config
self.quant_config = quant_config
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@@ -74,6 +83,134 @@ class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic):
"""Supports SiLU (gated) and RELU^2 (non-gated) activations."""
return activation in [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
@property
def expects_unquantized_inputs(self) -> bool:
return True
class TrtLlmBf16ExpertsModular(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsModular):
"""
BF16 unquantized TRTLLM-Gen MoE kernels. Supports modular interface.
"""
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return (
moe_parallel_config.use_all2all_kernels
and not moe_parallel_config.use_ag_rs_all2all_kernels
and not moe_parallel_config.use_deepep_ht_kernels
and not moe_parallel_config.enable_eplb
)
def moe_problem_size(
self,
a1: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_ids: torch.Tensor,
) -> tuple[int, int, int, int, int]:
"""Override to handle 4D BlockMajorK weights (E, K/bk, Mn, bk)."""
if w1.dim() == 4:
E = w1.shape[0]
N = w1.shape[2]
K = a1.size(-1)
M = a1.size(0) if a1.dim() == 2 else a1.size(1)
topk = topk_ids.size(1)
return E, M, N, K, topk
return super().moe_problem_size(a1, w1, w2, topk_ids)
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# The workspaces for this implementation are managed by FlashInfer.
workspace1 = (0,)
workspace2 = (0,)
output = (M, K)
return workspace1, workspace2, output
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
import flashinfer
from flashinfer.fused_moe import WeightLayout
# Pack topk ids and weights into format expected by the TRTLLM kernel.
packed_topk_ids = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
result = flashinfer.fused_moe.trtllm_bf16_routed_moe(
topk_ids=packed_topk_ids,
hidden_states=hidden_states,
gemm1_weights=w1,
gemm2_weights=w2,
num_experts=global_num_experts,
top_k=self.topk,
n_group=None,
topk_group=None,
intermediate_size=self.intermediate_size_per_partition,
local_expert_offset=self.ep_rank * self.local_num_experts,
local_num_experts=self.local_num_experts,
routed_scaling_factor=None,
routing_method_type=1, # not used
use_shuffled_weight=True,
weight_layout=WeightLayout.BlockMajorK,
do_finalize=True,
activation_type=activation_to_flashinfer_int(activation),
)
# FlashInfer's BF16 routed wrapper does not expose an output= argument.
output.copy_(result[0] if isinstance(result, list) else result)
class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonolithic):
"""
BF16 unquantized TRTLLM-Gen MoE kernels. Supports monolithic interface.
"""
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
"""Monolithic kernel supports no-all2all and AG/RS paths."""
return (
not moe_parallel_config.use_all2all_kernels
or moe_parallel_config.use_ag_rs_all2all_kernels
) and not moe_parallel_config.enable_eplb
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
@@ -89,27 +226,6 @@ class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic):
RoutingMethodType.Sigmoid,
]
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
"""Monolithic kernel so only use with naive DP/EP and TP."""
return (
not moe_parallel_config.use_all2all_kernels
or moe_parallel_config.use_ag_rs_all2all_kernels
) and not moe_parallel_config.enable_eplb
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
@property
def expects_unquantized_inputs(self) -> bool:
return True
def apply(
self,
hidden_states: torch.Tensor,
@@ -63,9 +63,11 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
this `moe_config`."""
@abstractmethod
def backend_to_kernel_cls(self, backend: BackendT) -> type[mk.FusedMoEExperts]:
def backend_to_kernel_cls(
self, backend: BackendT
) -> list[type[mk.FusedMoEExperts]]:
"""Map a backend enum value to its concrete `FusedMoEExperts`
subclass."""
subclasses, in selection priority order."""
@abstractmethod
def map_backend(self, runner_backend: MoEBackend) -> BackendT:
@@ -101,46 +101,47 @@ def _get_priority_backends(moe_config: FusedMoEConfig) -> list[UnquantizedMoeBac
def backend_to_kernel_cls(
backend: UnquantizedMoeBackend,
) -> type[mk.FusedMoEExperts]:
) -> list[type[mk.FusedMoEExperts]]:
if backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
TrtLlmBf16Experts,
TrtLlmBf16ExpertsModular,
TrtLlmBf16ExpertsMonolithic,
)
return TrtLlmBf16Experts
return [TrtLlmBf16ExpertsMonolithic, TrtLlmBf16ExpertsModular]
elif backend == UnquantizedMoeBackend.FLASHINFER_CUTLASS:
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( # noqa: E501
FlashInferExperts,
)
return FlashInferExperts
return [FlashInferExperts]
elif backend == UnquantizedMoeBackend.AITER:
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
)
return AiterExperts
return [AiterExperts]
elif backend == UnquantizedMoeBackend.TRITON:
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
TritonExperts,
)
return TritonExperts
return [TritonExperts]
elif backend == UnquantizedMoeBackend.BATCHED_TRITON:
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
BatchedTritonExperts,
)
return BatchedTritonExperts
return [BatchedTritonExperts]
elif backend == UnquantizedMoeBackend.XPU:
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExperts
return XPUExperts
return [XPUExperts]
else:
raise ValueError(f"Unknown unquantized MoE backend: {backend.value}")
@@ -183,7 +184,7 @@ def select_unquantized_moe_backend(
if moe_config.is_lora_enabled:
return UnquantizedMoeBackend.TRITON, backend_to_kernel_cls(
UnquantizedMoeBackend.TRITON
)
)[0]
# NOTE: the kernels are selected in the following order.
AVAILABLE_BACKENDS = _get_priority_backends(moe_config)
@@ -222,13 +223,14 @@ def select_unquantized_moe_backend(
config: FusedMoEConfig,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[UnquantizedMoeBackend, type[mk.FusedMoEExperts] | None]:
k_cls = backend_to_kernel_cls(backend)
supported, reason = k_cls.is_supported_config(
k_cls, config, None, None, activation_format
)
if supported:
logger.info_once(_make_log_backend(backend))
return backend, k_cls
reason = None
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls, config, None, None, activation_format
)
if supported:
logger.info_once(_make_log_backend(backend))
return backend, k_cls
raise ValueError(_make_log_unsupported(backend, reason))
runner_backend = moe_config.moe_backend
@@ -254,15 +256,15 @@ def select_unquantized_moe_backend(
return _return_or_raise(backend, moe_config, activation_format)
for backend in AVAILABLE_BACKENDS:
k_cls = backend_to_kernel_cls(backend)
supported, reason = k_cls.is_supported_config(
k_cls, moe_config, None, None, activation_format
)
if supported:
logger.info_once(_make_log_backend(backend))
return backend, k_cls
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls, moe_config, None, None, activation_format
)
if supported:
logger.info_once(_make_log_backend(backend))
return backend, k_cls
logger.debug_once(_make_log_unsupported(backend, reason))
logger.debug_once(_make_log_unsupported(backend, reason))
raise NotImplementedError(
"No Unquantized MoE backend supports the deployment configuration."
@@ -332,6 +334,7 @@ def make_unquantized_moe_kernel(
assert prepare_finalize is not None
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
logger.info_once("Using %s MoE backend", experts_cls.__name__)
# Create Experts
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
@@ -383,7 +386,7 @@ class UnquantizedMoEKernelOracle(MoEKernelOracle[UnquantizedMoeBackend]):
def backend_to_kernel_cls(
self, backend: UnquantizedMoeBackend
) -> type[mk.FusedMoEExperts]:
) -> list[type[mk.FusedMoEExperts]]:
return backend_to_kernel_cls(backend)
def map_backend(self, runner_backend: MoEBackend) -> UnquantizedMoeBackend: