[Perf] Integrate TRTLLM BF16 MoE Modular Kernel (#45182)

Signed-off-by: Kaihang Jiang <kaihangj@login-lyris02.lyris.clusters.nvidia.com>
This commit is contained in:
Kaihang Jiang
2026-07-09 01:36:14 +04:00
committed by GitHub
parent a5d19cbb95
commit 089e412878
5 changed files with 487 additions and 57 deletions
+145
View File
@@ -0,0 +1,145 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for the FlashInfer TRTLLM BF16 MoE backend
(`TrtLlmBf16ExpertsModular`).
This mirrors the TRTLLM NvFP4 modular test shape: construct the modular
expert wrapper directly, pass production-format BlockMajorK weights, and
compare against a torch MoE reference using the original BF16 weights.
"""
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.utils import torch_moe
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FUSED_MOE_UNQUANTIZED_CONFIG,
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
TrtLlmBf16ExpertsModular,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
convert_moe_weights_to_flashinfer_trtllm_block_layout,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import set_random_seed
if pytest and (
not has_flashinfer_trtllm_fused_moe()
or not current_platform.has_device_capability(100)
):
pytest.skip(
"Requires flashinfer TRTLLM fused MoE BF16 backend (SM100)",
allow_module_level=True,
)
# (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim).
# Covers larger NvFP4-like shapes while keeping BF16's FlashInfer TRTLLM
# intermediate-size multiple-of-128 requirement.
MNK_FACTORS = [
(2, 1024, 1024),
(64, 2048, 1536),
(64, 1024, 4096),
]
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
@pytest.mark.parametrize("e", [128])
@pytest.mark.parametrize("topk", [8])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@torch.inference_mode()
def test_trtllm_bf16_moe_modular_no_graph(
m: int,
n: int,
k: int,
e: int,
topk: int,
dtype: torch.dtype,
workspace_init,
):
set_random_seed(7)
with set_current_vllm_config(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
score = torch.randn((m, e), device="cuda", dtype=dtype)
scores = torch.softmax(score, dim=-1, dtype=torch.float32)
topk_weights, topk_ids = torch.topk(scores, topk)
topk_weights = topk_weights.contiguous()
topk_ids = topk_ids.to(torch.int32).contiguous()
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
num_local_experts=e,
num_logical_experts=e,
activation=MoEActivation.SILU,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=dtype,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
trtllm_w1, trtllm_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout(
{},
w1,
w2,
)
trtllm_experts = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
allow_new_interface=True,
use_monolithic=False,
),
TrtLlmBf16ExpertsModular(
moe_config=moe_config,
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
),
)
trtllm_output = trtllm_experts.apply(
hidden_states=a,
w1=trtllm_w1,
w2=trtllm_w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SILU,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
)
torch_output = torch_moe(
a,
w1,
w2,
score,
topk,
activation=MoEActivation.SILU,
)
torch.testing.assert_close(
torch_output,
trtllm_output,
atol=1e-1,
rtol=2e-1,
)
@@ -5,6 +5,7 @@ from unittest.mock import patch
import pytest
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
select_unquantized_moe_backend,
@@ -117,13 +118,15 @@ def test_select_rocm_aiter_backend(mock_aiter_enabled, mock_has_flashinfer):
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16Experts.is_supported_config",
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config",
return_value=(True, None),
)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
)
def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm):
def test_select_cuda_flashinfer_trtllm_backend(
mock_is_supported_trtllm_monolithic,
):
"""Test CUDA backend selection when FlashInfer TRTLLM is available and enabled."""
with (
patch.object(current_platform, "is_cuda", return_value=True),
@@ -146,6 +149,162 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm):
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
assert experts_cls is not None
assert experts_cls.__name__ == "TrtLlmBf16ExpertsMonolithic"
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config",
return_value=(False, "monolithic unsupported"),
)
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsModular.is_supported_config",
return_value=(True, None),
)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
)
def test_select_cuda_flashinfer_trtllm_modular_backend(
mock_is_supported_trtllm_modular,
mock_is_supported_trtllm_monolithic,
):
"""Test CUDA backend selection falls back to FlashInfer TRTLLM modular."""
with (
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "is_rocm", return_value=False),
patch.object(current_platform, "is_cpu", return_value=False),
patch.object(current_platform, "is_xpu", return_value=False),
patch.object(current_platform, "is_tpu", return_value=False),
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(current_platform, "has_device_capability", return_value=True),
):
moe_config = make_dummy_moe_config()
moe_config.moe_backend = "flashinfer_trtllm"
moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.use_dp = False
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
assert experts_cls is not None
assert experts_cls.__name__ == "TrtLlmBf16ExpertsModular"
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
return_value=True,
)
@pytest.mark.parametrize(
"all2all_backend",
[
"mori_high_throughput",
"mori_low_latency",
"flashinfer_nvlink_two_sided",
"flashinfer_nvlink_one_sided",
],
)
def test_select_cuda_flashinfer_trtllm_modular_for_standard_all2all(
mock_supports_current_device,
all2all_backend,
):
"""Test non-AG/RS standard-format all2all backends select modular BF16."""
with (
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "is_rocm", return_value=False),
patch.object(current_platform, "is_cpu", return_value=False),
patch.object(current_platform, "is_xpu", return_value=False),
patch.object(current_platform, "is_tpu", return_value=False),
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(
current_platform, "is_device_capability_family", return_value=False
),
):
moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2)
moe_config.moe_backend = "flashinfer_trtllm"
moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.dp_size = 2
moe_config.moe_parallel_config.ep_size = 2
moe_config.moe_parallel_config.all2all_backend = all2all_backend
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
assert experts_cls is not None
assert experts_cls.__name__ == "TrtLlmBf16ExpertsModular"
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
return_value=True,
)
def test_select_cuda_deepep_ht_falls_back_from_trtllm(
mock_supports_current_device,
):
"""Test DeepEP HT avoids the unsupported BF16 TRTLLM modular path."""
with (
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "is_rocm", return_value=False),
patch.object(current_platform, "is_cpu", return_value=False),
patch.object(current_platform, "is_xpu", return_value=False),
patch.object(current_platform, "is_tpu", return_value=False),
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(
current_platform, "is_device_capability_family", return_value=False
),
):
moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2)
moe_config.moe_backend = "auto"
moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.dp_size = 2
moe_config.moe_parallel_config.ep_size = 2
moe_config.moe_parallel_config.all2all_backend = "deepep_high_throughput"
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.TRITON
assert experts_cls is not None
assert experts_cls.__name__ == "TritonExperts"
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
return_value=True,
)
def test_select_cuda_flashinfer_trtllm_ag_rs_uses_monolithic(
mock_supports_current_device,
):
"""Test AG/RS stays on BF16 TRTLLM monolithic when TRTLLM is supported."""
with (
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "is_rocm", return_value=False),
patch.object(current_platform, "is_cpu", return_value=False),
patch.object(current_platform, "is_xpu", return_value=False),
patch.object(current_platform, "is_tpu", return_value=False),
patch.object(current_platform, "is_out_of_tree", return_value=False),
patch.object(
current_platform, "is_device_capability_family", return_value=False
),
):
moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2)
moe_config.moe_backend = "flashinfer_trtllm"
moe_config.routing_method = RoutingMethodType.Renormalize
moe_config.moe_parallel_config.use_ep = True
moe_config.moe_parallel_config.dp_size = 2
moe_config.moe_parallel_config.ep_size = 2
moe_config.moe_parallel_config.all2all_backend = "allgather_reducescatter"
selected_backend, experts_cls = select_unquantized_moe_backend(
moe_config=moe_config
)
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
assert experts_cls is not None
assert experts_cls.__name__ == "TrtLlmBf16ExpertsMonolithic"
@patch(
@@ -153,7 +312,11 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm):
return_value=True,
)
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16Experts.is_supported_config",
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config",
return_value=(False, None),
)
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsModular.is_supported_config",
return_value=(False, None),
)
@patch(
@@ -164,9 +327,10 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm):
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
)
def test_select_cuda_flashinfer_cutlass_backend(
mock_has_flashinfer,
mock_is_supported_trtllm,
mock_is_supported_cutlass,
mock_is_supported_trtllm_modular,
mock_is_supported_trtllm_monolithic,
mock_has_flashinfer,
):
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
and FlashInfer CUTLASS is available."""