[Perf] Integrate TRTLLM BF16 MoE Modular Kernel (#45182)
Signed-off-by: Kaihang Jiang <kaihangj@login-lyris02.lyris.clusters.nvidia.com>
This commit is contained in:
@@ -0,0 +1,145 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for the FlashInfer TRTLLM BF16 MoE backend
|
||||
(`TrtLlmBf16ExpertsModular`).
|
||||
|
||||
This mirrors the TRTLLM NvFP4 modular test shape: construct the modular
|
||||
expert wrapper directly, pass production-format BlockMajorK weights, and
|
||||
compare against a torch MoE reference using the original BF16 weights.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.utils import torch_moe
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
|
||||
TrtLlmBf16ExpertsModular,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
convert_moe_weights_to_flashinfer_trtllm_block_layout,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if pytest and (
|
||||
not has_flashinfer_trtllm_fused_moe()
|
||||
or not current_platform.has_device_capability(100)
|
||||
):
|
||||
pytest.skip(
|
||||
"Requires flashinfer TRTLLM fused MoE BF16 backend (SM100)",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
# (m, n, k) = (tokens, intermediate_size_per_partition, hidden_dim).
|
||||
# Covers larger NvFP4-like shapes while keeping BF16's FlashInfer TRTLLM
|
||||
# intermediate-size multiple-of-128 requirement.
|
||||
MNK_FACTORS = [
|
||||
(2, 1024, 1024),
|
||||
(64, 2048, 1536),
|
||||
(64, 1024, 4096),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
|
||||
@pytest.mark.parametrize("e", [128])
|
||||
@pytest.mark.parametrize("topk", [8])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_bf16_moe_modular_no_graph(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
e: int,
|
||||
topk: int,
|
||||
dtype: torch.dtype,
|
||||
workspace_init,
|
||||
):
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
|
||||
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
scores = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weights, topk_ids = torch.topk(scores, topk)
|
||||
topk_weights = topk_weights.contiguous()
|
||||
topk_ids = topk_ids.to(torch.int32).contiguous()
|
||||
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=MoEActivation.SILU,
|
||||
device="cuda",
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=dtype,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
|
||||
trtllm_w1, trtllm_w2 = convert_moe_weights_to_flashinfer_trtllm_block_layout(
|
||||
{},
|
||||
w1,
|
||||
w2,
|
||||
)
|
||||
|
||||
trtllm_experts = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
TrtLlmBf16ExpertsModular(
|
||||
moe_config=moe_config,
|
||||
quant_config=FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
),
|
||||
)
|
||||
|
||||
trtllm_output = trtllm_experts.apply(
|
||||
hidden_states=a,
|
||||
w1=trtllm_w1,
|
||||
w2=trtllm_w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
torch_output = torch_moe(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
score,
|
||||
topk,
|
||||
activation=MoEActivation.SILU,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
torch_output,
|
||||
trtllm_output,
|
||||
atol=1e-1,
|
||||
rtol=2e-1,
|
||||
)
|
||||
@@ -5,6 +5,7 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
select_unquantized_moe_backend,
|
||||
@@ -117,13 +118,15 @@ def test_select_rocm_aiter_backend(mock_aiter_enabled, mock_has_flashinfer):
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16Experts.is_supported_config",
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config",
|
||||
return_value=(True, None),
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
|
||||
)
|
||||
def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm):
|
||||
def test_select_cuda_flashinfer_trtllm_backend(
|
||||
mock_is_supported_trtllm_monolithic,
|
||||
):
|
||||
"""Test CUDA backend selection when FlashInfer TRTLLM is available and enabled."""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
@@ -146,6 +149,162 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm):
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
|
||||
assert experts_cls is not None
|
||||
assert experts_cls.__name__ == "TrtLlmBf16ExpertsMonolithic"
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config",
|
||||
return_value=(False, "monolithic unsupported"),
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsModular.is_supported_config",
|
||||
return_value=(True, None),
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
|
||||
)
|
||||
def test_select_cuda_flashinfer_trtllm_modular_backend(
|
||||
mock_is_supported_trtllm_modular,
|
||||
mock_is_supported_trtllm_monolithic,
|
||||
):
|
||||
"""Test CUDA backend selection falls back to FlashInfer TRTLLM modular."""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
patch.object(current_platform, "is_cpu", return_value=False),
|
||||
patch.object(current_platform, "is_xpu", return_value=False),
|
||||
patch.object(current_platform, "is_tpu", return_value=False),
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(current_platform, "has_device_capability", return_value=True),
|
||||
):
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.moe_backend = "flashinfer_trtllm"
|
||||
moe_config.moe_parallel_config.use_ep = True
|
||||
moe_config.moe_parallel_config.use_dp = False
|
||||
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
|
||||
assert experts_cls is not None
|
||||
assert experts_cls.__name__ == "TrtLlmBf16ExpertsModular"
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
|
||||
return_value=True,
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"all2all_backend",
|
||||
[
|
||||
"mori_high_throughput",
|
||||
"mori_low_latency",
|
||||
"flashinfer_nvlink_two_sided",
|
||||
"flashinfer_nvlink_one_sided",
|
||||
],
|
||||
)
|
||||
def test_select_cuda_flashinfer_trtllm_modular_for_standard_all2all(
|
||||
mock_supports_current_device,
|
||||
all2all_backend,
|
||||
):
|
||||
"""Test non-AG/RS standard-format all2all backends select modular BF16."""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
patch.object(current_platform, "is_cpu", return_value=False),
|
||||
patch.object(current_platform, "is_xpu", return_value=False),
|
||||
patch.object(current_platform, "is_tpu", return_value=False),
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(
|
||||
current_platform, "is_device_capability_family", return_value=False
|
||||
),
|
||||
):
|
||||
moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2)
|
||||
moe_config.moe_backend = "flashinfer_trtllm"
|
||||
moe_config.moe_parallel_config.use_ep = True
|
||||
moe_config.moe_parallel_config.dp_size = 2
|
||||
moe_config.moe_parallel_config.ep_size = 2
|
||||
moe_config.moe_parallel_config.all2all_backend = all2all_backend
|
||||
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
|
||||
assert experts_cls is not None
|
||||
assert experts_cls.__name__ == "TrtLlmBf16ExpertsModular"
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
|
||||
return_value=True,
|
||||
)
|
||||
def test_select_cuda_deepep_ht_falls_back_from_trtllm(
|
||||
mock_supports_current_device,
|
||||
):
|
||||
"""Test DeepEP HT avoids the unsupported BF16 TRTLLM modular path."""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
patch.object(current_platform, "is_cpu", return_value=False),
|
||||
patch.object(current_platform, "is_xpu", return_value=False),
|
||||
patch.object(current_platform, "is_tpu", return_value=False),
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(
|
||||
current_platform, "is_device_capability_family", return_value=False
|
||||
),
|
||||
):
|
||||
moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2)
|
||||
moe_config.moe_backend = "auto"
|
||||
moe_config.moe_parallel_config.use_ep = True
|
||||
moe_config.moe_parallel_config.dp_size = 2
|
||||
moe_config.moe_parallel_config.ep_size = 2
|
||||
moe_config.moe_parallel_config.all2all_backend = "deepep_high_throughput"
|
||||
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.TRITON
|
||||
assert experts_cls is not None
|
||||
assert experts_cls.__name__ == "TritonExperts"
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
|
||||
return_value=True,
|
||||
)
|
||||
def test_select_cuda_flashinfer_trtllm_ag_rs_uses_monolithic(
|
||||
mock_supports_current_device,
|
||||
):
|
||||
"""Test AG/RS stays on BF16 TRTLLM monolithic when TRTLLM is supported."""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
patch.object(current_platform, "is_cpu", return_value=False),
|
||||
patch.object(current_platform, "is_xpu", return_value=False),
|
||||
patch.object(current_platform, "is_tpu", return_value=False),
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(
|
||||
current_platform, "is_device_capability_family", return_value=False
|
||||
),
|
||||
):
|
||||
moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2)
|
||||
moe_config.moe_backend = "flashinfer_trtllm"
|
||||
moe_config.routing_method = RoutingMethodType.Renormalize
|
||||
moe_config.moe_parallel_config.use_ep = True
|
||||
moe_config.moe_parallel_config.dp_size = 2
|
||||
moe_config.moe_parallel_config.ep_size = 2
|
||||
moe_config.moe_parallel_config.all2all_backend = "allgather_reducescatter"
|
||||
|
||||
selected_backend, experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM
|
||||
assert experts_cls is not None
|
||||
assert experts_cls.__name__ == "TrtLlmBf16ExpertsMonolithic"
|
||||
|
||||
|
||||
@patch(
|
||||
@@ -153,7 +312,11 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm):
|
||||
return_value=True,
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16Experts.is_supported_config",
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsMonolithic.is_supported_config",
|
||||
return_value=(False, None),
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsModular.is_supported_config",
|
||||
return_value=(False, None),
|
||||
)
|
||||
@patch(
|
||||
@@ -164,9 +327,10 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm):
|
||||
not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
|
||||
)
|
||||
def test_select_cuda_flashinfer_cutlass_backend(
|
||||
mock_has_flashinfer,
|
||||
mock_is_supported_trtllm,
|
||||
mock_is_supported_cutlass,
|
||||
mock_is_supported_trtllm_modular,
|
||||
mock_is_supported_trtllm_monolithic,
|
||||
mock_has_flashinfer,
|
||||
):
|
||||
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
|
||||
and FlashInfer CUTLASS is available."""
|
||||
|
||||
@@ -11,7 +11,13 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
fi_moe_largest_bucket,
|
||||
trtllm_moe_pack_topk_ids_weights,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
activation_to_flashinfer_int,
|
||||
)
|
||||
@@ -22,9 +28,10 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
|
||||
|
||||
class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic):
|
||||
class TrtLlmBf16ExpertsBase:
|
||||
"""
|
||||
BF16 unquantized TRTLLM-Gen MoE kernels. Supports monolithic interface.
|
||||
BF16 unquantized TRTLLM-Gen MoE kernels. Shared base for modular and
|
||||
monolithic interfaces.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -32,7 +39,6 @@ class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic):
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
self.routing_method_type = moe_config.routing_method
|
||||
self.topk = moe_config.experts_per_token
|
||||
self.intermediate_size_per_partition = (
|
||||
@@ -42,6 +48,9 @@ class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic):
|
||||
self.local_num_experts = moe_config.num_local_experts
|
||||
self.ep_rank = moe_config.moe_parallel_config.ep_rank
|
||||
|
||||
self.moe_config = moe_config
|
||||
self.quant_config = quant_config
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
@@ -74,6 +83,134 @@ class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic):
|
||||
"""Supports SiLU (gated) and RELU^2 (non-gated) activations."""
|
||||
return activation in [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL]
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
class TrtLlmBf16ExpertsModular(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
"""
|
||||
BF16 unquantized TRTLLM-Gen MoE kernels. Supports modular interface.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
return (
|
||||
moe_parallel_config.use_all2all_kernels
|
||||
and not moe_parallel_config.use_ag_rs_all2all_kernels
|
||||
and not moe_parallel_config.use_deepep_ht_kernels
|
||||
and not moe_parallel_config.enable_eplb
|
||||
)
|
||||
|
||||
def moe_problem_size(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
"""Override to handle 4D BlockMajorK weights (E, K/bk, Mn, bk)."""
|
||||
if w1.dim() == 4:
|
||||
E = w1.shape[0]
|
||||
N = w1.shape[2]
|
||||
K = a1.size(-1)
|
||||
M = a1.size(0) if a1.dim() == 2 else a1.size(1)
|
||||
topk = topk_ids.size(1)
|
||||
return E, M, N, K, topk
|
||||
return super().moe_problem_size(a1, w1, w2, topk_ids)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# The workspaces for this implementation are managed by FlashInfer.
|
||||
workspace1 = (0,)
|
||||
workspace2 = (0,)
|
||||
output = (M, K)
|
||||
|
||||
return workspace1, workspace2, output
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
import flashinfer
|
||||
from flashinfer.fused_moe import WeightLayout
|
||||
|
||||
# Pack topk ids and weights into format expected by the TRTLLM kernel.
|
||||
packed_topk_ids = trtllm_moe_pack_topk_ids_weights(topk_ids, topk_weights)
|
||||
|
||||
result = flashinfer.fused_moe.trtllm_bf16_routed_moe(
|
||||
topk_ids=packed_topk_ids,
|
||||
hidden_states=hidden_states,
|
||||
gemm1_weights=w1,
|
||||
gemm2_weights=w2,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
local_expert_offset=self.ep_rank * self.local_num_experts,
|
||||
local_num_experts=self.local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
routing_method_type=1, # not used
|
||||
use_shuffled_weight=True,
|
||||
weight_layout=WeightLayout.BlockMajorK,
|
||||
do_finalize=True,
|
||||
activation_type=activation_to_flashinfer_int(activation),
|
||||
)
|
||||
# FlashInfer's BF16 routed wrapper does not expose an output= argument.
|
||||
output.copy_(result[0] if isinstance(result, list) else result)
|
||||
|
||||
|
||||
class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonolithic):
|
||||
"""
|
||||
BF16 unquantized TRTLLM-Gen MoE kernels. Supports monolithic interface.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
"""Monolithic kernel supports no-all2all and AG/RS paths."""
|
||||
return (
|
||||
not moe_parallel_config.use_all2all_kernels
|
||||
or moe_parallel_config.use_ag_rs_all2all_kernels
|
||||
) and not moe_parallel_config.enable_eplb
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
routing_method: RoutingMethodType,
|
||||
@@ -89,27 +226,6 @@ class TrtLlmBf16Experts(mk.FusedMoEExpertsMonolithic):
|
||||
RoutingMethodType.Sigmoid,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
"""Monolithic kernel so only use with naive DP/EP and TP."""
|
||||
return (
|
||||
not moe_parallel_config.use_all2all_kernels
|
||||
or moe_parallel_config.use_ag_rs_all2all_kernels
|
||||
) and not moe_parallel_config.enable_eplb
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
|
||||
@@ -63,9 +63,11 @@ class MoEKernelOracle(ABC, Generic[BackendT]):
|
||||
this `moe_config`."""
|
||||
|
||||
@abstractmethod
|
||||
def backend_to_kernel_cls(self, backend: BackendT) -> type[mk.FusedMoEExperts]:
|
||||
def backend_to_kernel_cls(
|
||||
self, backend: BackendT
|
||||
) -> list[type[mk.FusedMoEExperts]]:
|
||||
"""Map a backend enum value to its concrete `FusedMoEExperts`
|
||||
subclass."""
|
||||
subclasses, in selection priority order."""
|
||||
|
||||
@abstractmethod
|
||||
def map_backend(self, runner_backend: MoEBackend) -> BackendT:
|
||||
|
||||
@@ -101,46 +101,47 @@ def _get_priority_backends(moe_config: FusedMoEConfig) -> list[UnquantizedMoeBac
|
||||
|
||||
def backend_to_kernel_cls(
|
||||
backend: UnquantizedMoeBackend,
|
||||
) -> type[mk.FusedMoEExperts]:
|
||||
) -> list[type[mk.FusedMoEExperts]]:
|
||||
if backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
|
||||
TrtLlmBf16Experts,
|
||||
TrtLlmBf16ExpertsModular,
|
||||
TrtLlmBf16ExpertsMonolithic,
|
||||
)
|
||||
|
||||
return TrtLlmBf16Experts
|
||||
return [TrtLlmBf16ExpertsMonolithic, TrtLlmBf16ExpertsModular]
|
||||
|
||||
elif backend == UnquantizedMoeBackend.FLASHINFER_CUTLASS:
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( # noqa: E501
|
||||
FlashInferExperts,
|
||||
)
|
||||
|
||||
return FlashInferExperts
|
||||
return [FlashInferExperts]
|
||||
|
||||
elif backend == UnquantizedMoeBackend.AITER:
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
)
|
||||
|
||||
return AiterExperts
|
||||
return [AiterExperts]
|
||||
|
||||
elif backend == UnquantizedMoeBackend.TRITON:
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import (
|
||||
TritonExperts,
|
||||
)
|
||||
|
||||
return TritonExperts
|
||||
return [TritonExperts]
|
||||
|
||||
elif backend == UnquantizedMoeBackend.BATCHED_TRITON:
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_batched_moe import (
|
||||
BatchedTritonExperts,
|
||||
)
|
||||
|
||||
return BatchedTritonExperts
|
||||
return [BatchedTritonExperts]
|
||||
|
||||
elif backend == UnquantizedMoeBackend.XPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExperts
|
||||
|
||||
return XPUExperts
|
||||
return [XPUExperts]
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown unquantized MoE backend: {backend.value}")
|
||||
@@ -183,7 +184,7 @@ def select_unquantized_moe_backend(
|
||||
if moe_config.is_lora_enabled:
|
||||
return UnquantizedMoeBackend.TRITON, backend_to_kernel_cls(
|
||||
UnquantizedMoeBackend.TRITON
|
||||
)
|
||||
)[0]
|
||||
|
||||
# NOTE: the kernels are selected in the following order.
|
||||
AVAILABLE_BACKENDS = _get_priority_backends(moe_config)
|
||||
@@ -222,13 +223,14 @@ def select_unquantized_moe_backend(
|
||||
config: FusedMoEConfig,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[UnquantizedMoeBackend, type[mk.FusedMoEExperts] | None]:
|
||||
k_cls = backend_to_kernel_cls(backend)
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, config, None, None, activation_format
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend))
|
||||
return backend, k_cls
|
||||
reason = None
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, config, None, None, activation_format
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend))
|
||||
return backend, k_cls
|
||||
raise ValueError(_make_log_unsupported(backend, reason))
|
||||
|
||||
runner_backend = moe_config.moe_backend
|
||||
@@ -254,15 +256,15 @@ def select_unquantized_moe_backend(
|
||||
return _return_or_raise(backend, moe_config, activation_format)
|
||||
|
||||
for backend in AVAILABLE_BACKENDS:
|
||||
k_cls = backend_to_kernel_cls(backend)
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, moe_config, None, None, activation_format
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend))
|
||||
return backend, k_cls
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, moe_config, None, None, activation_format
|
||||
)
|
||||
if supported:
|
||||
logger.info_once(_make_log_backend(backend))
|
||||
return backend, k_cls
|
||||
|
||||
logger.debug_once(_make_log_unsupported(backend, reason))
|
||||
logger.debug_once(_make_log_unsupported(backend, reason))
|
||||
|
||||
raise NotImplementedError(
|
||||
"No Unquantized MoE backend supports the deployment configuration."
|
||||
@@ -332,6 +334,7 @@ def make_unquantized_moe_kernel(
|
||||
assert prepare_finalize is not None
|
||||
|
||||
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
|
||||
logger.info_once("Using %s MoE backend", experts_cls.__name__)
|
||||
|
||||
# Create Experts
|
||||
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
@@ -383,7 +386,7 @@ class UnquantizedMoEKernelOracle(MoEKernelOracle[UnquantizedMoeBackend]):
|
||||
|
||||
def backend_to_kernel_cls(
|
||||
self, backend: UnquantizedMoeBackend
|
||||
) -> type[mk.FusedMoEExperts]:
|
||||
) -> list[type[mk.FusedMoEExperts]]:
|
||||
return backend_to_kernel_cls(backend)
|
||||
|
||||
def map_backend(self, runner_backend: MoEBackend) -> UnquantizedMoeBackend:
|
||||
|
||||
Reference in New Issue
Block a user