[MoE] Shape-based padding for FlashInfer TRT-LLM MXFP8 experts (gated and non-gated) (#61293)

Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
Jhao-Ting Chen
2026-10-15 13:44:11 -04:00
committed by GitHub
co-authored by Claude Opus 5.5 Robert Shaw
parent 7cd7982dfd
commit 4f49b55c1f
2 changed files with 216 additions and 5 deletions
+125 -3
View File
@@ -52,6 +52,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import input_to_float8
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
dequant_mxfp8_to_bf16,
mxfp8_e4m3_quantize,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -65,7 +66,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.model_executor.models.llama4 import Llama4MoE
from vllm.platforms import current_platform
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.math_utils import next_power_of_2, round_up
from vllm.utils.torch_utils import set_random_seed
try:
@@ -788,7 +789,7 @@ def test_trtllm_fp8_swiglu_clamp_support(
def _make_mxfp8_moe_weights(
e: int, n: int, k: int
e: int, n: int, k: int, is_gated: bool = True
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
q, s = zip(
@@ -796,8 +797,9 @@ def _make_mxfp8_moe_weights(
)
return torch.stack(q), torch.stack(s)
w1_rows = 2 * n if is_gated else n
w1, w1_scale = quantize(
torch.randn(e, 2 * n, k, device="cuda", dtype=torch.bfloat16) / 10
torch.randn(e, w1_rows, k, device="cuda", dtype=torch.bfloat16) / 10
)
w2, w2_scale = quantize(
torch.randn(e, k, n, device="cuda", dtype=torch.bfloat16) / 10
@@ -986,3 +988,123 @@ def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_in
)
torch.testing.assert_close(fused, reference, atol=2e-2, rtol=2e-2)
@pytest.mark.parametrize(
("activation", "n", "k"),
[
(MoEActivation.SILU, 1024, 1024),
(MoEActivation.SILU, 928, 1024),
(MoEActivation.SILU, 1024, 2688),
(MoEActivation.RELU2_NO_MUL, 1024, 1024),
# Nemotron-3.5 Lightning's non-gated MoE: intermediate 1856, hidden 2688.
(MoEActivation.RELU2_NO_MUL, 1856, 2688),
],
)
@pytest.mark.parametrize("m", [1, 32])
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
)
def test_trtllm_mxfp8_moe_pads_intermediate(
m: int, activation: MoEActivation, n: int, k: int, workspace_init
):
"""TRTLLM-Gen MXFP8 experts need the intermediate size to be a multiple
of 128 and the hidden size a multiple of 256. Misaligned experts, gated or
not, are padded at load and, with the activations padded and the output
cropped as the MoE runner does, must still match the unquantized reference
on the dequantized weights."""
e, topk = 8, 2
is_gated = activation.is_gated
block_shape = [1, 32]
set_random_seed(7)
with set_current_vllm_config(vllm_config):
w1, w2, w1_scale, w2_scale = _make_mxfp8_moe_weights(e, n, k, is_gated)
w1_ref = dequant_mxfp8_to_bf16(w1, w1_scale)
w2_ref = dequant_mxfp8_to_bf16(w2, w2_scale)
layer = SimpleNamespace(
weight_block_size=block_shape,
moe_config=SimpleNamespace(
is_act_and_mul=is_gated, intermediate_size_per_partition=n
),
activation=activation,
)
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
Fp8MoeBackend.FLASHINFER_TRTLLM,
layer,
w1,
w2,
w1_scale,
w2_scale,
w13_input_scale=None,
w2_input_scale=None,
)
padded_n = layer.moe_config.intermediate_size_per_partition
padded_k = layer.moe_config.hidden_dim
assert padded_n == round_up(n, 128)
assert padded_k == round_up(k, 256)
quant_config = make_fp8_moe_quant_config(
Fp8MoeBackend.FLASHINFER_TRTLLM,
w1_scale,
w2_scale,
a1_scale=None,
a2_scale=None,
block_shape=block_shape,
)
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=padded_k,
intermediate_size=padded_n,
num_local_experts=e,
num_logical_experts=e,
activation=activation,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
routing_method=RoutingMethodType.Renormalize,
max_num_tokens=next_power_of_2(m),
)
# The monolithic experts pass the activation type to the kernel; the
# modular ones always run it gated.
kernel = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=True,
),
TrtLlmFp8ExpertsMonolithic(
moe_config=moe_config, quant_config=quant_config
),
)
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
logits = torch.randn((m, e), device="cuda", dtype=torch.float32)
output = kernel.apply_monolithic(
hidden_states=torch.nn.functional.pad(a, (0, padded_k - k)),
w1=w1,
w2=w2,
router_logits=logits,
activation=activation,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
routed_scaling_factor=1.0,
routing_sink=None,
)[:, :k]
topk_weights, topk_ids, _ = fused_topk(a, logits, topk, renormalize=True)
reference = fused_experts(
a,
w1_ref,
w2_ref,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=activation,
global_num_experts=e,
expert_map=None,
)
check_accuracy(ref_output=reference, actual_output=output)