[MoE] Shape-based padding for FlashInfer TRT-LLM MXFP8 experts (gated and non-gated) (#61293)
Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Robert Shaw
parent
7cd7982dfd
commit
4f49b55c1f
@@ -52,6 +52,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import input_to_float8
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
dequant_mxfp8_to_bf16,
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
@@ -65,7 +66,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.model_executor.models.llama4 import Llama4MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.math_utils import next_power_of_2, round_up
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
try:
|
||||
@@ -788,7 +789,7 @@ def test_trtllm_fp8_swiglu_clamp_support(
|
||||
|
||||
|
||||
def _make_mxfp8_moe_weights(
|
||||
e: int, n: int, k: int
|
||||
e: int, n: int, k: int, is_gated: bool = True
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
q, s = zip(
|
||||
@@ -796,8 +797,9 @@ def _make_mxfp8_moe_weights(
|
||||
)
|
||||
return torch.stack(q), torch.stack(s)
|
||||
|
||||
w1_rows = 2 * n if is_gated else n
|
||||
w1, w1_scale = quantize(
|
||||
torch.randn(e, 2 * n, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||
torch.randn(e, w1_rows, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||
)
|
||||
w2, w2_scale = quantize(
|
||||
torch.randn(e, k, n, device="cuda", dtype=torch.bfloat16) / 10
|
||||
@@ -986,3 +988,123 @@ def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_in
|
||||
)
|
||||
|
||||
torch.testing.assert_close(fused, reference, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation", "n", "k"),
|
||||
[
|
||||
(MoEActivation.SILU, 1024, 1024),
|
||||
(MoEActivation.SILU, 928, 1024),
|
||||
(MoEActivation.SILU, 1024, 2688),
|
||||
(MoEActivation.RELU2_NO_MUL, 1024, 1024),
|
||||
# Nemotron-3.5 Lightning's non-gated MoE: intermediate 1856, hidden 2688.
|
||||
(MoEActivation.RELU2_NO_MUL, 1856, 2688),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("m", [1, 32])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
|
||||
)
|
||||
def test_trtllm_mxfp8_moe_pads_intermediate(
|
||||
m: int, activation: MoEActivation, n: int, k: int, workspace_init
|
||||
):
|
||||
"""TRTLLM-Gen MXFP8 experts need the intermediate size to be a multiple
|
||||
of 128 and the hidden size a multiple of 256. Misaligned experts, gated or
|
||||
not, are padded at load and, with the activations padded and the output
|
||||
cropped as the MoE runner does, must still match the unquantized reference
|
||||
on the dequantized weights."""
|
||||
e, topk = 8, 2
|
||||
is_gated = activation.is_gated
|
||||
block_shape = [1, 32]
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
w1, w2, w1_scale, w2_scale = _make_mxfp8_moe_weights(e, n, k, is_gated)
|
||||
w1_ref = dequant_mxfp8_to_bf16(w1, w1_scale)
|
||||
w2_ref = dequant_mxfp8_to_bf16(w2, w2_scale)
|
||||
|
||||
layer = SimpleNamespace(
|
||||
weight_block_size=block_shape,
|
||||
moe_config=SimpleNamespace(
|
||||
is_act_and_mul=is_gated, intermediate_size_per_partition=n
|
||||
),
|
||||
activation=activation,
|
||||
)
|
||||
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
layer,
|
||||
w1,
|
||||
w2,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
padded_n = layer.moe_config.intermediate_size_per_partition
|
||||
padded_k = layer.moe_config.hidden_dim
|
||||
assert padded_n == round_up(n, 128)
|
||||
assert padded_k == round_up(k, 256)
|
||||
|
||||
quant_config = make_fp8_moe_quant_config(
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=padded_k,
|
||||
intermediate_size=padded_n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=activation,
|
||||
device="cuda",
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
# The monolithic experts pass the activation type to the kernel; the
|
||||
# modular ones always run it gated.
|
||||
kernel = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=True,
|
||||
),
|
||||
TrtLlmFp8ExpertsMonolithic(
|
||||
moe_config=moe_config, quant_config=quant_config
|
||||
),
|
||||
)
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
|
||||
logits = torch.randn((m, e), device="cuda", dtype=torch.float32)
|
||||
output = kernel.apply_monolithic(
|
||||
hidden_states=torch.nn.functional.pad(a, (0, padded_k - k)),
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
router_logits=logits,
|
||||
activation=activation,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
routed_scaling_factor=1.0,
|
||||
routing_sink=None,
|
||||
)[:, :k]
|
||||
topk_weights, topk_ids, _ = fused_topk(a, logits, topk, renormalize=True)
|
||||
reference = fused_experts(
|
||||
a,
|
||||
w1_ref,
|
||||
w2_ref,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
check_accuracy(ref_output=reference, actual_output=output)
|
||||
|
||||
Reference in New Issue
Block a user