[MoE] Shape-based padding for FlashInfer TRT-LLM MXFP8 experts (gated and non-gated) (#61293)

Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
Jhao-Ting Chen
2026-10-15 13:44:11 -04:00
committed by GitHub
co-authored by Claude Opus 5.5 Robert Shaw
parent 7cd7982dfd
commit 4f49b55c1f
2 changed files with 216 additions and 5 deletions
@@ -379,6 +379,79 @@ def align_moe_weights_for_fi(
return padded_w13, padded_w2, padded_intermediate
def align_mxfp8_moe_weights_for_fi_trtllm(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
is_act_and_mul: bool,
min_alignment: int,
hidden_alignment: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int, int]:
"""align_moe_weights_for_fi for MXFP8 weights and their UE8M0 block scales
for the FlashInfer TRT-LLM MoE kernel, which also pads the hidden size to a
multiple of hidden_alignment.
Padded weights are zero and padded scales are 127 (UE8M0 for 1.0), so the
padded rows and columns contribute nothing. Scales may be 3D
[E, rows, cols] or flattened [E, rows * cols]; their layout is kept.
Returns the padded intermediate and hidden sizes.
"""
num_experts, hidden_size, intermediate = w2.shape
padded_hidden = round_up(hidden_size, hidden_alignment)
w13, w2, padded_intermediate = align_moe_weights_for_fi(
w13, w2, is_act_and_mul, min_alignment
)
if padded_intermediate == intermediate and padded_hidden == hidden_size:
return w13, w2, w13_scale, w2_scale, intermediate, hidden_size
if padded_hidden != hidden_size:
logger.info_once(
"Padding hidden size from %d to %d for MXFP8 TRTLLM MoE kernel.",
hidden_size,
padded_hidden,
)
padded_w13 = w13.new_zeros((num_experts, w13.shape[1], padded_hidden))
padded_w13[:, :, :hidden_size] = w13
padded_w2 = w2.new_zeros((num_experts, padded_hidden, w2.shape[2]))
padded_w2[:, :hidden_size] = w2
w13, w2 = padded_w13, padded_w2
up_mult = 2 if is_act_and_mul else 1
w13_scale_3d = w13_scale.reshape(num_experts, up_mult * intermediate, -1)
w2_scale_3d = w2_scale.reshape(num_experts, hidden_size, -1)
block = hidden_size // w13_scale_3d.shape[2]
assert padded_hidden % block == 0 and padded_intermediate % block == 0
# Pad each of the gate and up halves (or the single non-gated block) on its
# own, like the weights.
padded_w13_scale = w13_scale_3d.new_full(
(num_experts, up_mult * padded_intermediate, padded_hidden // block), 127
)
for i in range(up_mult):
dst = i * padded_intermediate
padded_w13_scale[:, dst : dst + intermediate, : w13_scale_3d.shape[2]] = (
w13_scale_3d[:, i * intermediate : (i + 1) * intermediate]
)
padded_w2_scale = w2_scale_3d.new_full(
(num_experts, padded_hidden, padded_intermediate // block), 127
)
padded_w2_scale[:, :hidden_size, : w2_scale_3d.shape[2]] = w2_scale_3d
if w13_scale.ndim == 2:
padded_w13_scale = padded_w13_scale.reshape(num_experts, -1)
if w2_scale.ndim == 2:
padded_w2_scale = padded_w2_scale.reshape(num_experts, -1)
return (
w13,
w2,
padded_w13_scale,
padded_w2_scale,
padded_intermediate,
padded_hidden,
)
def _shuffle_deepseek_fp8_moe_weights(
w13: torch.Tensor,
w2: torch.Tensor,
@@ -486,8 +559,9 @@ def prepare_fp8_moe_layer_for_fi(
Note that for trtllm we update the model state dict
with the scale format needed for these kernels.
Note that for per-tensor, we update the layer's
intermediate size if the weights needed padding.
Note that for per-tensor and for MXFP8 TRT-LLM, we update the layer's
intermediate size (and for MXFP8 TRT-LLM its hidden size) if the weights
needed padding.
"""
assert hasattr(layer.moe_config, "is_act_and_mul")
block_quant = (
@@ -499,6 +573,21 @@ def prepare_fp8_moe_layer_for_fi(
# MXFP8 TRT-LLM requires W31 swap + reorder + shuffle.
if is_mxfp8 and is_trtllm:
# The kernel needs intermediate % 128 == 0 and hidden % 256 == 0. The
# MoE runner pads the activations to the padded hidden size.
w13, w2, w13_scale, w2_scale, new_intermediate, new_hidden = (
align_mxfp8_moe_weights_for_fi_trtllm(
w13,
w2,
w13_scale,
w2_scale,
layer.moe_config.is_act_and_mul,
min_alignment=128,
hidden_alignment=256,
)
)
layer.moe_config.intermediate_size_per_partition = new_intermediate
layer.moe_config.hidden_dim = new_hidden
# FlashInfer TRT-LLM SwiGLU expects [up; gate] but vLLM stores
# [gate; up]. Swap both weights and scales before interleaving.
if layer.moe_config.is_act_and_mul: