[MoE] Shape-based padding for FlashInfer TRT-LLM MXFP8 experts (gated and non-gated) (#61293)
Signed-off-by: Jhao-Ting Chen <jhaotingc@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Robert Shaw
parent
7cd7982dfd
commit
4f49b55c1f
@@ -379,6 +379,79 @@ def align_moe_weights_for_fi(
|
||||
return padded_w13, padded_w2, padded_intermediate
|
||||
|
||||
|
||||
def align_mxfp8_moe_weights_for_fi_trtllm(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
is_act_and_mul: bool,
|
||||
min_alignment: int,
|
||||
hidden_alignment: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int, int]:
|
||||
"""align_moe_weights_for_fi for MXFP8 weights and their UE8M0 block scales
|
||||
for the FlashInfer TRT-LLM MoE kernel, which also pads the hidden size to a
|
||||
multiple of hidden_alignment.
|
||||
|
||||
Padded weights are zero and padded scales are 127 (UE8M0 for 1.0), so the
|
||||
padded rows and columns contribute nothing. Scales may be 3D
|
||||
[E, rows, cols] or flattened [E, rows * cols]; their layout is kept.
|
||||
Returns the padded intermediate and hidden sizes.
|
||||
"""
|
||||
num_experts, hidden_size, intermediate = w2.shape
|
||||
padded_hidden = round_up(hidden_size, hidden_alignment)
|
||||
w13, w2, padded_intermediate = align_moe_weights_for_fi(
|
||||
w13, w2, is_act_and_mul, min_alignment
|
||||
)
|
||||
if padded_intermediate == intermediate and padded_hidden == hidden_size:
|
||||
return w13, w2, w13_scale, w2_scale, intermediate, hidden_size
|
||||
|
||||
if padded_hidden != hidden_size:
|
||||
logger.info_once(
|
||||
"Padding hidden size from %d to %d for MXFP8 TRTLLM MoE kernel.",
|
||||
hidden_size,
|
||||
padded_hidden,
|
||||
)
|
||||
padded_w13 = w13.new_zeros((num_experts, w13.shape[1], padded_hidden))
|
||||
padded_w13[:, :, :hidden_size] = w13
|
||||
padded_w2 = w2.new_zeros((num_experts, padded_hidden, w2.shape[2]))
|
||||
padded_w2[:, :hidden_size] = w2
|
||||
w13, w2 = padded_w13, padded_w2
|
||||
|
||||
up_mult = 2 if is_act_and_mul else 1
|
||||
w13_scale_3d = w13_scale.reshape(num_experts, up_mult * intermediate, -1)
|
||||
w2_scale_3d = w2_scale.reshape(num_experts, hidden_size, -1)
|
||||
block = hidden_size // w13_scale_3d.shape[2]
|
||||
assert padded_hidden % block == 0 and padded_intermediate % block == 0
|
||||
|
||||
# Pad each of the gate and up halves (or the single non-gated block) on its
|
||||
# own, like the weights.
|
||||
padded_w13_scale = w13_scale_3d.new_full(
|
||||
(num_experts, up_mult * padded_intermediate, padded_hidden // block), 127
|
||||
)
|
||||
for i in range(up_mult):
|
||||
dst = i * padded_intermediate
|
||||
padded_w13_scale[:, dst : dst + intermediate, : w13_scale_3d.shape[2]] = (
|
||||
w13_scale_3d[:, i * intermediate : (i + 1) * intermediate]
|
||||
)
|
||||
padded_w2_scale = w2_scale_3d.new_full(
|
||||
(num_experts, padded_hidden, padded_intermediate // block), 127
|
||||
)
|
||||
padded_w2_scale[:, :hidden_size, : w2_scale_3d.shape[2]] = w2_scale_3d
|
||||
|
||||
if w13_scale.ndim == 2:
|
||||
padded_w13_scale = padded_w13_scale.reshape(num_experts, -1)
|
||||
if w2_scale.ndim == 2:
|
||||
padded_w2_scale = padded_w2_scale.reshape(num_experts, -1)
|
||||
return (
|
||||
w13,
|
||||
w2,
|
||||
padded_w13_scale,
|
||||
padded_w2_scale,
|
||||
padded_intermediate,
|
||||
padded_hidden,
|
||||
)
|
||||
|
||||
|
||||
def _shuffle_deepseek_fp8_moe_weights(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
@@ -486,8 +559,9 @@ def prepare_fp8_moe_layer_for_fi(
|
||||
Note that for trtllm we update the model state dict
|
||||
with the scale format needed for these kernels.
|
||||
|
||||
Note that for per-tensor, we update the layer's
|
||||
intermediate size if the weights needed padding.
|
||||
Note that for per-tensor and for MXFP8 TRT-LLM, we update the layer's
|
||||
intermediate size (and for MXFP8 TRT-LLM its hidden size) if the weights
|
||||
needed padding.
|
||||
"""
|
||||
assert hasattr(layer.moe_config, "is_act_and_mul")
|
||||
block_quant = (
|
||||
@@ -499,6 +573,21 @@ def prepare_fp8_moe_layer_for_fi(
|
||||
|
||||
# MXFP8 TRT-LLM requires W31 swap + reorder + shuffle.
|
||||
if is_mxfp8 and is_trtllm:
|
||||
# The kernel needs intermediate % 128 == 0 and hidden % 256 == 0. The
|
||||
# MoE runner pads the activations to the padded hidden size.
|
||||
w13, w2, w13_scale, w2_scale, new_intermediate, new_hidden = (
|
||||
align_mxfp8_moe_weights_for_fi_trtllm(
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
layer.moe_config.is_act_and_mul,
|
||||
min_alignment=128,
|
||||
hidden_alignment=256,
|
||||
)
|
||||
)
|
||||
layer.moe_config.intermediate_size_per_partition = new_intermediate
|
||||
layer.moe_config.hidden_dim = new_hidden
|
||||
# FlashInfer TRT-LLM SwiGLU expects [up; gate] but vLLM stores
|
||||
# [gate; up]. Swap both weights and scales before interleaving.
|
||||
if layer.moe_config.is_act_and_mul:
|
||||
|
||||
Reference in New Issue
Block a user