[Kernel] Support TRTLLM GEN NVFP4 MoE for non-512-aligned hidden dims via weight padding (#39510)

Signed-off-by: root <root@lyris0017.lyris.clusters.nvidia.com>
Signed-off-by: Daniel Afrimi <dafrimi@nvidia.com>
Co-authored-by: root <root@lyris0017.lyris.clusters.nvidia.com>
This commit is contained in:
danielafrimi
2026-04-14 11:49:56 -07:00
committed by GitHub
co-authored by root
parent dc8df110bc
commit 240f2636ca
4 changed files with 124 additions and 3 deletions
@@ -10,6 +10,7 @@ import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
align_fp4_moe_weights_for_fi,
align_trtllm_fp4_moe_hidden_dim_for_fi,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
swizzle_blockscale,
@@ -341,6 +342,13 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
# Shuffle weights and scales for FI TRTLLM NVFP4 MoE kernels.
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM:
w13, w13_scale, w2, w2_scale, padded_hidden = (
align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale)
)
if layer.moe_config.hidden_dim_unpadded is None:
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
layer.moe_config.hidden_dim = padded_hidden
# Align weights for FI NVFP4 MoE kernels.
min_alignment = 16 if is_gated else 128
w13, w13_scale, w2, w2_scale, padded_intermediate = (
@@ -265,6 +265,48 @@ def align_fp4_moe_weights_for_fi(
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate
def align_trtllm_fp4_moe_hidden_dim_for_fi(
w13: torch.Tensor,
w13_scale: torch.Tensor,
w2: torch.Tensor,
w2_scale: torch.Tensor,
min_alignment: int = 256,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]:
num_experts, gate_up_dim, packed_hidden_size = w13.shape
hidden_size = packed_hidden_size * 2
padded_hidden_size = round_up(hidden_size, min_alignment)
if padded_hidden_size == hidden_size:
return w13, w13_scale, w2, w2_scale, hidden_size
logger.warning_once(
"Padding hidden size from %d to %d for TRTLLM NVFP4 MoE weights. "
"This requires activation slicing at runtime and may cause "
"performance degradation.",
hidden_size,
padded_hidden_size,
scope="local",
)
padded_w13 = w13.new_zeros((num_experts, gate_up_dim, padded_hidden_size // 2))
padded_w13[:, :, :packed_hidden_size] = w13
padded_w13_scale = w13_scale.new_zeros(
(num_experts, gate_up_dim, padded_hidden_size // 16)
)
padded_w13_scale[:, :, : w13_scale.shape[2]] = w13_scale
padded_w2 = w2.new_zeros((num_experts, padded_hidden_size, w2.shape[2]))
padded_w2[:, : w2.shape[1], :] = w2
padded_w2_scale = w2_scale.new_zeros(
(num_experts, padded_hidden_size, w2_scale.shape[2])
)
padded_w2_scale[:, : w2_scale.shape[1], :] = w2_scale
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_hidden_size
def align_fp8_moe_weights_for_fi(
w13: torch.Tensor, w2: torch.Tensor, is_act_and_mul: bool, min_alignment: int = 16
) -> tuple[torch.Tensor, torch.Tensor, int]: