[quantization] Add per-token NVFP4 quantization MoE support for ReLU2 (#56740)

Signed-off-by: Xuanteng Huang <xuantengh@nvidia.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Xuanteng Huang
2026-10-07 22:01:09 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 OpenAI Codex
parent b803edbc9f
commit f38c6778ce
18 changed files with 450 additions and 61 deletions
@@ -344,6 +344,7 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
w2_scale_2: torch.Tensor,
a2_scale: torch.Tensor,
is_act_and_mul: bool,
trtllm_hidden_alignment: int = 256,
) -> tuple[
torch.Tensor,
torch.Tensor,
@@ -398,7 +399,11 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
# Shuffle weights and scales for FI TRTLLM NVFP4 MoE kernels.
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM:
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
w13, w13_scale, w2, w2_scale
w13,
w13_scale,
w2,
w2_scale,
min_alignment=trtllm_hidden_alignment,
)
if layer.moe_config.hidden_dim_unpadded is None:
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
@@ -280,6 +280,20 @@ def align_fp4_moe_weights_for_fi(
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate
def trtllm_nvfp4_hidden_alignment(
per_token_activation: bool, is_act_and_mul: bool
) -> int:
"""Hidden-dim alignment for the TRTLLM-Gen NVFP4 MoE cubins.
The per-token (dynamic activation scale) non-gated variant has far fewer
tile configs for 256-aligned K and its heuristic default tactic fails with
"No valid config found" (e.g. Nemotron's 2688 -> 2816), so pad to 512.
"""
if per_token_activation and not is_act_and_mul:
return 512
return 256
def align_fp4_moe_hidden_dim_for_fi(
w13: torch.Tensor,
w13_scale: torch.Tensor,