[quantization] Add per-token NVFP4 quantization MoE support for ReLU2 (#56740)
Signed-off-by: Xuanteng Huang <xuantengh@nvidia.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
OpenAI Codex
parent
b803edbc9f
commit
f38c6778ce
@@ -344,6 +344,7 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
w2_scale_2: torch.Tensor,
|
||||
a2_scale: torch.Tensor,
|
||||
is_act_and_mul: bool,
|
||||
trtllm_hidden_alignment: int = 256,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
@@ -398,7 +399,11 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
# Shuffle weights and scales for FI TRTLLM NVFP4 MoE kernels.
|
||||
if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM:
|
||||
w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi(
|
||||
w13, w13_scale, w2, w2_scale
|
||||
w13,
|
||||
w13_scale,
|
||||
w2,
|
||||
w2_scale,
|
||||
min_alignment=trtllm_hidden_alignment,
|
||||
)
|
||||
if layer.moe_config.hidden_dim_unpadded is None:
|
||||
layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim
|
||||
|
||||
@@ -280,6 +280,20 @@ def align_fp4_moe_weights_for_fi(
|
||||
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate
|
||||
|
||||
|
||||
def trtllm_nvfp4_hidden_alignment(
|
||||
per_token_activation: bool, is_act_and_mul: bool
|
||||
) -> int:
|
||||
"""Hidden-dim alignment for the TRTLLM-Gen NVFP4 MoE cubins.
|
||||
|
||||
The per-token (dynamic activation scale) non-gated variant has far fewer
|
||||
tile configs for 256-aligned K and its heuristic default tactic fails with
|
||||
"No valid config found" (e.g. Nemotron's 2688 -> 2816), so pad to 512.
|
||||
"""
|
||||
if per_token_activation and not is_act_and_mul:
|
||||
return 512
|
||||
return 256
|
||||
|
||||
|
||||
def align_fp4_moe_hidden_dim_for_fi(
|
||||
w13: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user