[quantization] Add per-token NVFP4 quantization MoE support for ReLU2 (#56740)

Signed-off-by: Xuanteng Huang <xuantengh@nvidia.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Xuanteng Huang
2026-10-07 22:01:09 +00:00
committed by GitHub
co-authored by Claude Fable 5.1 OpenAI Codex
parent b803edbc9f
commit f38c6778ce
18 changed files with 450 additions and 61 deletions
@@ -0,0 +1,14 @@
model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
accuracy_threshold: 0.93
num_questions: 1319
num_fewshot: 5
startup_max_wait_seconds: 1200
server_args: >-
--enforce-eager
--max-model-len 4096
--tensor-parallel-size 2
--enable-expert-parallel
--mamba-backend flashinfer
--moe-backend flashinfer_trtllm
--quantization-config.moe.activation nvfp4_per_token
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
@@ -1,3 +1,4 @@
Qwen3-Next-80B-A3B-NVFP4-EP2.yaml
Qwen3-Next-FP8-EP2.yaml
Nemotron-3-Super-120B-A12B-NVFP4.yaml
Nemotron-3-Super-120B-A12B-NVFP4-per-token.yaml