[quantization] Add per-token NVFP4 quantization MoE support for ReLU2 (#56740)
Signed-off-by: Xuanteng Huang <xuantengh@nvidia.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
OpenAI Codex
parent
b803edbc9f
commit
f38c6778ce
@@ -0,0 +1,14 @@
|
||||
model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
|
||||
accuracy_threshold: 0.93
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
startup_max_wait_seconds: 1200
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 4096
|
||||
--tensor-parallel-size 2
|
||||
--enable-expert-parallel
|
||||
--mamba-backend flashinfer
|
||||
--moe-backend flashinfer_trtllm
|
||||
--quantization-config.moe.activation nvfp4_per_token
|
||||
--speculative-config '{"method":"mtp","num_speculative_tokens":5}'
|
||||
@@ -1,3 +1,4 @@
|
||||
Qwen3-Next-80B-A3B-NVFP4-EP2.yaml
|
||||
Qwen3-Next-FP8-EP2.yaml
|
||||
Nemotron-3-Super-120B-A12B-NVFP4.yaml
|
||||
Nemotron-3-Super-120B-A12B-NVFP4-per-token.yaml
|
||||
|
||||
Reference in New Issue
Block a user