[Online quantization] Add online MXFP4 quantization support (#49347)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
fxmarty-amd
2026-08-07 12:39:46 -07:00
committed by GitHub
parent a671679e9f
commit 3518110f2f
18 changed files with 1423 additions and 35 deletions
@@ -0,0 +1,15 @@
model_name: "Qwen/Qwen3-30B-A3B"
accuracy_threshold: 0.89
tolerance: 0.03
num_questions: 1319
num_fewshot: 5
server_args: >-
--max-model-len 4096
--tensor-parallel-size 2
--gpu-memory-utilization 0.35
--moe-backend aiter
--quantization mxfp4
--quantization-config '{"linear":{"activation":null,"weight":null}}'
env:
VLLM_ROCM_USE_AITER: "1"
ENABLE_CK: "0" # Avoid AITER CK-based MHA JIT compilation to save time
@@ -0,0 +1,14 @@
# Note: this model was produced from Qwen3-30B-A3B (BF16)
model_name: "amd-quark/Qwen3-30B-A3B-MXFP4"
accuracy_threshold: 0.89
tolerance: 0.03
num_questions: 1319
num_fewshot: 5
server_args: >-
--max-model-len 4096
--tensor-parallel-size 2
--gpu-memory-utilization 0.35
--moe-backend aiter
env:
VLLM_ROCM_USE_AITER: "1"
ENABLE_CK: "0" # Avoid AITER CK-based MHA JIT compilation to save time
@@ -1,3 +1,4 @@
# NOTE: this model was produced from Qwen/Qwen3.5-35B-A3B-FP8
model_name: "amd/Qwen3.5-35B-A3B-MXFP4"
accuracy_threshold: 0.89
tolerance: 0.03
@@ -1,3 +1,4 @@
# NOTE: this model was produced from Qwen/Qwen3.5-35B-A3B-FP8
model_name: "amd/Qwen3.5-35B-A3B-MXFP4"
accuracy_threshold: 0.89
tolerance: 0.03