[Online quantization] Add online MXFP4 quantization support (#49347)
Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
@@ -0,0 +1,15 @@
|
||||
model_name: "Qwen/Qwen3-30B-A3B"
|
||||
accuracy_threshold: 0.89
|
||||
tolerance: 0.03
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--max-model-len 4096
|
||||
--tensor-parallel-size 2
|
||||
--gpu-memory-utilization 0.35
|
||||
--moe-backend aiter
|
||||
--quantization mxfp4
|
||||
--quantization-config '{"linear":{"activation":null,"weight":null}}'
|
||||
env:
|
||||
VLLM_ROCM_USE_AITER: "1"
|
||||
ENABLE_CK: "0" # Avoid AITER CK-based MHA JIT compilation to save time
|
||||
@@ -0,0 +1,14 @@
|
||||
# Note: this model was produced from Qwen3-30B-A3B (BF16)
|
||||
model_name: "amd-quark/Qwen3-30B-A3B-MXFP4"
|
||||
accuracy_threshold: 0.89
|
||||
tolerance: 0.03
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--max-model-len 4096
|
||||
--tensor-parallel-size 2
|
||||
--gpu-memory-utilization 0.35
|
||||
--moe-backend aiter
|
||||
env:
|
||||
VLLM_ROCM_USE_AITER: "1"
|
||||
ENABLE_CK: "0" # Avoid AITER CK-based MHA JIT compilation to save time
|
||||
@@ -1,3 +1,4 @@
|
||||
# NOTE: this model was produced from Qwen/Qwen3.5-35B-A3B-FP8
|
||||
model_name: "amd/Qwen3.5-35B-A3B-MXFP4"
|
||||
accuracy_threshold: 0.89
|
||||
tolerance: 0.03
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# NOTE: this model was produced from Qwen/Qwen3.5-35B-A3B-FP8
|
||||
model_name: "amd/Qwen3.5-35B-A3B-MXFP4"
|
||||
accuracy_threshold: 0.89
|
||||
tolerance: 0.03
|
||||
|
||||
Reference in New Issue
Block a user