[MoE Refactor][15/N] Apply Refactor to Fp8 (#31415)
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"
|
||||
accuracy_threshold: 0.92
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
|
||||
accuracy_threshold: 0.88
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_high_throughput"
|
||||
env:
|
||||
VLLM_USE_DEEP_GEMM: "1"
|
||||
VLLM_USE_DEEP_GEMM_MOE: "1"
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
|
||||
accuracy_threshold: 0.88
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --disable-uvicorn-access-log"
|
||||
env:
|
||||
VLLM_USE_DEEP_GEMM: "1"
|
||||
VLLM_USE_DEEP_GEMM_MOE: "1"
|
||||
VLLM_USE_DEEP_GEMM_E8M0: "0"
|
||||
@@ -0,0 +1,8 @@
|
||||
model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
|
||||
accuracy_threshold: 0.88
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"
|
||||
env:
|
||||
VLLM_USE_DEEP_GEMM: "1"
|
||||
VLLM_USE_DEEP_GEMM_MOE: "1"
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
|
||||
accuracy_threshold: 0.85
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_high_throughput"
|
||||
env:
|
||||
VLLM_USE_DEEP_GEMM: "1"
|
||||
VLLM_USE_DEEP_GEMM_MOE: "1"
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
|
||||
accuracy_threshold: 0.85
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --disable-uvicorn-access-log"
|
||||
env:
|
||||
VLLM_USE_DEEP_GEMM: "1"
|
||||
VLLM_USE_DEEP_GEMM_MOE: "1"
|
||||
VLLM_USE_DEEP_GEMM_E8M0: "0"
|
||||
@@ -0,0 +1,8 @@
|
||||
model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
|
||||
accuracy_threshold: 0.85
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"
|
||||
env:
|
||||
VLLM_USE_DEEP_GEMM: "1"
|
||||
VLLM_USE_DEEP_GEMM_MOE: "1"
|
||||
@@ -0,0 +1,8 @@
|
||||
model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"
|
||||
accuracy_threshold: 0.88
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"
|
||||
env:
|
||||
VLLM_USE_FLASHINFER_MOE_FP4: "1"
|
||||
VLLM_FLASHINFER_MOE_BACKEND: "throughput"
|
||||
@@ -0,0 +1,8 @@
|
||||
Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ht.yaml
|
||||
Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ll.yaml
|
||||
Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ht.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm.yaml
|
||||
Qwen3-30B-A3B-NvFp4-CT-fi-cutlass.yaml
|
||||
Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutlass-fi-a2av.yaml
|
||||
@@ -0,0 +1,5 @@
|
||||
model_name: "RedHatAI/Llama-4-Scout-17B-16E-Instruct-FP8-dynamic"
|
||||
accuracy_threshold: 0.92
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
|
||||
@@ -1,3 +1,4 @@
|
||||
Llama-4-Scout-Fp8-CT-vllm-cutlass.yaml
|
||||
Llama-4-Scout-Fp8-ModelOpt-fi-trtllm.yaml
|
||||
Qwen3-30B-A3B-Fp8-AutoFp8-fi-trtllm.yaml
|
||||
Qwen3-30B-A3B-NvFp4-CT-vllm-cutlass.yaml
|
||||
|
||||
@@ -5,7 +5,7 @@ Qwen3-30B-A3B-Fp8-AutoFp8-marlin.yaml
|
||||
Qwen3-30B-A3B-Fp8-AutoFp8-triton.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-marlin.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-vllm-cutlass.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-triton.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Channel-marlin.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Channel-vllm-cutlass.yaml
|
||||
Llama-4-Scout-Fp8-ModelOpt-fi-cutlass.yaml
|
||||
|
||||
@@ -61,6 +61,7 @@ def test_gsm8k_correctness(config_filename):
|
||||
server_args.extend(
|
||||
[
|
||||
"--trust-remote-code",
|
||||
"--disable-uvicorn-access-log",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user