[Quantization] add humming moe backend to all dense/moe oracles (#41652)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8"
|
||||
accuracy_threshold: 0.55
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--linear-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
|
||||
@@ -0,0 +1,8 @@
|
||||
model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8"
|
||||
accuracy_threshold: 0.55
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--linear-backend humming
|
||||
@@ -0,0 +1,10 @@
|
||||
model_name: "mgoin/Qwen3-0.6B-MXFP8"
|
||||
accuracy_threshold: 0.39
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--linear-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
|
||||
@@ -0,0 +1,8 @@
|
||||
model_name: "mgoin/Qwen3-0.6B-MXFP8"
|
||||
accuracy_threshold: 0.39
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--linear-backend humming
|
||||
@@ -0,0 +1,13 @@
|
||||
model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
--dtype bfloat16
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
|
||||
@@ -0,0 +1,13 @@
|
||||
model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
--dtype bfloat16
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
|
||||
@@ -0,0 +1,11 @@
|
||||
model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
--dtype bfloat16
|
||||
@@ -0,0 +1,11 @@
|
||||
model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
|
||||
accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
|
||||
@@ -0,0 +1,9 @@
|
||||
model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
|
||||
accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
@@ -0,0 +1,11 @@
|
||||
model_name: "nm-testing/Qwen3-30B-A3B-Fp8-v1"
|
||||
accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
|
||||
@@ -0,0 +1,9 @@
|
||||
model_name: "nm-testing/Qwen3-30B-A3B-Fp8-v1"
|
||||
accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
@@ -0,0 +1,11 @@
|
||||
model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
|
||||
accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
|
||||
@@ -0,0 +1,11 @@
|
||||
model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
|
||||
accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
|
||||
@@ -0,0 +1,9 @@
|
||||
model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
|
||||
accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
model_name: "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8"
|
||||
accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
model_name: "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8"
|
||||
accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
@@ -5,8 +5,7 @@ num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--tensor-parallel-size 1
|
||||
--quantization humming
|
||||
--kernel-config.enable_flashinfer_autotune=False
|
||||
--linear-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
|
||||
|
||||
@@ -5,6 +5,5 @@ num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--tensor-parallel-size 1
|
||||
--quantization humming
|
||||
--kernel-config.enable_flashinfer_autotune=False
|
||||
--linear-backend humming
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
|
||||
accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
@@ -0,0 +1,13 @@
|
||||
model_name: "Qwen/Qwen3-30B-A3B"
|
||||
accuracy_threshold: 0.80
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--tensor-parallel-size 1
|
||||
--quantization humming
|
||||
--kernel-config.enable_flashinfer_autotune=False
|
||||
env:
|
||||
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
|
||||
@@ -0,0 +1,13 @@
|
||||
model_name: "Qwen/Qwen3-30B-A3B"
|
||||
accuracy_threshold: 0.80
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--tensor-parallel-size 1
|
||||
--quantization humming
|
||||
--kernel-config.enable_flashinfer_autotune=False
|
||||
env:
|
||||
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
|
||||
@@ -0,0 +1,12 @@
|
||||
model_name: "Qwen/Qwen3-30B-A3B"
|
||||
accuracy_threshold: 0.80
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--tensor-parallel-size 1
|
||||
--quantization humming
|
||||
--kernel-config.enable_flashinfer_autotune=False
|
||||
env:
|
||||
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
|
||||
@@ -0,0 +1,12 @@
|
||||
model_name: "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
|
||||
@@ -0,0 +1,10 @@
|
||||
model_name: "Qwen/Qwen3.5-35B-A3B-FP8"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
@@ -0,0 +1,12 @@
|
||||
model_name: "Qwen/Qwen3.5-35B-A3B"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--quantization experts_int8
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
|
||||
@@ -0,0 +1,10 @@
|
||||
model_name: "Qwen/Qwen3.5-35B-A3B"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--quantization experts_int8
|
||||
@@ -0,0 +1,10 @@
|
||||
model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
|
||||
accuracy_threshold: 0.82
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
|
||||
@@ -0,0 +1,11 @@
|
||||
model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
|
||||
accuracy_threshold: 0.82
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--linear-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
|
||||
@@ -0,0 +1,9 @@
|
||||
model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
|
||||
accuracy_threshold: 0.82
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--linear-backend humming
|
||||
@@ -0,0 +1,10 @@
|
||||
model_name: "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
|
||||
accuracy_threshold: 0.91
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
@@ -1,2 +1,9 @@
|
||||
gpt-oss-20b-humming-act-fp8.yaml
|
||||
Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
|
||||
Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml
|
||||
Qwen3-0.6B-MXFP8-humming-act-fp8.yaml
|
||||
Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml
|
||||
Qwen3-30B-A3B-FP8-block-humming-act-fp8.yaml
|
||||
Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml
|
||||
Qwen3-30B-A3B-AWQ-humming.yaml
|
||||
Qwen3.5-35B-A3B-FP8-humming-act-fp8.yaml
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
|
||||
Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
|
||||
Qwen3-30B-A3B-AWQ-humming.yaml
|
||||
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
|
||||
@@ -0,0 +1,3 @@
|
||||
Qwen3-30B-A3B-int5wc-hadamard-humming.yaml
|
||||
Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml
|
||||
Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml
|
||||
@@ -1,2 +1,13 @@
|
||||
gpt-oss-20b-humming.yaml
|
||||
Qwen3-30B-A3B-MXFP4A16-humming.yaml
|
||||
Qwen2-1.5B-Instruct-FP8W8-humming.yaml
|
||||
Qwen3-0.6B-MXFP8-humming.yaml
|
||||
Qwen3.6-35B-A3B-NVFP4-humming.yaml
|
||||
Qwen3-30B-A3B-Fp8-v1-humming.yaml
|
||||
Qwen3-30B-A3B-FP8-block-humming.yaml
|
||||
Qwen3-30B-A3B-GPTQ-Int4-humming.yaml
|
||||
Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml
|
||||
Qwen3-30B-A3B-NVFP4-humming.yaml
|
||||
Qwen3-30B-A3B-AWQ-humming.yaml
|
||||
Qwen3.5-35B-A3B-FP8-humming.yaml
|
||||
Qwen3.5-35B-A3B-experts-int8-humming.yaml
|
||||
|
||||
@@ -5,7 +5,6 @@ num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--tensor-parallel-size 1
|
||||
--moe-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
|
||||
|
||||
@@ -5,5 +5,4 @@ num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--tensor-parallel-size 1
|
||||
--moe-backend humming
|
||||
|
||||
@@ -146,6 +146,7 @@ async def call_vllm_chat_api(
|
||||
def _build_gsm8k_prompts(
|
||||
num_questions: int = 1319,
|
||||
num_shots: int = 5,
|
||||
gen_prefix: str = "",
|
||||
) -> tuple[list[str], list[int]]:
|
||||
"""Build few-shot GSM8K completion prompts and ground-truth labels."""
|
||||
if num_questions == 0:
|
||||
@@ -157,14 +158,15 @@ def _build_gsm8k_prompts(
|
||||
for i in range(num_shots):
|
||||
few_shot_examples += (
|
||||
f"Question: {train_data[i]['question']}\n"
|
||||
f"Answer: {train_data[i]['answer']}\n\n"
|
||||
f"Answer:{gen_prefix} {train_data[i]['answer']}\n\n"
|
||||
)
|
||||
|
||||
prompts = []
|
||||
labels = []
|
||||
for i in range(num_questions):
|
||||
prompts.append(
|
||||
few_shot_examples + f"Question: {test_data[i]['question']}\nAnswer:"
|
||||
few_shot_examples
|
||||
+ f"Question: {test_data[i]['question']}\nAnswer:{gen_prefix}"
|
||||
)
|
||||
labels.append(get_answer_value(test_data[i]["answer"]))
|
||||
|
||||
@@ -213,6 +215,7 @@ def evaluate_gsm8k(
|
||||
temperature: float = 0.0,
|
||||
seed: int | None = 42,
|
||||
request_timeout_seconds: float = 600,
|
||||
gen_prefix: str = "",
|
||||
) -> dict[str, float | int]:
|
||||
"""
|
||||
Evaluate GSM8K accuracy using vLLM serve endpoint.
|
||||
@@ -220,7 +223,7 @@ def evaluate_gsm8k(
|
||||
Returns dict with accuracy, invalid_rate, latency, etc.
|
||||
"""
|
||||
base_url = f"{host}:{port}"
|
||||
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots)
|
||||
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
|
||||
num_questions = len(prompts)
|
||||
|
||||
async def run_async_evaluation():
|
||||
@@ -278,6 +281,7 @@ def evaluate_gsm8k_offline(
|
||||
num_shots: int = 5,
|
||||
max_tokens: int = 256,
|
||||
temperature: float = 0.0,
|
||||
gen_prefix: str = "",
|
||||
) -> dict[str, float | int]:
|
||||
"""Evaluate GSM8K accuracy using an offline vllm.LLM object.
|
||||
|
||||
@@ -286,7 +290,7 @@ def evaluate_gsm8k_offline(
|
||||
"""
|
||||
from vllm import SamplingParams
|
||||
|
||||
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots)
|
||||
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
|
||||
|
||||
sampling_params = SamplingParams(
|
||||
temperature=temperature,
|
||||
|
||||
@@ -70,6 +70,7 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
|
||||
host=host,
|
||||
port=port,
|
||||
request_timeout_seconds=request_timeout_seconds,
|
||||
gen_prefix=eval_config.get("gen_prefix", ""),
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
Reference in New Issue
Block a user