[Quantization] add humming moe backend to all dense/moe oracles (#41652)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Jinzhen Lin
2026-07-06 13:36:07 -07:00
committed by GitHub
co-authored by mgoin
parent 04adc8843b
commit d891b9bd51
73 changed files with 1337 additions and 123 deletions
@@ -0,0 +1,10 @@
model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8"
accuracy_threshold: 0.55
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,8 @@
model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8"
accuracy_threshold: 0.55
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--linear-backend humming
@@ -0,0 +1,10 @@
model_name: "mgoin/Qwen3-0.6B-MXFP8"
accuracy_threshold: 0.39
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,8 @@
model_name: "mgoin/Qwen3-0.6B-MXFP8"
accuracy_threshold: 0.39
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--linear-backend humming
@@ -0,0 +1,13 @@
model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
--dtype bfloat16
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,13 @@
model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
--dtype bfloat16
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -0,0 +1,11 @@
model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
--dtype bfloat16
@@ -0,0 +1,11 @@
model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,9 @@
model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -0,0 +1,11 @@
model_name: "nm-testing/Qwen3-30B-A3B-Fp8-v1"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,9 @@
model_name: "nm-testing/Qwen3-30B-A3B-Fp8-v1"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -0,0 +1,11 @@
model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,11 @@
model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -0,0 +1,9 @@
model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -0,0 +1,11 @@
model_name: "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -0,0 +1,9 @@
model_name: "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -5,8 +5,7 @@ num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -5,6 +5,5 @@ num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
--linear-backend humming
@@ -0,0 +1,9 @@
model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -0,0 +1,13 @@
model_name: "Qwen/Qwen3-30B-A3B"
accuracy_threshold: 0.80
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
env:
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,13 @@
model_name: "Qwen/Qwen3-30B-A3B"
accuracy_threshold: 0.80
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
env:
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -0,0 +1,12 @@
model_name: "Qwen/Qwen3-30B-A3B"
accuracy_threshold: 0.80
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
env:
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
@@ -0,0 +1,12 @@
model_name: "Qwen/Qwen3.5-35B-A3B-FP8"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,10 @@
model_name: "Qwen/Qwen3.5-35B-A3B-FP8"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -0,0 +1,12 @@
model_name: "Qwen/Qwen3.5-35B-A3B"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--quantization experts_int8
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -0,0 +1,10 @@
model_name: "Qwen/Qwen3.5-35B-A3B"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--quantization experts_int8
@@ -0,0 +1,10 @@
model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
accuracy_threshold: 0.82
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,11 @@
model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
accuracy_threshold: 0.82
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -0,0 +1,9 @@
model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
accuracy_threshold: 0.82
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--linear-backend humming
@@ -0,0 +1,10 @@
model_name: "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
accuracy_threshold: 0.91
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -1,2 +1,9 @@
gpt-oss-20b-humming-act-fp8.yaml
Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml
Qwen3-0.6B-MXFP8-humming-act-fp8.yaml
Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml
Qwen3-30B-A3B-FP8-block-humming-act-fp8.yaml
Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml
Qwen3-30B-A3B-AWQ-humming.yaml
Qwen3.5-35B-A3B-FP8-humming-act-fp8.yaml
@@ -0,0 +1,4 @@
Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
Qwen3-30B-A3B-AWQ-humming.yaml
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
@@ -0,0 +1,3 @@
Qwen3-30B-A3B-int5wc-hadamard-humming.yaml
Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml
Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml
@@ -1,2 +1,13 @@
gpt-oss-20b-humming.yaml
Qwen3-30B-A3B-MXFP4A16-humming.yaml
Qwen2-1.5B-Instruct-FP8W8-humming.yaml
Qwen3-0.6B-MXFP8-humming.yaml
Qwen3.6-35B-A3B-NVFP4-humming.yaml
Qwen3-30B-A3B-Fp8-v1-humming.yaml
Qwen3-30B-A3B-FP8-block-humming.yaml
Qwen3-30B-A3B-GPTQ-Int4-humming.yaml
Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml
Qwen3-30B-A3B-NVFP4-humming.yaml
Qwen3-30B-A3B-AWQ-humming.yaml
Qwen3.5-35B-A3B-FP8-humming.yaml
Qwen3.5-35B-A3B-experts-int8-humming.yaml
@@ -5,7 +5,6 @@ num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--moe-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -5,5 +5,4 @@ num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--moe-backend humming
+8 -4
View File
@@ -146,6 +146,7 @@ async def call_vllm_chat_api(
def _build_gsm8k_prompts(
num_questions: int = 1319,
num_shots: int = 5,
gen_prefix: str = "",
) -> tuple[list[str], list[int]]:
"""Build few-shot GSM8K completion prompts and ground-truth labels."""
if num_questions == 0:
@@ -157,14 +158,15 @@ def _build_gsm8k_prompts(
for i in range(num_shots):
few_shot_examples += (
f"Question: {train_data[i]['question']}\n"
f"Answer: {train_data[i]['answer']}\n\n"
f"Answer:{gen_prefix} {train_data[i]['answer']}\n\n"
)
prompts = []
labels = []
for i in range(num_questions):
prompts.append(
few_shot_examples + f"Question: {test_data[i]['question']}\nAnswer:"
few_shot_examples
+ f"Question: {test_data[i]['question']}\nAnswer:{gen_prefix}"
)
labels.append(get_answer_value(test_data[i]["answer"]))
@@ -213,6 +215,7 @@ def evaluate_gsm8k(
temperature: float = 0.0,
seed: int | None = 42,
request_timeout_seconds: float = 600,
gen_prefix: str = "",
) -> dict[str, float | int]:
"""
Evaluate GSM8K accuracy using vLLM serve endpoint.
@@ -220,7 +223,7 @@ def evaluate_gsm8k(
Returns dict with accuracy, invalid_rate, latency, etc.
"""
base_url = f"{host}:{port}"
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots)
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
num_questions = len(prompts)
async def run_async_evaluation():
@@ -278,6 +281,7 @@ def evaluate_gsm8k_offline(
num_shots: int = 5,
max_tokens: int = 256,
temperature: float = 0.0,
gen_prefix: str = "",
) -> dict[str, float | int]:
"""Evaluate GSM8K accuracy using an offline vllm.LLM object.
@@ -286,7 +290,7 @@ def evaluate_gsm8k_offline(
"""
from vllm import SamplingParams
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots)
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
sampling_params = SamplingParams(
temperature=temperature,
@@ -70,6 +70,7 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
host=host,
port=port,
request_timeout_seconds=request_timeout_seconds,
gen_prefix=eval_config.get("gen_prefix", ""),
)
return results