[Feature][Humming] Humming feature integration (#56685)
Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
27757dde02
commit
7d99c2c4fd
@@ -9,3 +9,5 @@ server_args: >-
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
--dtype bfloat16
|
||||
env:
|
||||
VLLM_HUMMING_MOE_GEMM_TYPE: "grouped_contiguous"
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
|
||||
accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
--data-parallel-size 2
|
||||
--enable-expert-parallel
|
||||
--all2all-backend deepep_high_throughput
|
||||
env:
|
||||
VLLM_HUMMING_MOE_GEMM_TYPE: "grouped_contiguous"
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
|
||||
accuracy_threshold: 0.86
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
--data-parallel-size 2
|
||||
--enable-expert-parallel
|
||||
--all2all-backend deepep_high_throughput
|
||||
env:
|
||||
VLLM_HUMMING_MOE_GEMM_TYPE: "indexed"
|
||||
@@ -7,3 +7,5 @@ server_args: >-
|
||||
--max-model-len 8192
|
||||
--quantization humming
|
||||
--linear-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_MOE_GEMM_TYPE: "grouped_contiguous"
|
||||
|
||||
@@ -7,3 +7,5 @@ server_args: >-
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
env:
|
||||
VLLM_HUMMING_MOE_GEMM_TYPE: "grouped_contiguous"
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
model_name: "Qwen/Qwen3-30B-A3B"
|
||||
accuracy_threshold: 0.80
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--tensor-parallel-size 1
|
||||
--quantization humming
|
||||
--kernel-config.enable_flashinfer_autotune=False
|
||||
env:
|
||||
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
|
||||
+4
-3
@@ -1,7 +1,8 @@
|
||||
model_name: "Qwen/Qwen3-30B-A3B"
|
||||
accuracy_threshold: 0.80
|
||||
model_name: "Qwen/Qwen3.5-35B-A3B"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
@@ -9,5 +10,5 @@ server_args: >-
|
||||
--quantization humming
|
||||
--kernel-config.enable_flashinfer_autotune=False
|
||||
env:
|
||||
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
|
||||
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int6","hadamard_block_size":-1}'
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
|
||||
+4
-3
@@ -1,7 +1,8 @@
|
||||
model_name: "Qwen/Qwen3-30B-A3B"
|
||||
accuracy_threshold: 0.80
|
||||
model_name: "Qwen/Qwen3.5-35B-A3B"
|
||||
accuracy_threshold: 0.90
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
@@ -9,4 +10,4 @@ server_args: >-
|
||||
--quantization humming
|
||||
--kernel-config.enable_flashinfer_autotune=False
|
||||
env:
|
||||
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
|
||||
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int6","hadamard_block_size":-1}'
|
||||
@@ -0,0 +1,13 @@
|
||||
model_name: "Qwen/Qwen3.5-9B"
|
||||
accuracy_threshold: 0.88
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--quantization humming
|
||||
--kernel-config.enable_flashinfer_autotune=False
|
||||
env:
|
||||
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int6","hadamard_block_size":-1}'
|
||||
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
|
||||
@@ -0,0 +1,12 @@
|
||||
model_name: "Qwen/Qwen3.5-9B"
|
||||
accuracy_threshold: 0.88
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
gen_prefix: " <think>\n\n</think>\n"
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--quantization humming
|
||||
--kernel-config.enable_flashinfer_autotune=False
|
||||
env:
|
||||
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int6","hadamard_block_size":-1}'
|
||||
@@ -0,0 +1,3 @@
|
||||
gpt-oss-20b-humming.yaml
|
||||
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
|
||||
Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml
|
||||
@@ -0,0 +1,4 @@
|
||||
Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
|
||||
Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
|
||||
Qwen3-30B-A3B-AWQ-humming.yaml
|
||||
Qwen3-4B-mixed-quant-RTN-humming.yaml
|
||||
@@ -1,4 +1,5 @@
|
||||
Qwen3.5-35B-A3B-experts-int8-humming.yaml
|
||||
Qwen3.5-35B-A3B-int6wc-hadamard-humming.yaml
|
||||
gpt-oss-20b-humming.yaml
|
||||
Qwen3-30B-A3B-GPTQ-Int4-humming.yaml
|
||||
Qwen3-30B-A3B-FP8-block-humming.yaml
|
||||
|
||||
@@ -3,4 +3,5 @@ Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
|
||||
Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
|
||||
Qwen3-30B-A3B-AWQ-humming.yaml
|
||||
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
|
||||
Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml
|
||||
Qwen3-4B-mixed-quant-RTN-humming.yaml
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
Qwen3-30B-A3B-FP8-block-humming-deepep-ht-grouped.yaml
|
||||
Qwen3-30B-A3B-FP8-block-humming-deepep-ht-indexed.yaml
|
||||
@@ -1,4 +1,5 @@
|
||||
Qwen3.5-35B-A3B-experts-int8-humming.yaml
|
||||
Qwen3.5-35B-A3B-int6wc-hadamard-humming.yaml
|
||||
Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml
|
||||
Qwen3-30B-A3B-Fp8-v1-humming.yaml
|
||||
Qwen3-30B-A3B-NVFP4-humming.yaml
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
Qwen3-30B-A3B-int5wc-hadamard-humming.yaml
|
||||
Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml
|
||||
Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml
|
||||
@@ -0,0 +1,9 @@
|
||||
Qwen2-1.5B-Instruct-FP8W8-humming.yaml
|
||||
Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml
|
||||
Qwen3-0.6B-MXFP8-humming.yaml
|
||||
Qwen3-0.6B-MXFP8-humming-act-fp8.yaml
|
||||
Qwen3-30B-A3B-AWQ-humming.yaml
|
||||
Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml
|
||||
gpt-oss-20b-humming.yaml
|
||||
Qwen3.5-9B-int6wc-hadamard-humming.yaml
|
||||
Qwen3.5-9B-int6wc-hadamard-humming-act-int8.yaml
|
||||
Reference in New Issue
Block a user