diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml
index 8f47e7b5c2..4cf7774ebd 100644
--- a/.buildkite/test_areas/lm_eval.yaml
+++ b/.buildkite/test_areas/lm_eval.yaml
@@ -150,9 +150,9 @@ steps:
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
-- label: LM Eval Humming (A100 - TEMPORARY)
- key: lm-eval-humming-a100
- timeout_in_minutes: 30
+- label: LM Eval Humming f16 (A100 - TEMPORARY)
+ key: lm-eval-humming-f16-a100
+ timeout_in_minutes: 120
device: a100
optional: true
num_devices: 1
@@ -160,13 +160,29 @@ steps:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
+ - vllm/model_executor/layers/fused_moe/oracle/
+ - vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
-- label: LM Eval Humming (H100 - TEMPORARY)
- key: lm-eval-humming-h100
- timeout_in_minutes: 30
+- label: LM Eval Humming Act int8 (A100 - TEMPORARY)
+ key: lm-eval-humming-act-a100
+ timeout_in_minutes: 120
+ device: a100
+ optional: true
+ num_devices: 1
+ source_file_dependencies:
+ - vllm/model_executor/layers/quantization/humming.py
+ - vllm/model_executor/layers/quantization/utils/humming_utils.py
+ - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
+ - vllm/model_executor/layers/fused_moe/oracle/
+ - vllm/model_executor/kernels/linear/
+ commands:
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
+
+- label: LM Eval Humming f16 (H100 - TEMPORARY)
+ key: lm-eval-humming-f16-h100
+ timeout_in_minutes: 120
device: h100
optional: true
num_devices: 1
@@ -174,14 +190,30 @@ steps:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
+ - vllm/model_executor/layers/fused_moe/oracle/
+ - vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
- - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
-- label: LM Eval Humming (B200 - TEMPORARY)
- key: lm-eval-humming-b200
- timeout_in_minutes: 30
+- label: LM Eval Humming Act fp8/int8 (H100 - TEMPORARY)
+ key: lm-eval-humming-act-h100
+ timeout_in_minutes: 120
+ device: h100
+ optional: true
+ num_devices: 1
+ source_file_dependencies:
+ - vllm/model_executor/layers/quantization/humming.py
+ - vllm/model_executor/layers/quantization/utils/humming_utils.py
+ - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
+ - vllm/model_executor/layers/fused_moe/oracle/
+ - vllm/model_executor/kernels/linear/
+ commands:
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
+
+- label: LM Eval Humming f16 (B200 - TEMPORARY)
+ key: lm-eval-humming-f16-b200
+ timeout_in_minutes: 120
device: b200-k8s
optional: true
num_devices: 1
@@ -189,10 +221,26 @@ steps:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
+ - vllm/model_executor/layers/fused_moe/oracle/
+ - vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
+
+- label: LM Eval Humming Act fp8/int8 (B200 - TEMPORARY)
+ key: lm-eval-humming-act-b200
+ timeout_in_minutes: 120
+ device: b200-k8s
+ optional: true
+ num_devices: 1
+ source_file_dependencies:
+ - vllm/model_executor/layers/quantization/humming.py
+ - vllm/model_executor/layers/quantization/utils/humming_utils.py
+ - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
+ - vllm/model_executor/layers/fused_moe/oracle/
+ - vllm/model_executor/kernels/linear/
+ commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- label: LM Eval TurboQuant KV Cache
key: lm-eval-turboquant-kv-cache
diff --git a/requirements/cuda.txt b/requirements/cuda.txt
index 595d61b863..f89854f632 100644
--- a/requirements/cuda.txt
+++ b/requirements/cuda.txt
@@ -28,4 +28,4 @@ quack-kernels>=0.3.3
tokenspeed-mla==0.1.2; platform_system == "Linux"
# Humming kernels for quantization gemm
-humming-kernels[cu13]==0.1.6
+humming-kernels[cu13]==0.1.10
diff --git a/tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml
new file mode 100644
index 0000000000..ba292eb972
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml
@@ -0,0 +1,10 @@
+model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8"
+accuracy_threshold: 0.55
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --linear-backend humming
+env:
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming.yaml
new file mode 100644
index 0000000000..3179c3251b
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming.yaml
@@ -0,0 +1,8 @@
+model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8"
+accuracy_threshold: 0.55
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --linear-backend humming
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming-act-fp8.yaml
new file mode 100644
index 0000000000..66888312ef
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming-act-fp8.yaml
@@ -0,0 +1,10 @@
+model_name: "mgoin/Qwen3-0.6B-MXFP8"
+accuracy_threshold: 0.39
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --linear-backend humming
+env:
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml
new file mode 100644
index 0000000000..b83d9e6a9e
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml
@@ -0,0 +1,8 @@
+model_name: "mgoin/Qwen3-0.6B-MXFP8"
+accuracy_threshold: 0.39
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --linear-backend humming
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-fp8.yaml
new file mode 100644
index 0000000000..d3de5b3792
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-fp8.yaml
@@ -0,0 +1,13 @@
+model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
+accuracy_threshold: 0.90
+num_questions: 1319
+num_fewshot: 5
+gen_prefix: " \n\n\n"
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
+ --dtype bfloat16
+env:
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-int8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-int8.yaml
new file mode 100644
index 0000000000..6b76efc4e2
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-int8.yaml
@@ -0,0 +1,13 @@
+model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
+accuracy_threshold: 0.90
+num_questions: 1319
+num_fewshot: 5
+gen_prefix: " \n\n\n"
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
+ --dtype bfloat16
+env:
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming.yaml
new file mode 100644
index 0000000000..310255f6bf
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming.yaml
@@ -0,0 +1,11 @@
+model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
+accuracy_threshold: 0.90
+num_questions: 1319
+num_fewshot: 5
+gen_prefix: " \n\n\n"
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
+ --dtype bfloat16
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming-act-fp8.yaml
new file mode 100644
index 0000000000..6d2702d8f4
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming-act-fp8.yaml
@@ -0,0 +1,11 @@
+model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
+accuracy_threshold: 0.86
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
+env:
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml
new file mode 100644
index 0000000000..8ca4777ce5
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml
@@ -0,0 +1,9 @@
+model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
+accuracy_threshold: 0.86
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml
new file mode 100644
index 0000000000..618e7fdcc3
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml
@@ -0,0 +1,11 @@
+model_name: "nm-testing/Qwen3-30B-A3B-Fp8-v1"
+accuracy_threshold: 0.86
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
+env:
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming.yaml
new file mode 100644
index 0000000000..71aabcef99
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming.yaml
@@ -0,0 +1,9 @@
+model_name: "nm-testing/Qwen3-30B-A3B-Fp8-v1"
+accuracy_threshold: 0.86
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml
new file mode 100644
index 0000000000..251fb252cf
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml
@@ -0,0 +1,11 @@
+model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
+accuracy_threshold: 0.86
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
+env:
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
new file mode 100644
index 0000000000..d813e05d6f
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
@@ -0,0 +1,11 @@
+model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
+accuracy_threshold: 0.86
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
+env:
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming.yaml
new file mode 100644
index 0000000000..ab89ebf515
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming.yaml
@@ -0,0 +1,9 @@
+model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
+accuracy_threshold: 0.86
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
new file mode 100644
index 0000000000..f77ee1173e
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
@@ -0,0 +1,11 @@
+model_name: "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8"
+accuracy_threshold: 0.86
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
+env:
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml
new file mode 100644
index 0000000000..7b4f82c845
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml
@@ -0,0 +1,9 @@
+model_name: "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8"
+accuracy_threshold: 0.86
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
index 9b77af6732..d98f91e1f9 100644
--- a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
@@ -5,8 +5,7 @@ num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
- --tensor-parallel-size 1
--quantization humming
- --kernel-config.enable_flashinfer_autotune=False
+ --linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming.yaml
index 0b1599ff94..67725fce64 100644
--- a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming.yaml
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming.yaml
@@ -5,6 +5,5 @@ num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
- --tensor-parallel-size 1
--quantization humming
- --kernel-config.enable_flashinfer_autotune=False
+ --linear-backend humming
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-NVFP4-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-NVFP4-humming.yaml
new file mode 100644
index 0000000000..a20c143310
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-NVFP4-humming.yaml
@@ -0,0 +1,9 @@
+model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
+accuracy_threshold: 0.86
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml
new file mode 100644
index 0000000000..b58c0d710e
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml
@@ -0,0 +1,13 @@
+model_name: "Qwen/Qwen3-30B-A3B"
+accuracy_threshold: 0.80
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --tensor-parallel-size 1
+ --quantization humming
+ --kernel-config.enable_flashinfer_autotune=False
+env:
+ VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml
new file mode 100644
index 0000000000..c932091dbe
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml
@@ -0,0 +1,13 @@
+model_name: "Qwen/Qwen3-30B-A3B"
+accuracy_threshold: 0.80
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --tensor-parallel-size 1
+ --quantization humming
+ --kernel-config.enable_flashinfer_autotune=False
+env:
+ VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming.yaml
new file mode 100644
index 0000000000..fa5095cc88
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming.yaml
@@ -0,0 +1,12 @@
+model_name: "Qwen/Qwen3-30B-A3B"
+accuracy_threshold: 0.80
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --tensor-parallel-size 1
+ --quantization humming
+ --kernel-config.enable_flashinfer_autotune=False
+env:
+ VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming-act-fp8.yaml
new file mode 100644
index 0000000000..3af5c03a24
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming-act-fp8.yaml
@@ -0,0 +1,12 @@
+model_name: "Qwen/Qwen3.5-35B-A3B-FP8"
+accuracy_threshold: 0.90
+num_questions: 1319
+num_fewshot: 5
+gen_prefix: " \n\n\n"
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
+env:
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming.yaml
new file mode 100644
index 0000000000..85fce24420
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming.yaml
@@ -0,0 +1,10 @@
+model_name: "Qwen/Qwen3.5-35B-A3B-FP8"
+accuracy_threshold: 0.90
+num_questions: 1319
+num_fewshot: 5
+gen_prefix: " \n\n\n"
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
new file mode 100644
index 0000000000..6a85ab384f
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
@@ -0,0 +1,12 @@
+model_name: "Qwen/Qwen3.5-35B-A3B"
+accuracy_threshold: 0.90
+num_questions: 1319
+num_fewshot: 5
+gen_prefix: " \n\n\n"
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --quantization experts_int8
+env:
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming.yaml
new file mode 100644
index 0000000000..d282fdc701
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming.yaml
@@ -0,0 +1,10 @@
+model_name: "Qwen/Qwen3.5-35B-A3B"
+accuracy_threshold: 0.90
+num_questions: 1319
+num_fewshot: 5
+gen_prefix: " \n\n\n"
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --quantization experts_int8
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-fp8.yaml
new file mode 100644
index 0000000000..2a118098fc
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-fp8.yaml
@@ -0,0 +1,10 @@
+model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
+accuracy_threshold: 0.82
+num_questions: 1319
+num_fewshot: 5
+gen_prefix: " \n\n\n"
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+env:
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-int8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-int8.yaml
new file mode 100644
index 0000000000..34f17a4b05
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-int8.yaml
@@ -0,0 +1,11 @@
+model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
+accuracy_threshold: 0.82
+num_questions: 1319
+num_fewshot: 5
+gen_prefix: " \n\n\n"
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --linear-backend humming
+env:
+ VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming.yaml
new file mode 100644
index 0000000000..b617c61eb3
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming.yaml
@@ -0,0 +1,9 @@
+model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
+accuracy_threshold: 0.82
+num_questions: 1319
+num_fewshot: 5
+gen_prefix: " \n\n\n"
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --linear-backend humming
diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.6-35B-A3B-NVFP4-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.6-35B-A3B-NVFP4-humming.yaml
new file mode 100644
index 0000000000..502ab776f4
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/Qwen3.6-35B-A3B-NVFP4-humming.yaml
@@ -0,0 +1,10 @@
+model_name: "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
+accuracy_threshold: 0.91
+num_questions: 1319
+num_fewshot: 5
+gen_prefix: " \n\n\n"
+server_args: >-
+ --enforce-eager
+ --max-model-len 8192
+ --moe-backend humming
+ --linear-backend humming
diff --git a/tests/evals/gsm8k/configs/humming/config-act-fp8.txt b/tests/evals/gsm8k/configs/humming/config-act-fp8.txt
index 05fb6a1583..42ff6be00e 100644
--- a/tests/evals/gsm8k/configs/humming/config-act-fp8.txt
+++ b/tests/evals/gsm8k/configs/humming/config-act-fp8.txt
@@ -1,2 +1,9 @@
gpt-oss-20b-humming-act-fp8.yaml
Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
+Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml
+Qwen3-0.6B-MXFP8-humming-act-fp8.yaml
+Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml
+Qwen3-30B-A3B-FP8-block-humming-act-fp8.yaml
+Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml
+Qwen3-30B-A3B-AWQ-humming.yaml
+Qwen3.5-35B-A3B-FP8-humming-act-fp8.yaml
diff --git a/tests/evals/gsm8k/configs/humming/config-act-int8.txt b/tests/evals/gsm8k/configs/humming/config-act-int8.txt
new file mode 100644
index 0000000000..b018b35cbf
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/config-act-int8.txt
@@ -0,0 +1,4 @@
+Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
+Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
+Qwen3-30B-A3B-AWQ-humming.yaml
+Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
diff --git a/tests/evals/gsm8k/configs/humming/config-int5wc-hadamard.txt b/tests/evals/gsm8k/configs/humming/config-int5wc-hadamard.txt
new file mode 100644
index 0000000000..2c10777a09
--- /dev/null
+++ b/tests/evals/gsm8k/configs/humming/config-int5wc-hadamard.txt
@@ -0,0 +1,3 @@
+Qwen3-30B-A3B-int5wc-hadamard-humming.yaml
+Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml
+Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml
diff --git a/tests/evals/gsm8k/configs/humming/config.txt b/tests/evals/gsm8k/configs/humming/config.txt
index 821025365c..144ed95993 100644
--- a/tests/evals/gsm8k/configs/humming/config.txt
+++ b/tests/evals/gsm8k/configs/humming/config.txt
@@ -1,2 +1,13 @@
gpt-oss-20b-humming.yaml
Qwen3-30B-A3B-MXFP4A16-humming.yaml
+Qwen2-1.5B-Instruct-FP8W8-humming.yaml
+Qwen3-0.6B-MXFP8-humming.yaml
+Qwen3.6-35B-A3B-NVFP4-humming.yaml
+Qwen3-30B-A3B-Fp8-v1-humming.yaml
+Qwen3-30B-A3B-FP8-block-humming.yaml
+Qwen3-30B-A3B-GPTQ-Int4-humming.yaml
+Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml
+Qwen3-30B-A3B-NVFP4-humming.yaml
+Qwen3-30B-A3B-AWQ-humming.yaml
+Qwen3.5-35B-A3B-FP8-humming.yaml
+Qwen3.5-35B-A3B-experts-int8-humming.yaml
diff --git a/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming-act-fp8.yaml
index 00ba9eccfd..8e0d953503 100644
--- a/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming-act-fp8.yaml
+++ b/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming-act-fp8.yaml
@@ -5,7 +5,6 @@ num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
- --tensor-parallel-size 1
--moe-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
diff --git a/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming.yaml b/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming.yaml
index e2beb3739b..7e9b6508a2 100644
--- a/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming.yaml
+++ b/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming.yaml
@@ -5,5 +5,4 @@ num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
- --tensor-parallel-size 1
--moe-backend humming
diff --git a/tests/evals/gsm8k/gsm8k_eval.py b/tests/evals/gsm8k/gsm8k_eval.py
index 89d91cb4a9..87235a9c37 100644
--- a/tests/evals/gsm8k/gsm8k_eval.py
+++ b/tests/evals/gsm8k/gsm8k_eval.py
@@ -146,6 +146,7 @@ async def call_vllm_chat_api(
def _build_gsm8k_prompts(
num_questions: int = 1319,
num_shots: int = 5,
+ gen_prefix: str = "",
) -> tuple[list[str], list[int]]:
"""Build few-shot GSM8K completion prompts and ground-truth labels."""
if num_questions == 0:
@@ -157,14 +158,15 @@ def _build_gsm8k_prompts(
for i in range(num_shots):
few_shot_examples += (
f"Question: {train_data[i]['question']}\n"
- f"Answer: {train_data[i]['answer']}\n\n"
+ f"Answer:{gen_prefix} {train_data[i]['answer']}\n\n"
)
prompts = []
labels = []
for i in range(num_questions):
prompts.append(
- few_shot_examples + f"Question: {test_data[i]['question']}\nAnswer:"
+ few_shot_examples
+ + f"Question: {test_data[i]['question']}\nAnswer:{gen_prefix}"
)
labels.append(get_answer_value(test_data[i]["answer"]))
@@ -213,6 +215,7 @@ def evaluate_gsm8k(
temperature: float = 0.0,
seed: int | None = 42,
request_timeout_seconds: float = 600,
+ gen_prefix: str = "",
) -> dict[str, float | int]:
"""
Evaluate GSM8K accuracy using vLLM serve endpoint.
@@ -220,7 +223,7 @@ def evaluate_gsm8k(
Returns dict with accuracy, invalid_rate, latency, etc.
"""
base_url = f"{host}:{port}"
- prompts, labels = _build_gsm8k_prompts(num_questions, num_shots)
+ prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
num_questions = len(prompts)
async def run_async_evaluation():
@@ -278,6 +281,7 @@ def evaluate_gsm8k_offline(
num_shots: int = 5,
max_tokens: int = 256,
temperature: float = 0.0,
+ gen_prefix: str = "",
) -> dict[str, float | int]:
"""Evaluate GSM8K accuracy using an offline vllm.LLM object.
@@ -286,7 +290,7 @@ def evaluate_gsm8k_offline(
"""
from vllm import SamplingParams
- prompts, labels = _build_gsm8k_prompts(num_questions, num_shots)
+ prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
sampling_params = SamplingParams(
temperature=temperature,
diff --git a/tests/evals/gsm8k/test_gsm8k_correctness.py b/tests/evals/gsm8k/test_gsm8k_correctness.py
index c9ec5ff66e..0c48af6d3c 100644
--- a/tests/evals/gsm8k/test_gsm8k_correctness.py
+++ b/tests/evals/gsm8k/test_gsm8k_correctness.py
@@ -70,6 +70,7 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
host=host,
port=port,
request_timeout_seconds=request_timeout_seconds,
+ gen_prefix=eval_config.get("gen_prefix", ""),
)
return results
diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py
index e98ab1b3e0..aff94fe03d 100644
--- a/vllm/config/kernel.py
+++ b/vllm/config/kernel.py
@@ -147,6 +147,7 @@ LinearBackend = Literal[
"flashinfer_cudnn",
"flashinfer_b12x",
"marlin",
+ "humming",
"triton",
"deep_gemm",
"torch",
diff --git a/vllm/config/model.py b/vllm/config/model.py
index 2ec754f934..c53686eaf0 100644
--- a/vllm/config/model.py
+++ b/vllm/config/model.py
@@ -999,6 +999,7 @@ class ModelConfig:
"modelopt",
"modelopt_fp4",
"modelopt_mxfp8",
+ "mxfp8",
"modelopt_mixed",
# Ensure heavy backends are probed last to avoid unnecessary
# imports during override detection (e.g., MXFP4 imports Triton)
diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py
index f5e3a16d71..5d1d370730 100644
--- a/vllm/model_executor/kernels/linear/__init__.py
+++ b/vllm/model_executor/kernels/linear/__init__.py
@@ -74,6 +74,9 @@ from vllm.model_executor.kernels.linear.mxfp4 import (
from vllm.model_executor.kernels.linear.mxfp4.flashinfer import (
FlashInferMxFp4LinearKernel,
)
+from vllm.model_executor.kernels.linear.mxfp4.humming import (
+ HummingMxFp4LinearKernel,
+)
from vllm.model_executor.kernels.linear.mxfp4.marlin import (
MarlinMxFp4LinearKernel,
)
@@ -91,6 +94,9 @@ from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
FlashInferCutedslMxfp8LinearKernel,
FlashInferCutlassMxfp8LinearKernel,
)
+from vllm.model_executor.kernels.linear.mxfp8.humming import (
+ HummingMxfp8LinearKernel,
+)
from vllm.model_executor.kernels.linear.mxfp8.marlin import (
MarlinMxfp8LinearKernel,
)
@@ -120,6 +126,9 @@ from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
FlashInferCutlassNvFp4LinearKernel,
FlashInferTrtllmNvFp4LinearKernel,
)
+from vllm.model_executor.kernels.linear.nvfp4.humming import (
+ HummingNvFp4LinearKernel,
+)
from vllm.model_executor.kernels.linear.nvfp4.marlin import (
MarlinNvFp4LinearKernel,
)
@@ -154,6 +163,10 @@ from vllm.model_executor.kernels.linear.scaled_mm.flashinfer import (
FlashInferFp8DeepGEMMDynamicBlockScaledKernel,
FlashInferFP8ScaledMMLinearKernel,
)
+from vllm.model_executor.kernels.linear.scaled_mm.humming import (
+ HummingFP8ScaledMMLinearKernel,
+ HummingInt8ScaledMMLinearKernel,
+)
from vllm.model_executor.kernels.linear.scaled_mm.marlin import (
MarlinFP8ScaledMMLinearKernel,
)
@@ -225,6 +238,14 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
"flashinfer_b12x": {
FlashInferB12xNvFp4LinearKernel,
},
+ "humming": {
+ HummingFP8ScaledMMLinearKernel,
+ HummingInt8ScaledMMLinearKernel,
+ HummingLinearKernel,
+ HummingMxfp8LinearKernel,
+ HummingMxFp4LinearKernel,
+ HummingNvFp4LinearKernel,
+ },
"marlin": {
MarlinFP8ScaledMMLinearKernel,
MarlinLinearKernel,
@@ -292,6 +313,7 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]
PlatformEnum.CUDA: [
CutlassInt8ScaledMMLinearKernel,
TritonInt8ScaledMMLinearKernel,
+ HummingInt8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [AiterInt8ScaledMMLinearKernel, TritonInt8ScaledMMLinearKernel],
}
@@ -304,6 +326,7 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
CutlassFP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
+ HummingFP8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
@@ -335,6 +358,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
CutlassFp8BlockScaledMMKernel,
MarlinFP8ScaledMMLinearKernel,
TritonFp8BlockScaledMMKernel,
+ HummingFP8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [
AiterFp8BlockScaledMMKernel,
@@ -351,6 +375,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
_POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
PlatformEnum.CUDA: [
+ HummingFP8ScaledMMLinearKernel,
MarlinFP8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [
@@ -371,10 +396,10 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
MacheteLinearKernel,
AllSparkLinearKernel,
MarlinLinearKernel,
- HummingLinearKernel,
ConchLinearKernel,
ExllamaLinearKernel,
TritonW4A16LinearKernel,
+ HummingLinearKernel,
],
PlatformEnum.ROCM: [
RDNA3W4A16LinearKernel,
@@ -400,6 +425,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
FlashInferCutlassMxfp8LinearKernel,
MarlinMxfp8LinearKernel,
EmulationMxfp8LinearKernel,
+ HummingMxfp8LinearKernel,
],
PlatformEnum.ROCM: [
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
@@ -426,6 +452,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
FlashInferCudnnNvFp4LinearKernel,
FbgemmNvFp4LinearKernel,
EmulationNvFp4LinearKernel,
+ HummingNvFp4LinearKernel,
],
PlatformEnum.ROCM: [
EmulationNvFp4LinearKernel,
@@ -436,6 +463,7 @@ _POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = {
PlatformEnum.CUDA: [
FlashInferMxFp4LinearKernel,
MarlinMxFp4LinearKernel,
+ HummingMxFp4LinearKernel,
],
PlatformEnum.XPU: [
XPUMxFp4LinearKernel,
diff --git a/vllm/model_executor/kernels/linear/mixed_precision/__init__.py b/vllm/model_executor/kernels/linear/mixed_precision/__init__.py
index c0b8c35bbd..f9b4bd6e43 100644
--- a/vllm/model_executor/kernels/linear/mixed_precision/__init__.py
+++ b/vllm/model_executor/kernels/linear/mixed_precision/__init__.py
@@ -19,6 +19,9 @@ from vllm.model_executor.kernels.linear.mixed_precision.dynamic_4bit import (
from vllm.model_executor.kernels.linear.mixed_precision.exllama import (
ExllamaLinearKernel,
)
+from vllm.model_executor.kernels.linear.mixed_precision.humming import (
+ HummingLinearKernel,
+)
from vllm.model_executor.kernels.linear.mixed_precision.machete import (
MacheteLinearKernel,
)
@@ -52,6 +55,7 @@ __all__ = [
"CutlassW4A8LinearKernel",
"Dynamic4bitLinearKernel",
"ExllamaLinearKernel",
+ "HummingLinearKernel",
"MacheteLinearKernel",
"MarlinLinearKernel",
"RDNA3W4A16LinearKernel",
diff --git a/vllm/model_executor/kernels/linear/mixed_precision/humming.py b/vllm/model_executor/kernels/linear/mixed_precision/humming.py
index 764c0f4227..7f1a9024ee 100644
--- a/vllm/model_executor/kernels/linear/mixed_precision/humming.py
+++ b/vllm/model_executor/kernels/linear/mixed_precision/humming.py
@@ -23,8 +23,6 @@ class HummingLinearKernel(MPLinearKernel):
return False, "Humming is not installed"
if c.has_g_idx:
return False, "Humming does not support act-order (g_idx)"
- if c.zero_points:
- return False, "Humming linear kernel only supports symmetric weights"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -41,6 +39,11 @@ class HummingLinearKernel(MPLinearKernel):
"group_size": 0 if group_size == -1 else group_size,
}
+ if self.config.zero_points:
+ assert self.w_zp_name is not None
+ name_map["zero_point"] = self.w_zp_name
+ quant_config["has_zero_point"] = True
+
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
diff --git a/vllm/model_executor/kernels/linear/mxfp4/humming.py b/vllm/model_executor/kernels/linear/mxfp4/humming.py
new file mode 100644
index 0000000000..d93f5d4815
--- /dev/null
+++ b/vllm/model_executor/kernels/linear/mxfp4/humming.py
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import torch
+
+from vllm.model_executor.layers.quantization.utils.humming_utils import (
+ convert_linear_layer_to_humming_standard,
+ prepare_humming_layer,
+)
+from vllm.platforms import current_platform
+
+from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
+
+
+class HummingMxFp4LinearKernel(MxFp4LinearKernel):
+ """Humming GEMM Kernel for MXFP4."""
+
+ @classmethod
+ def is_supported(
+ cls, compute_capability: int | None = None
+ ) -> tuple[bool, str | None]:
+ if not current_platform.is_cuda():
+ return False, "Humming only supported on CUDA"
+
+ if not current_platform.has_device_capability(75):
+ return False, "Humming only supported on SM75+"
+
+ return True, None
+
+ @classmethod
+ def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
+ return True, None
+
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
+ layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu)
+ name_map = {"weight": "weight", "weight_scale": "weight_scale"}
+
+ quant_config = {
+ "quant_method": "humming",
+ "dtype": "float4e2m1",
+ "scale_dtype": "float8e8m0",
+ "group_size": 32,
+ "weight_scale_type": "group",
+ }
+
+ convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
+ prepare_humming_layer(layer, quant_config)
+
+ def apply_weights(
+ self,
+ layer: torch.nn.Module,
+ x: torch.Tensor,
+ bias: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ from vllm.utils.humming import HummingMethod
+
+ flatten_inputs = x.view(-1, x.size(-1))
+ output = HummingMethod.forward_layer(
+ layer=layer,
+ inputs=flatten_inputs,
+ compute_config=layer.compute_config,
+ )
+ return output.view(*x.shape[:-1], output.size(-1))
diff --git a/vllm/model_executor/kernels/linear/mxfp8/humming.py b/vllm/model_executor/kernels/linear/mxfp8/humming.py
new file mode 100644
index 0000000000..ed1cd39cbd
--- /dev/null
+++ b/vllm/model_executor/kernels/linear/mxfp8/humming.py
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import torch
+
+from vllm.model_executor.layers.quantization.utils.humming_utils import (
+ convert_linear_layer_to_humming_standard,
+ prepare_humming_layer,
+)
+from vllm.platforms import current_platform
+
+from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
+
+
+class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
+ """Humming GEMM Kernel for MXFP8."""
+
+ @classmethod
+ def is_supported(
+ cls, compute_capability: int | None = None
+ ) -> tuple[bool, str | None]:
+ if not current_platform.is_cuda():
+ return False, "Humming only supported on CUDA"
+
+ if not current_platform.has_device_capability(75):
+ return False, "Humming only supported on SM75+"
+
+ return True, None
+
+ @classmethod
+ def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
+ return True, None
+
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
+ layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu)
+ name_map = {"weight": "weight", "weight_scale": "weight_scale"}
+
+ quant_config = {
+ "quant_method": "humming",
+ "dtype": "float8e4m3",
+ "scale_dtype": "float8e8m0",
+ "group_size": 32,
+ "weight_scale_type": "group",
+ }
+
+ convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
+ prepare_humming_layer(layer, quant_config)
+
+ def apply_weights(
+ self,
+ layer: torch.nn.Module,
+ x: torch.Tensor,
+ bias: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ from vllm.utils.humming import HummingMethod
+
+ flatten_inputs = x.view(-1, x.size(-1))
+ output = HummingMethod.forward_layer(
+ layer=layer,
+ inputs=flatten_inputs,
+ compute_config=layer.compute_config,
+ )
+ return output.view(*x.shape[:-1], output.size(-1))
diff --git a/vllm/model_executor/kernels/linear/nvfp4/humming.py b/vllm/model_executor/kernels/linear/nvfp4/humming.py
new file mode 100644
index 0000000000..7e39034385
--- /dev/null
+++ b/vllm/model_executor/kernels/linear/nvfp4/humming.py
@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import torch
+
+from vllm.logger import init_logger
+from vllm.model_executor.layers.quantization.utils.humming_utils import (
+ prepare_humming_layer,
+)
+from vllm.platforms import current_platform
+
+from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
+
+logger = init_logger(__name__)
+
+
+class HummingNvFp4LinearKernel(NvFp4LinearKernel):
+ """Humming GEMM Kernel for NVFP4."""
+
+ @classmethod
+ def is_supported(
+ cls, compute_capability: int | None = None
+ ) -> tuple[bool, str | None]:
+ if not current_platform.is_cuda():
+ return False, "Humming only supported on CUDA"
+
+ if not current_platform.has_device_capability(75):
+ return False, "Humming only supported on SM75+"
+
+ return True, None
+
+ @classmethod
+ def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]:
+ return True, None
+
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
+ # Route through humming's compressed-tensors nvfp4 loader (same path as
+ # the MoE oracle); the native group_tensor schema mishandles a scalar
+ # global scale.
+ quant_config = {
+ "quant_method": "compressed-tensors",
+ "format": "nvfp4-pack-quantized",
+ "type": "float",
+ "num_bits": 4,
+ "strategy": "group",
+ "group_size": 16,
+ }
+ # CT pack-quantized reads `weight_packed`; the scheme renamed it to `weight`.
+ if not hasattr(layer, "weight_packed"):
+ layer.weight_packed = layer.weight
+ del layer.weight
+ # The CT linear scheme inverts the global scale (1/scale) for
+ # marlin/cutlass; humming wants the original.
+ layer.weight_global_scale = torch.nn.Parameter(
+ 1.0 / layer.weight_global_scale, requires_grad=False
+ )
+ prepare_humming_layer(layer, quant_config)
+
+ def apply_weights(
+ self,
+ layer: torch.nn.Module,
+ x: torch.Tensor,
+ bias: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ from vllm.utils.humming import HummingMethod
+
+ flatten_inputs = x.view(-1, x.size(-1))
+ output = HummingMethod.forward_layer(
+ layer=layer,
+ inputs=flatten_inputs,
+ compute_config=layer.compute_config,
+ )
+ return output.view(*x.shape[:-1], output.size(-1))
diff --git a/vllm/model_executor/kernels/linear/scaled_mm/humming.py b/vllm/model_executor/kernels/linear/scaled_mm/humming.py
new file mode 100644
index 0000000000..7b8ed21fbd
--- /dev/null
+++ b/vllm/model_executor/kernels/linear/scaled_mm/humming.py
@@ -0,0 +1,156 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import torch
+
+from vllm.logger import init_logger
+from vllm.model_executor.layers.quantization.utils.humming_utils import (
+ convert_linear_layer_to_humming_standard,
+ prepare_humming_layer,
+)
+from vllm.platforms import current_platform
+
+from .ScaledMMLinearKernel import (
+ FP8ScaledMMLinearKernel,
+ FP8ScaledMMLinearLayerConfig,
+ Int8ScaledMMLinearKernel,
+ Int8ScaledMMLinearLayerConfig,
+)
+
+logger = init_logger(__name__)
+
+
+class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
+ """Humming GEMM Kernel for FP8."""
+
+ @classmethod
+ def is_supported(
+ cls, compute_capability: int | None = None
+ ) -> tuple[bool, str | None]:
+ if not current_platform.is_cuda():
+ return False, "Humming only supported on CUDA"
+
+ if not current_platform.has_device_capability(75):
+ return False, "Humming only supported on SM75+"
+
+ return True, None
+
+ @classmethod
+ def can_implement(
+ cls, config: FP8ScaledMMLinearLayerConfig
+ ) -> tuple[bool, str | None]:
+ return True, None
+
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
+ from vllm.utils.humming import dtypes
+
+ name_map = {"weight": "weight", "weight_scale": "weight_scale"}
+ scale_torch_dtype = self.config.weight_quant_key.scale.dtype
+ scale_dtype = dtypes.DataType.from_torch_dtype(scale_torch_dtype)
+
+ quant_config = {
+ "quant_method": "humming",
+ "dtype": "float8e4m3",
+ "scale_dtype": scale_dtype,
+ }
+
+ assert self.config.weight_quant_key.scale2 is None
+ scale_group_shape = self.config.weight_quant_key.scale.group_shape
+ if scale_group_shape.is_per_tensor():
+ quant_config["weight_scale_type"] = "tensor"
+ if not hasattr(layer, "global_scale") and hasattr(layer, "weight_scale"):
+ del name_map["weight_scale"]
+ name_map["global_scale"] = "weight_scale"
+ elif scale_group_shape.is_per_channel():
+ quant_config["weight_scale_type"] = "channel"
+ elif scale_group_shape.is_per_group():
+ quant_config["weight_scale_type"] = "group"
+ quant_config["group_size"] = scale_group_shape.col
+ else:
+ assert scale_group_shape.row > 0 and scale_group_shape.col > 0
+ quant_config["weight_scale_type"] = "block"
+ quant_config["weight_scale_group_size_n"] = scale_group_shape.row
+ quant_config["weight_scale_group_size"] = scale_group_shape.col
+
+ if hasattr(layer, "weight_scale_inv"):
+ name_map["weight_scale"] = "weight_scale_inv"
+
+ convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
+ prepare_humming_layer(layer, quant_config)
+
+ def apply_weights(
+ self,
+ layer: torch.nn.Module,
+ x: torch.Tensor,
+ bias: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ from vllm.utils.humming import HummingMethod
+
+ flatten_inputs = x.view(-1, x.size(-1))
+ output = HummingMethod.forward_layer(
+ layer=layer,
+ inputs=flatten_inputs,
+ compute_config=layer.compute_config,
+ )
+ return output.view(*x.shape[:-1], output.size(-1))
+
+ def apply_scaled_mm(
+ self,
+ *,
+ A: torch.Tensor,
+ B: torch.Tensor,
+ out_dtype: torch.dtype,
+ As: torch.Tensor,
+ Bs: torch.Tensor,
+ bias: torch.Tensor | None,
+ output_shape: list,
+ ) -> torch.Tensor:
+ pass
+
+
+class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
+ """Humming GEMM Kernel for INT8."""
+
+ @classmethod
+ def is_supported(
+ cls, compute_capability: int | None = None
+ ) -> tuple[bool, str | None]:
+ if not current_platform.is_cuda():
+ return False, "Humming only supported on CUDA"
+
+ if not current_platform.has_device_capability(75):
+ return False, "Humming only supported on SM75+"
+
+ return True, None
+
+ @classmethod
+ def can_implement(
+ cls, config: Int8ScaledMMLinearLayerConfig
+ ) -> tuple[bool, str | None]:
+ return True, None
+
+ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
+ weight_name, weight_scale_name, *_ = self.layer_param_names
+ name_map = {"weight": weight_name, "weight_scale": weight_scale_name}
+ quant_config = {"quant_method": "humming", "dtype": "int8"}
+ weight = getattr(layer, weight_name)
+ weight.data = weight.data + 128
+
+ convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
+ prepare_humming_layer(layer, quant_config)
+
+ def apply_weights(
+ self,
+ layer: torch.nn.Module,
+ x: torch.Tensor,
+ bias: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ from vllm.utils.humming import HummingMethod
+
+ flatten_inputs = x.view(-1, x.size(-1))
+ output = HummingMethod.forward_layer(
+ layer=layer,
+ inputs=flatten_inputs,
+ compute_config=layer.compute_config,
+ )
+ return output.view(*x.shape[:-1], output.size(-1))
diff --git a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
index 047ae46c0d..5f112380cf 100644
--- a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
+++ b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
@@ -38,15 +38,20 @@ from vllm.model_executor.layers.fused_moe.utils import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
+ kFp8Dynamic128Sym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
+ kFp8StaticTensorSym,
kInt4Static,
+ kInt8DynamicTokenSym,
kInt8Static,
+ kInt8StaticChannelSym,
kMxfp4Dynamic,
kMxfp4Static,
kMxfp8Dynamic,
kMxfp8Static,
+ kNvfp4Dynamic,
kNvfp4Static,
)
from vllm.platforms import current_platform
@@ -61,9 +66,9 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
-def get_humming_moe_gemm_type() -> str | None:
+def get_humming_moe_gemm_type() -> str:
env_gemm_type: str | None = envs.VLLM_HUMMING_MOE_GEMM_TYPE
- gemm_type = None
+ gemm_type = "indexed"
if env_gemm_type is not None:
env_gemm_type = env_gemm_type.lower()
if env_gemm_type == "indexed":
@@ -87,7 +92,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
num_dispatchers: int | None = None,
):
self.layer = layer
- self.num_experts = self.layer.num_experts
+ self.num_experts = self.layer.local_num_experts
self.global_num_experts = self.layer.global_num_experts
self.init_humming_moe()
@@ -186,6 +191,24 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
(kInt4Static, kFp8DynamicTokenSym),
(kInt8Static, None),
(kInt8Static, kFp8DynamicTokenSym),
+ # Checkpoint-driven (weight, activation) pairs the dense/MoE oracles
+ # pass. Humming defers input quant (see expects_unquantized_inputs),
+ # so the activation key does not constrain support.
+ # fp8 (compressed-tensors / native / modelopt)
+ (kFp8StaticChannelSym, kFp8StaticTensorSym),
+ (kFp8StaticChannelSym, kFp8Dynamic128Sym),
+ (kFp8StaticTensorSym, None),
+ (kFp8StaticTensorSym, kFp8DynamicTokenSym),
+ (kFp8StaticTensorSym, kFp8StaticTensorSym),
+ (kFp8StaticTensorSym, kFp8Dynamic128Sym),
+ (kFp8Static128BlockSym, kFp8Dynamic128Sym),
+ # int8 (compressed-tensors w8a8 / experts_int8)
+ (kInt8StaticChannelSym, None),
+ (kInt8StaticChannelSym, kInt8DynamicTokenSym),
+ # nvfp4 (compressed-tensors / modelopt / quark)
+ (kNvfp4Static, kNvfp4Dynamic),
+ # mxfp8 (compressed-tensors / modelopt / online)
+ (kMxfp8Static, kMxfp8Dynamic),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@@ -463,13 +486,12 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
assert hasattr(cls, "humming_gemm_type")
gemm_type = cls.humming_gemm_type().value.lower()
preferred_gemm_type = get_humming_moe_gemm_type()
- if preferred_gemm_type is not None:
- supported = preferred_gemm_type.lower() == gemm_type
- if not supported:
- reason = (
- f"preferred gemm type {preferred_gemm_type} != "
- f"supported gemm type {gemm_type}"
- )
+ supported = preferred_gemm_type.lower() == gemm_type
+ if not supported:
+ reason = (
+ f"preferred gemm type {preferred_gemm_type} != "
+ f"supported gemm type {gemm_type}"
+ )
return supported, reason
diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py
index 862f292009..c60e68232d 100644
--- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py
+++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
+from typing import Any
import torch
@@ -45,6 +46,7 @@ class Fp8MoeBackend(Enum):
DEEPGEMM = "DEEPGEMM"
BATCHED_DEEPGEMM = "BATCHED_DEEPGEMM"
MARLIN = "MARLIN"
+ HUMMING = "HUMMING"
TRITON = "TRITON"
BATCHED_TRITON = "BATCHED_TRITON"
AITER = "AITER"
@@ -83,6 +85,7 @@ def _get_priority_backends(
Fp8MoeBackend.VLLM_CUTLASS,
Fp8MoeBackend.TRITON,
Fp8MoeBackend.MARLIN,
+ Fp8MoeBackend.HUMMING,
Fp8MoeBackend.BATCHED_DEEPGEMM,
Fp8MoeBackend.BATCHED_VLLM_CUTLASS,
Fp8MoeBackend.BATCHED_TRITON,
@@ -162,6 +165,19 @@ def backend_to_kernel_cls(
return [BatchedDeepGemmExperts]
+ elif backend == Fp8MoeBackend.HUMMING:
+ from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
+ BatchedHummingGroupedExperts,
+ HummingGroupedExperts,
+ HummingIndexedExperts,
+ )
+
+ return [
+ BatchedHummingGroupedExperts,
+ HummingGroupedExperts,
+ HummingIndexedExperts,
+ ]
+
elif backend == Fp8MoeBackend.MARLIN:
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
MarlinExperts,
@@ -240,6 +256,7 @@ def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend:
"flashinfer_trtllm": Fp8MoeBackend.FLASHINFER_TRTLLM,
"flashinfer_cutlass": Fp8MoeBackend.FLASHINFER_CUTLASS,
"marlin": Fp8MoeBackend.MARLIN,
+ "humming": Fp8MoeBackend.HUMMING,
"aiter": Fp8MoeBackend.AITER,
"hpc": Fp8MoeBackend.HPC,
}
@@ -402,6 +419,41 @@ def select_fp8_moe_backend(
return Fp8MoeBackend.NONE, None
+def _humming_fp8_weight_schema(
+ layer: RoutedExperts, weight: torch.Tensor, weight_scale: torch.Tensor
+) -> dict[str, Any]:
+ """Build the humming weight schema from the canonical on-device fp8/mxfp8
+ tensors (scale dtype/shape, block size), not the producing quant method."""
+ # mxfp8: e8m0 group-32 scales (stored as uint8 bytes or e8m0). humming has
+ # no compressed-tensors mxfp8 loader; its modelopt schema fits both sources.
+ if weight_scale.dtype in (torch.uint8, torch.float8_e8m0fnu):
+ return {"quant_method": "modelopt", "quant_algo": "mxfp8"}
+
+ if hasattr(layer, "w13_weight_scale_inv"):
+ assert hasattr(layer, "weight_block_size")
+ return {"quant_method": "fp8", "weight_block_size": layer.weight_block_size}
+
+ # fp8 (e4m3): recover the strategy from the scale layout (block from
+ # weight_block_size, else channel vs tensor by per-expert scale count).
+ config: dict[str, Any] = {
+ "quant_method": "compressed-tensors",
+ "format": "float-quantized",
+ "type": "float",
+ "num_bits": 8,
+ "symmetric": True,
+ }
+ weight_block_size = getattr(layer, "weight_block_size", None)
+ num_experts, num_output = weight.shape[0], weight.shape[-2]
+ if weight_block_size is not None:
+ config["strategy"] = "block"
+ config["block_structure"] = list(weight_block_size)
+ elif weight_scale.numel() >= num_experts * num_output:
+ config["strategy"] = "channel"
+ else:
+ config["strategy"] = "tensor"
+ return config
+
+
def convert_to_fp8_moe_kernel_format(
fp8_backend: Fp8MoeBackend,
# TODO(bnell): replace layer with weight_block_size
@@ -429,6 +481,18 @@ def convert_to_fp8_moe_kernel_format(
w13, w2, w13_scale, w2_scale = rocm_aiter_ops.shuffle_mxfp8_moe_weights(
w13, w2, w13_scale, w2_scale
)
+ elif fp8_backend == Fp8MoeBackend.HUMMING:
+ from vllm.model_executor.layers.quantization.utils.humming_utils import (
+ convert_to_humming_moe_kernel_format,
+ )
+
+ convert_to_humming_moe_kernel_format(
+ layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale)
+ )
+ w13 = layer.w13_weight
+ w2 = layer.w2_weight
+ w13_scale = layer.w13_weight_scale
+ w2_scale = layer.w2_weight_scale
elif fp8_backend == Fp8MoeBackend.MARLIN:
weight_block_size = getattr(layer, "weight_block_size", None)
if weight_block_size == [1, 32]:
@@ -511,6 +575,7 @@ def make_fp8_moe_quant_config(
swiglu_limit: float | None = None,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
+ layer: torch.nn.Module | None = None,
) -> FusedMoEQuantConfig:
"""
Create FusedMoEQuantConfig for the specified FP8 Backend.
@@ -537,6 +602,14 @@ def make_fp8_moe_quant_config(
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=swiglu_limit,
)
+ elif fp8_backend == Fp8MoeBackend.HUMMING:
+ from vllm.model_executor.layers.fused_moe import RoutedExperts
+ from vllm.model_executor.layers.quantization.utils.humming_utils import (
+ get_humming_moe_quant_config,
+ )
+
+ assert isinstance(layer, RoutedExperts)
+ return get_humming_moe_quant_config(layer)
# Flashinfer CUTLASS or HPC per-tensor uses single dq scale
# (alpha = w_scale * a_scale) and inverse a2 scale.
@@ -600,6 +673,7 @@ def make_fp8_moe_kernel(
experts_cls: type[mk.FusedMoEExperts],
fp8_backend: Fp8MoeBackend,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
+ layer: torch.nn.Module | None = None,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
prepare_finalize = maybe_make_prepare_finalize(
@@ -613,6 +687,11 @@ def make_fp8_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
+ extra_kwargs = {}
+ if fp8_backend == Fp8MoeBackend.HUMMING:
+ assert layer is not None
+ extra_kwargs = {"layer": layer}
+
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -622,11 +701,13 @@ def make_fp8_moe_kernel(
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
+ **extra_kwargs,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
+ **extra_kwargs,
)
kernel = mk.FusedMoEKernel(
diff --git a/vllm/model_executor/layers/fused_moe/oracle/int8.py b/vllm/model_executor/layers/fused_moe/oracle/int8.py
index e31a3ca07e..5a2b4c3a75 100644
--- a/vllm/model_executor/layers/fused_moe/oracle/int8.py
+++ b/vllm/model_executor/layers/fused_moe/oracle/int8.py
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
+from typing import Any
import torch
@@ -22,6 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
)
+from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -29,6 +31,7 @@ logger = init_logger(__name__)
class Int8MoeBackend(Enum):
TRITON = "TRITON"
+ HUMMING = "HUMMING"
CPU = "CPU"
@@ -40,6 +43,7 @@ def _get_priority_backends(
"""
_AVAILABLE_BACKENDS = [
Int8MoeBackend.TRITON,
+ Int8MoeBackend.HUMMING,
Int8MoeBackend.CPU,
]
@@ -62,6 +66,19 @@ def backend_to_kernel_cls(
return [TritonExperts]
+ elif backend == Int8MoeBackend.HUMMING:
+ from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
+ BatchedHummingGroupedExperts,
+ HummingGroupedExperts,
+ HummingIndexedExperts,
+ )
+
+ return [
+ BatchedHummingGroupedExperts,
+ HummingGroupedExperts,
+ HummingIndexedExperts,
+ ]
+
elif backend == Int8MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUExpertsInt8,
@@ -77,6 +94,7 @@ def map_int8_backend(runner_backend: MoEBackend) -> Int8MoeBackend:
"""Map user's MoEBackend to Int8MoeBackend."""
mapping = {
"triton": Int8MoeBackend.TRITON,
+ "humming": Int8MoeBackend.HUMMING,
}
if backend := mapping.get(runner_backend):
return backend
@@ -163,6 +181,7 @@ def select_int8_moe_backend(
def make_int8_moe_quant_config(
+ int8_backend: Int8MoeBackend,
w1_scale: torch.Tensor,
w2_scale: torch.Tensor,
a1_scale: torch.Tensor | None = None,
@@ -170,11 +189,21 @@ def make_int8_moe_quant_config(
w1_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
per_act_token_quant: bool = False,
+ layer: torch.nn.Module | None = None,
) -> FusedMoEQuantConfig:
assert (a1_scale is None and a2_scale is None) or (
a1_scale is not None and a2_scale is not None
), "a1_scale and a2_scale must both be provided or both be None"
+ if int8_backend == Int8MoeBackend.HUMMING:
+ from vllm.model_executor.layers.fused_moe import RoutedExperts
+ from vllm.model_executor.layers.quantization.utils.humming_utils import (
+ get_humming_moe_quant_config,
+ )
+
+ assert isinstance(layer, RoutedExperts)
+ return get_humming_moe_quant_config(layer)
+
if a1_scale is None or a2_scale is None:
return int8_w8a16_moe_quant_config(
w1_scale=w1_scale,
@@ -196,13 +225,56 @@ def make_int8_moe_quant_config(
)
+def _humming_int8_weight_schema(
+ weight: torch.Tensor, weight_scale: torch.Tensor
+) -> dict[str, Any]:
+ """Build the humming compressed-tensors int8 schema from the canonical
+ on-device tensors; humming does the signed-int8 -> native conversion."""
+ config: dict[str, Any] = {
+ "quant_method": "compressed-tensors",
+ "format": "int-quantized",
+ "type": "int",
+ "num_bits": 8,
+ "symmetric": True,
+ "strategy": "channel",
+ }
+ num_experts, num_output = weight.shape[0], weight.shape[-2]
+ if weight_scale.numel() < num_experts * num_output:
+ config["strategy"] = "tensor"
+ return config
+
+
def convert_to_int8_moe_kernel_format(
int8_backend: Int8MoeBackend,
w13: torch.Tensor,
w2: torch.Tensor,
+ layer: torch.nn.Module | None = None,
+ w13_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Convert INT8 MoE weights to backend-specific kernel format."""
- if int8_backend == Int8MoeBackend.CPU:
+ if int8_backend == Int8MoeBackend.HUMMING:
+ from vllm.model_executor.layers.quantization.utils.humming_utils import (
+ convert_to_humming_moe_kernel_format,
+ )
+
+ assert layer is not None
+ # Humming reads canonical CT scales (w*_weight_scale) from the layer.
+ # Online int8 produces per-channel (E, N) w*_scale; expose them as the
+ # (E, N, 1) w*_weight_scale humming's loader expects.
+ for sub in ("w13", "w2"):
+ if hasattr(layer, f"{sub}_weight_scale"):
+ continue
+ scale = getattr(layer, f"{sub}_scale").data
+ if scale.dim() < 3:
+ scale = scale.unsqueeze(-1)
+ replace_parameter(layer, f"{sub}_weight_scale", scale)
+ delattr(layer, f"{sub}_scale")
+ convert_to_humming_moe_kernel_format(
+ layer,
+ quant_config=_humming_int8_weight_schema(w13, layer.w13_weight_scale),
+ )
+ return layer.w13_weight, layer.w2_weight
+ elif int8_backend == Int8MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_int8_moe_layer_for_cpu,
)
@@ -215,10 +287,12 @@ def convert_to_int8_moe_kernel_format(
def make_int8_moe_kernel(
+ int8_backend: Int8MoeBackend,
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
experts_cls: type[mk.FusedMoEExperts],
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
+ layer: torch.nn.Module | None = None,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
prepare_finalize = maybe_make_prepare_finalize(
@@ -232,6 +306,11 @@ def make_int8_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
+ extra_kwargs = {}
+ if int8_backend == Int8MoeBackend.HUMMING:
+ assert layer is not None
+ extra_kwargs = {"layer": layer}
+
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -241,11 +320,13 @@ def make_int8_moe_kernel(
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
+ **extra_kwargs,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
+ **extra_kwargs,
)
kernel = mk.FusedMoEKernel(
diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py
index 0cf1382d40..d0cc08ea14 100644
--- a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py
+++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py
@@ -11,6 +11,7 @@ from compressed_tensors.quantization import (
import vllm._custom_ops as ops
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
+from vllm.config.kernel import MoEBackend
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
@@ -46,6 +47,7 @@ logger = init_logger(__name__)
class WNA16MoEBackend(Enum):
MARLIN = "MARLIN"
BATCHED_MARLIN = "BATCHED_MARLIN"
+ HUMMING = "HUMMING"
CPU = "CPU"
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
XPU = "XPU"
@@ -55,7 +57,19 @@ def backend_to_kernel_cls(
backend: WNA16MoEBackend,
) -> list[type[mk.FusedMoEExperts]]:
"""Return the experts class for the given backend, or None for NONE."""
- if backend == WNA16MoEBackend.MARLIN:
+ if backend == WNA16MoEBackend.HUMMING:
+ from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
+ BatchedHummingGroupedExperts,
+ HummingGroupedExperts,
+ HummingIndexedExperts,
+ )
+
+ return [
+ BatchedHummingGroupedExperts,
+ HummingGroupedExperts,
+ HummingIndexedExperts,
+ ]
+ elif backend == WNA16MoEBackend.MARLIN:
return [MarlinExperts]
elif backend == WNA16MoEBackend.BATCHED_MARLIN:
return [BatchedMarlinExperts]
@@ -90,10 +104,26 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
WNA16MoEBackend.FLASHINFER_TRTLLM,
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
+ WNA16MoEBackend.HUMMING,
]
return _AVAILABLE_BACKENDS
+def map_wna16_backend(runner_backend: MoEBackend) -> WNA16MoEBackend:
+ """Map user's MoEBackend to WNA16MoEBackend."""
+ mapping = {
+ "marlin": WNA16MoEBackend.MARLIN,
+ "humming": WNA16MoEBackend.HUMMING,
+ "flashinfer_trtllm": WNA16MoEBackend.FLASHINFER_TRTLLM,
+ }
+ if backend := mapping.get(runner_backend):
+ return backend
+ raise ValueError(
+ f"moe_backend='{runner_backend}' is not supported for WNA16 MoE. "
+ f"Expected one of {list(mapping.keys())}."
+ )
+
+
def select_wna16_moe_backend(
config: FusedMoEConfig,
weight_key: QuantKey,
@@ -146,6 +176,14 @@ def select_wna16_moe_backend(
return backend, k_cls
raise ValueError(_make_log_unsupported(backend, reason))
+ # Handle explicit moe_backend from user.
+ runner_backend = config.moe_backend
+ if runner_backend != "auto":
+ requested_backend = map_wna16_backend(runner_backend)
+ return _return_or_raise(
+ requested_backend, config, weight_key, None, activation_format
+ )
+
# Select kernels in order of backend.
AVAILABLE_BACKENDS = _get_priority_backends()
@@ -210,6 +248,8 @@ def make_wna16_moe_kernel(
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
experts_cls: type[mk.FusedMoEExperts],
+ backend: WNA16MoEBackend = WNA16MoEBackend.MARLIN,
+ layer: torch.nn.Module | None = None,
is_k_full: bool = False,
w13_g_idx: torch.Tensor | None = None,
w2_g_idx: torch.Tensor | None = None,
@@ -228,14 +268,18 @@ def make_wna16_moe_kernel(
)
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
- # BatchedMarlinExperts, XPUExpertsWNA16, and CPUExpertsInt4
- assert experts_cls in (
+ # BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, and the Humming
+ # grouped/indexed experts.
+ allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = (
MarlinExperts,
BatchedMarlinExperts,
TrtLlmMxint4ExpertsMonolithic,
XPUExpertsWNA16,
CPUExpertsInt4,
)
+ if backend == WNA16MoEBackend.HUMMING:
+ allowed_experts += tuple(backend_to_kernel_cls(WNA16MoEBackend.HUMMING))
+ assert experts_cls in allowed_experts
is_monolithic = experts_cls.is_monolithic()
@@ -251,7 +295,10 @@ def make_wna16_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__, scope="local")
extra_args: dict[str, Any] = {}
- if issubclass(experts_cls, MarlinExpertsBase):
+ if backend == WNA16MoEBackend.HUMMING:
+ assert layer is not None
+ extra_args = {"layer": layer}
+ elif issubclass(experts_cls, MarlinExpertsBase):
extra_args = {
"w13_g_idx": w13_g_idx,
"w2_g_idx": w2_g_idx,
@@ -941,6 +988,35 @@ def _process_weights_xpu(
)
+def _humming_wna16_weight_schema(
+ quant_config: QuantizationConfig | QuantizationArgs | None,
+) -> dict[str, Any]:
+ """Humming weight schema for a WNA16 checkpoint, derived from the quant
+ config rather than the running kernel."""
+ from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
+ from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
+
+ if isinstance(quant_config, AutoAWQConfig):
+ return {
+ "quant_method": "awq",
+ "bits": quant_config.weight_bits,
+ "group_size": quant_config.group_size,
+ "zero_point": quant_config.zero_point,
+ }
+ if isinstance(quant_config, AutoGPTQConfig):
+ return {
+ "quant_method": "gptq",
+ "bits": quant_config.weight_bits,
+ "group_size": quant_config.group_size,
+ "desc_act": quant_config.desc_act,
+ "sym": quant_config.is_sym,
+ }
+ raise TypeError(
+ "Humming WNA16 MoE requires AutoAWQConfig or AutoGPTQConfig, "
+ f"got {type(quant_config).__name__}."
+ )
+
+
def convert_to_wna16_moe_kernel_format(
backend: WNA16MoEBackend,
layer: torch.nn.Module,
@@ -956,26 +1032,30 @@ def convert_to_wna16_moe_kernel_format(
w2_qzeros: torch.Tensor | None = None,
w13_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
-) -> tuple[
- torch.Tensor, # w13_qweight
- torch.Tensor, # w2_qweight
- torch.Tensor, # w13_scales
- torch.Tensor, # w2_scales
- torch.Tensor | None, # w13_g_idx
- torch.Tensor | None, # w2_g_idx
- torch.Tensor | None, # w13_g_idx_sort_indices
- torch.Tensor | None, # w2_g_idx_sort_indices
- torch.Tensor | None, # w13_qzeros
- torch.Tensor | None, # w2_qzeros
- torch.Tensor | None, # w13_input_global_scale
- torch.Tensor | None, # w2_input_global_scale
- torch.Tensor | None, # w13_bias
- torch.Tensor | None, # w2_bias
-]:
+) -> (
+ tuple[
+ torch.Tensor, # w13_qweight
+ torch.Tensor, # w2_qweight
+ torch.Tensor, # w13_scales
+ torch.Tensor, # w2_scales
+ torch.Tensor | None, # w13_g_idx
+ torch.Tensor | None, # w2_g_idx
+ torch.Tensor | None, # w13_g_idx_sort_indices
+ torch.Tensor | None, # w2_g_idx_sort_indices
+ torch.Tensor | None, # w13_qzeros
+ torch.Tensor | None, # w2_qzeros
+ torch.Tensor | None, # w13_input_global_scale
+ torch.Tensor | None, # w2_input_global_scale
+ torch.Tensor | None, # w13_bias
+ torch.Tensor | None, # w2_bias
+ ]
+ | None
+):
"""Dispatch weight post-processing to the appropriate per-backend handler.
To add a new backend, implement a ``_process_weights_`` helper and
- add a branch here.
+ add a branch here. Backends that rewrite the layer's parameters in place
+ (e.g. Humming) return ``None``; the caller then skips the param scatter.
Args:
backend: the selected ``WNA16MoEBackend``.
@@ -983,6 +1063,16 @@ def convert_to_wna16_moe_kernel_format(
quant_config: the ``QuantizationConfig`` for this layer.
input_dtype: optional activation dtype, usually should be 16 bit.
"""
+ if backend == WNA16MoEBackend.HUMMING:
+ from vllm.model_executor.layers.quantization.utils.humming_utils import (
+ convert_to_humming_moe_kernel_format,
+ )
+
+ convert_to_humming_moe_kernel_format(
+ layer, quant_config=_humming_wna16_weight_schema(quant_config)
+ )
+ return None
+
if backend in (
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py
index 06b622a6c4..b9086cfa48 100644
--- a/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py
+++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py
@@ -25,6 +25,7 @@ _SUPPORTED_BACKENDS = (
# is_supported_config passes (gfx950 + flydsl installed + not EP). On other
# devices / no flydsl / EP it is skipped and native is used.
Fp8MoeBackend.AITER_MXFP8,
+ Fp8MoeBackend.HUMMING,
)
_BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
@@ -34,6 +35,7 @@ _BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
"xpu": Fp8MoeBackend.XPU,
"aiter": Fp8MoeBackend.AITER_MXFP8,
"triton": Fp8MoeBackend.TRITON_MXFP8,
+ "humming": Fp8MoeBackend.HUMMING,
}
diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py
index 408c69fea0..f295163568 100644
--- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py
+++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py
@@ -43,6 +43,7 @@ class NvFp4MoeBackend(Enum):
FLASHINFER_B12X = "FLASHINFER_B12X"
VLLM_CUTLASS = "VLLM_CUTLASS"
MARLIN = "MARLIN"
+ HUMMING = "HUMMING"
EMULATION = "EMULATION"
@@ -119,6 +120,18 @@ def backend_to_kernel_cls(
)
return [MarlinExperts]
+ elif backend == NvFp4MoeBackend.HUMMING:
+ from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
+ BatchedHummingGroupedExperts,
+ HummingGroupedExperts,
+ HummingIndexedExperts,
+ )
+
+ return [
+ BatchedHummingGroupedExperts,
+ HummingGroupedExperts,
+ HummingIndexedExperts,
+ ]
elif backend == NvFp4MoeBackend.EMULATION:
from vllm.model_executor.layers.fused_moe.experts.nvfp4_emulation_moe import (
Nvfp4QuantizationEmulationTritonExperts,
@@ -138,6 +151,7 @@ def map_nvfp4_backend(runner_backend: MoEBackend) -> NvFp4MoeBackend:
"flashinfer_cutedsl": NvFp4MoeBackend.FLASHINFER_CUTEDSL,
"flashinfer_b12x": NvFp4MoeBackend.FLASHINFER_B12X,
"marlin": NvFp4MoeBackend.MARLIN,
+ "humming": NvFp4MoeBackend.HUMMING,
"emulation": NvFp4MoeBackend.EMULATION,
}
if backend := mapping.get(runner_backend):
@@ -169,6 +183,7 @@ def select_nvfp4_moe_backend(
NvFp4MoeBackend.FLASHINFER_CUTLASS,
NvFp4MoeBackend.VLLM_CUTLASS,
NvFp4MoeBackend.MARLIN,
+ NvFp4MoeBackend.HUMMING,
NvFp4MoeBackend.EMULATION,
]
@@ -346,6 +361,41 @@ def convert_to_nvfp4_moe_kernel_format(
a2_scale=a2_scale,
is_act_and_mul=is_act_and_mul,
)
+ elif nvfp4_backend == NvFp4MoeBackend.HUMMING:
+ from vllm.model_executor.layers.quantization.utils.humming_utils import (
+ convert_to_humming_moe_kernel_format,
+ )
+
+ # Discriminate the source checkpoint layout by its global-scale param:
+ # compressed-tensors uses *_weight_global_scale, modelopt *_weight_scale_2.
+ # The logical schema is identical (nvfp4 group-16); only the on-layer
+ # param names differ. TODO: normalize both methods to a single canonical
+ # layout upstream so the oracle needs neither the probe nor the re-alias.
+ if hasattr(layer, "w13_weight_global_scale"):
+ quant_config = {
+ "quant_method": "compressed-tensors",
+ "format": "nvfp4-pack-quantized",
+ "type": "float",
+ "num_bits": 4,
+ "strategy": "group",
+ "group_size": 16,
+ }
+ # CT pack-quantized reads `weight_packed`; the method renamed it to
+ # `weight`. Re-alias (convert replaces all params anyway).
+ layer.w13_weight_packed = layer.w13_weight
+ layer.w2_weight_packed = layer.w2_weight
+ else:
+ quant_config = {"quant_method": "modelopt", "quant_algo": "nvfp4"}
+
+ convert_to_humming_moe_kernel_format(layer, quant_config=quant_config)
+ a13_scale = None
+ a2_scale = None
+ w13 = layer.w13_weight
+ w13_scale = layer.w13_weight_scale
+ w13_scale_2 = getattr(layer, "w13_global_scale", None)
+ w2 = layer.w2_weight
+ w2_scale = layer.w2_weight_scale
+ w2_scale_2 = getattr(layer, "w2_global_scale", None)
elif nvfp4_backend == NvFp4MoeBackend.MARLIN:
a13_scale = None
a2_scale = None
@@ -418,8 +468,17 @@ def make_nvfp4_moe_quant_config(
a13_scale: torch.Tensor,
a2_scale: torch.Tensor,
swiglu_limit: float | None = None,
+ layer: torch.nn.Module | None = None,
) -> FusedMoEQuantConfig:
- if backend == NvFp4MoeBackend.MARLIN:
+ if backend == NvFp4MoeBackend.HUMMING:
+ from vllm.model_executor.layers.fused_moe import RoutedExperts
+ from vllm.model_executor.layers.quantization.utils.humming_utils import (
+ get_humming_moe_quant_config,
+ )
+
+ assert isinstance(layer, RoutedExperts)
+ return get_humming_moe_quant_config(layer)
+ elif backend == NvFp4MoeBackend.MARLIN:
return nvfp4_w4a16_moe_quant_config(
g1_alphas=w13_scale_2,
g2_alphas=w2_scale_2,
@@ -467,7 +526,9 @@ def make_nvfp4_moe_kernel(
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
experts_cls: type[mk.FusedMoEExperts],
+ backend: NvFp4MoeBackend,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
+ layer: torch.nn.Module | None = None,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
prepare_finalize = maybe_make_prepare_finalize(
@@ -481,6 +542,11 @@ def make_nvfp4_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
+ extra_kwargs = {}
+ if backend == NvFp4MoeBackend.HUMMING:
+ assert layer is not None
+ extra_kwargs = {"layer": layer}
+
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -490,11 +556,13 @@ def make_nvfp4_moe_kernel(
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
+ **extra_kwargs,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
+ **extra_kwargs,
)
kernel = mk.FusedMoEKernel(
diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py
index cda0eaf730..639fce5234 100644
--- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py
+++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py
@@ -232,7 +232,9 @@ def select_unquantized_moe_backend(
raise ValueError(_make_log_unsupported(backend, reason))
runner_backend = moe_config.moe_backend
- if runner_backend != "auto":
+ # 'humming' is quantization-only; an unquantized layer (e.g. excluded via
+ # modules_to_not_convert) falls through to auto instead of erroring.
+ if runner_backend not in ["auto", "humming"]:
requested_backend = map_unquantized_backend(runner_backend)
if (
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
diff --git a/vllm/model_executor/layers/quantization/auto_awq.py b/vllm/model_executor/layers/quantization/auto_awq.py
index cebfad7e59..58104fa7d2 100644
--- a/vllm/model_executor/layers/quantization/auto_awq.py
+++ b/vllm/model_executor/layers/quantization/auto_awq.py
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe import (
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
+ WNA16MoEBackend,
convert_to_wna16_moe_kernel_format,
make_wna16_moe_kernel,
make_wna16_moe_quant_config,
@@ -663,6 +664,26 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
layer.workspace = marlin_make_workspace_new(device, 4)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
+ converted = convert_to_wna16_moe_kernel_format(
+ backend=self.wna16_moe_backend,
+ layer=layer,
+ quant_config=self.quant_config,
+ input_dtype=self.input_dtype,
+ w13=layer.w13_qweight,
+ w2=layer.w2_qweight,
+ w13_scale=layer.w13_scales,
+ w2_scale=layer.w2_scales,
+ w13_qzeros=layer.w13_qzeros,
+ w2_qzeros=layer.w2_qzeros,
+ w13_bias=getattr(layer, "w13_bias", None),
+ w2_bias=getattr(layer, "w2_bias", None),
+ )
+
+ if converted is None:
+ # Backend rewrote the layer's params in place (e.g. Humming).
+ self._setup_kernel(layer)
+ return
+
(
w13,
w2,
@@ -678,20 +699,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
w2_input_global_scale,
w13_bias,
w2_bias,
- ) = convert_to_wna16_moe_kernel_format(
- backend=self.wna16_moe_backend,
- layer=layer,
- quant_config=self.quant_config,
- input_dtype=self.input_dtype,
- w13=layer.w13_qweight,
- w2=layer.w2_qweight,
- w13_scale=layer.w13_scales,
- w2_scale=layer.w2_scales,
- w13_qzeros=layer.w13_qzeros,
- w2_qzeros=layer.w2_qzeros,
- w13_bias=getattr(layer, "w13_bias", None),
- w2_bias=getattr(layer, "w2_bias", None),
- )
+ ) = converted
replace_parameter(layer, "w13_qweight", w13)
replace_parameter(layer, "w2_qweight", w2)
@@ -734,6 +742,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
+ backend=self.wna16_moe_backend,
+ layer=layer,
is_k_full=self.is_k_full,
w13_g_idx=getattr(layer, "w13_g_idx", None),
w2_g_idx=getattr(layer, "w2_g_idx", None),
@@ -743,6 +753,12 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
)
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
+ if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
+ from vllm.model_executor.layers.quantization.utils.humming_utils import (
+ get_humming_moe_quant_config,
+ )
+
+ return get_humming_moe_quant_config(layer)
return make_wna16_moe_quant_config(
w1_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
@@ -783,8 +799,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
assert self.moe_kernel is not None
return self.moe_kernel.apply(
hidden_states=x,
- w1=layer.w13_qweight,
- w2=layer.w2_qweight,
+ w1=layer.w13_weight,
+ w2=layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
@@ -806,8 +822,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
hidden_states=x,
- w1=layer.w13_qweight,
- w2=layer.w2_qweight,
+ w1=layer.w13_weight,
+ w2=layer.w2_weight,
router_logits=router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
diff --git a/vllm/model_executor/layers/quantization/auto_gptq.py b/vllm/model_executor/layers/quantization/auto_gptq.py
index aca76162f9..b056f5222a 100644
--- a/vllm/model_executor/layers/quantization/auto_gptq.py
+++ b/vllm/model_executor/layers/quantization/auto_gptq.py
@@ -658,6 +658,26 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
"W8A8-INT8 is not supported by marlin kernel."
)
+ converted = convert_to_wna16_moe_kernel_format(
+ backend=self.wna16_moe_backend,
+ layer=layer,
+ quant_config=self.quant_config,
+ input_dtype=self.input_dtype,
+ w13=layer.w13_qweight,
+ w2=layer.w2_qweight,
+ w13_scale=layer.w13_scales,
+ w2_scale=layer.w2_scales,
+ w13_g_idx=layer.w13_g_idx,
+ w2_g_idx=layer.w2_g_idx,
+ w13_bias=getattr(layer, "w13_bias", None),
+ w2_bias=getattr(layer, "w2_bias", None),
+ )
+
+ if converted is None:
+ # Backend rewrote the layer's params in place (e.g. Humming).
+ self._setup_kernel(layer)
+ return
+
(
w13,
w2,
@@ -673,20 +693,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
w2_input_global_scale,
w13_bias,
w2_bias,
- ) = convert_to_wna16_moe_kernel_format(
- backend=self.wna16_moe_backend,
- layer=layer,
- quant_config=self.quant_config,
- input_dtype=self.input_dtype,
- w13=layer.w13_qweight,
- w2=layer.w2_qweight,
- w13_scale=layer.w13_scales,
- w2_scale=layer.w2_scales,
- w13_g_idx=layer.w13_g_idx,
- w2_g_idx=layer.w2_g_idx,
- w13_bias=getattr(layer, "w13_bias", None),
- w2_bias=getattr(layer, "w2_bias", None),
- )
+ ) = converted
replace_parameter(layer, "w13_qweight", w13)
replace_parameter(layer, "w2_qweight", w2)
@@ -733,6 +740,10 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
"w2_bias", torch.nn.Parameter(w2_bias, requires_grad=False)
)
+ # The modular kernel reads w13_weight/w2_weight; marlin keeps *_qweight.
+ layer.w13_weight = layer.w13_qweight
+ layer.w2_weight = layer.w2_qweight
+
self._setup_kernel(layer)
def _setup_kernel(self, layer: RoutedExperts) -> None:
@@ -743,15 +754,24 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
+ backend=self.wna16_moe_backend,
+ layer=layer,
is_k_full=self.is_k_full,
- w13_g_idx=layer.w13_g_idx,
- w2_g_idx=layer.w2_g_idx,
+ w13_g_idx=getattr(layer, "w13_g_idx", None),
+ w2_g_idx=getattr(layer, "w2_g_idx", None),
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
routing_tables=layer._expert_routing_tables(),
)
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
+ if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
+ from vllm.model_executor.layers.quantization.utils.humming_utils import (
+ get_humming_moe_quant_config,
+ )
+
+ return get_humming_moe_quant_config(layer)
+
from vllm.model_executor.layers.fused_moe.config import (
gptq_marlin_moe_quant_config,
)
@@ -795,8 +815,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
assert self.moe_kernel is not None
return self.moe_kernel.apply(
hidden_states=x,
- w1=layer.w13_qweight,
- w2=layer.w2_qweight,
+ w1=layer.w13_weight,
+ w2=layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
@@ -818,8 +838,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
hidden_states=x,
- w1=layer.w13_qweight,
- w2=layer.w2_qweight,
+ w1=layer.w13_weight,
+ w2=layer.w2_weight,
router_logits=router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py
index 9a051c038f..f74ed2d7b3 100644
--- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py
+++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py
@@ -236,7 +236,9 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
+ backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
+ layer=layer,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -259,6 +261,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
a13_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
swiglu_limit=getattr(layer, "swiglu_limit", None),
+ layer=layer,
)
def apply_monolithic(
diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py
index 14ef8bf614..35d2a37842 100644
--- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py
+++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py
@@ -338,6 +338,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
+ layer=layer,
)
def maybe_make_prepare_finalize(
@@ -355,12 +356,13 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
fp8_backend=self.fp8_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
- a1_scale=layer.w13_input_scale,
- a2_scale=layer.w2_input_scale,
+ a1_scale=getattr(layer, "w13_input_scale", None),
+ a2_scale=getattr(layer, "w2_input_scale", None),
per_act_token_quant=is_per_token,
per_out_ch_quant=is_per_token,
block_shape=self.weight_block_size,
swiglu_limit=getattr(layer, "swiglu_limit", None),
+ layer=layer,
)
def apply_monolithic(
diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py
index c29472cfc6..d304ca56bf 100644
--- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py
+++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py
@@ -146,6 +146,8 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
int8_backend=self.int8_backend,
w13=layer.w13_weight,
w2=layer.w2_weight,
+ layer=layer,
+ w13_scale=layer.w13_weight_scale,
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
@@ -153,10 +155,12 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.experts_cls is not None
self.moe_kernel = make_int8_moe_kernel(
+ int8_backend=self.int8_backend,
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
+ layer=layer,
)
def maybe_make_prepare_finalize(
@@ -170,11 +174,13 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
return make_int8_moe_quant_config(
+ int8_backend=self.int8_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
a1_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
per_act_token_quant=True,
+ layer=layer,
)
def apply(
diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py
index 2e6e01ca76..468aed29a9 100644
--- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py
+++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py
@@ -140,6 +140,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
+ layer=layer,
)
def get_fused_moe_quant_config(
@@ -155,6 +156,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
+ layer=layer,
)
def maybe_make_prepare_finalize(
diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py
index 46fa36180d..8af36bcb10 100644
--- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py
+++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py
@@ -416,6 +416,27 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Process weights using the shared oracle infrastructure
is_flashinfer = self.wna16_backend == WNA16MoEBackend.FLASHINFER_TRTLLM
+ converted = convert_to_wna16_moe_kernel_format(
+ backend=self.wna16_backend,
+ layer=layer,
+ quant_config=self.weight_quant,
+ input_dtype=self.marlin_input_dtype,
+ w13=layer.w13_weight_packed,
+ w2=layer.w2_weight_packed,
+ w13_scale=layer.w13_weight_scale,
+ w2_scale=layer.w2_weight_scale,
+ w13_g_idx=layer.w13_weight_g_idx,
+ w2_g_idx=layer.w2_weight_g_idx,
+ w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
+ w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
+ )
+ if converted is None:
+ # In-place backends (e.g. Humming) are not wired through this
+ # marlin-only method; fail clearly rather than unpacking None.
+ raise NotImplementedError(
+ f"{type(self).__name__} does not support the "
+ f"{self.wna16_backend.value} MoE backend."
+ )
(
w13_qweight,
w2_qweight,
@@ -431,20 +452,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
w2_input_global_scale,
_, # w13_bias
_, # w2_bias
- ) = convert_to_wna16_moe_kernel_format(
- backend=self.wna16_backend,
- layer=layer,
- quant_config=self.weight_quant,
- input_dtype=self.marlin_input_dtype,
- w13=layer.w13_weight_packed,
- w2=layer.w2_weight_packed,
- w13_scale=layer.w13_weight_scale,
- w2_scale=layer.w2_weight_scale,
- w13_g_idx=layer.w13_weight_g_idx,
- w2_g_idx=layer.w2_weight_g_idx,
- w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
- w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
- )
+ ) = converted
# Replace common parameters
replace_parameter(layer, "w13_weight_packed", w13_qweight)
diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
index 1301c98f45..23e3510614 100644
--- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
+++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py
@@ -137,8 +137,16 @@ class CompressedTensorsW8A16Fp8(CompressedTensorsScheme):
"weight_scale",
convert_to_channelwise(layer.weight_scale, layer.logical_widths),
)
+ self.strategy = QuantizationStrategy.CHANNEL
+ self.weight_quant_key = STRATEGY_TO_WEIGHT_QUANT_KEY[self.strategy]
+ self.linear_kernel.config.weight_quant_key = self.weight_quant_key
+
# Canonicalize to (K, N) for the kernel.
replace_parameter(layer, "weight", layer.weight.t())
+ # Preserve the dim tags dropped by the transpose so layout-aware
+ # kernels see (K, N).
+ layer.weight.input_dim = 0
+ layer.weight.output_dim = 1
self.linear_kernel.process_weights_after_loading(layer)
diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
index 1a240f6540..e50778f0ac 100644
--- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
@@ -181,6 +181,10 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
# required by torch.compile to be torch.nn.Parameter
layer.weight = Parameter(weight.data, requires_grad=False)
+ # Preserve the dim tags dropped by the transpose so layout-aware
+ # kernels (humming) see (K, N) instead of assuming (N, K).
+ layer.weight.input_dim = 0
+ layer.weight.output_dim = 1
layer.weight_scale = Parameter(weight_scale.data, requires_grad=False)
if input_scale is not None:
layer.input_scale = Parameter(input_scale.data, requires_grad=False)
diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py
index 86818ed4b7..626fc83cdf 100644
--- a/vllm/model_executor/layers/quantization/fp8.py
+++ b/vllm/model_executor/layers/quantization/fp8.py
@@ -714,6 +714,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
+ layer=layer,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
@@ -786,6 +787,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
+ layer=layer,
)
# Inject biases into the quant config if the model has them
diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py
index 8fa1cb4d54..4a078f1357 100644
--- a/vllm/model_executor/layers/quantization/modelopt.py
+++ b/vllm/model_executor/layers/quantization/modelopt.py
@@ -905,6 +905,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
+ layer=layer,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
@@ -951,6 +952,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
a1_scale=a1_scale,
a2_scale=a2_scale,
swiglu_limit=getattr(layer, "swiglu_limit", None),
+ layer=layer,
)
def apply_monolithic(
@@ -1602,7 +1604,9 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
+ backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
+ layer=layer,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -1616,6 +1620,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
a13_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
swiglu_limit=getattr(layer, "swiglu_limit", None),
+ layer=layer,
)
@property
@@ -2162,6 +2167,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
fp8_backend=self.mxfp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
+ layer=layer,
)
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
@@ -2207,6 +2213,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
+ layer=layer,
)
def apply_monolithic(
diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py
index 4d3a315879..e63c285aa3 100644
--- a/vllm/model_executor/layers/quantization/online/fp8.py
+++ b/vllm/model_executor/layers/quantization/online/fp8.py
@@ -458,6 +458,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
+ layer=layer,
)
def get_fused_moe_quant_config(
@@ -486,6 +487,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
+ layer=layer,
)
diff --git a/vllm/model_executor/layers/quantization/online/int8.py b/vllm/model_executor/layers/quantization/online/int8.py
index 4274b16d55..ef76594164 100644
--- a/vllm/model_executor/layers/quantization/online/int8.py
+++ b/vllm/model_executor/layers/quantization/online/int8.py
@@ -13,6 +13,7 @@ if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
+ convert_to_int8_moe_kernel_format,
make_int8_moe_kernel,
make_int8_moe_quant_config,
select_int8_moe_backend,
@@ -92,22 +93,36 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
replace_parameter(layer, "w2_scale", w2_scale)
def _setup_kernel(self, layer: RoutedExperts) -> None:
+ w13, w2 = convert_to_int8_moe_kernel_format(
+ int8_backend=self.int8_backend,
+ w13=layer.w13_weight,
+ w2=layer.w2_weight,
+ layer=layer,
+ w13_scale=layer.w13_scale,
+ )
+ replace_parameter(layer, "w13_weight", w13)
+ replace_parameter(layer, "w2_weight", w2)
+
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
assert self.experts_cls is not None
self.moe_kernel = make_int8_moe_kernel(
+ int8_backend=self.int8_backend,
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
+ layer=layer,
)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> "FusedMoEQuantConfig | None":
return make_int8_moe_quant_config(
- w1_scale=layer.w13_scale,
- w2_scale=layer.w2_scale,
+ int8_backend=self.int8_backend,
+ w1_scale=getattr(layer, "w13_scale", None),
+ w2_scale=getattr(layer, "w2_scale", None),
w1_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
+ layer=layer,
)
diff --git a/vllm/model_executor/layers/quantization/online/mxfp8.py b/vllm/model_executor/layers/quantization/online/mxfp8.py
index 09d581a073..84a81bd906 100644
--- a/vllm/model_executor/layers/quantization/online/mxfp8.py
+++ b/vllm/model_executor/layers/quantization/online/mxfp8.py
@@ -200,6 +200,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
+ layer=layer,
)
def get_fused_moe_quant_config(
@@ -226,6 +227,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
+ layer=layer,
)
def process_weights_after_loading(self, layer: Module) -> None:
diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py
index bce888415c..7bdf963b51 100644
--- a/vllm/model_executor/layers/quantization/quark/quark_moe.py
+++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py
@@ -1562,7 +1562,9 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
+ backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
+ layer=layer,
)
def get_fused_moe_quant_config(
@@ -1576,6 +1578,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
w2_scale_2=layer.w2_weight_scale_2,
a13_scale=layer.w13_input_scale_2,
a2_scale=layer.w2_input_scale_2,
+ layer=layer,
)
def apply(
diff --git a/vllm/model_executor/layers/quantization/utils/humming_utils.py b/vllm/model_executor/layers/quantization/utils/humming_utils.py
index d84a2e12f5..2d9e7aca76 100644
--- a/vllm/model_executor/layers/quantization/utils/humming_utils.py
+++ b/vllm/model_executor/layers/quantization/utils/humming_utils.py
@@ -764,7 +764,7 @@ def _convert_sublayer_to_humming(
Returns:
Tuple of (converted_weight_schema, converted_input_schema)
"""
- from humming.schema import HummingWeightSchema
+ from vllm.utils.humming import HummingWeightSchema
if isinstance(weight_schema, HummingWeightSchema):
# Already in Humming format
@@ -814,7 +814,7 @@ def _prepare_and_transform_sublayer(
This calls Humming's prepare_layer_meta and transform_humming_layer.
"""
- from humming.layer import HummingMethod
+ from vllm.utils.humming import HummingMethod
HummingMethod.prepare_layer_meta(
layer=layer,
@@ -866,7 +866,7 @@ def _process_single_sublayer(
Returns:
Tuple of (final_weight_schema, final_input_schema)
"""
- from humming.schema import HummingWeightSchema
+ from vllm.utils.humming import HummingWeightSchema
# Step 1: Convert from checkpoint format to humming format if needed
current_weight_schema, current_input_schema = _convert_sublayer_to_humming(
@@ -958,12 +958,10 @@ def convert_to_humming_moe_kernel_format(
"Must provide either weight_schema/input_schema or quant_config"
)
- from humming.layer import HummingInputSchema
- from humming.schema import BaseWeightSchema
-
from vllm.model_executor.layers.quantization.utils.humming_utils import (
humming_is_layer_skipped,
)
+ from vllm.utils.humming import BaseWeightSchema, HummingInputSchema
if weight_schema is None:
weight_schema = BaseWeightSchema.from_config(quant_config)
diff --git a/vllm/model_executor/warmup/deep_gemm_warmup.py b/vllm/model_executor/warmup/deep_gemm_warmup.py
index d41604fc7a..cfff491ab2 100644
--- a/vllm/model_executor/warmup/deep_gemm_warmup.py
+++ b/vllm/model_executor/warmup/deep_gemm_warmup.py
@@ -11,6 +11,9 @@ from tqdm import tqdm
import vllm.envs as envs
from vllm.distributed.parallel_state import get_dp_group, is_global_first_rank
+from vllm.model_executor.kernels.linear.scaled_mm.deep_gemm import (
+ DeepGemmFp8BlockScaledMMKernel,
+)
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
compute_aligned_M_and_alignment,
@@ -147,6 +150,12 @@ def _fp8_linear_may_use_deep_gemm(module: torch.nn.Module) -> bool:
):
return False
+ if not isinstance(
+ getattr(module.quant_method, "fp8_linear", None),
+ DeepGemmFp8BlockScaledMMKernel,
+ ):
+ return False
+
w, _, block_sizes = _extract_data_from_linear_base_module(module)
return (
block_sizes == get_mk_alignment_for_contiguous_layout()