diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 8f47e7b5c2..4cf7774ebd 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -150,9 +150,9 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt -- label: LM Eval Humming (A100 - TEMPORARY) - key: lm-eval-humming-a100 - timeout_in_minutes: 30 +- label: LM Eval Humming f16 (A100 - TEMPORARY) + key: lm-eval-humming-f16-a100 + timeout_in_minutes: 120 device: a100 optional: true num_devices: 1 @@ -160,13 +160,29 @@ steps: - vllm/model_executor/layers/quantization/humming.py - vllm/model_executor/layers/quantization/utils/humming_utils.py - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py - - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py + - vllm/model_executor/layers/fused_moe/oracle/ + - vllm/model_executor/kernels/linear/ commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt -- label: LM Eval Humming (H100 - TEMPORARY) - key: lm-eval-humming-h100 - timeout_in_minutes: 30 +- label: LM Eval Humming Act int8 (A100 - TEMPORARY) + key: lm-eval-humming-act-a100 + timeout_in_minutes: 120 + device: a100 + optional: true + num_devices: 1 + source_file_dependencies: + - vllm/model_executor/layers/quantization/humming.py + - vllm/model_executor/layers/quantization/utils/humming_utils.py + - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py + - vllm/model_executor/layers/fused_moe/oracle/ + - vllm/model_executor/kernels/linear/ + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt + +- label: LM Eval Humming f16 (H100 - TEMPORARY) + key: lm-eval-humming-f16-h100 + timeout_in_minutes: 120 device: h100 optional: true num_devices: 1 @@ -174,14 +190,30 @@ steps: - vllm/model_executor/layers/quantization/humming.py - vllm/model_executor/layers/quantization/utils/humming_utils.py - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py - - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py + - vllm/model_executor/layers/fused_moe/oracle/ + - vllm/model_executor/kernels/linear/ commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt -- label: LM Eval Humming (B200 - TEMPORARY) - key: lm-eval-humming-b200 - timeout_in_minutes: 30 +- label: LM Eval Humming Act fp8/int8 (H100 - TEMPORARY) + key: lm-eval-humming-act-h100 + timeout_in_minutes: 120 + device: h100 + optional: true + num_devices: 1 + source_file_dependencies: + - vllm/model_executor/layers/quantization/humming.py + - vllm/model_executor/layers/quantization/utils/humming_utils.py + - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py + - vllm/model_executor/layers/fused_moe/oracle/ + - vllm/model_executor/kernels/linear/ + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt + +- label: LM Eval Humming f16 (B200 - TEMPORARY) + key: lm-eval-humming-f16-b200 + timeout_in_minutes: 120 device: b200-k8s optional: true num_devices: 1 @@ -189,10 +221,26 @@ steps: - vllm/model_executor/layers/quantization/humming.py - vllm/model_executor/layers/quantization/utils/humming_utils.py - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py - - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py + - vllm/model_executor/layers/fused_moe/oracle/ + - vllm/model_executor/kernels/linear/ commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt + +- label: LM Eval Humming Act fp8/int8 (B200 - TEMPORARY) + key: lm-eval-humming-act-b200 + timeout_in_minutes: 120 + device: b200-k8s + optional: true + num_devices: 1 + source_file_dependencies: + - vllm/model_executor/layers/quantization/humming.py + - vllm/model_executor/layers/quantization/utils/humming_utils.py + - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py + - vllm/model_executor/layers/fused_moe/oracle/ + - vllm/model_executor/kernels/linear/ + commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt - label: LM Eval TurboQuant KV Cache key: lm-eval-turboquant-kv-cache diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 595d61b863..f89854f632 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -28,4 +28,4 @@ quack-kernels>=0.3.3 tokenspeed-mla==0.1.2; platform_system == "Linux" # Humming kernels for quantization gemm -humming-kernels[cu13]==0.1.6 +humming-kernels[cu13]==0.1.10 diff --git a/tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml new file mode 100644 index 0000000000..ba292eb972 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml @@ -0,0 +1,10 @@ +model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8" +accuracy_threshold: 0.55 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --linear-backend humming +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming.yaml new file mode 100644 index 0000000000..3179c3251b --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen2-1.5B-Instruct-FP8W8-humming.yaml @@ -0,0 +1,8 @@ +model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8" +accuracy_threshold: 0.55 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming-act-fp8.yaml new file mode 100644 index 0000000000..66888312ef --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming-act-fp8.yaml @@ -0,0 +1,10 @@ +model_name: "mgoin/Qwen3-0.6B-MXFP8" +accuracy_threshold: 0.39 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --linear-backend humming +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml new file mode 100644 index 0000000000..b83d9e6a9e --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-0.6B-MXFP8-humming.yaml @@ -0,0 +1,8 @@ +model_name: "mgoin/Qwen3-0.6B-MXFP8" +accuracy_threshold: 0.39 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-fp8.yaml new file mode 100644 index 0000000000..d3de5b3792 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-fp8.yaml @@ -0,0 +1,13 @@ +model_name: "QuixiAI/Qwen3-30B-A3B-AWQ" +accuracy_threshold: 0.90 +num_questions: 1319 +num_fewshot: 5 +gen_prefix: " \n\n\n" +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming + --dtype bfloat16 +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-int8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-int8.yaml new file mode 100644 index 0000000000..6b76efc4e2 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming-act-int8.yaml @@ -0,0 +1,13 @@ +model_name: "QuixiAI/Qwen3-30B-A3B-AWQ" +accuracy_threshold: 0.90 +num_questions: 1319 +num_fewshot: 5 +gen_prefix: " \n\n\n" +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming + --dtype bfloat16 +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming.yaml new file mode 100644 index 0000000000..310255f6bf --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-AWQ-humming.yaml @@ -0,0 +1,11 @@ +model_name: "QuixiAI/Qwen3-30B-A3B-AWQ" +accuracy_threshold: 0.90 +num_questions: 1319 +num_fewshot: 5 +gen_prefix: " \n\n\n" +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming + --dtype bfloat16 diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming-act-fp8.yaml new file mode 100644 index 0000000000..6d2702d8f4 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming-act-fp8.yaml @@ -0,0 +1,11 @@ +model_name: "nm-testing/Qwen3-30B-A3B-FP8-block" +accuracy_threshold: 0.86 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml new file mode 100644 index 0000000000..8ca4777ce5 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-FP8-block-humming.yaml @@ -0,0 +1,9 @@ +model_name: "nm-testing/Qwen3-30B-A3B-FP8-block" +accuracy_threshold: 0.86 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml new file mode 100644 index 0000000000..618e7fdcc3 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml @@ -0,0 +1,11 @@ +model_name: "nm-testing/Qwen3-30B-A3B-Fp8-v1" +accuracy_threshold: 0.86 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming.yaml new file mode 100644 index 0000000000..71aabcef99 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Fp8-v1-humming.yaml @@ -0,0 +1,9 @@ +model_name: "nm-testing/Qwen3-30B-A3B-Fp8-v1" +accuracy_threshold: 0.86 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml new file mode 100644 index 0000000000..251fb252cf --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml @@ -0,0 +1,11 @@ +model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4" +accuracy_threshold: 0.86 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml new file mode 100644 index 0000000000..d813e05d6f --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml @@ -0,0 +1,11 @@ +model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4" +accuracy_threshold: 0.86 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming.yaml new file mode 100644 index 0000000000..ab89ebf515 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-GPTQ-Int4-humming.yaml @@ -0,0 +1,9 @@ +model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4" +accuracy_threshold: 0.86 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml new file mode 100644 index 0000000000..f77ee1173e --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml @@ -0,0 +1,11 @@ +model_name: "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8" +accuracy_threshold: 0.86 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml new file mode 100644 index 0000000000..7b4f82c845 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml @@ -0,0 +1,9 @@ +model_name: "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8" +accuracy_threshold: 0.86 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml index 9b77af6732..d98f91e1f9 100644 --- a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml @@ -5,8 +5,7 @@ num_fewshot: 5 server_args: >- --enforce-eager --max-model-len 8192 - --tensor-parallel-size 1 --quantization humming - --kernel-config.enable_flashinfer_autotune=False + --linear-backend humming env: VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming.yaml index 0b1599ff94..67725fce64 100644 --- a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming.yaml +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-MXFP4A16-humming.yaml @@ -5,6 +5,5 @@ num_fewshot: 5 server_args: >- --enforce-eager --max-model-len 8192 - --tensor-parallel-size 1 --quantization humming - --kernel-config.enable_flashinfer_autotune=False + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-NVFP4-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-NVFP4-humming.yaml new file mode 100644 index 0000000000..a20c143310 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-NVFP4-humming.yaml @@ -0,0 +1,9 @@ +model_name: "nvidia/Qwen3-30B-A3B-NVFP4" +accuracy_threshold: 0.86 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml new file mode 100644 index 0000000000..b58c0d710e --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml @@ -0,0 +1,13 @@ +model_name: "Qwen/Qwen3-30B-A3B" +accuracy_threshold: 0.80 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --tensor-parallel-size 1 + --quantization humming + --kernel-config.enable_flashinfer_autotune=False +env: + VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}' + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml new file mode 100644 index 0000000000..c932091dbe --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml @@ -0,0 +1,13 @@ +model_name: "Qwen/Qwen3-30B-A3B" +accuracy_threshold: 0.80 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --tensor-parallel-size 1 + --quantization humming + --kernel-config.enable_flashinfer_autotune=False +env: + VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}' + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming.yaml new file mode 100644 index 0000000000..fa5095cc88 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3-30B-A3B-int5wc-hadamard-humming.yaml @@ -0,0 +1,12 @@ +model_name: "Qwen/Qwen3-30B-A3B" +accuracy_threshold: 0.80 +num_questions: 1319 +num_fewshot: 5 +server_args: >- + --enforce-eager + --max-model-len 8192 + --tensor-parallel-size 1 + --quantization humming + --kernel-config.enable_flashinfer_autotune=False +env: + VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming-act-fp8.yaml new file mode 100644 index 0000000000..3af5c03a24 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming-act-fp8.yaml @@ -0,0 +1,12 @@ +model_name: "Qwen/Qwen3.5-35B-A3B-FP8" +accuracy_threshold: 0.90 +num_questions: 1319 +num_fewshot: 5 +gen_prefix: " \n\n\n" +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming.yaml new file mode 100644 index 0000000000..85fce24420 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-FP8-humming.yaml @@ -0,0 +1,10 @@ +model_name: "Qwen/Qwen3.5-35B-A3B-FP8" +accuracy_threshold: 0.90 +num_questions: 1319 +num_fewshot: 5 +gen_prefix: " \n\n\n" +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml new file mode 100644 index 0000000000..6a85ab384f --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml @@ -0,0 +1,12 @@ +model_name: "Qwen/Qwen3.5-35B-A3B" +accuracy_threshold: 0.90 +num_questions: 1319 +num_fewshot: 5 +gen_prefix: " \n\n\n" +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --quantization experts_int8 +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming.yaml new file mode 100644 index 0000000000..d282fdc701 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-35B-A3B-experts-int8-humming.yaml @@ -0,0 +1,10 @@ +model_name: "Qwen/Qwen3.5-35B-A3B" +accuracy_threshold: 0.90 +num_questions: 1319 +num_fewshot: 5 +gen_prefix: " \n\n\n" +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --quantization experts_int8 diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-fp8.yaml new file mode 100644 index 0000000000..2a118098fc --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-fp8.yaml @@ -0,0 +1,10 @@ +model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16" +accuracy_threshold: 0.82 +num_questions: 1319 +num_fewshot: 5 +gen_prefix: " \n\n\n" +server_args: >- + --enforce-eager + --max-model-len 8192 +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-int8.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-int8.yaml new file mode 100644 index 0000000000..34f17a4b05 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming-act-int8.yaml @@ -0,0 +1,11 @@ +model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16" +accuracy_threshold: 0.82 +num_questions: 1319 +num_fewshot: 5 +gen_prefix: " \n\n\n" +server_args: >- + --enforce-eager + --max-model-len 8192 + --linear-backend humming +env: + VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}' diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming.yaml new file mode 100644 index 0000000000..b617c61eb3 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3.5-4B-quantized.w4a16-humming.yaml @@ -0,0 +1,9 @@ +model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16" +accuracy_threshold: 0.82 +num_questions: 1319 +num_fewshot: 5 +gen_prefix: " \n\n\n" +server_args: >- + --enforce-eager + --max-model-len 8192 + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/Qwen3.6-35B-A3B-NVFP4-humming.yaml b/tests/evals/gsm8k/configs/humming/Qwen3.6-35B-A3B-NVFP4-humming.yaml new file mode 100644 index 0000000000..502ab776f4 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/Qwen3.6-35B-A3B-NVFP4-humming.yaml @@ -0,0 +1,10 @@ +model_name: "RedHatAI/Qwen3.6-35B-A3B-NVFP4" +accuracy_threshold: 0.91 +num_questions: 1319 +num_fewshot: 5 +gen_prefix: " \n\n\n" +server_args: >- + --enforce-eager + --max-model-len 8192 + --moe-backend humming + --linear-backend humming diff --git a/tests/evals/gsm8k/configs/humming/config-act-fp8.txt b/tests/evals/gsm8k/configs/humming/config-act-fp8.txt index 05fb6a1583..42ff6be00e 100644 --- a/tests/evals/gsm8k/configs/humming/config-act-fp8.txt +++ b/tests/evals/gsm8k/configs/humming/config-act-fp8.txt @@ -1,2 +1,9 @@ gpt-oss-20b-humming-act-fp8.yaml Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml +Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml +Qwen3-0.6B-MXFP8-humming-act-fp8.yaml +Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml +Qwen3-30B-A3B-FP8-block-humming-act-fp8.yaml +Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml +Qwen3-30B-A3B-AWQ-humming.yaml +Qwen3.5-35B-A3B-FP8-humming-act-fp8.yaml diff --git a/tests/evals/gsm8k/configs/humming/config-act-int8.txt b/tests/evals/gsm8k/configs/humming/config-act-int8.txt new file mode 100644 index 0000000000..b018b35cbf --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/config-act-int8.txt @@ -0,0 +1,4 @@ +Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml +Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml +Qwen3-30B-A3B-AWQ-humming.yaml +Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml diff --git a/tests/evals/gsm8k/configs/humming/config-int5wc-hadamard.txt b/tests/evals/gsm8k/configs/humming/config-int5wc-hadamard.txt new file mode 100644 index 0000000000..2c10777a09 --- /dev/null +++ b/tests/evals/gsm8k/configs/humming/config-int5wc-hadamard.txt @@ -0,0 +1,3 @@ +Qwen3-30B-A3B-int5wc-hadamard-humming.yaml +Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml +Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml diff --git a/tests/evals/gsm8k/configs/humming/config.txt b/tests/evals/gsm8k/configs/humming/config.txt index 821025365c..144ed95993 100644 --- a/tests/evals/gsm8k/configs/humming/config.txt +++ b/tests/evals/gsm8k/configs/humming/config.txt @@ -1,2 +1,13 @@ gpt-oss-20b-humming.yaml Qwen3-30B-A3B-MXFP4A16-humming.yaml +Qwen2-1.5B-Instruct-FP8W8-humming.yaml +Qwen3-0.6B-MXFP8-humming.yaml +Qwen3.6-35B-A3B-NVFP4-humming.yaml +Qwen3-30B-A3B-Fp8-v1-humming.yaml +Qwen3-30B-A3B-FP8-block-humming.yaml +Qwen3-30B-A3B-GPTQ-Int4-humming.yaml +Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml +Qwen3-30B-A3B-NVFP4-humming.yaml +Qwen3-30B-A3B-AWQ-humming.yaml +Qwen3.5-35B-A3B-FP8-humming.yaml +Qwen3.5-35B-A3B-experts-int8-humming.yaml diff --git a/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming-act-fp8.yaml b/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming-act-fp8.yaml index 00ba9eccfd..8e0d953503 100644 --- a/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming-act-fp8.yaml +++ b/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming-act-fp8.yaml @@ -5,7 +5,6 @@ num_fewshot: 5 server_args: >- --enforce-eager --max-model-len 8192 - --tensor-parallel-size 1 --moe-backend humming env: VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}' diff --git a/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming.yaml b/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming.yaml index e2beb3739b..7e9b6508a2 100644 --- a/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming.yaml +++ b/tests/evals/gsm8k/configs/humming/gpt-oss-20b-humming.yaml @@ -5,5 +5,4 @@ num_fewshot: 5 server_args: >- --enforce-eager --max-model-len 8192 - --tensor-parallel-size 1 --moe-backend humming diff --git a/tests/evals/gsm8k/gsm8k_eval.py b/tests/evals/gsm8k/gsm8k_eval.py index 89d91cb4a9..87235a9c37 100644 --- a/tests/evals/gsm8k/gsm8k_eval.py +++ b/tests/evals/gsm8k/gsm8k_eval.py @@ -146,6 +146,7 @@ async def call_vllm_chat_api( def _build_gsm8k_prompts( num_questions: int = 1319, num_shots: int = 5, + gen_prefix: str = "", ) -> tuple[list[str], list[int]]: """Build few-shot GSM8K completion prompts and ground-truth labels.""" if num_questions == 0: @@ -157,14 +158,15 @@ def _build_gsm8k_prompts( for i in range(num_shots): few_shot_examples += ( f"Question: {train_data[i]['question']}\n" - f"Answer: {train_data[i]['answer']}\n\n" + f"Answer:{gen_prefix} {train_data[i]['answer']}\n\n" ) prompts = [] labels = [] for i in range(num_questions): prompts.append( - few_shot_examples + f"Question: {test_data[i]['question']}\nAnswer:" + few_shot_examples + + f"Question: {test_data[i]['question']}\nAnswer:{gen_prefix}" ) labels.append(get_answer_value(test_data[i]["answer"])) @@ -213,6 +215,7 @@ def evaluate_gsm8k( temperature: float = 0.0, seed: int | None = 42, request_timeout_seconds: float = 600, + gen_prefix: str = "", ) -> dict[str, float | int]: """ Evaluate GSM8K accuracy using vLLM serve endpoint. @@ -220,7 +223,7 @@ def evaluate_gsm8k( Returns dict with accuracy, invalid_rate, latency, etc. """ base_url = f"{host}:{port}" - prompts, labels = _build_gsm8k_prompts(num_questions, num_shots) + prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix) num_questions = len(prompts) async def run_async_evaluation(): @@ -278,6 +281,7 @@ def evaluate_gsm8k_offline( num_shots: int = 5, max_tokens: int = 256, temperature: float = 0.0, + gen_prefix: str = "", ) -> dict[str, float | int]: """Evaluate GSM8K accuracy using an offline vllm.LLM object. @@ -286,7 +290,7 @@ def evaluate_gsm8k_offline( """ from vllm import SamplingParams - prompts, labels = _build_gsm8k_prompts(num_questions, num_shots) + prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix) sampling_params = SamplingParams( temperature=temperature, diff --git a/tests/evals/gsm8k/test_gsm8k_correctness.py b/tests/evals/gsm8k/test_gsm8k_correctness.py index c9ec5ff66e..0c48af6d3c 100644 --- a/tests/evals/gsm8k/test_gsm8k_correctness.py +++ b/tests/evals/gsm8k/test_gsm8k_correctness.py @@ -70,6 +70,7 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict: host=host, port=port, request_timeout_seconds=request_timeout_seconds, + gen_prefix=eval_config.get("gen_prefix", ""), ) return results diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index e98ab1b3e0..aff94fe03d 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -147,6 +147,7 @@ LinearBackend = Literal[ "flashinfer_cudnn", "flashinfer_b12x", "marlin", + "humming", "triton", "deep_gemm", "torch", diff --git a/vllm/config/model.py b/vllm/config/model.py index 2ec754f934..c53686eaf0 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -999,6 +999,7 @@ class ModelConfig: "modelopt", "modelopt_fp4", "modelopt_mxfp8", + "mxfp8", "modelopt_mixed", # Ensure heavy backends are probed last to avoid unnecessary # imports during override detection (e.g., MXFP4 imports Triton) diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index f5e3a16d71..5d1d370730 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -74,6 +74,9 @@ from vllm.model_executor.kernels.linear.mxfp4 import ( from vllm.model_executor.kernels.linear.mxfp4.flashinfer import ( FlashInferMxFp4LinearKernel, ) +from vllm.model_executor.kernels.linear.mxfp4.humming import ( + HummingMxFp4LinearKernel, +) from vllm.model_executor.kernels.linear.mxfp4.marlin import ( MarlinMxFp4LinearKernel, ) @@ -91,6 +94,9 @@ from vllm.model_executor.kernels.linear.mxfp8.flashinfer import ( FlashInferCutedslMxfp8LinearKernel, FlashInferCutlassMxfp8LinearKernel, ) +from vllm.model_executor.kernels.linear.mxfp8.humming import ( + HummingMxfp8LinearKernel, +) from vllm.model_executor.kernels.linear.mxfp8.marlin import ( MarlinMxfp8LinearKernel, ) @@ -120,6 +126,9 @@ from vllm.model_executor.kernels.linear.nvfp4.flashinfer import ( FlashInferCutlassNvFp4LinearKernel, FlashInferTrtllmNvFp4LinearKernel, ) +from vllm.model_executor.kernels.linear.nvfp4.humming import ( + HummingNvFp4LinearKernel, +) from vllm.model_executor.kernels.linear.nvfp4.marlin import ( MarlinNvFp4LinearKernel, ) @@ -154,6 +163,10 @@ from vllm.model_executor.kernels.linear.scaled_mm.flashinfer import ( FlashInferFp8DeepGEMMDynamicBlockScaledKernel, FlashInferFP8ScaledMMLinearKernel, ) +from vllm.model_executor.kernels.linear.scaled_mm.humming import ( + HummingFP8ScaledMMLinearKernel, + HummingInt8ScaledMMLinearKernel, +) from vllm.model_executor.kernels.linear.scaled_mm.marlin import ( MarlinFP8ScaledMMLinearKernel, ) @@ -225,6 +238,14 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = { "flashinfer_b12x": { FlashInferB12xNvFp4LinearKernel, }, + "humming": { + HummingFP8ScaledMMLinearKernel, + HummingInt8ScaledMMLinearKernel, + HummingLinearKernel, + HummingMxfp8LinearKernel, + HummingMxFp4LinearKernel, + HummingNvFp4LinearKernel, + }, "marlin": { MarlinFP8ScaledMMLinearKernel, MarlinLinearKernel, @@ -292,6 +313,7 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]] PlatformEnum.CUDA: [ CutlassInt8ScaledMMLinearKernel, TritonInt8ScaledMMLinearKernel, + HummingInt8ScaledMMLinearKernel, ], PlatformEnum.ROCM: [AiterInt8ScaledMMLinearKernel, TritonInt8ScaledMMLinearKernel], } @@ -304,6 +326,7 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = CutlassFP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel, ChannelWiseTorchFP8ScaledMMLinearKernel, + HummingFP8ScaledMMLinearKernel, ], PlatformEnum.ROCM: [ AiterHipbMMPerTokenFp8ScaledMMLinearKernel, @@ -335,6 +358,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[ CutlassFp8BlockScaledMMKernel, MarlinFP8ScaledMMLinearKernel, TritonFp8BlockScaledMMKernel, + HummingFP8ScaledMMLinearKernel, ], PlatformEnum.ROCM: [ AiterFp8BlockScaledMMKernel, @@ -351,6 +375,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[ _POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = { PlatformEnum.CUDA: [ + HummingFP8ScaledMMLinearKernel, MarlinFP8ScaledMMLinearKernel, ], PlatformEnum.ROCM: [ @@ -371,10 +396,10 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = { MacheteLinearKernel, AllSparkLinearKernel, MarlinLinearKernel, - HummingLinearKernel, ConchLinearKernel, ExllamaLinearKernel, TritonW4A16LinearKernel, + HummingLinearKernel, ], PlatformEnum.ROCM: [ RDNA3W4A16LinearKernel, @@ -400,6 +425,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = { FlashInferCutlassMxfp8LinearKernel, MarlinMxfp8LinearKernel, EmulationMxfp8LinearKernel, + HummingMxfp8LinearKernel, ], PlatformEnum.ROCM: [ # Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and @@ -426,6 +452,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = { FlashInferCudnnNvFp4LinearKernel, FbgemmNvFp4LinearKernel, EmulationNvFp4LinearKernel, + HummingNvFp4LinearKernel, ], PlatformEnum.ROCM: [ EmulationNvFp4LinearKernel, @@ -436,6 +463,7 @@ _POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = { PlatformEnum.CUDA: [ FlashInferMxFp4LinearKernel, MarlinMxFp4LinearKernel, + HummingMxFp4LinearKernel, ], PlatformEnum.XPU: [ XPUMxFp4LinearKernel, diff --git a/vllm/model_executor/kernels/linear/mixed_precision/__init__.py b/vllm/model_executor/kernels/linear/mixed_precision/__init__.py index c0b8c35bbd..f9b4bd6e43 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/__init__.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/__init__.py @@ -19,6 +19,9 @@ from vllm.model_executor.kernels.linear.mixed_precision.dynamic_4bit import ( from vllm.model_executor.kernels.linear.mixed_precision.exllama import ( ExllamaLinearKernel, ) +from vllm.model_executor.kernels.linear.mixed_precision.humming import ( + HummingLinearKernel, +) from vllm.model_executor.kernels.linear.mixed_precision.machete import ( MacheteLinearKernel, ) @@ -52,6 +55,7 @@ __all__ = [ "CutlassW4A8LinearKernel", "Dynamic4bitLinearKernel", "ExllamaLinearKernel", + "HummingLinearKernel", "MacheteLinearKernel", "MarlinLinearKernel", "RDNA3W4A16LinearKernel", diff --git a/vllm/model_executor/kernels/linear/mixed_precision/humming.py b/vllm/model_executor/kernels/linear/mixed_precision/humming.py index 764c0f4227..7f1a9024ee 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/humming.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/humming.py @@ -23,8 +23,6 @@ class HummingLinearKernel(MPLinearKernel): return False, "Humming is not installed" if c.has_g_idx: return False, "Humming does not support act-order (g_idx)" - if c.zero_points: - return False, "Humming linear kernel only supports symmetric weights" return True, None def process_weights_after_loading(self, layer: torch.nn.Module) -> None: @@ -41,6 +39,11 @@ class HummingLinearKernel(MPLinearKernel): "group_size": 0 if group_size == -1 else group_size, } + if self.config.zero_points: + assert self.w_zp_name is not None + name_map["zero_point"] = self.w_zp_name + quant_config["has_zero_point"] = True + convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) prepare_humming_layer(layer, quant_config) diff --git a/vllm/model_executor/kernels/linear/mxfp4/humming.py b/vllm/model_executor/kernels/linear/mxfp4/humming.py new file mode 100644 index 0000000000..d93f5d4815 --- /dev/null +++ b/vllm/model_executor/kernels/linear/mxfp4/humming.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.model_executor.layers.quantization.utils.humming_utils import ( + convert_linear_layer_to_humming_standard, + prepare_humming_layer, +) +from vllm.platforms import current_platform + +from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig + + +class HummingMxFp4LinearKernel(MxFp4LinearKernel): + """Humming GEMM Kernel for MXFP4.""" + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not current_platform.is_cuda(): + return False, "Humming only supported on CUDA" + + if not current_platform.has_device_capability(75): + return False, "Humming only supported on SM75+" + + return True, None + + @classmethod + def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]: + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu) + name_map = {"weight": "weight", "weight_scale": "weight_scale"} + + quant_config = { + "quant_method": "humming", + "dtype": "float4e2m1", + "scale_dtype": "float8e8m0", + "group_size": 32, + "weight_scale_type": "group", + } + + convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) + prepare_humming_layer(layer, quant_config) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + from vllm.utils.humming import HummingMethod + + flatten_inputs = x.view(-1, x.size(-1)) + output = HummingMethod.forward_layer( + layer=layer, + inputs=flatten_inputs, + compute_config=layer.compute_config, + ) + return output.view(*x.shape[:-1], output.size(-1)) diff --git a/vllm/model_executor/kernels/linear/mxfp8/humming.py b/vllm/model_executor/kernels/linear/mxfp8/humming.py new file mode 100644 index 0000000000..ed1cd39cbd --- /dev/null +++ b/vllm/model_executor/kernels/linear/mxfp8/humming.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.model_executor.layers.quantization.utils.humming_utils import ( + convert_linear_layer_to_humming_standard, + prepare_humming_layer, +) +from vllm.platforms import current_platform + +from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig + + +class HummingMxfp8LinearKernel(Mxfp8LinearKernel): + """Humming GEMM Kernel for MXFP8.""" + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not current_platform.is_cuda(): + return False, "Humming only supported on CUDA" + + if not current_platform.has_device_capability(75): + return False, "Humming only supported on SM75+" + + return True, None + + @classmethod + def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]: + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu) + name_map = {"weight": "weight", "weight_scale": "weight_scale"} + + quant_config = { + "quant_method": "humming", + "dtype": "float8e4m3", + "scale_dtype": "float8e8m0", + "group_size": 32, + "weight_scale_type": "group", + } + + convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) + prepare_humming_layer(layer, quant_config) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + from vllm.utils.humming import HummingMethod + + flatten_inputs = x.view(-1, x.size(-1)) + output = HummingMethod.forward_layer( + layer=layer, + inputs=flatten_inputs, + compute_config=layer.compute_config, + ) + return output.view(*x.shape[:-1], output.size(-1)) diff --git a/vllm/model_executor/kernels/linear/nvfp4/humming.py b/vllm/model_executor/kernels/linear/nvfp4/humming.py new file mode 100644 index 0000000000..7e39034385 --- /dev/null +++ b/vllm/model_executor/kernels/linear/nvfp4/humming.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.utils.humming_utils import ( + prepare_humming_layer, +) +from vllm.platforms import current_platform + +from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig + +logger = init_logger(__name__) + + +class HummingNvFp4LinearKernel(NvFp4LinearKernel): + """Humming GEMM Kernel for NVFP4.""" + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not current_platform.is_cuda(): + return False, "Humming only supported on CUDA" + + if not current_platform.has_device_capability(75): + return False, "Humming only supported on SM75+" + + return True, None + + @classmethod + def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]: + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # Route through humming's compressed-tensors nvfp4 loader (same path as + # the MoE oracle); the native group_tensor schema mishandles a scalar + # global scale. + quant_config = { + "quant_method": "compressed-tensors", + "format": "nvfp4-pack-quantized", + "type": "float", + "num_bits": 4, + "strategy": "group", + "group_size": 16, + } + # CT pack-quantized reads `weight_packed`; the scheme renamed it to `weight`. + if not hasattr(layer, "weight_packed"): + layer.weight_packed = layer.weight + del layer.weight + # The CT linear scheme inverts the global scale (1/scale) for + # marlin/cutlass; humming wants the original. + layer.weight_global_scale = torch.nn.Parameter( + 1.0 / layer.weight_global_scale, requires_grad=False + ) + prepare_humming_layer(layer, quant_config) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + from vllm.utils.humming import HummingMethod + + flatten_inputs = x.view(-1, x.size(-1)) + output = HummingMethod.forward_layer( + layer=layer, + inputs=flatten_inputs, + compute_config=layer.compute_config, + ) + return output.view(*x.shape[:-1], output.size(-1)) diff --git a/vllm/model_executor/kernels/linear/scaled_mm/humming.py b/vllm/model_executor/kernels/linear/scaled_mm/humming.py new file mode 100644 index 0000000000..7b8ed21fbd --- /dev/null +++ b/vllm/model_executor/kernels/linear/scaled_mm/humming.py @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.utils.humming_utils import ( + convert_linear_layer_to_humming_standard, + prepare_humming_layer, +) +from vllm.platforms import current_platform + +from .ScaledMMLinearKernel import ( + FP8ScaledMMLinearKernel, + FP8ScaledMMLinearLayerConfig, + Int8ScaledMMLinearKernel, + Int8ScaledMMLinearLayerConfig, +) + +logger = init_logger(__name__) + + +class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): + """Humming GEMM Kernel for FP8.""" + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not current_platform.is_cuda(): + return False, "Humming only supported on CUDA" + + if not current_platform.has_device_capability(75): + return False, "Humming only supported on SM75+" + + return True, None + + @classmethod + def can_implement( + cls, config: FP8ScaledMMLinearLayerConfig + ) -> tuple[bool, str | None]: + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + from vllm.utils.humming import dtypes + + name_map = {"weight": "weight", "weight_scale": "weight_scale"} + scale_torch_dtype = self.config.weight_quant_key.scale.dtype + scale_dtype = dtypes.DataType.from_torch_dtype(scale_torch_dtype) + + quant_config = { + "quant_method": "humming", + "dtype": "float8e4m3", + "scale_dtype": scale_dtype, + } + + assert self.config.weight_quant_key.scale2 is None + scale_group_shape = self.config.weight_quant_key.scale.group_shape + if scale_group_shape.is_per_tensor(): + quant_config["weight_scale_type"] = "tensor" + if not hasattr(layer, "global_scale") and hasattr(layer, "weight_scale"): + del name_map["weight_scale"] + name_map["global_scale"] = "weight_scale" + elif scale_group_shape.is_per_channel(): + quant_config["weight_scale_type"] = "channel" + elif scale_group_shape.is_per_group(): + quant_config["weight_scale_type"] = "group" + quant_config["group_size"] = scale_group_shape.col + else: + assert scale_group_shape.row > 0 and scale_group_shape.col > 0 + quant_config["weight_scale_type"] = "block" + quant_config["weight_scale_group_size_n"] = scale_group_shape.row + quant_config["weight_scale_group_size"] = scale_group_shape.col + + if hasattr(layer, "weight_scale_inv"): + name_map["weight_scale"] = "weight_scale_inv" + + convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) + prepare_humming_layer(layer, quant_config) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + from vllm.utils.humming import HummingMethod + + flatten_inputs = x.view(-1, x.size(-1)) + output = HummingMethod.forward_layer( + layer=layer, + inputs=flatten_inputs, + compute_config=layer.compute_config, + ) + return output.view(*x.shape[:-1], output.size(-1)) + + def apply_scaled_mm( + self, + *, + A: torch.Tensor, + B: torch.Tensor, + out_dtype: torch.dtype, + As: torch.Tensor, + Bs: torch.Tensor, + bias: torch.Tensor | None, + output_shape: list, + ) -> torch.Tensor: + pass + + +class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel): + """Humming GEMM Kernel for INT8.""" + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not current_platform.is_cuda(): + return False, "Humming only supported on CUDA" + + if not current_platform.has_device_capability(75): + return False, "Humming only supported on SM75+" + + return True, None + + @classmethod + def can_implement( + cls, config: Int8ScaledMMLinearLayerConfig + ) -> tuple[bool, str | None]: + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + weight_name, weight_scale_name, *_ = self.layer_param_names + name_map = {"weight": weight_name, "weight_scale": weight_scale_name} + quant_config = {"quant_method": "humming", "dtype": "int8"} + weight = getattr(layer, weight_name) + weight.data = weight.data + 128 + + convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) + prepare_humming_layer(layer, quant_config) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + ) -> torch.Tensor: + from vllm.utils.humming import HummingMethod + + flatten_inputs = x.view(-1, x.size(-1)) + output = HummingMethod.forward_layer( + layer=layer, + inputs=flatten_inputs, + compute_config=layer.compute_config, + ) + return output.view(*x.shape[:-1], output.size(-1)) diff --git a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py index 047ae46c0d..5f112380cf 100644 --- a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py @@ -38,15 +38,20 @@ from vllm.model_executor.layers.fused_moe.utils import ( ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, + kFp8Dynamic128Sym, kFp8DynamicTokenSym, kFp8Static128BlockSym, kFp8StaticChannelSym, + kFp8StaticTensorSym, kInt4Static, + kInt8DynamicTokenSym, kInt8Static, + kInt8StaticChannelSym, kMxfp4Dynamic, kMxfp4Static, kMxfp8Dynamic, kMxfp8Static, + kNvfp4Dynamic, kNvfp4Static, ) from vllm.platforms import current_platform @@ -61,9 +66,9 @@ if TYPE_CHECKING: logger = init_logger(__name__) -def get_humming_moe_gemm_type() -> str | None: +def get_humming_moe_gemm_type() -> str: env_gemm_type: str | None = envs.VLLM_HUMMING_MOE_GEMM_TYPE - gemm_type = None + gemm_type = "indexed" if env_gemm_type is not None: env_gemm_type = env_gemm_type.lower() if env_gemm_type == "indexed": @@ -87,7 +92,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): num_dispatchers: int | None = None, ): self.layer = layer - self.num_experts = self.layer.num_experts + self.num_experts = self.layer.local_num_experts self.global_num_experts = self.layer.global_num_experts self.init_humming_moe() @@ -186,6 +191,24 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): (kInt4Static, kFp8DynamicTokenSym), (kInt8Static, None), (kInt8Static, kFp8DynamicTokenSym), + # Checkpoint-driven (weight, activation) pairs the dense/MoE oracles + # pass. Humming defers input quant (see expects_unquantized_inputs), + # so the activation key does not constrain support. + # fp8 (compressed-tensors / native / modelopt) + (kFp8StaticChannelSym, kFp8StaticTensorSym), + (kFp8StaticChannelSym, kFp8Dynamic128Sym), + (kFp8StaticTensorSym, None), + (kFp8StaticTensorSym, kFp8DynamicTokenSym), + (kFp8StaticTensorSym, kFp8StaticTensorSym), + (kFp8StaticTensorSym, kFp8Dynamic128Sym), + (kFp8Static128BlockSym, kFp8Dynamic128Sym), + # int8 (compressed-tensors w8a8 / experts_int8) + (kInt8StaticChannelSym, None), + (kInt8StaticChannelSym, kInt8DynamicTokenSym), + # nvfp4 (compressed-tensors / modelopt / quark) + (kNvfp4Static, kNvfp4Dynamic), + # mxfp8 (compressed-tensors / modelopt / online) + (kMxfp8Static, kMxfp8Dynamic), ] return (weight_key, activation_key) in SUPPORTED_W_A @@ -463,13 +486,12 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): assert hasattr(cls, "humming_gemm_type") gemm_type = cls.humming_gemm_type().value.lower() preferred_gemm_type = get_humming_moe_gemm_type() - if preferred_gemm_type is not None: - supported = preferred_gemm_type.lower() == gemm_type - if not supported: - reason = ( - f"preferred gemm type {preferred_gemm_type} != " - f"supported gemm type {gemm_type}" - ) + supported = preferred_gemm_type.lower() == gemm_type + if not supported: + reason = ( + f"preferred gemm type {preferred_gemm_type} != " + f"supported gemm type {gemm_type}" + ) return supported, reason diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 862f292009..c60e68232d 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from enum import Enum +from typing import Any import torch @@ -45,6 +46,7 @@ class Fp8MoeBackend(Enum): DEEPGEMM = "DEEPGEMM" BATCHED_DEEPGEMM = "BATCHED_DEEPGEMM" MARLIN = "MARLIN" + HUMMING = "HUMMING" TRITON = "TRITON" BATCHED_TRITON = "BATCHED_TRITON" AITER = "AITER" @@ -83,6 +85,7 @@ def _get_priority_backends( Fp8MoeBackend.VLLM_CUTLASS, Fp8MoeBackend.TRITON, Fp8MoeBackend.MARLIN, + Fp8MoeBackend.HUMMING, Fp8MoeBackend.BATCHED_DEEPGEMM, Fp8MoeBackend.BATCHED_VLLM_CUTLASS, Fp8MoeBackend.BATCHED_TRITON, @@ -162,6 +165,19 @@ def backend_to_kernel_cls( return [BatchedDeepGemmExperts] + elif backend == Fp8MoeBackend.HUMMING: + from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + ) + + return [ + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + ] + elif backend == Fp8MoeBackend.MARLIN: from vllm.model_executor.layers.fused_moe.experts.marlin_moe import ( MarlinExperts, @@ -240,6 +256,7 @@ def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend: "flashinfer_trtllm": Fp8MoeBackend.FLASHINFER_TRTLLM, "flashinfer_cutlass": Fp8MoeBackend.FLASHINFER_CUTLASS, "marlin": Fp8MoeBackend.MARLIN, + "humming": Fp8MoeBackend.HUMMING, "aiter": Fp8MoeBackend.AITER, "hpc": Fp8MoeBackend.HPC, } @@ -402,6 +419,41 @@ def select_fp8_moe_backend( return Fp8MoeBackend.NONE, None +def _humming_fp8_weight_schema( + layer: RoutedExperts, weight: torch.Tensor, weight_scale: torch.Tensor +) -> dict[str, Any]: + """Build the humming weight schema from the canonical on-device fp8/mxfp8 + tensors (scale dtype/shape, block size), not the producing quant method.""" + # mxfp8: e8m0 group-32 scales (stored as uint8 bytes or e8m0). humming has + # no compressed-tensors mxfp8 loader; its modelopt schema fits both sources. + if weight_scale.dtype in (torch.uint8, torch.float8_e8m0fnu): + return {"quant_method": "modelopt", "quant_algo": "mxfp8"} + + if hasattr(layer, "w13_weight_scale_inv"): + assert hasattr(layer, "weight_block_size") + return {"quant_method": "fp8", "weight_block_size": layer.weight_block_size} + + # fp8 (e4m3): recover the strategy from the scale layout (block from + # weight_block_size, else channel vs tensor by per-expert scale count). + config: dict[str, Any] = { + "quant_method": "compressed-tensors", + "format": "float-quantized", + "type": "float", + "num_bits": 8, + "symmetric": True, + } + weight_block_size = getattr(layer, "weight_block_size", None) + num_experts, num_output = weight.shape[0], weight.shape[-2] + if weight_block_size is not None: + config["strategy"] = "block" + config["block_structure"] = list(weight_block_size) + elif weight_scale.numel() >= num_experts * num_output: + config["strategy"] = "channel" + else: + config["strategy"] = "tensor" + return config + + def convert_to_fp8_moe_kernel_format( fp8_backend: Fp8MoeBackend, # TODO(bnell): replace layer with weight_block_size @@ -429,6 +481,18 @@ def convert_to_fp8_moe_kernel_format( w13, w2, w13_scale, w2_scale = rocm_aiter_ops.shuffle_mxfp8_moe_weights( w13, w2, w13_scale, w2_scale ) + elif fp8_backend == Fp8MoeBackend.HUMMING: + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + convert_to_humming_moe_kernel_format, + ) + + convert_to_humming_moe_kernel_format( + layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale) + ) + w13 = layer.w13_weight + w2 = layer.w2_weight + w13_scale = layer.w13_weight_scale + w2_scale = layer.w2_weight_scale elif fp8_backend == Fp8MoeBackend.MARLIN: weight_block_size = getattr(layer, "weight_block_size", None) if weight_block_size == [1, 32]: @@ -511,6 +575,7 @@ def make_fp8_moe_quant_config( swiglu_limit: float | None = None, gemm1_alpha: float | None = None, gemm1_beta: float | None = None, + layer: torch.nn.Module | None = None, ) -> FusedMoEQuantConfig: """ Create FusedMoEQuantConfig for the specified FP8 Backend. @@ -537,6 +602,14 @@ def make_fp8_moe_quant_config( gemm1_beta=gemm1_beta, gemm1_clamp_limit=swiglu_limit, ) + elif fp8_backend == Fp8MoeBackend.HUMMING: + from vllm.model_executor.layers.fused_moe import RoutedExperts + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + get_humming_moe_quant_config, + ) + + assert isinstance(layer, RoutedExperts) + return get_humming_moe_quant_config(layer) # Flashinfer CUTLASS or HPC per-tensor uses single dq scale # (alpha = w_scale * a_scale) and inverse a2 scale. @@ -600,6 +673,7 @@ def make_fp8_moe_kernel( experts_cls: type[mk.FusedMoEExperts], fp8_backend: Fp8MoeBackend, routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, + layer: torch.nn.Module | None = None, ) -> mk.FusedMoEKernel: # Create Prepare/Finalize. prepare_finalize = maybe_make_prepare_finalize( @@ -613,6 +687,11 @@ def make_fp8_moe_kernel( logger.info_once("Using %s", prepare_finalize.__class__.__name__) + extra_kwargs = {} + if fp8_backend == Fp8MoeBackend.HUMMING: + assert layer is not None + extra_kwargs = {"layer": layer} + # Create Experts. if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: max_num_tokens = prepare_finalize.max_num_tokens_per_rank() @@ -622,11 +701,13 @@ def make_fp8_moe_kernel( quant_config=moe_quant_config, max_num_tokens=max_num_tokens, num_dispatchers=prepare_finalize.num_dispatchers(), + **extra_kwargs, ) else: experts = experts_cls( moe_config=moe_config, quant_config=moe_quant_config, + **extra_kwargs, ) kernel = mk.FusedMoEKernel( diff --git a/vllm/model_executor/layers/fused_moe/oracle/int8.py b/vllm/model_executor/layers/fused_moe/oracle/int8.py index e31a3ca07e..5a2b4c3a75 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int8.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from enum import Enum +from typing import Any import torch @@ -22,6 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kInt8DynamicTokenSym, kInt8StaticChannelSym, ) +from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform logger = init_logger(__name__) @@ -29,6 +31,7 @@ logger = init_logger(__name__) class Int8MoeBackend(Enum): TRITON = "TRITON" + HUMMING = "HUMMING" CPU = "CPU" @@ -40,6 +43,7 @@ def _get_priority_backends( """ _AVAILABLE_BACKENDS = [ Int8MoeBackend.TRITON, + Int8MoeBackend.HUMMING, Int8MoeBackend.CPU, ] @@ -62,6 +66,19 @@ def backend_to_kernel_cls( return [TritonExperts] + elif backend == Int8MoeBackend.HUMMING: + from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + ) + + return [ + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + ] + elif backend == Int8MoeBackend.CPU: from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( CPUExpertsInt8, @@ -77,6 +94,7 @@ def map_int8_backend(runner_backend: MoEBackend) -> Int8MoeBackend: """Map user's MoEBackend to Int8MoeBackend.""" mapping = { "triton": Int8MoeBackend.TRITON, + "humming": Int8MoeBackend.HUMMING, } if backend := mapping.get(runner_backend): return backend @@ -163,6 +181,7 @@ def select_int8_moe_backend( def make_int8_moe_quant_config( + int8_backend: Int8MoeBackend, w1_scale: torch.Tensor, w2_scale: torch.Tensor, a1_scale: torch.Tensor | None = None, @@ -170,11 +189,21 @@ def make_int8_moe_quant_config( w1_bias: torch.Tensor | None = None, w2_bias: torch.Tensor | None = None, per_act_token_quant: bool = False, + layer: torch.nn.Module | None = None, ) -> FusedMoEQuantConfig: assert (a1_scale is None and a2_scale is None) or ( a1_scale is not None and a2_scale is not None ), "a1_scale and a2_scale must both be provided or both be None" + if int8_backend == Int8MoeBackend.HUMMING: + from vllm.model_executor.layers.fused_moe import RoutedExperts + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + get_humming_moe_quant_config, + ) + + assert isinstance(layer, RoutedExperts) + return get_humming_moe_quant_config(layer) + if a1_scale is None or a2_scale is None: return int8_w8a16_moe_quant_config( w1_scale=w1_scale, @@ -196,13 +225,56 @@ def make_int8_moe_quant_config( ) +def _humming_int8_weight_schema( + weight: torch.Tensor, weight_scale: torch.Tensor +) -> dict[str, Any]: + """Build the humming compressed-tensors int8 schema from the canonical + on-device tensors; humming does the signed-int8 -> native conversion.""" + config: dict[str, Any] = { + "quant_method": "compressed-tensors", + "format": "int-quantized", + "type": "int", + "num_bits": 8, + "symmetric": True, + "strategy": "channel", + } + num_experts, num_output = weight.shape[0], weight.shape[-2] + if weight_scale.numel() < num_experts * num_output: + config["strategy"] = "tensor" + return config + + def convert_to_int8_moe_kernel_format( int8_backend: Int8MoeBackend, w13: torch.Tensor, w2: torch.Tensor, + layer: torch.nn.Module | None = None, + w13_scale: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: """Convert INT8 MoE weights to backend-specific kernel format.""" - if int8_backend == Int8MoeBackend.CPU: + if int8_backend == Int8MoeBackend.HUMMING: + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + convert_to_humming_moe_kernel_format, + ) + + assert layer is not None + # Humming reads canonical CT scales (w*_weight_scale) from the layer. + # Online int8 produces per-channel (E, N) w*_scale; expose them as the + # (E, N, 1) w*_weight_scale humming's loader expects. + for sub in ("w13", "w2"): + if hasattr(layer, f"{sub}_weight_scale"): + continue + scale = getattr(layer, f"{sub}_scale").data + if scale.dim() < 3: + scale = scale.unsqueeze(-1) + replace_parameter(layer, f"{sub}_weight_scale", scale) + delattr(layer, f"{sub}_scale") + convert_to_humming_moe_kernel_format( + layer, + quant_config=_humming_int8_weight_schema(w13, layer.w13_weight_scale), + ) + return layer.w13_weight, layer.w2_weight + elif int8_backend == Int8MoeBackend.CPU: from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( prepare_int8_moe_layer_for_cpu, ) @@ -215,10 +287,12 @@ def convert_to_int8_moe_kernel_format( def make_int8_moe_kernel( + int8_backend: Int8MoeBackend, moe_quant_config: FusedMoEQuantConfig, moe_config: FusedMoEConfig, experts_cls: type[mk.FusedMoEExperts], routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, + layer: torch.nn.Module | None = None, ) -> mk.FusedMoEKernel: # Create Prepare/Finalize. prepare_finalize = maybe_make_prepare_finalize( @@ -232,6 +306,11 @@ def make_int8_moe_kernel( logger.info_once("Using %s", prepare_finalize.__class__.__name__) + extra_kwargs = {} + if int8_backend == Int8MoeBackend.HUMMING: + assert layer is not None + extra_kwargs = {"layer": layer} + # Create Experts. if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: max_num_tokens = prepare_finalize.max_num_tokens_per_rank() @@ -241,11 +320,13 @@ def make_int8_moe_kernel( quant_config=moe_quant_config, max_num_tokens=max_num_tokens, num_dispatchers=prepare_finalize.num_dispatchers(), + **extra_kwargs, ) else: experts = experts_cls( moe_config=moe_config, quant_config=moe_quant_config, + **extra_kwargs, ) kernel = mk.FusedMoEKernel( diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py index 0cf1382d40..d0cc08ea14 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py @@ -11,6 +11,7 @@ from compressed_tensors.quantization import ( import vllm._custom_ops as ops import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.config.kernel import MoEBackend from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, @@ -46,6 +47,7 @@ logger = init_logger(__name__) class WNA16MoEBackend(Enum): MARLIN = "MARLIN" BATCHED_MARLIN = "BATCHED_MARLIN" + HUMMING = "HUMMING" CPU = "CPU" FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM" XPU = "XPU" @@ -55,7 +57,19 @@ def backend_to_kernel_cls( backend: WNA16MoEBackend, ) -> list[type[mk.FusedMoEExperts]]: """Return the experts class for the given backend, or None for NONE.""" - if backend == WNA16MoEBackend.MARLIN: + if backend == WNA16MoEBackend.HUMMING: + from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + ) + + return [ + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + ] + elif backend == WNA16MoEBackend.MARLIN: return [MarlinExperts] elif backend == WNA16MoEBackend.BATCHED_MARLIN: return [BatchedMarlinExperts] @@ -90,10 +104,26 @@ def _get_priority_backends() -> list[WNA16MoEBackend]: WNA16MoEBackend.FLASHINFER_TRTLLM, WNA16MoEBackend.MARLIN, WNA16MoEBackend.BATCHED_MARLIN, + WNA16MoEBackend.HUMMING, ] return _AVAILABLE_BACKENDS +def map_wna16_backend(runner_backend: MoEBackend) -> WNA16MoEBackend: + """Map user's MoEBackend to WNA16MoEBackend.""" + mapping = { + "marlin": WNA16MoEBackend.MARLIN, + "humming": WNA16MoEBackend.HUMMING, + "flashinfer_trtllm": WNA16MoEBackend.FLASHINFER_TRTLLM, + } + if backend := mapping.get(runner_backend): + return backend + raise ValueError( + f"moe_backend='{runner_backend}' is not supported for WNA16 MoE. " + f"Expected one of {list(mapping.keys())}." + ) + + def select_wna16_moe_backend( config: FusedMoEConfig, weight_key: QuantKey, @@ -146,6 +176,14 @@ def select_wna16_moe_backend( return backend, k_cls raise ValueError(_make_log_unsupported(backend, reason)) + # Handle explicit moe_backend from user. + runner_backend = config.moe_backend + if runner_backend != "auto": + requested_backend = map_wna16_backend(runner_backend) + return _return_or_raise( + requested_backend, config, weight_key, None, activation_format + ) + # Select kernels in order of backend. AVAILABLE_BACKENDS = _get_priority_backends() @@ -210,6 +248,8 @@ def make_wna16_moe_kernel( moe_quant_config: FusedMoEQuantConfig, moe_config: FusedMoEConfig, experts_cls: type[mk.FusedMoEExperts], + backend: WNA16MoEBackend = WNA16MoEBackend.MARLIN, + layer: torch.nn.Module | None = None, is_k_full: bool = False, w13_g_idx: torch.Tensor | None = None, w2_g_idx: torch.Tensor | None = None, @@ -228,14 +268,18 @@ def make_wna16_moe_kernel( ) # Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts, - # BatchedMarlinExperts, XPUExpertsWNA16, and CPUExpertsInt4 - assert experts_cls in ( + # BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, and the Humming + # grouped/indexed experts. + allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = ( MarlinExperts, BatchedMarlinExperts, TrtLlmMxint4ExpertsMonolithic, XPUExpertsWNA16, CPUExpertsInt4, ) + if backend == WNA16MoEBackend.HUMMING: + allowed_experts += tuple(backend_to_kernel_cls(WNA16MoEBackend.HUMMING)) + assert experts_cls in allowed_experts is_monolithic = experts_cls.is_monolithic() @@ -251,7 +295,10 @@ def make_wna16_moe_kernel( logger.info_once("Using %s", prepare_finalize.__class__.__name__, scope="local") extra_args: dict[str, Any] = {} - if issubclass(experts_cls, MarlinExpertsBase): + if backend == WNA16MoEBackend.HUMMING: + assert layer is not None + extra_args = {"layer": layer} + elif issubclass(experts_cls, MarlinExpertsBase): extra_args = { "w13_g_idx": w13_g_idx, "w2_g_idx": w2_g_idx, @@ -941,6 +988,35 @@ def _process_weights_xpu( ) +def _humming_wna16_weight_schema( + quant_config: QuantizationConfig | QuantizationArgs | None, +) -> dict[str, Any]: + """Humming weight schema for a WNA16 checkpoint, derived from the quant + config rather than the running kernel.""" + from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig + from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig + + if isinstance(quant_config, AutoAWQConfig): + return { + "quant_method": "awq", + "bits": quant_config.weight_bits, + "group_size": quant_config.group_size, + "zero_point": quant_config.zero_point, + } + if isinstance(quant_config, AutoGPTQConfig): + return { + "quant_method": "gptq", + "bits": quant_config.weight_bits, + "group_size": quant_config.group_size, + "desc_act": quant_config.desc_act, + "sym": quant_config.is_sym, + } + raise TypeError( + "Humming WNA16 MoE requires AutoAWQConfig or AutoGPTQConfig, " + f"got {type(quant_config).__name__}." + ) + + def convert_to_wna16_moe_kernel_format( backend: WNA16MoEBackend, layer: torch.nn.Module, @@ -956,26 +1032,30 @@ def convert_to_wna16_moe_kernel_format( w2_qzeros: torch.Tensor | None = None, w13_bias: torch.Tensor | None = None, w2_bias: torch.Tensor | None = None, -) -> tuple[ - torch.Tensor, # w13_qweight - torch.Tensor, # w2_qweight - torch.Tensor, # w13_scales - torch.Tensor, # w2_scales - torch.Tensor | None, # w13_g_idx - torch.Tensor | None, # w2_g_idx - torch.Tensor | None, # w13_g_idx_sort_indices - torch.Tensor | None, # w2_g_idx_sort_indices - torch.Tensor | None, # w13_qzeros - torch.Tensor | None, # w2_qzeros - torch.Tensor | None, # w13_input_global_scale - torch.Tensor | None, # w2_input_global_scale - torch.Tensor | None, # w13_bias - torch.Tensor | None, # w2_bias -]: +) -> ( + tuple[ + torch.Tensor, # w13_qweight + torch.Tensor, # w2_qweight + torch.Tensor, # w13_scales + torch.Tensor, # w2_scales + torch.Tensor | None, # w13_g_idx + torch.Tensor | None, # w2_g_idx + torch.Tensor | None, # w13_g_idx_sort_indices + torch.Tensor | None, # w2_g_idx_sort_indices + torch.Tensor | None, # w13_qzeros + torch.Tensor | None, # w2_qzeros + torch.Tensor | None, # w13_input_global_scale + torch.Tensor | None, # w2_input_global_scale + torch.Tensor | None, # w13_bias + torch.Tensor | None, # w2_bias + ] + | None +): """Dispatch weight post-processing to the appropriate per-backend handler. To add a new backend, implement a ``_process_weights_`` helper and - add a branch here. + add a branch here. Backends that rewrite the layer's parameters in place + (e.g. Humming) return ``None``; the caller then skips the param scatter. Args: backend: the selected ``WNA16MoEBackend``. @@ -983,6 +1063,16 @@ def convert_to_wna16_moe_kernel_format( quant_config: the ``QuantizationConfig`` for this layer. input_dtype: optional activation dtype, usually should be 16 bit. """ + if backend == WNA16MoEBackend.HUMMING: + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + convert_to_humming_moe_kernel_format, + ) + + convert_to_humming_moe_kernel_format( + layer, quant_config=_humming_wna16_weight_schema(quant_config) + ) + return None + if backend in ( WNA16MoEBackend.MARLIN, WNA16MoEBackend.BATCHED_MARLIN, diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py index 06b622a6c4..b9086cfa48 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py @@ -25,6 +25,7 @@ _SUPPORTED_BACKENDS = ( # is_supported_config passes (gfx950 + flydsl installed + not EP). On other # devices / no flydsl / EP it is skipped and native is used. Fp8MoeBackend.AITER_MXFP8, + Fp8MoeBackend.HUMMING, ) _BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = { @@ -34,6 +35,7 @@ _BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = { "xpu": Fp8MoeBackend.XPU, "aiter": Fp8MoeBackend.AITER_MXFP8, "triton": Fp8MoeBackend.TRITON_MXFP8, + "humming": Fp8MoeBackend.HUMMING, } diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index 408c69fea0..f295163568 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -43,6 +43,7 @@ class NvFp4MoeBackend(Enum): FLASHINFER_B12X = "FLASHINFER_B12X" VLLM_CUTLASS = "VLLM_CUTLASS" MARLIN = "MARLIN" + HUMMING = "HUMMING" EMULATION = "EMULATION" @@ -119,6 +120,18 @@ def backend_to_kernel_cls( ) return [MarlinExperts] + elif backend == NvFp4MoeBackend.HUMMING: + from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + ) + + return [ + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + ] elif backend == NvFp4MoeBackend.EMULATION: from vllm.model_executor.layers.fused_moe.experts.nvfp4_emulation_moe import ( Nvfp4QuantizationEmulationTritonExperts, @@ -138,6 +151,7 @@ def map_nvfp4_backend(runner_backend: MoEBackend) -> NvFp4MoeBackend: "flashinfer_cutedsl": NvFp4MoeBackend.FLASHINFER_CUTEDSL, "flashinfer_b12x": NvFp4MoeBackend.FLASHINFER_B12X, "marlin": NvFp4MoeBackend.MARLIN, + "humming": NvFp4MoeBackend.HUMMING, "emulation": NvFp4MoeBackend.EMULATION, } if backend := mapping.get(runner_backend): @@ -169,6 +183,7 @@ def select_nvfp4_moe_backend( NvFp4MoeBackend.FLASHINFER_CUTLASS, NvFp4MoeBackend.VLLM_CUTLASS, NvFp4MoeBackend.MARLIN, + NvFp4MoeBackend.HUMMING, NvFp4MoeBackend.EMULATION, ] @@ -346,6 +361,41 @@ def convert_to_nvfp4_moe_kernel_format( a2_scale=a2_scale, is_act_and_mul=is_act_and_mul, ) + elif nvfp4_backend == NvFp4MoeBackend.HUMMING: + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + convert_to_humming_moe_kernel_format, + ) + + # Discriminate the source checkpoint layout by its global-scale param: + # compressed-tensors uses *_weight_global_scale, modelopt *_weight_scale_2. + # The logical schema is identical (nvfp4 group-16); only the on-layer + # param names differ. TODO: normalize both methods to a single canonical + # layout upstream so the oracle needs neither the probe nor the re-alias. + if hasattr(layer, "w13_weight_global_scale"): + quant_config = { + "quant_method": "compressed-tensors", + "format": "nvfp4-pack-quantized", + "type": "float", + "num_bits": 4, + "strategy": "group", + "group_size": 16, + } + # CT pack-quantized reads `weight_packed`; the method renamed it to + # `weight`. Re-alias (convert replaces all params anyway). + layer.w13_weight_packed = layer.w13_weight + layer.w2_weight_packed = layer.w2_weight + else: + quant_config = {"quant_method": "modelopt", "quant_algo": "nvfp4"} + + convert_to_humming_moe_kernel_format(layer, quant_config=quant_config) + a13_scale = None + a2_scale = None + w13 = layer.w13_weight + w13_scale = layer.w13_weight_scale + w13_scale_2 = getattr(layer, "w13_global_scale", None) + w2 = layer.w2_weight + w2_scale = layer.w2_weight_scale + w2_scale_2 = getattr(layer, "w2_global_scale", None) elif nvfp4_backend == NvFp4MoeBackend.MARLIN: a13_scale = None a2_scale = None @@ -418,8 +468,17 @@ def make_nvfp4_moe_quant_config( a13_scale: torch.Tensor, a2_scale: torch.Tensor, swiglu_limit: float | None = None, + layer: torch.nn.Module | None = None, ) -> FusedMoEQuantConfig: - if backend == NvFp4MoeBackend.MARLIN: + if backend == NvFp4MoeBackend.HUMMING: + from vllm.model_executor.layers.fused_moe import RoutedExperts + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + get_humming_moe_quant_config, + ) + + assert isinstance(layer, RoutedExperts) + return get_humming_moe_quant_config(layer) + elif backend == NvFp4MoeBackend.MARLIN: return nvfp4_w4a16_moe_quant_config( g1_alphas=w13_scale_2, g2_alphas=w2_scale_2, @@ -467,7 +526,9 @@ def make_nvfp4_moe_kernel( moe_quant_config: FusedMoEQuantConfig, moe_config: FusedMoEConfig, experts_cls: type[mk.FusedMoEExperts], + backend: NvFp4MoeBackend, routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, + layer: torch.nn.Module | None = None, ) -> mk.FusedMoEKernel: # Create Prepare/Finalize. prepare_finalize = maybe_make_prepare_finalize( @@ -481,6 +542,11 @@ def make_nvfp4_moe_kernel( logger.info_once("Using %s", prepare_finalize.__class__.__name__) + extra_kwargs = {} + if backend == NvFp4MoeBackend.HUMMING: + assert layer is not None + extra_kwargs = {"layer": layer} + # Create Experts. if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: max_num_tokens = prepare_finalize.max_num_tokens_per_rank() @@ -490,11 +556,13 @@ def make_nvfp4_moe_kernel( quant_config=moe_quant_config, max_num_tokens=max_num_tokens, num_dispatchers=prepare_finalize.num_dispatchers(), + **extra_kwargs, ) else: experts = experts_cls( moe_config=moe_config, quant_config=moe_quant_config, + **extra_kwargs, ) kernel = mk.FusedMoEKernel( diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index cda0eaf730..639fce5234 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -232,7 +232,9 @@ def select_unquantized_moe_backend( raise ValueError(_make_log_unsupported(backend, reason)) runner_backend = moe_config.moe_backend - if runner_backend != "auto": + # 'humming' is quantization-only; an unquantized layer (e.g. excluded via + # modules_to_not_convert) falls through to auto instead of erroring. + if runner_backend not in ["auto", "humming"]: requested_backend = map_unquantized_backend(runner_backend) if ( activation_format == mk.FusedMoEActivationFormat.BatchedExperts diff --git a/vllm/model_executor/layers/quantization/auto_awq.py b/vllm/model_executor/layers/quantization/auto_awq.py index cebfad7e59..58104fa7d2 100644 --- a/vllm/model_executor/layers/quantization/auto_awq.py +++ b/vllm/model_executor/layers/quantization/auto_awq.py @@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe import ( UnquantizedFusedMoEMethod, ) from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import ( + WNA16MoEBackend, convert_to_wna16_moe_kernel_format, make_wna16_moe_kernel, make_wna16_moe_quant_config, @@ -663,6 +664,26 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): layer.workspace = marlin_make_workspace_new(device, 4) def process_weights_after_loading(self, layer: RoutedExperts) -> None: + converted = convert_to_wna16_moe_kernel_format( + backend=self.wna16_moe_backend, + layer=layer, + quant_config=self.quant_config, + input_dtype=self.input_dtype, + w13=layer.w13_qweight, + w2=layer.w2_qweight, + w13_scale=layer.w13_scales, + w2_scale=layer.w2_scales, + w13_qzeros=layer.w13_qzeros, + w2_qzeros=layer.w2_qzeros, + w13_bias=getattr(layer, "w13_bias", None), + w2_bias=getattr(layer, "w2_bias", None), + ) + + if converted is None: + # Backend rewrote the layer's params in place (e.g. Humming). + self._setup_kernel(layer) + return + ( w13, w2, @@ -678,20 +699,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): w2_input_global_scale, w13_bias, w2_bias, - ) = convert_to_wna16_moe_kernel_format( - backend=self.wna16_moe_backend, - layer=layer, - quant_config=self.quant_config, - input_dtype=self.input_dtype, - w13=layer.w13_qweight, - w2=layer.w2_qweight, - w13_scale=layer.w13_scales, - w2_scale=layer.w2_scales, - w13_qzeros=layer.w13_qzeros, - w2_qzeros=layer.w2_qzeros, - w13_bias=getattr(layer, "w13_bias", None), - w2_bias=getattr(layer, "w2_bias", None), - ) + ) = converted replace_parameter(layer, "w13_qweight", w13) replace_parameter(layer, "w2_qweight", w2) @@ -734,6 +742,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): moe_quant_config=self.moe_quant_config, moe_config=self.moe, experts_cls=self.experts_cls, + backend=self.wna16_moe_backend, + layer=layer, is_k_full=self.is_k_full, w13_g_idx=getattr(layer, "w13_g_idx", None), w2_g_idx=getattr(layer, "w2_g_idx", None), @@ -743,6 +753,12 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): ) def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig: + if self.wna16_moe_backend == WNA16MoEBackend.HUMMING: + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + get_humming_moe_quant_config, + ) + + return get_humming_moe_quant_config(layer) return make_wna16_moe_quant_config( w1_scale=layer.w13_scales, w2_scale=layer.w2_scales, @@ -783,8 +799,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): assert self.moe_kernel is not None return self.moe_kernel.apply( hidden_states=x, - w1=layer.w13_qweight, - w2=layer.w2_qweight, + w1=layer.w13_weight, + w2=layer.w2_weight, topk_weights=topk_weights, topk_ids=topk_ids, activation=layer.activation, @@ -806,8 +822,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): assert self.moe_kernel is not None return self.moe_kernel.apply_monolithic( hidden_states=x, - w1=layer.w13_qweight, - w2=layer.w2_qweight, + w1=layer.w13_weight, + w2=layer.w2_weight, router_logits=router_logits, activation=layer.activation, global_num_experts=layer.global_num_experts, diff --git a/vllm/model_executor/layers/quantization/auto_gptq.py b/vllm/model_executor/layers/quantization/auto_gptq.py index aca76162f9..b056f5222a 100644 --- a/vllm/model_executor/layers/quantization/auto_gptq.py +++ b/vllm/model_executor/layers/quantization/auto_gptq.py @@ -658,6 +658,26 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): "W8A8-INT8 is not supported by marlin kernel." ) + converted = convert_to_wna16_moe_kernel_format( + backend=self.wna16_moe_backend, + layer=layer, + quant_config=self.quant_config, + input_dtype=self.input_dtype, + w13=layer.w13_qweight, + w2=layer.w2_qweight, + w13_scale=layer.w13_scales, + w2_scale=layer.w2_scales, + w13_g_idx=layer.w13_g_idx, + w2_g_idx=layer.w2_g_idx, + w13_bias=getattr(layer, "w13_bias", None), + w2_bias=getattr(layer, "w2_bias", None), + ) + + if converted is None: + # Backend rewrote the layer's params in place (e.g. Humming). + self._setup_kernel(layer) + return + ( w13, w2, @@ -673,20 +693,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): w2_input_global_scale, w13_bias, w2_bias, - ) = convert_to_wna16_moe_kernel_format( - backend=self.wna16_moe_backend, - layer=layer, - quant_config=self.quant_config, - input_dtype=self.input_dtype, - w13=layer.w13_qweight, - w2=layer.w2_qweight, - w13_scale=layer.w13_scales, - w2_scale=layer.w2_scales, - w13_g_idx=layer.w13_g_idx, - w2_g_idx=layer.w2_g_idx, - w13_bias=getattr(layer, "w13_bias", None), - w2_bias=getattr(layer, "w2_bias", None), - ) + ) = converted replace_parameter(layer, "w13_qweight", w13) replace_parameter(layer, "w2_qweight", w2) @@ -733,6 +740,10 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): "w2_bias", torch.nn.Parameter(w2_bias, requires_grad=False) ) + # The modular kernel reads w13_weight/w2_weight; marlin keeps *_qweight. + layer.w13_weight = layer.w13_qweight + layer.w2_weight = layer.w2_qweight + self._setup_kernel(layer) def _setup_kernel(self, layer: RoutedExperts) -> None: @@ -743,15 +754,24 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): moe_quant_config=self.moe_quant_config, moe_config=self.moe, experts_cls=self.experts_cls, + backend=self.wna16_moe_backend, + layer=layer, is_k_full=self.is_k_full, - w13_g_idx=layer.w13_g_idx, - w2_g_idx=layer.w2_g_idx, + w13_g_idx=getattr(layer, "w13_g_idx", None), + w2_g_idx=getattr(layer, "w2_g_idx", None), w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None), w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None), routing_tables=layer._expert_routing_tables(), ) def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig: + if self.wna16_moe_backend == WNA16MoEBackend.HUMMING: + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + get_humming_moe_quant_config, + ) + + return get_humming_moe_quant_config(layer) + from vllm.model_executor.layers.fused_moe.config import ( gptq_marlin_moe_quant_config, ) @@ -795,8 +815,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): assert self.moe_kernel is not None return self.moe_kernel.apply( hidden_states=x, - w1=layer.w13_qweight, - w2=layer.w2_qweight, + w1=layer.w13_weight, + w2=layer.w2_weight, topk_weights=topk_weights, topk_ids=topk_ids, activation=layer.activation, @@ -818,8 +838,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): assert self.moe_kernel is not None return self.moe_kernel.apply_monolithic( hidden_states=x, - w1=layer.w13_qweight, - w2=layer.w2_qweight, + w1=layer.w13_weight, + w2=layer.w2_weight, router_logits=router_logits, activation=layer.activation, global_num_experts=layer.global_num_experts, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py index 9a051c038f..f74ed2d7b3 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py @@ -236,7 +236,9 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod): moe_quant_config=self.moe_quant_config, moe_config=self.moe, experts_cls=self.experts_cls, + backend=self.nvfp4_backend, routing_tables=layer._expert_routing_tables(), + layer=layer, ) self.moe_kernel.fused_experts.process_weights_after_loading(layer) @@ -259,6 +261,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod): a13_scale=layer.w13_input_scale, a2_scale=layer.w2_input_scale, swiglu_limit=getattr(layer, "swiglu_limit", None), + layer=layer, ) def apply_monolithic( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py index 14ef8bf614..35d2a37842 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py @@ -338,6 +338,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod): fp8_backend=self.fp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), + layer=layer, ) def maybe_make_prepare_finalize( @@ -355,12 +356,13 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod): fp8_backend=self.fp8_backend, w1_scale=layer.w13_weight_scale, w2_scale=layer.w2_weight_scale, - a1_scale=layer.w13_input_scale, - a2_scale=layer.w2_input_scale, + a1_scale=getattr(layer, "w13_input_scale", None), + a2_scale=getattr(layer, "w2_input_scale", None), per_act_token_quant=is_per_token, per_out_ch_quant=is_per_token, block_shape=self.weight_block_size, swiglu_limit=getattr(layer, "swiglu_limit", None), + layer=layer, ) def apply_monolithic( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py index c29472cfc6..d304ca56bf 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py @@ -146,6 +146,8 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): int8_backend=self.int8_backend, w13=layer.w13_weight, w2=layer.w2_weight, + layer=layer, + w13_scale=layer.w13_weight_scale, ) replace_parameter(layer, "w13_weight", w13) replace_parameter(layer, "w2_weight", w2) @@ -153,10 +155,12 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): self.moe_quant_config = self.get_fused_moe_quant_config(layer) assert self.experts_cls is not None self.moe_kernel = make_int8_moe_kernel( + int8_backend=self.int8_backend, moe_quant_config=self.moe_quant_config, moe_config=self.moe, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), + layer=layer, ) def maybe_make_prepare_finalize( @@ -170,11 +174,13 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: return make_int8_moe_quant_config( + int8_backend=self.int8_backend, w1_scale=layer.w13_weight_scale, w2_scale=layer.w2_weight_scale, a1_scale=layer.w13_input_scale, a2_scale=layer.w2_input_scale, per_act_token_quant=True, + layer=layer, ) def apply( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py index 2e6e01ca76..468aed29a9 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py @@ -140,6 +140,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod): fp8_backend=self.fp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), + layer=layer, ) def get_fused_moe_quant_config( @@ -155,6 +156,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod): swiglu_limit=getattr(layer, "swiglu_limit", None), gemm1_alpha=getattr(layer, "swiglu_alpha", None), gemm1_beta=getattr(layer, "swiglu_beta", None), + layer=layer, ) def maybe_make_prepare_finalize( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py index 46fa36180d..8af36bcb10 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py @@ -416,6 +416,27 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod): def process_weights_after_loading(self, layer: torch.nn.Module) -> None: # Process weights using the shared oracle infrastructure is_flashinfer = self.wna16_backend == WNA16MoEBackend.FLASHINFER_TRTLLM + converted = convert_to_wna16_moe_kernel_format( + backend=self.wna16_backend, + layer=layer, + quant_config=self.weight_quant, + input_dtype=self.marlin_input_dtype, + w13=layer.w13_weight_packed, + w2=layer.w2_weight_packed, + w13_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + w13_g_idx=layer.w13_weight_g_idx, + w2_g_idx=layer.w2_weight_g_idx, + w13_qzeros=getattr(layer, "w13_weight_zero_point", None), + w2_qzeros=getattr(layer, "w2_weight_zero_point", None), + ) + if converted is None: + # In-place backends (e.g. Humming) are not wired through this + # marlin-only method; fail clearly rather than unpacking None. + raise NotImplementedError( + f"{type(self).__name__} does not support the " + f"{self.wna16_backend.value} MoE backend." + ) ( w13_qweight, w2_qweight, @@ -431,20 +452,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod): w2_input_global_scale, _, # w13_bias _, # w2_bias - ) = convert_to_wna16_moe_kernel_format( - backend=self.wna16_backend, - layer=layer, - quant_config=self.weight_quant, - input_dtype=self.marlin_input_dtype, - w13=layer.w13_weight_packed, - w2=layer.w2_weight_packed, - w13_scale=layer.w13_weight_scale, - w2_scale=layer.w2_weight_scale, - w13_g_idx=layer.w13_weight_g_idx, - w2_g_idx=layer.w2_weight_g_idx, - w13_qzeros=getattr(layer, "w13_weight_zero_point", None), - w2_qzeros=getattr(layer, "w2_weight_zero_point", None), - ) + ) = converted # Replace common parameters replace_parameter(layer, "w13_weight_packed", w13_qweight) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py index 1301c98f45..23e3510614 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py @@ -137,8 +137,16 @@ class CompressedTensorsW8A16Fp8(CompressedTensorsScheme): "weight_scale", convert_to_channelwise(layer.weight_scale, layer.logical_widths), ) + self.strategy = QuantizationStrategy.CHANNEL + self.weight_quant_key = STRATEGY_TO_WEIGHT_QUANT_KEY[self.strategy] + self.linear_kernel.config.weight_quant_key = self.weight_quant_key + # Canonicalize to (K, N) for the kernel. replace_parameter(layer, "weight", layer.weight.t()) + # Preserve the dim tags dropped by the transpose so layout-aware + # kernels see (K, N). + layer.weight.input_dim = 0 + layer.weight.output_dim = 1 self.linear_kernel.process_weights_after_loading(layer) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py index 1a240f6540..e50778f0ac 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py @@ -181,6 +181,10 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme): # required by torch.compile to be torch.nn.Parameter layer.weight = Parameter(weight.data, requires_grad=False) + # Preserve the dim tags dropped by the transpose so layout-aware + # kernels (humming) see (K, N) instead of assuming (N, K). + layer.weight.input_dim = 0 + layer.weight.output_dim = 1 layer.weight_scale = Parameter(weight_scale.data, requires_grad=False) if input_scale is not None: layer.input_scale = Parameter(input_scale.data, requires_grad=False) diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 86818ed4b7..626fc83cdf 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -714,6 +714,7 @@ class Fp8MoEMethod(FusedMoEMethodBase): fp8_backend=self.fp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), + layer=layer, ) def process_weights_after_loading(self, layer: RoutedExperts) -> None: @@ -786,6 +787,7 @@ class Fp8MoEMethod(FusedMoEMethodBase): swiglu_limit=getattr(layer, "swiglu_limit", None), gemm1_alpha=getattr(layer, "swiglu_alpha", None), gemm1_beta=getattr(layer, "swiglu_beta", None), + layer=layer, ) # Inject biases into the quant config if the model has them diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index 8fa1cb4d54..4a078f1357 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -905,6 +905,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): fp8_backend=self.fp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), + layer=layer, ) def process_weights_after_loading(self, layer: RoutedExperts) -> None: @@ -951,6 +952,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): a1_scale=a1_scale, a2_scale=a2_scale, swiglu_limit=getattr(layer, "swiglu_limit", None), + layer=layer, ) def apply_monolithic( @@ -1602,7 +1604,9 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase): moe_quant_config=self.moe_quant_config, moe_config=self.moe, experts_cls=self.experts_cls, + backend=self.nvfp4_backend, routing_tables=layer._expert_routing_tables(), + layer=layer, ) self.moe_kernel.fused_experts.process_weights_after_loading(layer) @@ -1616,6 +1620,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase): a13_scale=layer.w13_input_scale, a2_scale=layer.w2_input_scale, swiglu_limit=getattr(layer, "swiglu_limit", None), + layer=layer, ) @property @@ -2162,6 +2167,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase): fp8_backend=self.mxfp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), + layer=layer, ) # No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation @@ -2207,6 +2213,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase): swiglu_limit=getattr(layer, "swiglu_limit", None), gemm1_alpha=getattr(layer, "swiglu_alpha", None), gemm1_beta=getattr(layer, "swiglu_beta", None), + layer=layer, ) def apply_monolithic( diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index 4d3a315879..e63c285aa3 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -458,6 +458,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase): fp8_backend=self.fp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), + layer=layer, ) def get_fused_moe_quant_config( @@ -486,6 +487,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase): swiglu_limit=getattr(layer, "swiglu_limit", None), gemm1_alpha=getattr(layer, "swiglu_alpha", None), gemm1_beta=getattr(layer, "swiglu_beta", None), + layer=layer, ) diff --git a/vllm/model_executor/layers/quantization/online/int8.py b/vllm/model_executor/layers/quantization/online/int8.py index 4274b16d55..ef76594164 100644 --- a/vllm/model_executor/layers/quantization/online/int8.py +++ b/vllm/model_executor/layers/quantization/online/int8.py @@ -13,6 +13,7 @@ if TYPE_CHECKING: from vllm.model_executor.layers.fused_moe import RoutedExperts from vllm.model_executor.layers.fused_moe.oracle.int8 import ( + convert_to_int8_moe_kernel_format, make_int8_moe_kernel, make_int8_moe_quant_config, select_int8_moe_backend, @@ -92,22 +93,36 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase): replace_parameter(layer, "w2_scale", w2_scale) def _setup_kernel(self, layer: RoutedExperts) -> None: + w13, w2 = convert_to_int8_moe_kernel_format( + int8_backend=self.int8_backend, + w13=layer.w13_weight, + w2=layer.w2_weight, + layer=layer, + w13_scale=layer.w13_scale, + ) + replace_parameter(layer, "w13_weight", w13) + replace_parameter(layer, "w2_weight", w2) + self.moe_quant_config = self.get_fused_moe_quant_config(layer) assert self.moe_quant_config is not None assert self.experts_cls is not None self.moe_kernel = make_int8_moe_kernel( + int8_backend=self.int8_backend, moe_quant_config=self.moe_quant_config, moe_config=self.moe, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), + layer=layer, ) def get_fused_moe_quant_config( self, layer: torch.nn.Module ) -> "FusedMoEQuantConfig | None": return make_int8_moe_quant_config( - w1_scale=layer.w13_scale, - w2_scale=layer.w2_scale, + int8_backend=self.int8_backend, + w1_scale=getattr(layer, "w13_scale", None), + w2_scale=getattr(layer, "w2_scale", None), w1_bias=getattr(layer, "w13_bias", None), w2_bias=getattr(layer, "w2_bias", None), + layer=layer, ) diff --git a/vllm/model_executor/layers/quantization/online/mxfp8.py b/vllm/model_executor/layers/quantization/online/mxfp8.py index 09d581a073..84a81bd906 100644 --- a/vllm/model_executor/layers/quantization/online/mxfp8.py +++ b/vllm/model_executor/layers/quantization/online/mxfp8.py @@ -200,6 +200,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase): fp8_backend=self.fp8_backend, experts_cls=self.experts_cls, routing_tables=layer._expert_routing_tables(), + layer=layer, ) def get_fused_moe_quant_config( @@ -226,6 +227,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase): swiglu_limit=getattr(layer, "swiglu_limit", None), gemm1_alpha=getattr(layer, "swiglu_alpha", None), gemm1_beta=getattr(layer, "swiglu_beta", None), + layer=layer, ) def process_weights_after_loading(self, layer: Module) -> None: diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index bce888415c..7bdf963b51 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -1562,7 +1562,9 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod): moe_quant_config=self.moe_quant_config, moe_config=self.moe, experts_cls=self.experts_cls, + backend=self.nvfp4_backend, routing_tables=layer._expert_routing_tables(), + layer=layer, ) def get_fused_moe_quant_config( @@ -1576,6 +1578,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod): w2_scale_2=layer.w2_weight_scale_2, a13_scale=layer.w13_input_scale_2, a2_scale=layer.w2_input_scale_2, + layer=layer, ) def apply( diff --git a/vllm/model_executor/layers/quantization/utils/humming_utils.py b/vllm/model_executor/layers/quantization/utils/humming_utils.py index d84a2e12f5..2d9e7aca76 100644 --- a/vllm/model_executor/layers/quantization/utils/humming_utils.py +++ b/vllm/model_executor/layers/quantization/utils/humming_utils.py @@ -764,7 +764,7 @@ def _convert_sublayer_to_humming( Returns: Tuple of (converted_weight_schema, converted_input_schema) """ - from humming.schema import HummingWeightSchema + from vllm.utils.humming import HummingWeightSchema if isinstance(weight_schema, HummingWeightSchema): # Already in Humming format @@ -814,7 +814,7 @@ def _prepare_and_transform_sublayer( This calls Humming's prepare_layer_meta and transform_humming_layer. """ - from humming.layer import HummingMethod + from vllm.utils.humming import HummingMethod HummingMethod.prepare_layer_meta( layer=layer, @@ -866,7 +866,7 @@ def _process_single_sublayer( Returns: Tuple of (final_weight_schema, final_input_schema) """ - from humming.schema import HummingWeightSchema + from vllm.utils.humming import HummingWeightSchema # Step 1: Convert from checkpoint format to humming format if needed current_weight_schema, current_input_schema = _convert_sublayer_to_humming( @@ -958,12 +958,10 @@ def convert_to_humming_moe_kernel_format( "Must provide either weight_schema/input_schema or quant_config" ) - from humming.layer import HummingInputSchema - from humming.schema import BaseWeightSchema - from vllm.model_executor.layers.quantization.utils.humming_utils import ( humming_is_layer_skipped, ) + from vllm.utils.humming import BaseWeightSchema, HummingInputSchema if weight_schema is None: weight_schema = BaseWeightSchema.from_config(quant_config) diff --git a/vllm/model_executor/warmup/deep_gemm_warmup.py b/vllm/model_executor/warmup/deep_gemm_warmup.py index d41604fc7a..cfff491ab2 100644 --- a/vllm/model_executor/warmup/deep_gemm_warmup.py +++ b/vllm/model_executor/warmup/deep_gemm_warmup.py @@ -11,6 +11,9 @@ from tqdm import tqdm import vllm.envs as envs from vllm.distributed.parallel_state import get_dp_group, is_global_first_rank +from vllm.model_executor.kernels.linear.scaled_mm.deep_gemm import ( + DeepGemmFp8BlockScaledMMKernel, +) from vllm.model_executor.layers.fused_moe import MoERunner from vllm.model_executor.layers.fused_moe.deep_gemm_utils import ( compute_aligned_M_and_alignment, @@ -147,6 +150,12 @@ def _fp8_linear_may_use_deep_gemm(module: torch.nn.Module) -> bool: ): return False + if not isinstance( + getattr(module.quant_method, "fp8_linear", None), + DeepGemmFp8BlockScaledMMKernel, + ): + return False + w, _, block_sizes = _extract_data_from_linear_base_module(module) return ( block_sizes == get_mk_alignment_for_contiguous_layout()