[Quantization] add humming moe backend to all dense/moe oracles (#41652)

Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Jinzhen Lin
2026-07-06 13:36:07 -07:00
committed by GitHub
co-authored by mgoin
parent 04adc8843b
commit d891b9bd51
73 changed files with 1337 additions and 123 deletions
+61 -13
View File
@@ -150,9 +150,9 @@ steps:
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
- label: LM Eval Humming (A100 - TEMPORARY)
key: lm-eval-humming-a100
timeout_in_minutes: 30
- label: LM Eval Humming f16 (A100 - TEMPORARY)
key: lm-eval-humming-f16-a100
timeout_in_minutes: 120
device: a100
optional: true
num_devices: 1
@@ -160,13 +160,29 @@ steps:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
- vllm/model_executor/layers/fused_moe/oracle/
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
- label: LM Eval Humming (H100 - TEMPORARY)
key: lm-eval-humming-h100
timeout_in_minutes: 30
- label: LM Eval Humming Act int8 (A100 - TEMPORARY)
key: lm-eval-humming-act-a100
timeout_in_minutes: 120
device: a100
optional: true
num_devices: 1
source_file_dependencies:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- vllm/model_executor/layers/fused_moe/oracle/
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- label: LM Eval Humming f16 (H100 - TEMPORARY)
key: lm-eval-humming-f16-h100
timeout_in_minutes: 120
device: h100
optional: true
num_devices: 1
@@ -174,14 +190,30 @@ steps:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
- vllm/model_executor/layers/fused_moe/oracle/
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
- label: LM Eval Humming (B200 - TEMPORARY)
key: lm-eval-humming-b200
timeout_in_minutes: 30
- label: LM Eval Humming Act fp8/int8 (H100 - TEMPORARY)
key: lm-eval-humming-act-h100
timeout_in_minutes: 120
device: h100
optional: true
num_devices: 1
source_file_dependencies:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- vllm/model_executor/layers/fused_moe/oracle/
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- label: LM Eval Humming f16 (B200 - TEMPORARY)
key: lm-eval-humming-f16-b200
timeout_in_minutes: 120
device: b200-k8s
optional: true
num_devices: 1
@@ -189,10 +221,26 @@ steps:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
- vllm/model_executor/layers/fused_moe/oracle/
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
- label: LM Eval Humming Act fp8/int8 (B200 - TEMPORARY)
key: lm-eval-humming-act-b200
timeout_in_minutes: 120
device: b200-k8s
optional: true
num_devices: 1
source_file_dependencies:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- vllm/model_executor/layers/fused_moe/oracle/
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- label: LM Eval TurboQuant KV Cache
key: lm-eval-turboquant-kv-cache
+1 -1
View File
@@ -28,4 +28,4 @@ quack-kernels>=0.3.3
tokenspeed-mla==0.1.2; platform_system == "Linux"
# Humming kernels for quantization gemm
humming-kernels[cu13]==0.1.6
humming-kernels[cu13]==0.1.10
@@ -0,0 +1,10 @@
model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8"
accuracy_threshold: 0.55
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,8 @@
model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8"
accuracy_threshold: 0.55
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--linear-backend humming
@@ -0,0 +1,10 @@
model_name: "mgoin/Qwen3-0.6B-MXFP8"
accuracy_threshold: 0.39
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,8 @@
model_name: "mgoin/Qwen3-0.6B-MXFP8"
accuracy_threshold: 0.39
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--linear-backend humming
@@ -0,0 +1,13 @@
model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
--dtype bfloat16
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,13 @@
model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
--dtype bfloat16
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -0,0 +1,11 @@
model_name: "QuixiAI/Qwen3-30B-A3B-AWQ"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
--dtype bfloat16
@@ -0,0 +1,11 @@
model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,9 @@
model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -0,0 +1,11 @@
model_name: "nm-testing/Qwen3-30B-A3B-Fp8-v1"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,9 @@
model_name: "nm-testing/Qwen3-30B-A3B-Fp8-v1"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -0,0 +1,11 @@
model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,11 @@
model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -0,0 +1,9 @@
model_name: "Qwen/Qwen3-30B-A3B-GPTQ-Int4"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -0,0 +1,11 @@
model_name: "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -0,0 +1,9 @@
model_name: "RedHatAI/Qwen3-30B-A3B-Instruct-2507-quantized.w8a8"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -5,8 +5,7 @@ num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -5,6 +5,5 @@ num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
--linear-backend humming
@@ -0,0 +1,9 @@
model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -0,0 +1,13 @@
model_name: "Qwen/Qwen3-30B-A3B"
accuracy_threshold: 0.80
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
env:
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,13 @@
model_name: "Qwen/Qwen3-30B-A3B"
accuracy_threshold: 0.80
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
env:
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -0,0 +1,12 @@
model_name: "Qwen/Qwen3-30B-A3B"
accuracy_threshold: 0.80
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
env:
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
@@ -0,0 +1,12 @@
model_name: "Qwen/Qwen3.5-35B-A3B-FP8"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,10 @@
model_name: "Qwen/Qwen3.5-35B-A3B-FP8"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -0,0 +1,12 @@
model_name: "Qwen/Qwen3.5-35B-A3B"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--quantization experts_int8
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -0,0 +1,10 @@
model_name: "Qwen/Qwen3.5-35B-A3B"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--quantization experts_int8
@@ -0,0 +1,10 @@
model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
accuracy_threshold: 0.82
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -0,0 +1,11 @@
model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
accuracy_threshold: 0.82
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--linear-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -0,0 +1,9 @@
model_name: "RedHatAI/Qwen3.5-4B-quantized.w4a16"
accuracy_threshold: 0.82
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--linear-backend humming
@@ -0,0 +1,10 @@
model_name: "RedHatAI/Qwen3.6-35B-A3B-NVFP4"
accuracy_threshold: 0.91
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
@@ -1,2 +1,9 @@
gpt-oss-20b-humming-act-fp8.yaml
Qwen3-30B-A3B-MXFP4A16-humming-act-fp8.yaml
Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml
Qwen3-0.6B-MXFP8-humming-act-fp8.yaml
Qwen3-30B-A3B-Fp8-v1-humming-act-fp8.yaml
Qwen3-30B-A3B-FP8-block-humming-act-fp8.yaml
Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml
Qwen3-30B-A3B-AWQ-humming.yaml
Qwen3.5-35B-A3B-FP8-humming-act-fp8.yaml
@@ -0,0 +1,4 @@
Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
Qwen3-30B-A3B-AWQ-humming.yaml
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
@@ -0,0 +1,3 @@
Qwen3-30B-A3B-int5wc-hadamard-humming.yaml
Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml
Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml
@@ -1,2 +1,13 @@
gpt-oss-20b-humming.yaml
Qwen3-30B-A3B-MXFP4A16-humming.yaml
Qwen2-1.5B-Instruct-FP8W8-humming.yaml
Qwen3-0.6B-MXFP8-humming.yaml
Qwen3.6-35B-A3B-NVFP4-humming.yaml
Qwen3-30B-A3B-Fp8-v1-humming.yaml
Qwen3-30B-A3B-FP8-block-humming.yaml
Qwen3-30B-A3B-GPTQ-Int4-humming.yaml
Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml
Qwen3-30B-A3B-NVFP4-humming.yaml
Qwen3-30B-A3B-AWQ-humming.yaml
Qwen3.5-35B-A3B-FP8-humming.yaml
Qwen3.5-35B-A3B-experts-int8-humming.yaml
@@ -5,7 +5,6 @@ num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--moe-backend humming
env:
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -5,5 +5,4 @@ num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--moe-backend humming
+8 -4
View File
@@ -146,6 +146,7 @@ async def call_vllm_chat_api(
def _build_gsm8k_prompts(
num_questions: int = 1319,
num_shots: int = 5,
gen_prefix: str = "",
) -> tuple[list[str], list[int]]:
"""Build few-shot GSM8K completion prompts and ground-truth labels."""
if num_questions == 0:
@@ -157,14 +158,15 @@ def _build_gsm8k_prompts(
for i in range(num_shots):
few_shot_examples += (
f"Question: {train_data[i]['question']}\n"
f"Answer: {train_data[i]['answer']}\n\n"
f"Answer:{gen_prefix} {train_data[i]['answer']}\n\n"
)
prompts = []
labels = []
for i in range(num_questions):
prompts.append(
few_shot_examples + f"Question: {test_data[i]['question']}\nAnswer:"
few_shot_examples
+ f"Question: {test_data[i]['question']}\nAnswer:{gen_prefix}"
)
labels.append(get_answer_value(test_data[i]["answer"]))
@@ -213,6 +215,7 @@ def evaluate_gsm8k(
temperature: float = 0.0,
seed: int | None = 42,
request_timeout_seconds: float = 600,
gen_prefix: str = "",
) -> dict[str, float | int]:
"""
Evaluate GSM8K accuracy using vLLM serve endpoint.
@@ -220,7 +223,7 @@ def evaluate_gsm8k(
Returns dict with accuracy, invalid_rate, latency, etc.
"""
base_url = f"{host}:{port}"
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots)
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
num_questions = len(prompts)
async def run_async_evaluation():
@@ -278,6 +281,7 @@ def evaluate_gsm8k_offline(
num_shots: int = 5,
max_tokens: int = 256,
temperature: float = 0.0,
gen_prefix: str = "",
) -> dict[str, float | int]:
"""Evaluate GSM8K accuracy using an offline vllm.LLM object.
@@ -286,7 +290,7 @@ def evaluate_gsm8k_offline(
"""
from vllm import SamplingParams
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots)
prompts, labels = _build_gsm8k_prompts(num_questions, num_shots, gen_prefix)
sampling_params = SamplingParams(
temperature=temperature,
@@ -70,6 +70,7 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
host=host,
port=port,
request_timeout_seconds=request_timeout_seconds,
gen_prefix=eval_config.get("gen_prefix", ""),
)
return results
+1
View File
@@ -147,6 +147,7 @@ LinearBackend = Literal[
"flashinfer_cudnn",
"flashinfer_b12x",
"marlin",
"humming",
"triton",
"deep_gemm",
"torch",
+1
View File
@@ -999,6 +999,7 @@ class ModelConfig:
"modelopt",
"modelopt_fp4",
"modelopt_mxfp8",
"mxfp8",
"modelopt_mixed",
# Ensure heavy backends are probed last to avoid unnecessary
# imports during override detection (e.g., MXFP4 imports Triton)
+29 -1
View File
@@ -74,6 +74,9 @@ from vllm.model_executor.kernels.linear.mxfp4 import (
from vllm.model_executor.kernels.linear.mxfp4.flashinfer import (
FlashInferMxFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp4.humming import (
HummingMxFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp4.marlin import (
MarlinMxFp4LinearKernel,
)
@@ -91,6 +94,9 @@ from vllm.model_executor.kernels.linear.mxfp8.flashinfer import (
FlashInferCutedslMxfp8LinearKernel,
FlashInferCutlassMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.humming import (
HummingMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.marlin import (
MarlinMxfp8LinearKernel,
)
@@ -120,6 +126,9 @@ from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
FlashInferCutlassNvFp4LinearKernel,
FlashInferTrtllmNvFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.nvfp4.humming import (
HummingNvFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.nvfp4.marlin import (
MarlinNvFp4LinearKernel,
)
@@ -154,6 +163,10 @@ from vllm.model_executor.kernels.linear.scaled_mm.flashinfer import (
FlashInferFp8DeepGEMMDynamicBlockScaledKernel,
FlashInferFP8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.humming import (
HummingFP8ScaledMMLinearKernel,
HummingInt8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.marlin import (
MarlinFP8ScaledMMLinearKernel,
)
@@ -225,6 +238,14 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
"flashinfer_b12x": {
FlashInferB12xNvFp4LinearKernel,
},
"humming": {
HummingFP8ScaledMMLinearKernel,
HummingInt8ScaledMMLinearKernel,
HummingLinearKernel,
HummingMxfp8LinearKernel,
HummingMxFp4LinearKernel,
HummingNvFp4LinearKernel,
},
"marlin": {
MarlinFP8ScaledMMLinearKernel,
MarlinLinearKernel,
@@ -292,6 +313,7 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]
PlatformEnum.CUDA: [
CutlassInt8ScaledMMLinearKernel,
TritonInt8ScaledMMLinearKernel,
HummingInt8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [AiterInt8ScaledMMLinearKernel, TritonInt8ScaledMMLinearKernel],
}
@@ -304,6 +326,7 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] =
CutlassFP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
HummingFP8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
@@ -335,6 +358,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
CutlassFp8BlockScaledMMKernel,
MarlinFP8ScaledMMLinearKernel,
TritonFp8BlockScaledMMKernel,
HummingFP8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [
AiterFp8BlockScaledMMKernel,
@@ -351,6 +375,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[
_POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {
PlatformEnum.CUDA: [
HummingFP8ScaledMMLinearKernel,
MarlinFP8ScaledMMLinearKernel,
],
PlatformEnum.ROCM: [
@@ -371,10 +396,10 @@ _POSSIBLE_KERNELS: dict[PlatformEnum, list[type[MPLinearKernel]]] = {
MacheteLinearKernel,
AllSparkLinearKernel,
MarlinLinearKernel,
HummingLinearKernel,
ConchLinearKernel,
ExllamaLinearKernel,
TritonW4A16LinearKernel,
HummingLinearKernel,
],
PlatformEnum.ROCM: [
RDNA3W4A16LinearKernel,
@@ -400,6 +425,7 @@ _POSSIBLE_MXFP8_KERNELS: dict[PlatformEnum, list[type[Mxfp8LinearKernel]]] = {
FlashInferCutlassMxfp8LinearKernel,
MarlinMxfp8LinearKernel,
EmulationMxfp8LinearKernel,
HummingMxfp8LinearKernel,
],
PlatformEnum.ROCM: [
# Native CDNA4 (gfx950) MX linear; is_supported() gates to gfx95x and
@@ -426,6 +452,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
FlashInferCudnnNvFp4LinearKernel,
FbgemmNvFp4LinearKernel,
EmulationNvFp4LinearKernel,
HummingNvFp4LinearKernel,
],
PlatformEnum.ROCM: [
EmulationNvFp4LinearKernel,
@@ -436,6 +463,7 @@ _POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = {
PlatformEnum.CUDA: [
FlashInferMxFp4LinearKernel,
MarlinMxFp4LinearKernel,
HummingMxFp4LinearKernel,
],
PlatformEnum.XPU: [
XPUMxFp4LinearKernel,
@@ -19,6 +19,9 @@ from vllm.model_executor.kernels.linear.mixed_precision.dynamic_4bit import (
from vllm.model_executor.kernels.linear.mixed_precision.exllama import (
ExllamaLinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.humming import (
HummingLinearKernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.machete import (
MacheteLinearKernel,
)
@@ -52,6 +55,7 @@ __all__ = [
"CutlassW4A8LinearKernel",
"Dynamic4bitLinearKernel",
"ExllamaLinearKernel",
"HummingLinearKernel",
"MacheteLinearKernel",
"MarlinLinearKernel",
"RDNA3W4A16LinearKernel",
@@ -23,8 +23,6 @@ class HummingLinearKernel(MPLinearKernel):
return False, "Humming is not installed"
if c.has_g_idx:
return False, "Humming does not support act-order (g_idx)"
if c.zero_points:
return False, "Humming linear kernel only supports symmetric weights"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -41,6 +39,11 @@ class HummingLinearKernel(MPLinearKernel):
"group_size": 0 if group_size == -1 else group_size,
}
if self.config.zero_points:
assert self.w_zp_name is not None
name_map["zero_point"] = self.w_zp_name
quant_config["has_zero_point"] = True
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_linear_layer_to_humming_standard,
prepare_humming_layer,
)
from vllm.platforms import current_platform
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
class HummingMxFp4LinearKernel(MxFp4LinearKernel):
"""Humming GEMM Kernel for MXFP4."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
return True, None
@classmethod
def can_implement(cls, c: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu)
name_map = {"weight": "weight", "weight_scale": "weight_scale"}
quant_config = {
"quant_method": "humming",
"dtype": "float4e2m1",
"scale_dtype": "float8e8m0",
"group_size": 32,
"weight_scale_type": "group",
}
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_linear_layer_to_humming_standard,
prepare_humming_layer,
)
from vllm.platforms import current_platform
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
"""Humming GEMM Kernel for MXFP8."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
return True, None
@classmethod
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight_scale.data = layer.weight_scale.data.view(torch.float8_e8m0fnu)
name_map = {"weight": "weight", "weight_scale": "weight_scale"}
quant_config = {
"quant_method": "humming",
"dtype": "float8e4m3",
"scale_dtype": "float8e8m0",
"group_size": 32,
"weight_scale_type": "group",
}
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -0,0 +1,73 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.humming_utils import (
prepare_humming_layer,
)
from vllm.platforms import current_platform
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
logger = init_logger(__name__)
class HummingNvFp4LinearKernel(NvFp4LinearKernel):
"""Humming GEMM Kernel for NVFP4."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
return True, None
@classmethod
def can_implement(cls, config: NvFp4LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Route through humming's compressed-tensors nvfp4 loader (same path as
# the MoE oracle); the native group_tensor schema mishandles a scalar
# global scale.
quant_config = {
"quant_method": "compressed-tensors",
"format": "nvfp4-pack-quantized",
"type": "float",
"num_bits": 4,
"strategy": "group",
"group_size": 16,
}
# CT pack-quantized reads `weight_packed`; the scheme renamed it to `weight`.
if not hasattr(layer, "weight_packed"):
layer.weight_packed = layer.weight
del layer.weight
# The CT linear scheme inverts the global scale (1/scale) for
# marlin/cutlass; humming wants the original.
layer.weight_global_scale = torch.nn.Parameter(
1.0 / layer.weight_global_scale, requires_grad=False
)
prepare_humming_layer(layer, quant_config)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -0,0 +1,156 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_linear_layer_to_humming_standard,
prepare_humming_layer,
)
from vllm.platforms import current_platform
from .ScaledMMLinearKernel import (
FP8ScaledMMLinearKernel,
FP8ScaledMMLinearLayerConfig,
Int8ScaledMMLinearKernel,
Int8ScaledMMLinearLayerConfig,
)
logger = init_logger(__name__)
class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
"""Humming GEMM Kernel for FP8."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
return True, None
@classmethod
def can_implement(
cls, config: FP8ScaledMMLinearLayerConfig
) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
from vllm.utils.humming import dtypes
name_map = {"weight": "weight", "weight_scale": "weight_scale"}
scale_torch_dtype = self.config.weight_quant_key.scale.dtype
scale_dtype = dtypes.DataType.from_torch_dtype(scale_torch_dtype)
quant_config = {
"quant_method": "humming",
"dtype": "float8e4m3",
"scale_dtype": scale_dtype,
}
assert self.config.weight_quant_key.scale2 is None
scale_group_shape = self.config.weight_quant_key.scale.group_shape
if scale_group_shape.is_per_tensor():
quant_config["weight_scale_type"] = "tensor"
if not hasattr(layer, "global_scale") and hasattr(layer, "weight_scale"):
del name_map["weight_scale"]
name_map["global_scale"] = "weight_scale"
elif scale_group_shape.is_per_channel():
quant_config["weight_scale_type"] = "channel"
elif scale_group_shape.is_per_group():
quant_config["weight_scale_type"] = "group"
quant_config["group_size"] = scale_group_shape.col
else:
assert scale_group_shape.row > 0 and scale_group_shape.col > 0
quant_config["weight_scale_type"] = "block"
quant_config["weight_scale_group_size_n"] = scale_group_shape.row
quant_config["weight_scale_group_size"] = scale_group_shape.col
if hasattr(layer, "weight_scale_inv"):
name_map["weight_scale"] = "weight_scale_inv"
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
def apply_scaled_mm(
self,
*,
A: torch.Tensor,
B: torch.Tensor,
out_dtype: torch.dtype,
As: torch.Tensor,
Bs: torch.Tensor,
bias: torch.Tensor | None,
output_shape: list,
) -> torch.Tensor:
pass
class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
"""Humming GEMM Kernel for INT8."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_cuda():
return False, "Humming only supported on CUDA"
if not current_platform.has_device_capability(75):
return False, "Humming only supported on SM75+"
return True, None
@classmethod
def can_implement(
cls, config: Int8ScaledMMLinearLayerConfig
) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
weight_name, weight_scale_name, *_ = self.layer_param_names
name_map = {"weight": weight_name, "weight_scale": weight_scale_name}
quant_config = {"quant_method": "humming", "dtype": "int8"}
weight = getattr(layer, weight_name)
weight.data = weight.data + 128
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
prepare_humming_layer(layer, quant_config)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.utils.humming import HummingMethod
flatten_inputs = x.view(-1, x.size(-1))
output = HummingMethod.forward_layer(
layer=layer,
inputs=flatten_inputs,
compute_config=layer.compute_config,
)
return output.view(*x.shape[:-1], output.size(-1))
@@ -38,15 +38,20 @@ from vllm.model_executor.layers.fused_moe.utils import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTokenSym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4Static,
kInt8DynamicTokenSym,
kInt8Static,
kInt8StaticChannelSym,
kMxfp4Dynamic,
kMxfp4Static,
kMxfp8Dynamic,
kMxfp8Static,
kNvfp4Dynamic,
kNvfp4Static,
)
from vllm.platforms import current_platform
@@ -61,9 +66,9 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def get_humming_moe_gemm_type() -> str | None:
def get_humming_moe_gemm_type() -> str:
env_gemm_type: str | None = envs.VLLM_HUMMING_MOE_GEMM_TYPE
gemm_type = None
gemm_type = "indexed"
if env_gemm_type is not None:
env_gemm_type = env_gemm_type.lower()
if env_gemm_type == "indexed":
@@ -87,7 +92,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
num_dispatchers: int | None = None,
):
self.layer = layer
self.num_experts = self.layer.num_experts
self.num_experts = self.layer.local_num_experts
self.global_num_experts = self.layer.global_num_experts
self.init_humming_moe()
@@ -186,6 +191,24 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
(kInt4Static, kFp8DynamicTokenSym),
(kInt8Static, None),
(kInt8Static, kFp8DynamicTokenSym),
# Checkpoint-driven (weight, activation) pairs the dense/MoE oracles
# pass. Humming defers input quant (see expects_unquantized_inputs),
# so the activation key does not constrain support.
# fp8 (compressed-tensors / native / modelopt)
(kFp8StaticChannelSym, kFp8StaticTensorSym),
(kFp8StaticChannelSym, kFp8Dynamic128Sym),
(kFp8StaticTensorSym, None),
(kFp8StaticTensorSym, kFp8DynamicTokenSym),
(kFp8StaticTensorSym, kFp8StaticTensorSym),
(kFp8StaticTensorSym, kFp8Dynamic128Sym),
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
# int8 (compressed-tensors w8a8 / experts_int8)
(kInt8StaticChannelSym, None),
(kInt8StaticChannelSym, kInt8DynamicTokenSym),
# nvfp4 (compressed-tensors / modelopt / quark)
(kNvfp4Static, kNvfp4Dynamic),
# mxfp8 (compressed-tensors / modelopt / online)
(kMxfp8Static, kMxfp8Dynamic),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@@ -463,13 +486,12 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
assert hasattr(cls, "humming_gemm_type")
gemm_type = cls.humming_gemm_type().value.lower()
preferred_gemm_type = get_humming_moe_gemm_type()
if preferred_gemm_type is not None:
supported = preferred_gemm_type.lower() == gemm_type
if not supported:
reason = (
f"preferred gemm type {preferred_gemm_type} != "
f"supported gemm type {gemm_type}"
)
supported = preferred_gemm_type.lower() == gemm_type
if not supported:
reason = (
f"preferred gemm type {preferred_gemm_type} != "
f"supported gemm type {gemm_type}"
)
return supported, reason
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import Any
import torch
@@ -45,6 +46,7 @@ class Fp8MoeBackend(Enum):
DEEPGEMM = "DEEPGEMM"
BATCHED_DEEPGEMM = "BATCHED_DEEPGEMM"
MARLIN = "MARLIN"
HUMMING = "HUMMING"
TRITON = "TRITON"
BATCHED_TRITON = "BATCHED_TRITON"
AITER = "AITER"
@@ -83,6 +85,7 @@ def _get_priority_backends(
Fp8MoeBackend.VLLM_CUTLASS,
Fp8MoeBackend.TRITON,
Fp8MoeBackend.MARLIN,
Fp8MoeBackend.HUMMING,
Fp8MoeBackend.BATCHED_DEEPGEMM,
Fp8MoeBackend.BATCHED_VLLM_CUTLASS,
Fp8MoeBackend.BATCHED_TRITON,
@@ -162,6 +165,19 @@ def backend_to_kernel_cls(
return [BatchedDeepGemmExperts]
elif backend == Fp8MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
)
return [
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
]
elif backend == Fp8MoeBackend.MARLIN:
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
MarlinExperts,
@@ -240,6 +256,7 @@ def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend:
"flashinfer_trtllm": Fp8MoeBackend.FLASHINFER_TRTLLM,
"flashinfer_cutlass": Fp8MoeBackend.FLASHINFER_CUTLASS,
"marlin": Fp8MoeBackend.MARLIN,
"humming": Fp8MoeBackend.HUMMING,
"aiter": Fp8MoeBackend.AITER,
"hpc": Fp8MoeBackend.HPC,
}
@@ -402,6 +419,41 @@ def select_fp8_moe_backend(
return Fp8MoeBackend.NONE, None
def _humming_fp8_weight_schema(
layer: RoutedExperts, weight: torch.Tensor, weight_scale: torch.Tensor
) -> dict[str, Any]:
"""Build the humming weight schema from the canonical on-device fp8/mxfp8
tensors (scale dtype/shape, block size), not the producing quant method."""
# mxfp8: e8m0 group-32 scales (stored as uint8 bytes or e8m0). humming has
# no compressed-tensors mxfp8 loader; its modelopt schema fits both sources.
if weight_scale.dtype in (torch.uint8, torch.float8_e8m0fnu):
return {"quant_method": "modelopt", "quant_algo": "mxfp8"}
if hasattr(layer, "w13_weight_scale_inv"):
assert hasattr(layer, "weight_block_size")
return {"quant_method": "fp8", "weight_block_size": layer.weight_block_size}
# fp8 (e4m3): recover the strategy from the scale layout (block from
# weight_block_size, else channel vs tensor by per-expert scale count).
config: dict[str, Any] = {
"quant_method": "compressed-tensors",
"format": "float-quantized",
"type": "float",
"num_bits": 8,
"symmetric": True,
}
weight_block_size = getattr(layer, "weight_block_size", None)
num_experts, num_output = weight.shape[0], weight.shape[-2]
if weight_block_size is not None:
config["strategy"] = "block"
config["block_structure"] = list(weight_block_size)
elif weight_scale.numel() >= num_experts * num_output:
config["strategy"] = "channel"
else:
config["strategy"] = "tensor"
return config
def convert_to_fp8_moe_kernel_format(
fp8_backend: Fp8MoeBackend,
# TODO(bnell): replace layer with weight_block_size
@@ -429,6 +481,18 @@ def convert_to_fp8_moe_kernel_format(
w13, w2, w13_scale, w2_scale = rocm_aiter_ops.shuffle_mxfp8_moe_weights(
w13, w2, w13_scale, w2_scale
)
elif fp8_backend == Fp8MoeBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_to_humming_moe_kernel_format,
)
convert_to_humming_moe_kernel_format(
layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale)
)
w13 = layer.w13_weight
w2 = layer.w2_weight
w13_scale = layer.w13_weight_scale
w2_scale = layer.w2_weight_scale
elif fp8_backend == Fp8MoeBackend.MARLIN:
weight_block_size = getattr(layer, "weight_block_size", None)
if weight_block_size == [1, 32]:
@@ -511,6 +575,7 @@ def make_fp8_moe_quant_config(
swiglu_limit: float | None = None,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
layer: torch.nn.Module | None = None,
) -> FusedMoEQuantConfig:
"""
Create FusedMoEQuantConfig for the specified FP8 Backend.
@@ -537,6 +602,14 @@ def make_fp8_moe_quant_config(
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=swiglu_limit,
)
elif fp8_backend == Fp8MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
assert isinstance(layer, RoutedExperts)
return get_humming_moe_quant_config(layer)
# Flashinfer CUTLASS or HPC per-tensor uses single dq scale
# (alpha = w_scale * a_scale) and inverse a2 scale.
@@ -600,6 +673,7 @@ def make_fp8_moe_kernel(
experts_cls: type[mk.FusedMoEExperts],
fp8_backend: Fp8MoeBackend,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
layer: torch.nn.Module | None = None,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
prepare_finalize = maybe_make_prepare_finalize(
@@ -613,6 +687,11 @@ def make_fp8_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
extra_kwargs = {}
if fp8_backend == Fp8MoeBackend.HUMMING:
assert layer is not None
extra_kwargs = {"layer": layer}
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -622,11 +701,13 @@ def make_fp8_moe_kernel(
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
**extra_kwargs,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
**extra_kwargs,
)
kernel = mk.FusedMoEKernel(
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import Any
import torch
@@ -22,6 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -29,6 +31,7 @@ logger = init_logger(__name__)
class Int8MoeBackend(Enum):
TRITON = "TRITON"
HUMMING = "HUMMING"
CPU = "CPU"
@@ -40,6 +43,7 @@ def _get_priority_backends(
"""
_AVAILABLE_BACKENDS = [
Int8MoeBackend.TRITON,
Int8MoeBackend.HUMMING,
Int8MoeBackend.CPU,
]
@@ -62,6 +66,19 @@ def backend_to_kernel_cls(
return [TritonExperts]
elif backend == Int8MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
)
return [
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
]
elif backend == Int8MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUExpertsInt8,
@@ -77,6 +94,7 @@ def map_int8_backend(runner_backend: MoEBackend) -> Int8MoeBackend:
"""Map user's MoEBackend to Int8MoeBackend."""
mapping = {
"triton": Int8MoeBackend.TRITON,
"humming": Int8MoeBackend.HUMMING,
}
if backend := mapping.get(runner_backend):
return backend
@@ -163,6 +181,7 @@ def select_int8_moe_backend(
def make_int8_moe_quant_config(
int8_backend: Int8MoeBackend,
w1_scale: torch.Tensor,
w2_scale: torch.Tensor,
a1_scale: torch.Tensor | None = None,
@@ -170,11 +189,21 @@ def make_int8_moe_quant_config(
w1_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
per_act_token_quant: bool = False,
layer: torch.nn.Module | None = None,
) -> FusedMoEQuantConfig:
assert (a1_scale is None and a2_scale is None) or (
a1_scale is not None and a2_scale is not None
), "a1_scale and a2_scale must both be provided or both be None"
if int8_backend == Int8MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
assert isinstance(layer, RoutedExperts)
return get_humming_moe_quant_config(layer)
if a1_scale is None or a2_scale is None:
return int8_w8a16_moe_quant_config(
w1_scale=w1_scale,
@@ -196,13 +225,56 @@ def make_int8_moe_quant_config(
)
def _humming_int8_weight_schema(
weight: torch.Tensor, weight_scale: torch.Tensor
) -> dict[str, Any]:
"""Build the humming compressed-tensors int8 schema from the canonical
on-device tensors; humming does the signed-int8 -> native conversion."""
config: dict[str, Any] = {
"quant_method": "compressed-tensors",
"format": "int-quantized",
"type": "int",
"num_bits": 8,
"symmetric": True,
"strategy": "channel",
}
num_experts, num_output = weight.shape[0], weight.shape[-2]
if weight_scale.numel() < num_experts * num_output:
config["strategy"] = "tensor"
return config
def convert_to_int8_moe_kernel_format(
int8_backend: Int8MoeBackend,
w13: torch.Tensor,
w2: torch.Tensor,
layer: torch.nn.Module | None = None,
w13_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Convert INT8 MoE weights to backend-specific kernel format."""
if int8_backend == Int8MoeBackend.CPU:
if int8_backend == Int8MoeBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_to_humming_moe_kernel_format,
)
assert layer is not None
# Humming reads canonical CT scales (w*_weight_scale) from the layer.
# Online int8 produces per-channel (E, N) w*_scale; expose them as the
# (E, N, 1) w*_weight_scale humming's loader expects.
for sub in ("w13", "w2"):
if hasattr(layer, f"{sub}_weight_scale"):
continue
scale = getattr(layer, f"{sub}_scale").data
if scale.dim() < 3:
scale = scale.unsqueeze(-1)
replace_parameter(layer, f"{sub}_weight_scale", scale)
delattr(layer, f"{sub}_scale")
convert_to_humming_moe_kernel_format(
layer,
quant_config=_humming_int8_weight_schema(w13, layer.w13_weight_scale),
)
return layer.w13_weight, layer.w2_weight
elif int8_backend == Int8MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_int8_moe_layer_for_cpu,
)
@@ -215,10 +287,12 @@ def convert_to_int8_moe_kernel_format(
def make_int8_moe_kernel(
int8_backend: Int8MoeBackend,
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
experts_cls: type[mk.FusedMoEExperts],
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
layer: torch.nn.Module | None = None,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
prepare_finalize = maybe_make_prepare_finalize(
@@ -232,6 +306,11 @@ def make_int8_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
extra_kwargs = {}
if int8_backend == Int8MoeBackend.HUMMING:
assert layer is not None
extra_kwargs = {"layer": layer}
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -241,11 +320,13 @@ def make_int8_moe_kernel(
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
**extra_kwargs,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
**extra_kwargs,
)
kernel = mk.FusedMoEKernel(
@@ -11,6 +11,7 @@ from compressed_tensors.quantization import (
import vllm._custom_ops as ops
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config.kernel import MoEBackend
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
@@ -46,6 +47,7 @@ logger = init_logger(__name__)
class WNA16MoEBackend(Enum):
MARLIN = "MARLIN"
BATCHED_MARLIN = "BATCHED_MARLIN"
HUMMING = "HUMMING"
CPU = "CPU"
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
XPU = "XPU"
@@ -55,7 +57,19 @@ def backend_to_kernel_cls(
backend: WNA16MoEBackend,
) -> list[type[mk.FusedMoEExperts]]:
"""Return the experts class for the given backend, or None for NONE."""
if backend == WNA16MoEBackend.MARLIN:
if backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
)
return [
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
]
elif backend == WNA16MoEBackend.MARLIN:
return [MarlinExperts]
elif backend == WNA16MoEBackend.BATCHED_MARLIN:
return [BatchedMarlinExperts]
@@ -90,10 +104,26 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
WNA16MoEBackend.FLASHINFER_TRTLLM,
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
WNA16MoEBackend.HUMMING,
]
return _AVAILABLE_BACKENDS
def map_wna16_backend(runner_backend: MoEBackend) -> WNA16MoEBackend:
"""Map user's MoEBackend to WNA16MoEBackend."""
mapping = {
"marlin": WNA16MoEBackend.MARLIN,
"humming": WNA16MoEBackend.HUMMING,
"flashinfer_trtllm": WNA16MoEBackend.FLASHINFER_TRTLLM,
}
if backend := mapping.get(runner_backend):
return backend
raise ValueError(
f"moe_backend='{runner_backend}' is not supported for WNA16 MoE. "
f"Expected one of {list(mapping.keys())}."
)
def select_wna16_moe_backend(
config: FusedMoEConfig,
weight_key: QuantKey,
@@ -146,6 +176,14 @@ def select_wna16_moe_backend(
return backend, k_cls
raise ValueError(_make_log_unsupported(backend, reason))
# Handle explicit moe_backend from user.
runner_backend = config.moe_backend
if runner_backend != "auto":
requested_backend = map_wna16_backend(runner_backend)
return _return_or_raise(
requested_backend, config, weight_key, None, activation_format
)
# Select kernels in order of backend.
AVAILABLE_BACKENDS = _get_priority_backends()
@@ -210,6 +248,8 @@ def make_wna16_moe_kernel(
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
experts_cls: type[mk.FusedMoEExperts],
backend: WNA16MoEBackend = WNA16MoEBackend.MARLIN,
layer: torch.nn.Module | None = None,
is_k_full: bool = False,
w13_g_idx: torch.Tensor | None = None,
w2_g_idx: torch.Tensor | None = None,
@@ -228,14 +268,18 @@ def make_wna16_moe_kernel(
)
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
# BatchedMarlinExperts, XPUExpertsWNA16, and CPUExpertsInt4
assert experts_cls in (
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, and the Humming
# grouped/indexed experts.
allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = (
MarlinExperts,
BatchedMarlinExperts,
TrtLlmMxint4ExpertsMonolithic,
XPUExpertsWNA16,
CPUExpertsInt4,
)
if backend == WNA16MoEBackend.HUMMING:
allowed_experts += tuple(backend_to_kernel_cls(WNA16MoEBackend.HUMMING))
assert experts_cls in allowed_experts
is_monolithic = experts_cls.is_monolithic()
@@ -251,7 +295,10 @@ def make_wna16_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__, scope="local")
extra_args: dict[str, Any] = {}
if issubclass(experts_cls, MarlinExpertsBase):
if backend == WNA16MoEBackend.HUMMING:
assert layer is not None
extra_args = {"layer": layer}
elif issubclass(experts_cls, MarlinExpertsBase):
extra_args = {
"w13_g_idx": w13_g_idx,
"w2_g_idx": w2_g_idx,
@@ -941,6 +988,35 @@ def _process_weights_xpu(
)
def _humming_wna16_weight_schema(
quant_config: QuantizationConfig | QuantizationArgs | None,
) -> dict[str, Any]:
"""Humming weight schema for a WNA16 checkpoint, derived from the quant
config rather than the running kernel."""
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
if isinstance(quant_config, AutoAWQConfig):
return {
"quant_method": "awq",
"bits": quant_config.weight_bits,
"group_size": quant_config.group_size,
"zero_point": quant_config.zero_point,
}
if isinstance(quant_config, AutoGPTQConfig):
return {
"quant_method": "gptq",
"bits": quant_config.weight_bits,
"group_size": quant_config.group_size,
"desc_act": quant_config.desc_act,
"sym": quant_config.is_sym,
}
raise TypeError(
"Humming WNA16 MoE requires AutoAWQConfig or AutoGPTQConfig, "
f"got {type(quant_config).__name__}."
)
def convert_to_wna16_moe_kernel_format(
backend: WNA16MoEBackend,
layer: torch.nn.Module,
@@ -956,26 +1032,30 @@ def convert_to_wna16_moe_kernel_format(
w2_qzeros: torch.Tensor | None = None,
w13_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
) -> tuple[
torch.Tensor, # w13_qweight
torch.Tensor, # w2_qweight
torch.Tensor, # w13_scales
torch.Tensor, # w2_scales
torch.Tensor | None, # w13_g_idx
torch.Tensor | None, # w2_g_idx
torch.Tensor | None, # w13_g_idx_sort_indices
torch.Tensor | None, # w2_g_idx_sort_indices
torch.Tensor | None, # w13_qzeros
torch.Tensor | None, # w2_qzeros
torch.Tensor | None, # w13_input_global_scale
torch.Tensor | None, # w2_input_global_scale
torch.Tensor | None, # w13_bias
torch.Tensor | None, # w2_bias
]:
) -> (
tuple[
torch.Tensor, # w13_qweight
torch.Tensor, # w2_qweight
torch.Tensor, # w13_scales
torch.Tensor, # w2_scales
torch.Tensor | None, # w13_g_idx
torch.Tensor | None, # w2_g_idx
torch.Tensor | None, # w13_g_idx_sort_indices
torch.Tensor | None, # w2_g_idx_sort_indices
torch.Tensor | None, # w13_qzeros
torch.Tensor | None, # w2_qzeros
torch.Tensor | None, # w13_input_global_scale
torch.Tensor | None, # w2_input_global_scale
torch.Tensor | None, # w13_bias
torch.Tensor | None, # w2_bias
]
| None
):
"""Dispatch weight post-processing to the appropriate per-backend handler.
To add a new backend, implement a ``_process_weights_<name>`` helper and
add a branch here.
add a branch here. Backends that rewrite the layer's parameters in place
(e.g. Humming) return ``None``; the caller then skips the param scatter.
Args:
backend: the selected ``WNA16MoEBackend``.
@@ -983,6 +1063,16 @@ def convert_to_wna16_moe_kernel_format(
quant_config: the ``QuantizationConfig`` for this layer.
input_dtype: optional activation dtype, usually should be 16 bit.
"""
if backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_to_humming_moe_kernel_format,
)
convert_to_humming_moe_kernel_format(
layer, quant_config=_humming_wna16_weight_schema(quant_config)
)
return None
if backend in (
WNA16MoEBackend.MARLIN,
WNA16MoEBackend.BATCHED_MARLIN,
@@ -25,6 +25,7 @@ _SUPPORTED_BACKENDS = (
# is_supported_config passes (gfx950 + flydsl installed + not EP). On other
# devices / no flydsl / EP it is skipped and native is used.
Fp8MoeBackend.AITER_MXFP8,
Fp8MoeBackend.HUMMING,
)
_BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
@@ -34,6 +35,7 @@ _BACKEND_NAME_MAP: dict[str, Fp8MoeBackend] = {
"xpu": Fp8MoeBackend.XPU,
"aiter": Fp8MoeBackend.AITER_MXFP8,
"triton": Fp8MoeBackend.TRITON_MXFP8,
"humming": Fp8MoeBackend.HUMMING,
}
@@ -43,6 +43,7 @@ class NvFp4MoeBackend(Enum):
FLASHINFER_B12X = "FLASHINFER_B12X"
VLLM_CUTLASS = "VLLM_CUTLASS"
MARLIN = "MARLIN"
HUMMING = "HUMMING"
EMULATION = "EMULATION"
@@ -119,6 +120,18 @@ def backend_to_kernel_cls(
)
return [MarlinExperts]
elif backend == NvFp4MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
)
return [
BatchedHummingGroupedExperts,
HummingGroupedExperts,
HummingIndexedExperts,
]
elif backend == NvFp4MoeBackend.EMULATION:
from vllm.model_executor.layers.fused_moe.experts.nvfp4_emulation_moe import (
Nvfp4QuantizationEmulationTritonExperts,
@@ -138,6 +151,7 @@ def map_nvfp4_backend(runner_backend: MoEBackend) -> NvFp4MoeBackend:
"flashinfer_cutedsl": NvFp4MoeBackend.FLASHINFER_CUTEDSL,
"flashinfer_b12x": NvFp4MoeBackend.FLASHINFER_B12X,
"marlin": NvFp4MoeBackend.MARLIN,
"humming": NvFp4MoeBackend.HUMMING,
"emulation": NvFp4MoeBackend.EMULATION,
}
if backend := mapping.get(runner_backend):
@@ -169,6 +183,7 @@ def select_nvfp4_moe_backend(
NvFp4MoeBackend.FLASHINFER_CUTLASS,
NvFp4MoeBackend.VLLM_CUTLASS,
NvFp4MoeBackend.MARLIN,
NvFp4MoeBackend.HUMMING,
NvFp4MoeBackend.EMULATION,
]
@@ -346,6 +361,41 @@ def convert_to_nvfp4_moe_kernel_format(
a2_scale=a2_scale,
is_act_and_mul=is_act_and_mul,
)
elif nvfp4_backend == NvFp4MoeBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
convert_to_humming_moe_kernel_format,
)
# Discriminate the source checkpoint layout by its global-scale param:
# compressed-tensors uses *_weight_global_scale, modelopt *_weight_scale_2.
# The logical schema is identical (nvfp4 group-16); only the on-layer
# param names differ. TODO: normalize both methods to a single canonical
# layout upstream so the oracle needs neither the probe nor the re-alias.
if hasattr(layer, "w13_weight_global_scale"):
quant_config = {
"quant_method": "compressed-tensors",
"format": "nvfp4-pack-quantized",
"type": "float",
"num_bits": 4,
"strategy": "group",
"group_size": 16,
}
# CT pack-quantized reads `weight_packed`; the method renamed it to
# `weight`. Re-alias (convert replaces all params anyway).
layer.w13_weight_packed = layer.w13_weight
layer.w2_weight_packed = layer.w2_weight
else:
quant_config = {"quant_method": "modelopt", "quant_algo": "nvfp4"}
convert_to_humming_moe_kernel_format(layer, quant_config=quant_config)
a13_scale = None
a2_scale = None
w13 = layer.w13_weight
w13_scale = layer.w13_weight_scale
w13_scale_2 = getattr(layer, "w13_global_scale", None)
w2 = layer.w2_weight
w2_scale = layer.w2_weight_scale
w2_scale_2 = getattr(layer, "w2_global_scale", None)
elif nvfp4_backend == NvFp4MoeBackend.MARLIN:
a13_scale = None
a2_scale = None
@@ -418,8 +468,17 @@ def make_nvfp4_moe_quant_config(
a13_scale: torch.Tensor,
a2_scale: torch.Tensor,
swiglu_limit: float | None = None,
layer: torch.nn.Module | None = None,
) -> FusedMoEQuantConfig:
if backend == NvFp4MoeBackend.MARLIN:
if backend == NvFp4MoeBackend.HUMMING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
assert isinstance(layer, RoutedExperts)
return get_humming_moe_quant_config(layer)
elif backend == NvFp4MoeBackend.MARLIN:
return nvfp4_w4a16_moe_quant_config(
g1_alphas=w13_scale_2,
g2_alphas=w2_scale_2,
@@ -467,7 +526,9 @@ def make_nvfp4_moe_kernel(
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
experts_cls: type[mk.FusedMoEExperts],
backend: NvFp4MoeBackend,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
layer: torch.nn.Module | None = None,
) -> mk.FusedMoEKernel:
# Create Prepare/Finalize.
prepare_finalize = maybe_make_prepare_finalize(
@@ -481,6 +542,11 @@ def make_nvfp4_moe_kernel(
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
extra_kwargs = {}
if backend == NvFp4MoeBackend.HUMMING:
assert layer is not None
extra_kwargs = {"layer": layer}
# Create Experts.
if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
max_num_tokens = prepare_finalize.max_num_tokens_per_rank()
@@ -490,11 +556,13 @@ def make_nvfp4_moe_kernel(
quant_config=moe_quant_config,
max_num_tokens=max_num_tokens,
num_dispatchers=prepare_finalize.num_dispatchers(),
**extra_kwargs,
)
else:
experts = experts_cls(
moe_config=moe_config,
quant_config=moe_quant_config,
**extra_kwargs,
)
kernel = mk.FusedMoEKernel(
@@ -232,7 +232,9 @@ def select_unquantized_moe_backend(
raise ValueError(_make_log_unsupported(backend, reason))
runner_backend = moe_config.moe_backend
if runner_backend != "auto":
# 'humming' is quantization-only; an unquantized layer (e.g. excluded via
# modules_to_not_convert) falls through to auto instead of erroring.
if runner_backend not in ["auto", "humming"]:
requested_backend = map_unquantized_backend(runner_backend)
if (
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.fused_moe import (
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
WNA16MoEBackend,
convert_to_wna16_moe_kernel_format,
make_wna16_moe_kernel,
make_wna16_moe_quant_config,
@@ -663,6 +664,26 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
layer.workspace = marlin_make_workspace_new(device, 4)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_moe_backend,
layer=layer,
quant_config=self.quant_config,
input_dtype=self.input_dtype,
w13=layer.w13_qweight,
w2=layer.w2_qweight,
w13_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w13_qzeros=layer.w13_qzeros,
w2_qzeros=layer.w2_qzeros,
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
if converted is None:
# Backend rewrote the layer's params in place (e.g. Humming).
self._setup_kernel(layer)
return
(
w13,
w2,
@@ -678,20 +699,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
w2_input_global_scale,
w13_bias,
w2_bias,
) = convert_to_wna16_moe_kernel_format(
backend=self.wna16_moe_backend,
layer=layer,
quant_config=self.quant_config,
input_dtype=self.input_dtype,
w13=layer.w13_qweight,
w2=layer.w2_qweight,
w13_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w13_qzeros=layer.w13_qzeros,
w2_qzeros=layer.w2_qzeros,
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
) = converted
replace_parameter(layer, "w13_qweight", w13)
replace_parameter(layer, "w2_qweight", w2)
@@ -734,6 +742,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_moe_backend,
layer=layer,
is_k_full=self.is_k_full,
w13_g_idx=getattr(layer, "w13_g_idx", None),
w2_g_idx=getattr(layer, "w2_g_idx", None),
@@ -743,6 +753,12 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
)
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
return get_humming_moe_quant_config(layer)
return make_wna16_moe_quant_config(
w1_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
@@ -783,8 +799,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
assert self.moe_kernel is not None
return self.moe_kernel.apply(
hidden_states=x,
w1=layer.w13_qweight,
w2=layer.w2_qweight,
w1=layer.w13_weight,
w2=layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
@@ -806,8 +822,8 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
hidden_states=x,
w1=layer.w13_qweight,
w2=layer.w2_qweight,
w1=layer.w13_weight,
w2=layer.w2_weight,
router_logits=router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
@@ -658,6 +658,26 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
"W8A8-INT8 is not supported by marlin kernel."
)
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_moe_backend,
layer=layer,
quant_config=self.quant_config,
input_dtype=self.input_dtype,
w13=layer.w13_qweight,
w2=layer.w2_qweight,
w13_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w13_g_idx=layer.w13_g_idx,
w2_g_idx=layer.w2_g_idx,
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
if converted is None:
# Backend rewrote the layer's params in place (e.g. Humming).
self._setup_kernel(layer)
return
(
w13,
w2,
@@ -673,20 +693,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
w2_input_global_scale,
w13_bias,
w2_bias,
) = convert_to_wna16_moe_kernel_format(
backend=self.wna16_moe_backend,
layer=layer,
quant_config=self.quant_config,
input_dtype=self.input_dtype,
w13=layer.w13_qweight,
w2=layer.w2_qweight,
w13_scale=layer.w13_scales,
w2_scale=layer.w2_scales,
w13_g_idx=layer.w13_g_idx,
w2_g_idx=layer.w2_g_idx,
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
) = converted
replace_parameter(layer, "w13_qweight", w13)
replace_parameter(layer, "w2_qweight", w2)
@@ -733,6 +740,10 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
"w2_bias", torch.nn.Parameter(w2_bias, requires_grad=False)
)
# The modular kernel reads w13_weight/w2_weight; marlin keeps *_qweight.
layer.w13_weight = layer.w13_qweight
layer.w2_weight = layer.w2_qweight
self._setup_kernel(layer)
def _setup_kernel(self, layer: RoutedExperts) -> None:
@@ -743,15 +754,24 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.wna16_moe_backend,
layer=layer,
is_k_full=self.is_k_full,
w13_g_idx=layer.w13_g_idx,
w2_g_idx=layer.w2_g_idx,
w13_g_idx=getattr(layer, "w13_g_idx", None),
w2_g_idx=getattr(layer, "w2_g_idx", None),
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
routing_tables=layer._expert_routing_tables(),
)
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
if self.wna16_moe_backend == WNA16MoEBackend.HUMMING:
from vllm.model_executor.layers.quantization.utils.humming_utils import (
get_humming_moe_quant_config,
)
return get_humming_moe_quant_config(layer)
from vllm.model_executor.layers.fused_moe.config import (
gptq_marlin_moe_quant_config,
)
@@ -795,8 +815,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
assert self.moe_kernel is not None
return self.moe_kernel.apply(
hidden_states=x,
w1=layer.w13_qweight,
w2=layer.w2_qweight,
w1=layer.w13_weight,
w2=layer.w2_weight,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=layer.activation,
@@ -818,8 +838,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
hidden_states=x,
w1=layer.w13_qweight,
w2=layer.w2_qweight,
w1=layer.w13_weight,
w2=layer.w2_weight,
router_logits=router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
@@ -236,7 +236,9 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -259,6 +261,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
a13_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
swiglu_limit=getattr(layer, "swiglu_limit", None),
layer=layer,
)
def apply_monolithic(
@@ -338,6 +338,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def maybe_make_prepare_finalize(
@@ -355,12 +356,13 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
fp8_backend=self.fp8_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
a1_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
a1_scale=getattr(layer, "w13_input_scale", None),
a2_scale=getattr(layer, "w2_input_scale", None),
per_act_token_quant=is_per_token,
per_out_ch_quant=is_per_token,
block_shape=self.weight_block_size,
swiglu_limit=getattr(layer, "swiglu_limit", None),
layer=layer,
)
def apply_monolithic(
@@ -146,6 +146,8 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
int8_backend=self.int8_backend,
w13=layer.w13_weight,
w2=layer.w2_weight,
layer=layer,
w13_scale=layer.w13_weight_scale,
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
@@ -153,10 +155,12 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.experts_cls is not None
self.moe_kernel = make_int8_moe_kernel(
int8_backend=self.int8_backend,
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def maybe_make_prepare_finalize(
@@ -170,11 +174,13 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
return make_int8_moe_quant_config(
int8_backend=self.int8_backend,
w1_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
a1_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
per_act_token_quant=True,
layer=layer,
)
def apply(
@@ -140,6 +140,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -155,6 +156,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
layer=layer,
)
def maybe_make_prepare_finalize(
@@ -416,6 +416,27 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Process weights using the shared oracle infrastructure
is_flashinfer = self.wna16_backend == WNA16MoEBackend.FLASHINFER_TRTLLM
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
layer=layer,
quant_config=self.weight_quant,
input_dtype=self.marlin_input_dtype,
w13=layer.w13_weight_packed,
w2=layer.w2_weight_packed,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w13_g_idx=layer.w13_weight_g_idx,
w2_g_idx=layer.w2_weight_g_idx,
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
)
if converted is None:
# In-place backends (e.g. Humming) are not wired through this
# marlin-only method; fail clearly rather than unpacking None.
raise NotImplementedError(
f"{type(self).__name__} does not support the "
f"{self.wna16_backend.value} MoE backend."
)
(
w13_qweight,
w2_qweight,
@@ -431,20 +452,7 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
w2_input_global_scale,
_, # w13_bias
_, # w2_bias
) = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
layer=layer,
quant_config=self.weight_quant,
input_dtype=self.marlin_input_dtype,
w13=layer.w13_weight_packed,
w2=layer.w2_weight_packed,
w13_scale=layer.w13_weight_scale,
w2_scale=layer.w2_weight_scale,
w13_g_idx=layer.w13_weight_g_idx,
w2_g_idx=layer.w2_weight_g_idx,
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
)
) = converted
# Replace common parameters
replace_parameter(layer, "w13_weight_packed", w13_qweight)
@@ -137,8 +137,16 @@ class CompressedTensorsW8A16Fp8(CompressedTensorsScheme):
"weight_scale",
convert_to_channelwise(layer.weight_scale, layer.logical_widths),
)
self.strategy = QuantizationStrategy.CHANNEL
self.weight_quant_key = STRATEGY_TO_WEIGHT_QUANT_KEY[self.strategy]
self.linear_kernel.config.weight_quant_key = self.weight_quant_key
# Canonicalize to (K, N) for the kernel.
replace_parameter(layer, "weight", layer.weight.t())
# Preserve the dim tags dropped by the transpose so layout-aware
# kernels see (K, N).
layer.weight.input_dim = 0
layer.weight.output_dim = 1
self.linear_kernel.process_weights_after_loading(layer)
@@ -181,6 +181,10 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
# required by torch.compile to be torch.nn.Parameter
layer.weight = Parameter(weight.data, requires_grad=False)
# Preserve the dim tags dropped by the transpose so layout-aware
# kernels (humming) see (K, N) instead of assuming (N, K).
layer.weight.input_dim = 0
layer.weight.output_dim = 1
layer.weight_scale = Parameter(weight_scale.data, requires_grad=False)
if input_scale is not None:
layer.input_scale = Parameter(input_scale.data, requires_grad=False)
@@ -714,6 +714,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
@@ -786,6 +787,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
layer=layer,
)
# Inject biases into the quant config if the model has them
@@ -905,6 +905,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
@@ -951,6 +952,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
a1_scale=a1_scale,
a2_scale=a2_scale,
swiglu_limit=getattr(layer, "swiglu_limit", None),
layer=layer,
)
def apply_monolithic(
@@ -1602,7 +1604,9 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
@@ -1616,6 +1620,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
a13_scale=layer.w13_input_scale,
a2_scale=layer.w2_input_scale,
swiglu_limit=getattr(layer, "swiglu_limit", None),
layer=layer,
)
@property
@@ -2162,6 +2167,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
fp8_backend=self.mxfp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
@@ -2207,6 +2213,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
layer=layer,
)
def apply_monolithic(
@@ -458,6 +458,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -486,6 +487,7 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
layer=layer,
)
@@ -13,6 +13,7 @@ if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
convert_to_int8_moe_kernel_format,
make_int8_moe_kernel,
make_int8_moe_quant_config,
select_int8_moe_backend,
@@ -92,22 +93,36 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
replace_parameter(layer, "w2_scale", w2_scale)
def _setup_kernel(self, layer: RoutedExperts) -> None:
w13, w2 = convert_to_int8_moe_kernel_format(
int8_backend=self.int8_backend,
w13=layer.w13_weight,
w2=layer.w2_weight,
layer=layer,
w13_scale=layer.w13_scale,
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
assert self.experts_cls is not None
self.moe_kernel = make_int8_moe_kernel(
int8_backend=self.int8_backend,
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
self, layer: torch.nn.Module
) -> "FusedMoEQuantConfig | None":
return make_int8_moe_quant_config(
w1_scale=layer.w13_scale,
w2_scale=layer.w2_scale,
int8_backend=self.int8_backend,
w1_scale=getattr(layer, "w13_scale", None),
w2_scale=getattr(layer, "w2_scale", None),
w1_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
layer=layer,
)
@@ -200,6 +200,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
fp8_backend=self.fp8_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -226,6 +227,7 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
layer=layer,
)
def process_weights_after_loading(self, layer: Module) -> None:
@@ -1562,7 +1562,9 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
experts_cls=self.experts_cls,
backend=self.nvfp4_backend,
routing_tables=layer._expert_routing_tables(),
layer=layer,
)
def get_fused_moe_quant_config(
@@ -1576,6 +1578,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
w2_scale_2=layer.w2_weight_scale_2,
a13_scale=layer.w13_input_scale_2,
a2_scale=layer.w2_input_scale_2,
layer=layer,
)
def apply(
@@ -764,7 +764,7 @@ def _convert_sublayer_to_humming(
Returns:
Tuple of (converted_weight_schema, converted_input_schema)
"""
from humming.schema import HummingWeightSchema
from vllm.utils.humming import HummingWeightSchema
if isinstance(weight_schema, HummingWeightSchema):
# Already in Humming format
@@ -814,7 +814,7 @@ def _prepare_and_transform_sublayer(
This calls Humming's prepare_layer_meta and transform_humming_layer.
"""
from humming.layer import HummingMethod
from vllm.utils.humming import HummingMethod
HummingMethod.prepare_layer_meta(
layer=layer,
@@ -866,7 +866,7 @@ def _process_single_sublayer(
Returns:
Tuple of (final_weight_schema, final_input_schema)
"""
from humming.schema import HummingWeightSchema
from vllm.utils.humming import HummingWeightSchema
# Step 1: Convert from checkpoint format to humming format if needed
current_weight_schema, current_input_schema = _convert_sublayer_to_humming(
@@ -958,12 +958,10 @@ def convert_to_humming_moe_kernel_format(
"Must provide either weight_schema/input_schema or quant_config"
)
from humming.layer import HummingInputSchema
from humming.schema import BaseWeightSchema
from vllm.model_executor.layers.quantization.utils.humming_utils import (
humming_is_layer_skipped,
)
from vllm.utils.humming import BaseWeightSchema, HummingInputSchema
if weight_schema is None:
weight_schema = BaseWeightSchema.from_config(quant_config)
@@ -11,6 +11,9 @@ from tqdm import tqdm
import vllm.envs as envs
from vllm.distributed.parallel_state import get_dp_group, is_global_first_rank
from vllm.model_executor.kernels.linear.scaled_mm.deep_gemm import (
DeepGemmFp8BlockScaledMMKernel,
)
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
compute_aligned_M_and_alignment,
@@ -147,6 +150,12 @@ def _fp8_linear_may_use_deep_gemm(module: torch.nn.Module) -> bool:
):
return False
if not isinstance(
getattr(module.quant_method, "fp8_linear", None),
DeepGemmFp8BlockScaledMMKernel,
):
return False
w, _, block_sizes = _extract_data_from_linear_base_module(module)
return (
block_sizes == get_mk_alignment_for_contiguous_layout()