[Quantization] Prefer Humming before Marlin backends on SM90 (#56997)

Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jinzhen Lin
2026-10-06 12:31:35 -07:00
committed by GitHub
co-authored by Codex mgoin Claude Opus 5.5
parent bb87d227d4
commit d1f3d8b870
13 changed files with 159 additions and 15 deletions
+11 -3
View File
@@ -228,6 +228,10 @@ from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
ZentorchInt8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.quantization.utils.humming import (
prefers_humming,
prioritize_humming,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.platforms import PlatformEnum, current_platform
@@ -376,8 +380,9 @@ def _resolve_backend_kernels(
layer_desc: str,
*,
quantization: str,
compute_capability: int | None = None,
) -> list[type]:
"""Apply --linear-backend filtering to one layer type's kernel list.
"""Apply device priorities and --linear-backend filtering to a kernel list.
When the requested backend has no kernel for this layer type, fall back
to the unfiltered list (with a WARNING log) instead of failing engine
@@ -386,6 +391,7 @@ def _resolve_backend_kernels(
layer types (e.g. NVFP4 MoE projections next to FP8 attention
projections).
"""
kernels = prioritize_humming(kernels, compute_capability)
linear_backend = _get_linear_backend(quantization=quantization)
if linear_backend == "auto":
return kernels
@@ -682,6 +688,7 @@ def choose_scaled_mm_linear_kernel(
platform_kernels,
"scaled-mm",
quantization=quantization,
compute_capability=compute_capability,
)
for kernel in platform_kernels:
@@ -845,6 +852,7 @@ def choose_mp_linear_kernel(
platform_kernels,
"mixed-precision",
quantization="mixed_precision",
compute_capability=compute_capability,
)
failure_reasons = []
@@ -1121,13 +1129,13 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
_cc = current_platform.get_device_capability()
compute_capability = _cc.to_int() if _cc is not None else None
# Weight-only: prefer FlashInfer CuTe-DSL W4A16 on SM100/103,
# otherwise Marlin.
# Humming then Marlin where Humming is preferred, and Marlin elsewhere.
cutedsl_ok, _ = FlashInferCuteDslNvFp4W4A16LinearKernel.is_supported(
compute_capability
)
if compute_capability in (100, 103) and cutedsl_ok:
force_kernel = FlashInferCuteDslNvFp4W4A16LinearKernel
else:
elif not prefers_humming(compute_capability):
force_kernel = MarlinNvFp4LinearKernel
if force_kernel is not None:
@@ -4,11 +4,21 @@
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTokenSym,
kInt8DynamicTokenSym,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
# Per-token activation quantization requested through `act_type`.
_ACT_QUANT_KEYS = {
torch.int8: kInt8DynamicTokenSym,
torch.float8_e4m3fn: kFp8DynamicTokenSym,
}
class HummingLinearKernel(MPLinearKernel):
@classmethod
@@ -21,6 +31,8 @@ class HummingLinearKernel(MPLinearKernel):
return False, "Humming is only supported on CUDA"
if not has_humming():
return False, "Humming is not installed"
if c.act_type.itemsize == 1 and c.act_type not in _ACT_QUANT_KEYS:
return False, f"Activation type ({c.act_type}) not supported by Humming"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -28,6 +40,7 @@ class HummingLinearKernel(MPLinearKernel):
convert_linear_layer_to_humming_standard,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
quant_key_to_input_schema,
)
name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}
@@ -48,8 +61,14 @@ class HummingLinearKernel(MPLinearKernel):
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
input_quant_config = getattr(layer, "_humming_input_quant_config", None)
input_schema = None
if (act_key := _ACT_QUANT_KEYS.get(self.config.act_type)) is not None:
input_schema = quant_key_to_input_schema(act_key)
self.layer_config = prepare_humming_linear_layer_config(
layer, quant_config, input_quant_config=input_quant_config
layer,
quant_config,
input_quant_config=input_quant_config,
input_schema=input_schema,
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)