[Quantization] Prefer Humming before Marlin backends on SM90 (#56997)

Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jinzhen Lin
2026-10-06 12:31:35 -07:00
committed by GitHub
co-authored by Codex mgoin Claude Opus 5.5
parent bb87d227d4
commit d1f3d8b870
13 changed files with 159 additions and 15 deletions
+11 -3
View File
@@ -228,6 +228,10 @@ from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
ZentorchInt8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.quantization.utils.humming import (
prefers_humming,
prioritize_humming,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.platforms import PlatformEnum, current_platform
@@ -376,8 +380,9 @@ def _resolve_backend_kernels(
layer_desc: str,
*,
quantization: str,
compute_capability: int | None = None,
) -> list[type]:
"""Apply --linear-backend filtering to one layer type's kernel list.
"""Apply device priorities and --linear-backend filtering to a kernel list.
When the requested backend has no kernel for this layer type, fall back
to the unfiltered list (with a WARNING log) instead of failing engine
@@ -386,6 +391,7 @@ def _resolve_backend_kernels(
layer types (e.g. NVFP4 MoE projections next to FP8 attention
projections).
"""
kernels = prioritize_humming(kernels, compute_capability)
linear_backend = _get_linear_backend(quantization=quantization)
if linear_backend == "auto":
return kernels
@@ -682,6 +688,7 @@ def choose_scaled_mm_linear_kernel(
platform_kernels,
"scaled-mm",
quantization=quantization,
compute_capability=compute_capability,
)
for kernel in platform_kernels:
@@ -845,6 +852,7 @@ def choose_mp_linear_kernel(
platform_kernels,
"mixed-precision",
quantization="mixed_precision",
compute_capability=compute_capability,
)
failure_reasons = []
@@ -1121,13 +1129,13 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
_cc = current_platform.get_device_capability()
compute_capability = _cc.to_int() if _cc is not None else None
# Weight-only: prefer FlashInfer CuTe-DSL W4A16 on SM100/103,
# otherwise Marlin.
# Humming then Marlin where Humming is preferred, and Marlin elsewhere.
cutedsl_ok, _ = FlashInferCuteDslNvFp4W4A16LinearKernel.is_supported(
compute_capability
)
if compute_capability in (100, 103) and cutedsl_ok:
force_kernel = FlashInferCuteDslNvFp4W4A16LinearKernel
else:
elif not prefers_humming(compute_capability):
force_kernel = MarlinNvFp4LinearKernel
if force_kernel is not None:
@@ -4,11 +4,21 @@
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTokenSym,
kInt8DynamicTokenSym,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
# Per-token activation quantization requested through `act_type`.
_ACT_QUANT_KEYS = {
torch.int8: kInt8DynamicTokenSym,
torch.float8_e4m3fn: kFp8DynamicTokenSym,
}
class HummingLinearKernel(MPLinearKernel):
@classmethod
@@ -21,6 +31,8 @@ class HummingLinearKernel(MPLinearKernel):
return False, "Humming is only supported on CUDA"
if not has_humming():
return False, "Humming is not installed"
if c.act_type.itemsize == 1 and c.act_type not in _ACT_QUANT_KEYS:
return False, f"Activation type ({c.act_type}) not supported by Humming"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -28,6 +40,7 @@ class HummingLinearKernel(MPLinearKernel):
convert_linear_layer_to_humming_standard,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
quant_key_to_input_schema,
)
name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}
@@ -48,8 +61,14 @@ class HummingLinearKernel(MPLinearKernel):
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
input_quant_config = getattr(layer, "_humming_input_quant_config", None)
input_schema = None
if (act_key := _ACT_QUANT_KEYS.get(self.config.act_type)) is not None:
input_schema = quant_key_to_input_schema(act_key)
self.layer_config = prepare_humming_linear_layer_config(
layer, quant_config, input_quant_config=input_quant_config
layer,
quant_config,
input_quant_config=input_quant_config,
input_schema=input_schema,
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
prepare_fp8_moe_layer_for_deepgemm,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
prepare_fp8_moe_layer_for_marlin,
)
@@ -37,6 +38,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
@@ -98,6 +100,7 @@ def _get_priority_backends(
Fp8MoeBackend.CPU,
Fp8MoeBackend.HPC,
]
_AVAILABLE_BACKENDS = prioritize_humming(_AVAILABLE_BACKENDS)
def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None:
backends.insert(0, backends.pop(backends.index(backend)))
@@ -607,6 +610,21 @@ def convert_to_fp8_moe_kernel_format(
convert_to_humming_moe_kernel_format,
)
# Online quantization has not installed the quantized tensors yet.
scale_name = (
"weight_scale_inv"
if hasattr(layer, "w13_weight_scale_inv")
else "weight_scale"
)
for prefix, weight, scale, input_scale in (
("w13", w13, w13_scale, w13_input_scale),
("w2", w2, w2_scale, w2_input_scale),
):
replace_parameter(layer, f"{prefix}_weight", weight)
replace_parameter(layer, f"{prefix}_{scale_name}", scale)
if input_scale is not None:
replace_parameter(layer, f"{prefix}_input_scale", input_scale)
convert_to_humming_moe_kernel_format(
layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale)
)
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import (
TrtLlmMxint4ExpertsMonolithic,
)
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_config,
marlin_act_int8_process_scales,
@@ -129,7 +130,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
if current_platform.is_xpu():
return [WNA16MoEBackend.XPU]
return [
backends = [
# Native HIP kernel, gated on gfx1100 by _supports_current_device().
WNA16MoEBackend.RDNA3,
WNA16MoEBackend.FLASHINFER_TRTLLM,
@@ -139,6 +140,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
WNA16MoEBackend.HUMMING,
WNA16MoEBackend.EMULATION,
]
return prioritize_humming(backends)
def _backend_incompatibility_reason(
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
swap_w13_to_w31,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
_swizzle_mxfp4,
)
@@ -382,6 +383,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
# TRITON_UNFUSED
Mxfp4MoeBackend.MARLIN,
Mxfp4MoeBackend.BATCHED_MARLIN,
Mxfp4MoeBackend.HUMMING,
Mxfp4MoeBackend.XPU,
Mxfp4MoeBackend.CPU,
Mxfp4MoeBackend.EMULATION,
@@ -395,12 +397,12 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
_AVAILABLE_BACKENDS.index(Mxfp4MoeBackend.BATCHED_MARLIN) + 1,
Mxfp4MoeBackend.TRITON,
)
return _AVAILABLE_BACKENDS
return prioritize_humming(_AVAILABLE_BACKENDS)
def _get_priority_backends() -> list[Mxfp4MoeBackend]:
"""Get available backends in priority order. SM100+ prefers DeepGEMM FP4 /
TRTLLM MXFP8; SM90 falls through to Triton_unfused or Marlin (the
TRTLLM MXFP8; SM90 falls through to Humming or Marlin (the
backend-level ``is_supported_config`` check filters by device capability).
"""
if current_platform.is_rocm():
@@ -420,8 +422,9 @@ def _get_priority_backends() -> list[Mxfp4MoeBackend]:
# TRITON_UNFUSED
Mxfp4MoeBackend.MARLIN,
Mxfp4MoeBackend.BATCHED_MARLIN,
Mxfp4MoeBackend.HUMMING,
]
return _AVAILABLE_BACKENDS
return prioritize_humming(_AVAILABLE_BACKENDS)
def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
backend_to_kernel_cls,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp8Dynamic,
kMxfp8Static,
@@ -129,7 +130,8 @@ def select_mxfp8_moe_backend(
return backend, _select_kernel_cls(backend, config)
# Auto-select: pick the first supported backend.
for backend in _SUPPORTED_BACKENDS:
backends = prioritize_humming(list(_SUPPORTED_BACKENDS))
for backend in backends:
try:
experts_cls = _select_kernel_cls(backend, config)
except ValueError:
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_nvfp4_moe_layer_for_marlin,
)
@@ -209,6 +210,7 @@ def select_nvfp4_moe_backend(
NvFp4MoeBackend.HUMMING,
NvFp4MoeBackend.EMULATION,
]
AVAILABLE_BACKENDS = prioritize_humming(AVAILABLE_BACKENDS)
NVFP4_BACKENDS_WITH_CLAMP = {
NvFp4MoeBackend.B12X,
@@ -19,6 +19,10 @@ from vllm.model_executor.layers.quantization.utils.humming.moe import (
make_humming_moe_quant_config,
select_humming_moe_experts,
)
from vllm.model_executor.layers.quantization.utils.humming.priority import (
prefers_humming,
prioritize_humming,
)
from vllm.model_executor.layers.quantization.utils.humming.schema import (
check_and_fallback_input_schema,
humming_is_layer_skipped,
@@ -41,6 +45,8 @@ __all__ = [
"convert_linear_layer_to_humming_standard",
"get_humming_linear_compute_config",
"prepare_humming_linear_layer_config",
"prefers_humming",
"prioritize_humming",
"make_humming_moe_kernel",
"select_humming_moe_experts",
"input_schema_to_quant_key",
@@ -87,7 +87,7 @@ def prepare_humming_linear_layer_config(
shape_n_stacks = layer.output_partition_sizes
# Step 1: convert weight and input schemas to humming standard format
source_tensors = dict(layer.named_parameters())
source_tensors = dict(layer.named_parameters(recurse=False))
weight_schema, tensors = weight_schema.convert_humming(
tensors=source_tensors,
shape_n_stacks=shape_n_stacks,
@@ -143,7 +143,7 @@ def prepare_humming_linear_layer_config(
if "bias" in tensors:
zero_bias = torch.zeros_like(tensors["bias"])
layer.register_buffer("zero_bias", zero_bias)
for name, _ in list(layer.named_parameters()):
for name, _ in list(layer.named_parameters(recurse=False)):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Humming kernel selection priority."""
from enum import Enum
from typing import TypeVar
_KernelT = TypeVar("_KernelT", bound=type | Enum)
# Compute capabilities where Humming outranks Marlin.
_HUMMING_PREFERRED_CAPABILITIES = frozenset({90})
def prefers_humming(compute_capability: int | None = None) -> bool:
"""Whether Humming outranks Marlin. A missing compute capability uses the
current device."""
if compute_capability is None:
from vllm.platforms import current_platform
if current_platform.is_cuda():
cc = current_platform.get_device_capability()
compute_capability = cc.to_int() if cc is not None else None
return compute_capability in _HUMMING_PREFERRED_CAPABILITIES
def prioritize_humming(
kernels: list[_KernelT],
compute_capability: int | None = None,
) -> list[_KernelT]:
"""Move Humming directly ahead of Marlin where Humming is preferred.
Every other entry keeps its relative order and the input list is not
modified. Match kernel class names or MoE backend enum names.
"""
if not prefers_humming(compute_capability):
return kernels
names = [
(kernel.name if isinstance(kernel, Enum) else kernel.__name__).lower()
for kernel in kernels
]
humming = next((i for i, name in enumerate(names) if "humming" in name), None)
marlin = next((i for i, name in enumerate(names) if "marlin" in name), None)
if humming is None or marlin is None or humming < marlin:
return kernels
kernels = kernels.copy()
kernels.insert(marlin, kernels.pop(humming))
return kernels