[Quantization] Prefer Humming before Marlin backends on SM90 (#56997)

Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jinzhen Lin
2026-10-06 12:31:35 -07:00
committed by GitHub
co-authored by Codex mgoin Claude Opus 5.5
parent bb87d227d4
commit d1f3d8b870
13 changed files with 159 additions and 15 deletions
@@ -14,6 +14,8 @@ import torch
from vllm.config import KernelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.kernels.linear import (
_POSSIBLE_KERNELS,
_POSSIBLE_NVFP4_KERNELS,
AiterInt8ScaledMMLinearKernel,
CPUInt8ScaledMMLinearKernel,
HummingFP8ScaledMMLinearKernel,
@@ -26,6 +28,7 @@ from vllm.model_executor.kernels.linear import (
init_int8_linear_kernel,
register_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
@@ -197,3 +200,21 @@ def test_register_oot_linear_kernel(platform_mock):
assert isinstance(kernel, OOTInt8ScaledMMLinearKernel), (
"init_int8_linear_kernel should return an instance of the registered kernel"
)
@pytest.mark.parametrize(
"possible_kernels", [_POSSIBLE_KERNELS, _POSSIBLE_NVFP4_KERNELS]
)
def test_humming_priority_only_reorders_humming_and_marlin(possible_kernels):
"""Preferring Humming must not demote Marlin below unrelated kernels."""
kernels = possible_kernels[PlatformEnum.CUDA]
original = list(kernels)
assert prioritize_humming(kernels, compute_capability=80) == original
names = [k.__name__ for k in prioritize_humming(kernels, compute_capability=90)]
humming = next(i for i, name in enumerate(names) if "Humming" in name)
assert "Marlin" in names[humming + 1]
del names[humming]
assert names == [k.__name__ for k in original if "Humming" not in k.__name__]
assert kernels == original
+17 -3
View File
@@ -688,7 +688,8 @@ def test_modelopt_linear_method_builder_registry_override(monkeypatch):
@pytest.mark.parametrize(
("linear_backend", "kernel_cls"),
[
("auto", MarlinNvFp4LinearKernel),
("auto", None),
("marlin", MarlinNvFp4LinearKernel),
("humming", HummingNvFp4LinearKernel),
("flashinfer_cutedsl", FlashInferCuteDslNvFp4W4A16LinearKernel),
],
@@ -696,7 +697,7 @@ def test_modelopt_linear_method_builder_registry_override(monkeypatch):
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
def test_modelopt_w4a16_respects_linear_backend(linear_backend, kernel_cls):
"""W4A16 (`activation=None`) kernel selection honors ``--linear-backend``:
``use_a16=True`` defaults to Marlin, but an explicit backend wins. The
``use_a16=True`` follows platform priorities, but an explicit backend wins. The
generic method routes this through ``select_linear_kernel``."""
from vllm.config.quantization import QuantSpec
from vllm.model_executor.layers.quantization.modelopt import (
@@ -704,7 +705,20 @@ def test_modelopt_w4a16_respects_linear_backend(linear_backend, kernel_cls):
select_linear_kernel,
)
if linear_backend != "auto":
if linear_backend == "auto":
capability = current_platform.get_device_capability()
assert capability is not None
cc = capability.to_int()
if (
cc in (100, 103)
and FlashInferCuteDslNvFp4W4A16LinearKernel.is_supported()[0]
):
kernel_cls = FlashInferCuteDslNvFp4W4A16LinearKernel
elif cc == 90 and HummingNvFp4LinearKernel.is_supported()[0]:
kernel_cls = HummingNvFp4LinearKernel
else:
kernel_cls = MarlinNvFp4LinearKernel
else:
is_supported, reason = kernel_cls.is_supported()
if not is_supported:
pytest.skip(reason)
+2 -1
View File
@@ -34,6 +34,7 @@ from vllm.forward_context import set_forward_context
from vllm.model_executor.kernels.linear.mxfp8.emulation import (
EmulationMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.humming import HummingMxfp8LinearKernel
from vllm.model_executor.kernels.linear.mxfp8.marlin import (
MarlinMxfp8LinearKernel,
)
@@ -1060,7 +1061,7 @@ def test_online_quantization(
assert isinstance(moe._quant_method, expected_moe_cls)
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance(
o_proj.quant_method.kernel, MarlinMxfp8LinearKernel
o_proj.quant_method.kernel, (MarlinMxfp8LinearKernel, HummingMxfp8LinearKernel)
):
assert o_proj.weight.dtype == torch.int32
elif model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance(
+11 -3
View File
@@ -228,6 +228,10 @@ from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
ZentorchInt8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.quantization.utils.humming import (
prefers_humming,
prioritize_humming,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.platforms import PlatformEnum, current_platform
@@ -376,8 +380,9 @@ def _resolve_backend_kernels(
layer_desc: str,
*,
quantization: str,
compute_capability: int | None = None,
) -> list[type]:
"""Apply --linear-backend filtering to one layer type's kernel list.
"""Apply device priorities and --linear-backend filtering to a kernel list.
When the requested backend has no kernel for this layer type, fall back
to the unfiltered list (with a WARNING log) instead of failing engine
@@ -386,6 +391,7 @@ def _resolve_backend_kernels(
layer types (e.g. NVFP4 MoE projections next to FP8 attention
projections).
"""
kernels = prioritize_humming(kernels, compute_capability)
linear_backend = _get_linear_backend(quantization=quantization)
if linear_backend == "auto":
return kernels
@@ -682,6 +688,7 @@ def choose_scaled_mm_linear_kernel(
platform_kernels,
"scaled-mm",
quantization=quantization,
compute_capability=compute_capability,
)
for kernel in platform_kernels:
@@ -845,6 +852,7 @@ def choose_mp_linear_kernel(
platform_kernels,
"mixed-precision",
quantization="mixed_precision",
compute_capability=compute_capability,
)
failure_reasons = []
@@ -1121,13 +1129,13 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
_cc = current_platform.get_device_capability()
compute_capability = _cc.to_int() if _cc is not None else None
# Weight-only: prefer FlashInfer CuTe-DSL W4A16 on SM100/103,
# otherwise Marlin.
# Humming then Marlin where Humming is preferred, and Marlin elsewhere.
cutedsl_ok, _ = FlashInferCuteDslNvFp4W4A16LinearKernel.is_supported(
compute_capability
)
if compute_capability in (100, 103) and cutedsl_ok:
force_kernel = FlashInferCuteDslNvFp4W4A16LinearKernel
else:
elif not prefers_humming(compute_capability):
force_kernel = MarlinNvFp4LinearKernel
if force_kernel is not None:
@@ -4,11 +4,21 @@
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTokenSym,
kInt8DynamicTokenSym,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
# Per-token activation quantization requested through `act_type`.
_ACT_QUANT_KEYS = {
torch.int8: kInt8DynamicTokenSym,
torch.float8_e4m3fn: kFp8DynamicTokenSym,
}
class HummingLinearKernel(MPLinearKernel):
@classmethod
@@ -21,6 +31,8 @@ class HummingLinearKernel(MPLinearKernel):
return False, "Humming is only supported on CUDA"
if not has_humming():
return False, "Humming is not installed"
if c.act_type.itemsize == 1 and c.act_type not in _ACT_QUANT_KEYS:
return False, f"Activation type ({c.act_type}) not supported by Humming"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -28,6 +40,7 @@ class HummingLinearKernel(MPLinearKernel):
convert_linear_layer_to_humming_standard,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
quant_key_to_input_schema,
)
name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}
@@ -48,8 +61,14 @@ class HummingLinearKernel(MPLinearKernel):
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
input_quant_config = getattr(layer, "_humming_input_quant_config", None)
input_schema = None
if (act_key := _ACT_QUANT_KEYS.get(self.config.act_type)) is not None:
input_schema = quant_key_to_input_schema(act_key)
self.layer_config = prepare_humming_linear_layer_config(
layer, quant_config, input_quant_config=input_quant_config
layer,
quant_config,
input_quant_config=input_quant_config,
input_schema=input_schema,
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
prepare_fp8_moe_layer_for_deepgemm,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
prepare_fp8_moe_layer_for_marlin,
)
@@ -37,6 +38,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
@@ -98,6 +100,7 @@ def _get_priority_backends(
Fp8MoeBackend.CPU,
Fp8MoeBackend.HPC,
]
_AVAILABLE_BACKENDS = prioritize_humming(_AVAILABLE_BACKENDS)
def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None:
backends.insert(0, backends.pop(backends.index(backend)))
@@ -607,6 +610,21 @@ def convert_to_fp8_moe_kernel_format(
convert_to_humming_moe_kernel_format,
)
# Online quantization has not installed the quantized tensors yet.
scale_name = (
"weight_scale_inv"
if hasattr(layer, "w13_weight_scale_inv")
else "weight_scale"
)
for prefix, weight, scale, input_scale in (
("w13", w13, w13_scale, w13_input_scale),
("w2", w2, w2_scale, w2_input_scale),
):
replace_parameter(layer, f"{prefix}_weight", weight)
replace_parameter(layer, f"{prefix}_{scale_name}", scale)
if input_scale is not None:
replace_parameter(layer, f"{prefix}_input_scale", input_scale)
convert_to_humming_moe_kernel_format(
layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale)
)
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import (
TrtLlmMxint4ExpertsMonolithic,
)
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_config,
marlin_act_int8_process_scales,
@@ -129,7 +130,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
if current_platform.is_xpu():
return [WNA16MoEBackend.XPU]
return [
backends = [
# Native HIP kernel, gated on gfx1100 by _supports_current_device().
WNA16MoEBackend.RDNA3,
WNA16MoEBackend.FLASHINFER_TRTLLM,
@@ -139,6 +140,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
WNA16MoEBackend.HUMMING,
WNA16MoEBackend.EMULATION,
]
return prioritize_humming(backends)
def _backend_incompatibility_reason(
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
swap_w13_to_w31,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
_swizzle_mxfp4,
)
@@ -382,6 +383,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
# TRITON_UNFUSED
Mxfp4MoeBackend.MARLIN,
Mxfp4MoeBackend.BATCHED_MARLIN,
Mxfp4MoeBackend.HUMMING,
Mxfp4MoeBackend.XPU,
Mxfp4MoeBackend.CPU,
Mxfp4MoeBackend.EMULATION,
@@ -395,12 +397,12 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
_AVAILABLE_BACKENDS.index(Mxfp4MoeBackend.BATCHED_MARLIN) + 1,
Mxfp4MoeBackend.TRITON,
)
return _AVAILABLE_BACKENDS
return prioritize_humming(_AVAILABLE_BACKENDS)
def _get_priority_backends() -> list[Mxfp4MoeBackend]:
"""Get available backends in priority order. SM100+ prefers DeepGEMM FP4 /
TRTLLM MXFP8; SM90 falls through to Triton_unfused or Marlin (the
TRTLLM MXFP8; SM90 falls through to Humming or Marlin (the
backend-level ``is_supported_config`` check filters by device capability).
"""
if current_platform.is_rocm():
@@ -420,8 +422,9 @@ def _get_priority_backends() -> list[Mxfp4MoeBackend]:
# TRITON_UNFUSED
Mxfp4MoeBackend.MARLIN,
Mxfp4MoeBackend.BATCHED_MARLIN,
Mxfp4MoeBackend.HUMMING,
]
return _AVAILABLE_BACKENDS
return prioritize_humming(_AVAILABLE_BACKENDS)
def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
backend_to_kernel_cls,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp8Dynamic,
kMxfp8Static,
@@ -129,7 +130,8 @@ def select_mxfp8_moe_backend(
return backend, _select_kernel_cls(backend, config)
# Auto-select: pick the first supported backend.
for backend in _SUPPORTED_BACKENDS:
backends = prioritize_humming(list(_SUPPORTED_BACKENDS))
for backend in backends:
try:
experts_cls = _select_kernel_cls(backend, config)
except ValueError:
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_nvfp4_moe_layer_for_marlin,
)
@@ -209,6 +210,7 @@ def select_nvfp4_moe_backend(
NvFp4MoeBackend.HUMMING,
NvFp4MoeBackend.EMULATION,
]
AVAILABLE_BACKENDS = prioritize_humming(AVAILABLE_BACKENDS)
NVFP4_BACKENDS_WITH_CLAMP = {
NvFp4MoeBackend.B12X,
@@ -19,6 +19,10 @@ from vllm.model_executor.layers.quantization.utils.humming.moe import (
make_humming_moe_quant_config,
select_humming_moe_experts,
)
from vllm.model_executor.layers.quantization.utils.humming.priority import (
prefers_humming,
prioritize_humming,
)
from vllm.model_executor.layers.quantization.utils.humming.schema import (
check_and_fallback_input_schema,
humming_is_layer_skipped,
@@ -41,6 +45,8 @@ __all__ = [
"convert_linear_layer_to_humming_standard",
"get_humming_linear_compute_config",
"prepare_humming_linear_layer_config",
"prefers_humming",
"prioritize_humming",
"make_humming_moe_kernel",
"select_humming_moe_experts",
"input_schema_to_quant_key",
@@ -87,7 +87,7 @@ def prepare_humming_linear_layer_config(
shape_n_stacks = layer.output_partition_sizes
# Step 1: convert weight and input schemas to humming standard format
source_tensors = dict(layer.named_parameters())
source_tensors = dict(layer.named_parameters(recurse=False))
weight_schema, tensors = weight_schema.convert_humming(
tensors=source_tensors,
shape_n_stacks=shape_n_stacks,
@@ -143,7 +143,7 @@ def prepare_humming_linear_layer_config(
if "bias" in tensors:
zero_bias = torch.zeros_like(tensors["bias"])
layer.register_buffer("zero_bias", zero_bias)
for name, _ in list(layer.named_parameters()):
for name, _ in list(layer.named_parameters(recurse=False)):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Humming kernel selection priority."""
from enum import Enum
from typing import TypeVar
_KernelT = TypeVar("_KernelT", bound=type | Enum)
# Compute capabilities where Humming outranks Marlin.
_HUMMING_PREFERRED_CAPABILITIES = frozenset({90})
def prefers_humming(compute_capability: int | None = None) -> bool:
"""Whether Humming outranks Marlin. A missing compute capability uses the
current device."""
if compute_capability is None:
from vllm.platforms import current_platform
if current_platform.is_cuda():
cc = current_platform.get_device_capability()
compute_capability = cc.to_int() if cc is not None else None
return compute_capability in _HUMMING_PREFERRED_CAPABILITIES
def prioritize_humming(
kernels: list[_KernelT],
compute_capability: int | None = None,
) -> list[_KernelT]:
"""Move Humming directly ahead of Marlin where Humming is preferred.
Every other entry keeps its relative order and the input list is not
modified. Match kernel class names or MoE backend enum names.
"""
if not prefers_humming(compute_capability):
return kernels
names = [
(kernel.name if isinstance(kernel, Enum) else kernel.__name__).lower()
for kernel in kernels
]
humming = next((i for i, name in enumerate(names) if "humming" in name), None)
marlin = next((i for i, name in enumerate(names) if "marlin" in name), None)
if humming is None or marlin is None or humming < marlin:
return kernels
kernels = kernels.copy()
kernels.insert(marlin, kernels.pop(humming))
return kernels