From d1f3d8b87083a95f914c674c477796950e2ba5fe Mon Sep 17 00:00:00 2001 From: Jinzhen Lin Date: Wed, 7 Oct 2026 03:31:35 +0800 Subject: [PATCH] [Quantization] Prefer Humming before Marlin backends on SM90 (#56997) Signed-off-by: jinzhen.ljz Signed-off-by: mgoin Co-authored-by: Codex Co-authored-by: mgoin Co-authored-by: Claude Opus 5.5 --- .../test_scaled_mm_kernel_selection.py | 21 ++++++++ tests/quantization/test_modelopt.py | 20 ++++++-- tests/quantization/test_online.py | 3 +- .../model_executor/kernels/linear/__init__.py | 14 ++++-- .../kernels/linear/mixed_precision/humming.py | 21 +++++++- .../layers/fused_moe/oracle/fp8.py | 18 +++++++ .../layers/fused_moe/oracle/int_wna16.py | 4 +- .../layers/fused_moe/oracle/mxfp4.py | 9 ++-- .../layers/fused_moe/oracle/mxfp8.py | 4 +- .../layers/fused_moe/oracle/nvfp4.py | 2 + .../quantization/utils/humming/__init__.py | 6 +++ .../quantization/utils/humming/linear.py | 4 +- .../quantization/utils/humming/priority.py | 48 +++++++++++++++++++ 13 files changed, 159 insertions(+), 15 deletions(-) create mode 100644 vllm/model_executor/layers/quantization/utils/humming/priority.py diff --git a/tests/kernels/quantization/test_scaled_mm_kernel_selection.py b/tests/kernels/quantization/test_scaled_mm_kernel_selection.py index f8380dcecf..641c458608 100644 --- a/tests/kernels/quantization/test_scaled_mm_kernel_selection.py +++ b/tests/kernels/quantization/test_scaled_mm_kernel_selection.py @@ -14,6 +14,8 @@ import torch from vllm.config import KernelConfig, VllmConfig, set_current_vllm_config from vllm.model_executor.kernels.linear import ( + _POSSIBLE_KERNELS, + _POSSIBLE_NVFP4_KERNELS, AiterInt8ScaledMMLinearKernel, CPUInt8ScaledMMLinearKernel, HummingFP8ScaledMMLinearKernel, @@ -26,6 +28,7 @@ from vllm.model_executor.kernels.linear import ( init_int8_linear_kernel, register_linear_kernel, ) +from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8StaticTensorSym, ) @@ -197,3 +200,21 @@ def test_register_oot_linear_kernel(platform_mock): assert isinstance(kernel, OOTInt8ScaledMMLinearKernel), ( "init_int8_linear_kernel should return an instance of the registered kernel" ) + + +@pytest.mark.parametrize( + "possible_kernels", [_POSSIBLE_KERNELS, _POSSIBLE_NVFP4_KERNELS] +) +def test_humming_priority_only_reorders_humming_and_marlin(possible_kernels): + """Preferring Humming must not demote Marlin below unrelated kernels.""" + kernels = possible_kernels[PlatformEnum.CUDA] + original = list(kernels) + + assert prioritize_humming(kernels, compute_capability=80) == original + + names = [k.__name__ for k in prioritize_humming(kernels, compute_capability=90)] + humming = next(i for i, name in enumerate(names) if "Humming" in name) + assert "Marlin" in names[humming + 1] + del names[humming] + assert names == [k.__name__ for k in original if "Humming" not in k.__name__] + assert kernels == original diff --git a/tests/quantization/test_modelopt.py b/tests/quantization/test_modelopt.py index eea4b6218d..7b1131a00e 100644 --- a/tests/quantization/test_modelopt.py +++ b/tests/quantization/test_modelopt.py @@ -688,7 +688,8 @@ def test_modelopt_linear_method_builder_registry_override(monkeypatch): @pytest.mark.parametrize( ("linear_backend", "kernel_cls"), [ - ("auto", MarlinNvFp4LinearKernel), + ("auto", None), + ("marlin", MarlinNvFp4LinearKernel), ("humming", HummingNvFp4LinearKernel), ("flashinfer_cutedsl", FlashInferCuteDslNvFp4W4A16LinearKernel), ], @@ -696,7 +697,7 @@ def test_modelopt_linear_method_builder_registry_override(monkeypatch): @pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only") def test_modelopt_w4a16_respects_linear_backend(linear_backend, kernel_cls): """W4A16 (`activation=None`) kernel selection honors ``--linear-backend``: - ``use_a16=True`` defaults to Marlin, but an explicit backend wins. The + ``use_a16=True`` follows platform priorities, but an explicit backend wins. The generic method routes this through ``select_linear_kernel``.""" from vllm.config.quantization import QuantSpec from vllm.model_executor.layers.quantization.modelopt import ( @@ -704,7 +705,20 @@ def test_modelopt_w4a16_respects_linear_backend(linear_backend, kernel_cls): select_linear_kernel, ) - if linear_backend != "auto": + if linear_backend == "auto": + capability = current_platform.get_device_capability() + assert capability is not None + cc = capability.to_int() + if ( + cc in (100, 103) + and FlashInferCuteDslNvFp4W4A16LinearKernel.is_supported()[0] + ): + kernel_cls = FlashInferCuteDslNvFp4W4A16LinearKernel + elif cc == 90 and HummingNvFp4LinearKernel.is_supported()[0]: + kernel_cls = HummingNvFp4LinearKernel + else: + kernel_cls = MarlinNvFp4LinearKernel + else: is_supported, reason = kernel_cls.is_supported() if not is_supported: pytest.skip(reason) diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index f3b72707f3..3f27757b90 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -34,6 +34,7 @@ from vllm.forward_context import set_forward_context from vllm.model_executor.kernels.linear.mxfp8.emulation import ( EmulationMxfp8LinearKernel, ) +from vllm.model_executor.kernels.linear.mxfp8.humming import HummingMxfp8LinearKernel from vllm.model_executor.kernels.linear.mxfp8.marlin import ( MarlinMxfp8LinearKernel, ) @@ -1060,7 +1061,7 @@ def test_online_quantization( assert isinstance(moe._quant_method, expected_moe_cls) if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance( - o_proj.quant_method.kernel, MarlinMxfp8LinearKernel + o_proj.quant_method.kernel, (MarlinMxfp8LinearKernel, HummingMxfp8LinearKernel) ): assert o_proj.weight.dtype == torch.int32 elif model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance( diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index e5fed78c40..484143ffc1 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -228,6 +228,10 @@ from vllm.model_executor.kernels.linear.scaled_mm.xpu import ( from vllm.model_executor.kernels.linear.scaled_mm.zentorch import ( ZentorchInt8ScaledMMLinearKernel, ) +from vllm.model_executor.layers.quantization.utils.humming import ( + prefers_humming, + prioritize_humming, +) from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey from vllm.platforms import PlatformEnum, current_platform @@ -376,8 +380,9 @@ def _resolve_backend_kernels( layer_desc: str, *, quantization: str, + compute_capability: int | None = None, ) -> list[type]: - """Apply --linear-backend filtering to one layer type's kernel list. + """Apply device priorities and --linear-backend filtering to a kernel list. When the requested backend has no kernel for this layer type, fall back to the unfiltered list (with a WARNING log) instead of failing engine @@ -386,6 +391,7 @@ def _resolve_backend_kernels( layer types (e.g. NVFP4 MoE projections next to FP8 attention projections). """ + kernels = prioritize_humming(kernels, compute_capability) linear_backend = _get_linear_backend(quantization=quantization) if linear_backend == "auto": return kernels @@ -682,6 +688,7 @@ def choose_scaled_mm_linear_kernel( platform_kernels, "scaled-mm", quantization=quantization, + compute_capability=compute_capability, ) for kernel in platform_kernels: @@ -845,6 +852,7 @@ def choose_mp_linear_kernel( platform_kernels, "mixed-precision", quantization="mixed_precision", + compute_capability=compute_capability, ) failure_reasons = [] @@ -1121,13 +1129,13 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel: _cc = current_platform.get_device_capability() compute_capability = _cc.to_int() if _cc is not None else None # Weight-only: prefer FlashInfer CuTe-DSL W4A16 on SM100/103, - # otherwise Marlin. + # Humming then Marlin where Humming is preferred, and Marlin elsewhere. cutedsl_ok, _ = FlashInferCuteDslNvFp4W4A16LinearKernel.is_supported( compute_capability ) if compute_capability in (100, 103) and cutedsl_ok: force_kernel = FlashInferCuteDslNvFp4W4A16LinearKernel - else: + elif not prefers_humming(compute_capability): force_kernel = MarlinNvFp4LinearKernel if force_kernel is not None: diff --git a/vllm/model_executor/kernels/linear/mixed_precision/humming.py b/vllm/model_executor/kernels/linear/mixed_precision/humming.py index 4655181f78..a2d88705d2 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/humming.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/humming.py @@ -4,11 +4,21 @@ import torch +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kFp8DynamicTokenSym, + kInt8DynamicTokenSym, +) from vllm.platforms import current_platform from vllm.utils.import_utils import has_humming from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig +# Per-token activation quantization requested through `act_type`. +_ACT_QUANT_KEYS = { + torch.int8: kInt8DynamicTokenSym, + torch.float8_e4m3fn: kFp8DynamicTokenSym, +} + class HummingLinearKernel(MPLinearKernel): @classmethod @@ -21,6 +31,8 @@ class HummingLinearKernel(MPLinearKernel): return False, "Humming is only supported on CUDA" if not has_humming(): return False, "Humming is not installed" + if c.act_type.itemsize == 1 and c.act_type not in _ACT_QUANT_KEYS: + return False, f"Activation type ({c.act_type}) not supported by Humming" return True, None def process_weights_after_loading(self, layer: torch.nn.Module) -> None: @@ -28,6 +40,7 @@ class HummingLinearKernel(MPLinearKernel): convert_linear_layer_to_humming_standard, get_humming_linear_compute_config, prepare_humming_linear_layer_config, + quant_key_to_input_schema, ) name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name} @@ -48,8 +61,14 @@ class HummingLinearKernel(MPLinearKernel): convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) input_quant_config = getattr(layer, "_humming_input_quant_config", None) + input_schema = None + if (act_key := _ACT_QUANT_KEYS.get(self.config.act_type)) is not None: + input_schema = quant_key_to_input_schema(act_key) self.layer_config = prepare_humming_linear_layer_config( - layer, quant_config, input_quant_config=input_quant_config + layer, + quant_config, + input_quant_config=input_quant_config, + input_schema=input_schema, ) self.compute_config = get_humming_linear_compute_config() self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 5219fff617..446825c590 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( from vllm.model_executor.layers.quantization.utils.fp8_utils import ( prepare_fp8_moe_layer_for_deepgemm, ) +from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import ( prepare_fp8_moe_layer_for_marlin, ) @@ -37,6 +38,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Dynamic128Sym, kFp8Static128BlockSym, ) +from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform from vllm.utils.math_utils import round_up @@ -98,6 +100,7 @@ def _get_priority_backends( Fp8MoeBackend.CPU, Fp8MoeBackend.HPC, ] + _AVAILABLE_BACKENDS = prioritize_humming(_AVAILABLE_BACKENDS) def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None: backends.insert(0, backends.pop(backends.index(backend))) @@ -607,6 +610,21 @@ def convert_to_fp8_moe_kernel_format( convert_to_humming_moe_kernel_format, ) + # Online quantization has not installed the quantized tensors yet. + scale_name = ( + "weight_scale_inv" + if hasattr(layer, "w13_weight_scale_inv") + else "weight_scale" + ) + for prefix, weight, scale, input_scale in ( + ("w13", w13, w13_scale, w13_input_scale), + ("w2", w2, w2_scale, w2_input_scale), + ): + replace_parameter(layer, f"{prefix}_weight", weight) + replace_parameter(layer, f"{prefix}_{scale_name}", scale) + if input_scale is not None: + replace_parameter(layer, f"{prefix}_input_scale", input_scale) + convert_to_humming_moe_kernel_format( layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale) ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py index 239fe1d55a..3e91fd9e04 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py @@ -32,6 +32,7 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import ( TrtLlmMxint4ExpertsMonolithic, ) from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming from vllm.model_executor.layers.quantization.utils.marlin_utils import ( check_moe_marlin_supports_config, marlin_act_int8_process_scales, @@ -129,7 +130,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]: if current_platform.is_xpu(): return [WNA16MoEBackend.XPU] - return [ + backends = [ # Native HIP kernel, gated on gfx1100 by _supports_current_device(). WNA16MoEBackend.RDNA3, WNA16MoEBackend.FLASHINFER_TRTLLM, @@ -139,6 +140,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]: WNA16MoEBackend.HUMMING, WNA16MoEBackend.EMULATION, ] + return prioritize_humming(backends) def _backend_incompatibility_reason( diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 691d849228..5463853cab 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -35,6 +35,7 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( swap_w13_to_w31, ) +from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( _swizzle_mxfp4, ) @@ -382,6 +383,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]: # TRITON_UNFUSED Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN, + Mxfp4MoeBackend.HUMMING, Mxfp4MoeBackend.XPU, Mxfp4MoeBackend.CPU, Mxfp4MoeBackend.EMULATION, @@ -395,12 +397,12 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]: _AVAILABLE_BACKENDS.index(Mxfp4MoeBackend.BATCHED_MARLIN) + 1, Mxfp4MoeBackend.TRITON, ) - return _AVAILABLE_BACKENDS + return prioritize_humming(_AVAILABLE_BACKENDS) def _get_priority_backends() -> list[Mxfp4MoeBackend]: """Get available backends in priority order. SM100+ prefers DeepGEMM FP4 / - TRTLLM MXFP8; SM90 falls through to Triton_unfused or Marlin (the + TRTLLM MXFP8; SM90 falls through to Humming or Marlin (the backend-level ``is_supported_config`` check filters by device capability). """ if current_platform.is_rocm(): @@ -420,8 +422,9 @@ def _get_priority_backends() -> list[Mxfp4MoeBackend]: # TRITON_UNFUSED Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN, + Mxfp4MoeBackend.HUMMING, ] - return _AVAILABLE_BACKENDS + return prioritize_humming(_AVAILABLE_BACKENDS) def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None: diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py index 49f61764f0..c4417cdd76 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp8.py @@ -8,6 +8,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( Fp8MoeBackend, backend_to_kernel_cls, ) +from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming from vllm.model_executor.layers.quantization.utils.quant_utils import ( kMxfp8Dynamic, kMxfp8Static, @@ -129,7 +130,8 @@ def select_mxfp8_moe_backend( return backend, _select_kernel_cls(backend, config) # Auto-select: pick the first supported backend. - for backend in _SUPPORTED_BACKENDS: + backends = prioritize_humming(list(_SUPPORTED_BACKENDS)) + for backend in backends: try: experts_cls = _select_kernel_cls(backend, config) except ValueError: diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index 68ba62782c..330e7b6aa1 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import ( prepare_nvfp4_moe_layer_for_fi_or_cutlass, prepare_nvfp4_moe_layer_for_flashinfer_cutedsl, ) +from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import ( prepare_nvfp4_moe_layer_for_marlin, ) @@ -209,6 +210,7 @@ def select_nvfp4_moe_backend( NvFp4MoeBackend.HUMMING, NvFp4MoeBackend.EMULATION, ] + AVAILABLE_BACKENDS = prioritize_humming(AVAILABLE_BACKENDS) NVFP4_BACKENDS_WITH_CLAMP = { NvFp4MoeBackend.B12X, diff --git a/vllm/model_executor/layers/quantization/utils/humming/__init__.py b/vllm/model_executor/layers/quantization/utils/humming/__init__.py index 4303f74221..91a5713bbf 100644 --- a/vllm/model_executor/layers/quantization/utils/humming/__init__.py +++ b/vllm/model_executor/layers/quantization/utils/humming/__init__.py @@ -19,6 +19,10 @@ from vllm.model_executor.layers.quantization.utils.humming.moe import ( make_humming_moe_quant_config, select_humming_moe_experts, ) +from vllm.model_executor.layers.quantization.utils.humming.priority import ( + prefers_humming, + prioritize_humming, +) from vllm.model_executor.layers.quantization.utils.humming.schema import ( check_and_fallback_input_schema, humming_is_layer_skipped, @@ -41,6 +45,8 @@ __all__ = [ "convert_linear_layer_to_humming_standard", "get_humming_linear_compute_config", "prepare_humming_linear_layer_config", + "prefers_humming", + "prioritize_humming", "make_humming_moe_kernel", "select_humming_moe_experts", "input_schema_to_quant_key", diff --git a/vllm/model_executor/layers/quantization/utils/humming/linear.py b/vllm/model_executor/layers/quantization/utils/humming/linear.py index 0c64645c43..fcce3dabfe 100644 --- a/vllm/model_executor/layers/quantization/utils/humming/linear.py +++ b/vllm/model_executor/layers/quantization/utils/humming/linear.py @@ -87,7 +87,7 @@ def prepare_humming_linear_layer_config( shape_n_stacks = layer.output_partition_sizes # Step 1: convert weight and input schemas to humming standard format - source_tensors = dict(layer.named_parameters()) + source_tensors = dict(layer.named_parameters(recurse=False)) weight_schema, tensors = weight_schema.convert_humming( tensors=source_tensors, shape_n_stacks=shape_n_stacks, @@ -143,7 +143,7 @@ def prepare_humming_linear_layer_config( if "bias" in tensors: zero_bias = torch.zeros_like(tensors["bias"]) layer.register_buffer("zero_bias", zero_bias) - for name, _ in list(layer.named_parameters()): + for name, _ in list(layer.named_parameters(recurse=False)): delattr(layer, name) for name, tensor in tensors.items(): param = torch.nn.Parameter(tensor, requires_grad=False) diff --git a/vllm/model_executor/layers/quantization/utils/humming/priority.py b/vllm/model_executor/layers/quantization/utils/humming/priority.py new file mode 100644 index 0000000000..4a1452c2a9 --- /dev/null +++ b/vllm/model_executor/layers/quantization/utils/humming/priority.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Humming kernel selection priority.""" + +from enum import Enum +from typing import TypeVar + +_KernelT = TypeVar("_KernelT", bound=type | Enum) + +# Compute capabilities where Humming outranks Marlin. +_HUMMING_PREFERRED_CAPABILITIES = frozenset({90}) + + +def prefers_humming(compute_capability: int | None = None) -> bool: + """Whether Humming outranks Marlin. A missing compute capability uses the + current device.""" + if compute_capability is None: + from vllm.platforms import current_platform + + if current_platform.is_cuda(): + cc = current_platform.get_device_capability() + compute_capability = cc.to_int() if cc is not None else None + return compute_capability in _HUMMING_PREFERRED_CAPABILITIES + + +def prioritize_humming( + kernels: list[_KernelT], + compute_capability: int | None = None, +) -> list[_KernelT]: + """Move Humming directly ahead of Marlin where Humming is preferred. + + Every other entry keeps its relative order and the input list is not + modified. Match kernel class names or MoE backend enum names. + """ + if not prefers_humming(compute_capability): + return kernels + + names = [ + (kernel.name if isinstance(kernel, Enum) else kernel.__name__).lower() + for kernel in kernels + ] + humming = next((i for i, name in enumerate(names) if "humming" in name), None) + marlin = next((i for i, name in enumerate(names) if "marlin" in name), None) + if humming is None or marlin is None or humming < marlin: + return kernels + kernels = kernels.copy() + kernels.insert(marlin, kernels.pop(humming)) + return kernels