[Quantization] Prefer Humming before Marlin backends on SM90 (#56997)

Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jinzhen Lin
2026-10-06 12:31:35 -07:00
committed by GitHub
co-authored by Codex mgoin Claude Opus 5.5
parent bb87d227d4
commit d1f3d8b870
13 changed files with 159 additions and 15 deletions
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
prepare_fp8_moe_layer_for_deepgemm,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
prepare_fp8_moe_layer_for_marlin,
)
@@ -37,6 +38,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.utils.math_utils import round_up
@@ -98,6 +100,7 @@ def _get_priority_backends(
Fp8MoeBackend.CPU,
Fp8MoeBackend.HPC,
]
_AVAILABLE_BACKENDS = prioritize_humming(_AVAILABLE_BACKENDS)
def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None:
backends.insert(0, backends.pop(backends.index(backend)))
@@ -607,6 +610,21 @@ def convert_to_fp8_moe_kernel_format(
convert_to_humming_moe_kernel_format,
)
# Online quantization has not installed the quantized tensors yet.
scale_name = (
"weight_scale_inv"
if hasattr(layer, "w13_weight_scale_inv")
else "weight_scale"
)
for prefix, weight, scale, input_scale in (
("w13", w13, w13_scale, w13_input_scale),
("w2", w2, w2_scale, w2_input_scale),
):
replace_parameter(layer, f"{prefix}_weight", weight)
replace_parameter(layer, f"{prefix}_{scale_name}", scale)
if input_scale is not None:
replace_parameter(layer, f"{prefix}_input_scale", input_scale)
convert_to_humming_moe_kernel_format(
layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale)
)
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import (
TrtLlmMxint4ExpertsMonolithic,
)
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_config,
marlin_act_int8_process_scales,
@@ -129,7 +130,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
if current_platform.is_xpu():
return [WNA16MoEBackend.XPU]
return [
backends = [
# Native HIP kernel, gated on gfx1100 by _supports_current_device().
WNA16MoEBackend.RDNA3,
WNA16MoEBackend.FLASHINFER_TRTLLM,
@@ -139,6 +140,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
WNA16MoEBackend.HUMMING,
WNA16MoEBackend.EMULATION,
]
return prioritize_humming(backends)
def _backend_incompatibility_reason(
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
swap_w13_to_w31,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
_swizzle_mxfp4,
)
@@ -382,6 +383,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
# TRITON_UNFUSED
Mxfp4MoeBackend.MARLIN,
Mxfp4MoeBackend.BATCHED_MARLIN,
Mxfp4MoeBackend.HUMMING,
Mxfp4MoeBackend.XPU,
Mxfp4MoeBackend.CPU,
Mxfp4MoeBackend.EMULATION,
@@ -395,12 +397,12 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
_AVAILABLE_BACKENDS.index(Mxfp4MoeBackend.BATCHED_MARLIN) + 1,
Mxfp4MoeBackend.TRITON,
)
return _AVAILABLE_BACKENDS
return prioritize_humming(_AVAILABLE_BACKENDS)
def _get_priority_backends() -> list[Mxfp4MoeBackend]:
"""Get available backends in priority order. SM100+ prefers DeepGEMM FP4 /
TRTLLM MXFP8; SM90 falls through to Triton_unfused or Marlin (the
TRTLLM MXFP8; SM90 falls through to Humming or Marlin (the
backend-level ``is_supported_config`` check filters by device capability).
"""
if current_platform.is_rocm():
@@ -420,8 +422,9 @@ def _get_priority_backends() -> list[Mxfp4MoeBackend]:
# TRITON_UNFUSED
Mxfp4MoeBackend.MARLIN,
Mxfp4MoeBackend.BATCHED_MARLIN,
Mxfp4MoeBackend.HUMMING,
]
return _AVAILABLE_BACKENDS
return prioritize_humming(_AVAILABLE_BACKENDS)
def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
backend_to_kernel_cls,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp8Dynamic,
kMxfp8Static,
@@ -129,7 +130,8 @@ def select_mxfp8_moe_backend(
return backend, _select_kernel_cls(backend, config)
# Auto-select: pick the first supported backend.
for backend in _SUPPORTED_BACKENDS:
backends = prioritize_humming(list(_SUPPORTED_BACKENDS))
for backend in backends:
try:
experts_cls = _select_kernel_cls(backend, config)
except ValueError:
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_nvfp4_moe_layer_for_marlin,
)
@@ -209,6 +210,7 @@ def select_nvfp4_moe_backend(
NvFp4MoeBackend.HUMMING,
NvFp4MoeBackend.EMULATION,
]
AVAILABLE_BACKENDS = prioritize_humming(AVAILABLE_BACKENDS)
NVFP4_BACKENDS_WITH_CLAMP = {
NvFp4MoeBackend.B12X,
@@ -19,6 +19,10 @@ from vllm.model_executor.layers.quantization.utils.humming.moe import (
make_humming_moe_quant_config,
select_humming_moe_experts,
)
from vllm.model_executor.layers.quantization.utils.humming.priority import (
prefers_humming,
prioritize_humming,
)
from vllm.model_executor.layers.quantization.utils.humming.schema import (
check_and_fallback_input_schema,
humming_is_layer_skipped,
@@ -41,6 +45,8 @@ __all__ = [
"convert_linear_layer_to_humming_standard",
"get_humming_linear_compute_config",
"prepare_humming_linear_layer_config",
"prefers_humming",
"prioritize_humming",
"make_humming_moe_kernel",
"select_humming_moe_experts",
"input_schema_to_quant_key",
@@ -87,7 +87,7 @@ def prepare_humming_linear_layer_config(
shape_n_stacks = layer.output_partition_sizes
# Step 1: convert weight and input schemas to humming standard format
source_tensors = dict(layer.named_parameters())
source_tensors = dict(layer.named_parameters(recurse=False))
weight_schema, tensors = weight_schema.convert_humming(
tensors=source_tensors,
shape_n_stacks=shape_n_stacks,
@@ -143,7 +143,7 @@ def prepare_humming_linear_layer_config(
if "bias" in tensors:
zero_bias = torch.zeros_like(tensors["bias"])
layer.register_buffer("zero_bias", zero_bias)
for name, _ in list(layer.named_parameters()):
for name, _ in list(layer.named_parameters(recurse=False)):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Humming kernel selection priority."""
from enum import Enum
from typing import TypeVar
_KernelT = TypeVar("_KernelT", bound=type | Enum)
# Compute capabilities where Humming outranks Marlin.
_HUMMING_PREFERRED_CAPABILITIES = frozenset({90})
def prefers_humming(compute_capability: int | None = None) -> bool:
"""Whether Humming outranks Marlin. A missing compute capability uses the
current device."""
if compute_capability is None:
from vllm.platforms import current_platform
if current_platform.is_cuda():
cc = current_platform.get_device_capability()
compute_capability = cc.to_int() if cc is not None else None
return compute_capability in _HUMMING_PREFERRED_CAPABILITIES
def prioritize_humming(
kernels: list[_KernelT],
compute_capability: int | None = None,
) -> list[_KernelT]:
"""Move Humming directly ahead of Marlin where Humming is preferred.
Every other entry keeps its relative order and the input list is not
modified. Match kernel class names or MoE backend enum names.
"""
if not prefers_humming(compute_capability):
return kernels
names = [
(kernel.name if isinstance(kernel, Enum) else kernel.__name__).lower()
for kernel in kernels
]
humming = next((i for i, name in enumerate(names) if "humming" in name), None)
marlin = next((i for i, name in enumerate(names) if "marlin" in name), None)
if humming is None or marlin is None or humming < marlin:
return kernels
kernels = kernels.copy()
kernels.insert(marlin, kernels.pop(humming))
return kernels