[Quantization] Prefer Humming before Marlin backends on SM90 (#56997)

Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jinzhen Lin
2026-10-06 12:31:35 -07:00
committed by GitHub
co-authored by Codex mgoin Claude Opus 5.5
parent bb87d227d4
commit d1f3d8b870
13 changed files with 159 additions and 15 deletions
@@ -19,6 +19,10 @@ from vllm.model_executor.layers.quantization.utils.humming.moe import (
make_humming_moe_quant_config,
select_humming_moe_experts,
)
from vllm.model_executor.layers.quantization.utils.humming.priority import (
prefers_humming,
prioritize_humming,
)
from vllm.model_executor.layers.quantization.utils.humming.schema import (
check_and_fallback_input_schema,
humming_is_layer_skipped,
@@ -41,6 +45,8 @@ __all__ = [
"convert_linear_layer_to_humming_standard",
"get_humming_linear_compute_config",
"prepare_humming_linear_layer_config",
"prefers_humming",
"prioritize_humming",
"make_humming_moe_kernel",
"select_humming_moe_experts",
"input_schema_to_quant_key",
@@ -87,7 +87,7 @@ def prepare_humming_linear_layer_config(
shape_n_stacks = layer.output_partition_sizes
# Step 1: convert weight and input schemas to humming standard format
source_tensors = dict(layer.named_parameters())
source_tensors = dict(layer.named_parameters(recurse=False))
weight_schema, tensors = weight_schema.convert_humming(
tensors=source_tensors,
shape_n_stacks=shape_n_stacks,
@@ -143,7 +143,7 @@ def prepare_humming_linear_layer_config(
if "bias" in tensors:
zero_bias = torch.zeros_like(tensors["bias"])
layer.register_buffer("zero_bias", zero_bias)
for name, _ in list(layer.named_parameters()):
for name, _ in list(layer.named_parameters(recurse=False)):
delattr(layer, name)
for name, tensor in tensors.items():
param = torch.nn.Parameter(tensor, requires_grad=False)
@@ -0,0 +1,48 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Humming kernel selection priority."""
from enum import Enum
from typing import TypeVar
_KernelT = TypeVar("_KernelT", bound=type | Enum)
# Compute capabilities where Humming outranks Marlin.
_HUMMING_PREFERRED_CAPABILITIES = frozenset({90})
def prefers_humming(compute_capability: int | None = None) -> bool:
"""Whether Humming outranks Marlin. A missing compute capability uses the
current device."""
if compute_capability is None:
from vllm.platforms import current_platform
if current_platform.is_cuda():
cc = current_platform.get_device_capability()
compute_capability = cc.to_int() if cc is not None else None
return compute_capability in _HUMMING_PREFERRED_CAPABILITIES
def prioritize_humming(
kernels: list[_KernelT],
compute_capability: int | None = None,
) -> list[_KernelT]:
"""Move Humming directly ahead of Marlin where Humming is preferred.
Every other entry keeps its relative order and the input list is not
modified. Match kernel class names or MoE backend enum names.
"""
if not prefers_humming(compute_capability):
return kernels
names = [
(kernel.name if isinstance(kernel, Enum) else kernel.__name__).lower()
for kernel in kernels
]
humming = next((i for i, name in enumerate(names) if "humming" in name), None)
marlin = next((i for i, name in enumerate(names) if "marlin" in name), None)
if humming is None or marlin is None or humming < marlin:
return kernels
kernels = kernels.copy()
kernels.insert(marlin, kernels.pop(humming))
return kernels