[Quantization] Prefer Humming before Marlin backends on SM90 (#56997)
Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
mgoin
Claude Opus 5.5
parent
bb87d227d4
commit
d1f3d8b870
@@ -19,6 +19,10 @@ from vllm.model_executor.layers.quantization.utils.humming.moe import (
|
||||
make_humming_moe_quant_config,
|
||||
select_humming_moe_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming.priority import (
|
||||
prefers_humming,
|
||||
prioritize_humming,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming.schema import (
|
||||
check_and_fallback_input_schema,
|
||||
humming_is_layer_skipped,
|
||||
@@ -41,6 +45,8 @@ __all__ = [
|
||||
"convert_linear_layer_to_humming_standard",
|
||||
"get_humming_linear_compute_config",
|
||||
"prepare_humming_linear_layer_config",
|
||||
"prefers_humming",
|
||||
"prioritize_humming",
|
||||
"make_humming_moe_kernel",
|
||||
"select_humming_moe_experts",
|
||||
"input_schema_to_quant_key",
|
||||
|
||||
@@ -87,7 +87,7 @@ def prepare_humming_linear_layer_config(
|
||||
shape_n_stacks = layer.output_partition_sizes
|
||||
|
||||
# Step 1: convert weight and input schemas to humming standard format
|
||||
source_tensors = dict(layer.named_parameters())
|
||||
source_tensors = dict(layer.named_parameters(recurse=False))
|
||||
weight_schema, tensors = weight_schema.convert_humming(
|
||||
tensors=source_tensors,
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
@@ -143,7 +143,7 @@ def prepare_humming_linear_layer_config(
|
||||
if "bias" in tensors:
|
||||
zero_bias = torch.zeros_like(tensors["bias"])
|
||||
layer.register_buffer("zero_bias", zero_bias)
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
for name, _ in list(layer.named_parameters(recurse=False)):
|
||||
delattr(layer, name)
|
||||
for name, tensor in tensors.items():
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Humming kernel selection priority."""
|
||||
|
||||
from enum import Enum
|
||||
from typing import TypeVar
|
||||
|
||||
_KernelT = TypeVar("_KernelT", bound=type | Enum)
|
||||
|
||||
# Compute capabilities where Humming outranks Marlin.
|
||||
_HUMMING_PREFERRED_CAPABILITIES = frozenset({90})
|
||||
|
||||
|
||||
def prefers_humming(compute_capability: int | None = None) -> bool:
|
||||
"""Whether Humming outranks Marlin. A missing compute capability uses the
|
||||
current device."""
|
||||
if compute_capability is None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_cuda():
|
||||
cc = current_platform.get_device_capability()
|
||||
compute_capability = cc.to_int() if cc is not None else None
|
||||
return compute_capability in _HUMMING_PREFERRED_CAPABILITIES
|
||||
|
||||
|
||||
def prioritize_humming(
|
||||
kernels: list[_KernelT],
|
||||
compute_capability: int | None = None,
|
||||
) -> list[_KernelT]:
|
||||
"""Move Humming directly ahead of Marlin where Humming is preferred.
|
||||
|
||||
Every other entry keeps its relative order and the input list is not
|
||||
modified. Match kernel class names or MoE backend enum names.
|
||||
"""
|
||||
if not prefers_humming(compute_capability):
|
||||
return kernels
|
||||
|
||||
names = [
|
||||
(kernel.name if isinstance(kernel, Enum) else kernel.__name__).lower()
|
||||
for kernel in kernels
|
||||
]
|
||||
humming = next((i for i, name in enumerate(names) if "humming" in name), None)
|
||||
marlin = next((i for i, name in enumerate(names) if "marlin" in name), None)
|
||||
if humming is None or marlin is None or humming < marlin:
|
||||
return kernels
|
||||
kernels = kernels.copy()
|
||||
kernels.insert(marlin, kernels.pop(humming))
|
||||
return kernels
|
||||
Reference in New Issue
Block a user