[Quantization] Prefer Humming before Marlin backends on SM90 (#56997)
Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
mgoin
Claude Opus 5.5
parent
bb87d227d4
commit
d1f3d8b870
@@ -228,6 +228,10 @@ from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
|
||||
ZentorchInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
prefers_humming,
|
||||
prioritize_humming,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.platforms import PlatformEnum, current_platform
|
||||
|
||||
@@ -376,8 +380,9 @@ def _resolve_backend_kernels(
|
||||
layer_desc: str,
|
||||
*,
|
||||
quantization: str,
|
||||
compute_capability: int | None = None,
|
||||
) -> list[type]:
|
||||
"""Apply --linear-backend filtering to one layer type's kernel list.
|
||||
"""Apply device priorities and --linear-backend filtering to a kernel list.
|
||||
|
||||
When the requested backend has no kernel for this layer type, fall back
|
||||
to the unfiltered list (with a WARNING log) instead of failing engine
|
||||
@@ -386,6 +391,7 @@ def _resolve_backend_kernels(
|
||||
layer types (e.g. NVFP4 MoE projections next to FP8 attention
|
||||
projections).
|
||||
"""
|
||||
kernels = prioritize_humming(kernels, compute_capability)
|
||||
linear_backend = _get_linear_backend(quantization=quantization)
|
||||
if linear_backend == "auto":
|
||||
return kernels
|
||||
@@ -682,6 +688,7 @@ def choose_scaled_mm_linear_kernel(
|
||||
platform_kernels,
|
||||
"scaled-mm",
|
||||
quantization=quantization,
|
||||
compute_capability=compute_capability,
|
||||
)
|
||||
|
||||
for kernel in platform_kernels:
|
||||
@@ -845,6 +852,7 @@ def choose_mp_linear_kernel(
|
||||
platform_kernels,
|
||||
"mixed-precision",
|
||||
quantization="mixed_precision",
|
||||
compute_capability=compute_capability,
|
||||
)
|
||||
|
||||
failure_reasons = []
|
||||
@@ -1121,13 +1129,13 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
||||
_cc = current_platform.get_device_capability()
|
||||
compute_capability = _cc.to_int() if _cc is not None else None
|
||||
# Weight-only: prefer FlashInfer CuTe-DSL W4A16 on SM100/103,
|
||||
# otherwise Marlin.
|
||||
# Humming then Marlin where Humming is preferred, and Marlin elsewhere.
|
||||
cutedsl_ok, _ = FlashInferCuteDslNvFp4W4A16LinearKernel.is_supported(
|
||||
compute_capability
|
||||
)
|
||||
if compute_capability in (100, 103) and cutedsl_ok:
|
||||
force_kernel = FlashInferCuteDslNvFp4W4A16LinearKernel
|
||||
else:
|
||||
elif not prefers_humming(compute_capability):
|
||||
force_kernel = MarlinNvFp4LinearKernel
|
||||
|
||||
if force_kernel is not None:
|
||||
|
||||
@@ -4,11 +4,21 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTokenSym,
|
||||
kInt8DynamicTokenSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
|
||||
|
||||
# Per-token activation quantization requested through `act_type`.
|
||||
_ACT_QUANT_KEYS = {
|
||||
torch.int8: kInt8DynamicTokenSym,
|
||||
torch.float8_e4m3fn: kFp8DynamicTokenSym,
|
||||
}
|
||||
|
||||
|
||||
class HummingLinearKernel(MPLinearKernel):
|
||||
@classmethod
|
||||
@@ -21,6 +31,8 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
return False, "Humming is only supported on CUDA"
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
if c.act_type.itemsize == 1 and c.act_type not in _ACT_QUANT_KEYS:
|
||||
return False, f"Activation type ({c.act_type}) not supported by Humming"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
@@ -28,6 +40,7 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
convert_linear_layer_to_humming_standard,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
quant_key_to_input_schema,
|
||||
)
|
||||
|
||||
name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}
|
||||
@@ -48,8 +61,14 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
input_quant_config = getattr(layer, "_humming_input_quant_config", None)
|
||||
input_schema = None
|
||||
if (act_key := _ACT_QUANT_KEYS.get(self.config.act_type)) is not None:
|
||||
input_schema = quant_key_to_input_schema(act_key)
|
||||
self.layer_config = prepare_humming_linear_layer_config(
|
||||
layer, quant_config, input_quant_config=input_quant_config
|
||||
layer,
|
||||
quant_config,
|
||||
input_quant_config=input_quant_config,
|
||||
input_schema=input_schema,
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
prepare_fp8_moe_layer_for_deepgemm,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
prepare_fp8_moe_layer_for_marlin,
|
||||
)
|
||||
@@ -37,6 +38,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8Static128BlockSym,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
@@ -98,6 +100,7 @@ def _get_priority_backends(
|
||||
Fp8MoeBackend.CPU,
|
||||
Fp8MoeBackend.HPC,
|
||||
]
|
||||
_AVAILABLE_BACKENDS = prioritize_humming(_AVAILABLE_BACKENDS)
|
||||
|
||||
def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None:
|
||||
backends.insert(0, backends.pop(backends.index(backend)))
|
||||
@@ -607,6 +610,21 @@ def convert_to_fp8_moe_kernel_format(
|
||||
convert_to_humming_moe_kernel_format,
|
||||
)
|
||||
|
||||
# Online quantization has not installed the quantized tensors yet.
|
||||
scale_name = (
|
||||
"weight_scale_inv"
|
||||
if hasattr(layer, "w13_weight_scale_inv")
|
||||
else "weight_scale"
|
||||
)
|
||||
for prefix, weight, scale, input_scale in (
|
||||
("w13", w13, w13_scale, w13_input_scale),
|
||||
("w2", w2, w2_scale, w2_input_scale),
|
||||
):
|
||||
replace_parameter(layer, f"{prefix}_weight", weight)
|
||||
replace_parameter(layer, f"{prefix}_{scale_name}", scale)
|
||||
if input_scale is not None:
|
||||
replace_parameter(layer, f"{prefix}_input_scale", input_scale)
|
||||
|
||||
convert_to_humming_moe_kernel_format(
|
||||
layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale)
|
||||
)
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_mxint4_moe import (
|
||||
TrtLlmMxint4ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_moe_marlin_supports_config,
|
||||
marlin_act_int8_process_scales,
|
||||
@@ -129,7 +130,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
|
||||
if current_platform.is_xpu():
|
||||
return [WNA16MoEBackend.XPU]
|
||||
|
||||
return [
|
||||
backends = [
|
||||
# Native HIP kernel, gated on gfx1100 by _supports_current_device().
|
||||
WNA16MoEBackend.RDNA3,
|
||||
WNA16MoEBackend.FLASHINFER_TRTLLM,
|
||||
@@ -139,6 +140,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
|
||||
WNA16MoEBackend.HUMMING,
|
||||
WNA16MoEBackend.EMULATION,
|
||||
]
|
||||
return prioritize_humming(backends)
|
||||
|
||||
|
||||
def _backend_incompatibility_reason(
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
swap_w13_to_w31,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
_swizzle_mxfp4,
|
||||
)
|
||||
@@ -382,6 +383,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
|
||||
# TRITON_UNFUSED
|
||||
Mxfp4MoeBackend.MARLIN,
|
||||
Mxfp4MoeBackend.BATCHED_MARLIN,
|
||||
Mxfp4MoeBackend.HUMMING,
|
||||
Mxfp4MoeBackend.XPU,
|
||||
Mxfp4MoeBackend.CPU,
|
||||
Mxfp4MoeBackend.EMULATION,
|
||||
@@ -395,12 +397,12 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
|
||||
_AVAILABLE_BACKENDS.index(Mxfp4MoeBackend.BATCHED_MARLIN) + 1,
|
||||
Mxfp4MoeBackend.TRITON,
|
||||
)
|
||||
return _AVAILABLE_BACKENDS
|
||||
return prioritize_humming(_AVAILABLE_BACKENDS)
|
||||
|
||||
|
||||
def _get_priority_backends() -> list[Mxfp4MoeBackend]:
|
||||
"""Get available backends in priority order. SM100+ prefers DeepGEMM FP4 /
|
||||
TRTLLM MXFP8; SM90 falls through to Triton_unfused or Marlin (the
|
||||
TRTLLM MXFP8; SM90 falls through to Humming or Marlin (the
|
||||
backend-level ``is_supported_config`` check filters by device capability).
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
@@ -420,8 +422,9 @@ def _get_priority_backends() -> list[Mxfp4MoeBackend]:
|
||||
# TRITON_UNFUSED
|
||||
Mxfp4MoeBackend.MARLIN,
|
||||
Mxfp4MoeBackend.BATCHED_MARLIN,
|
||||
Mxfp4MoeBackend.HUMMING,
|
||||
]
|
||||
return _AVAILABLE_BACKENDS
|
||||
return prioritize_humming(_AVAILABLE_BACKENDS)
|
||||
|
||||
|
||||
def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
|
||||
|
||||
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
backend_to_kernel_cls,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
@@ -129,7 +130,8 @@ def select_mxfp8_moe_backend(
|
||||
return backend, _select_kernel_cls(backend, config)
|
||||
|
||||
# Auto-select: pick the first supported backend.
|
||||
for backend in _SUPPORTED_BACKENDS:
|
||||
backends = prioritize_humming(list(_SUPPORTED_BACKENDS))
|
||||
for backend in backends:
|
||||
try:
|
||||
experts_cls = _select_kernel_cls(backend, config)
|
||||
except ValueError:
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_nvfp4_moe_layer_for_fi_or_cutlass,
|
||||
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
prepare_nvfp4_moe_layer_for_marlin,
|
||||
)
|
||||
@@ -209,6 +210,7 @@ def select_nvfp4_moe_backend(
|
||||
NvFp4MoeBackend.HUMMING,
|
||||
NvFp4MoeBackend.EMULATION,
|
||||
]
|
||||
AVAILABLE_BACKENDS = prioritize_humming(AVAILABLE_BACKENDS)
|
||||
|
||||
NVFP4_BACKENDS_WITH_CLAMP = {
|
||||
NvFp4MoeBackend.B12X,
|
||||
|
||||
@@ -19,6 +19,10 @@ from vllm.model_executor.layers.quantization.utils.humming.moe import (
|
||||
make_humming_moe_quant_config,
|
||||
select_humming_moe_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming.priority import (
|
||||
prefers_humming,
|
||||
prioritize_humming,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming.schema import (
|
||||
check_and_fallback_input_schema,
|
||||
humming_is_layer_skipped,
|
||||
@@ -41,6 +45,8 @@ __all__ = [
|
||||
"convert_linear_layer_to_humming_standard",
|
||||
"get_humming_linear_compute_config",
|
||||
"prepare_humming_linear_layer_config",
|
||||
"prefers_humming",
|
||||
"prioritize_humming",
|
||||
"make_humming_moe_kernel",
|
||||
"select_humming_moe_experts",
|
||||
"input_schema_to_quant_key",
|
||||
|
||||
@@ -87,7 +87,7 @@ def prepare_humming_linear_layer_config(
|
||||
shape_n_stacks = layer.output_partition_sizes
|
||||
|
||||
# Step 1: convert weight and input schemas to humming standard format
|
||||
source_tensors = dict(layer.named_parameters())
|
||||
source_tensors = dict(layer.named_parameters(recurse=False))
|
||||
weight_schema, tensors = weight_schema.convert_humming(
|
||||
tensors=source_tensors,
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
@@ -143,7 +143,7 @@ def prepare_humming_linear_layer_config(
|
||||
if "bias" in tensors:
|
||||
zero_bias = torch.zeros_like(tensors["bias"])
|
||||
layer.register_buffer("zero_bias", zero_bias)
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
for name, _ in list(layer.named_parameters(recurse=False)):
|
||||
delattr(layer, name)
|
||||
for name, tensor in tensors.items():
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Humming kernel selection priority."""
|
||||
|
||||
from enum import Enum
|
||||
from typing import TypeVar
|
||||
|
||||
_KernelT = TypeVar("_KernelT", bound=type | Enum)
|
||||
|
||||
# Compute capabilities where Humming outranks Marlin.
|
||||
_HUMMING_PREFERRED_CAPABILITIES = frozenset({90})
|
||||
|
||||
|
||||
def prefers_humming(compute_capability: int | None = None) -> bool:
|
||||
"""Whether Humming outranks Marlin. A missing compute capability uses the
|
||||
current device."""
|
||||
if compute_capability is None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_cuda():
|
||||
cc = current_platform.get_device_capability()
|
||||
compute_capability = cc.to_int() if cc is not None else None
|
||||
return compute_capability in _HUMMING_PREFERRED_CAPABILITIES
|
||||
|
||||
|
||||
def prioritize_humming(
|
||||
kernels: list[_KernelT],
|
||||
compute_capability: int | None = None,
|
||||
) -> list[_KernelT]:
|
||||
"""Move Humming directly ahead of Marlin where Humming is preferred.
|
||||
|
||||
Every other entry keeps its relative order and the input list is not
|
||||
modified. Match kernel class names or MoE backend enum names.
|
||||
"""
|
||||
if not prefers_humming(compute_capability):
|
||||
return kernels
|
||||
|
||||
names = [
|
||||
(kernel.name if isinstance(kernel, Enum) else kernel.__name__).lower()
|
||||
for kernel in kernels
|
||||
]
|
||||
humming = next((i for i, name in enumerate(names) if "humming" in name), None)
|
||||
marlin = next((i for i, name in enumerate(names) if "marlin" in name), None)
|
||||
if humming is None or marlin is None or humming < marlin:
|
||||
return kernels
|
||||
kernels = kernels.copy()
|
||||
kernels.insert(marlin, kernels.pop(humming))
|
||||
return kernels
|
||||
Reference in New Issue
Block a user