[Quantization] Prefer Humming before Marlin backends on SM90 (#56997)

Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jinzhen Lin
2026-10-06 12:31:35 -07:00
committed by GitHub
co-authored by Codex mgoin Claude Opus 5.5
parent bb87d227d4
commit d1f3d8b870
13 changed files with 159 additions and 15 deletions
@@ -14,6 +14,8 @@ import torch
from vllm.config import KernelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.kernels.linear import (
_POSSIBLE_KERNELS,
_POSSIBLE_NVFP4_KERNELS,
AiterInt8ScaledMMLinearKernel,
CPUInt8ScaledMMLinearKernel,
HummingFP8ScaledMMLinearKernel,
@@ -26,6 +28,7 @@ from vllm.model_executor.kernels.linear import (
init_int8_linear_kernel,
register_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
@@ -197,3 +200,21 @@ def test_register_oot_linear_kernel(platform_mock):
assert isinstance(kernel, OOTInt8ScaledMMLinearKernel), (
"init_int8_linear_kernel should return an instance of the registered kernel"
)
@pytest.mark.parametrize(
"possible_kernels", [_POSSIBLE_KERNELS, _POSSIBLE_NVFP4_KERNELS]
)
def test_humming_priority_only_reorders_humming_and_marlin(possible_kernels):
"""Preferring Humming must not demote Marlin below unrelated kernels."""
kernels = possible_kernels[PlatformEnum.CUDA]
original = list(kernels)
assert prioritize_humming(kernels, compute_capability=80) == original
names = [k.__name__ for k in prioritize_humming(kernels, compute_capability=90)]
humming = next(i for i, name in enumerate(names) if "Humming" in name)
assert "Marlin" in names[humming + 1]
del names[humming]
assert names == [k.__name__ for k in original if "Humming" not in k.__name__]
assert kernels == original