[Quantization] Prefer Humming before Marlin backends on SM90 (#56997)
Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
mgoin
Claude Opus 5.5
parent
bb87d227d4
commit
d1f3d8b870
@@ -228,6 +228,10 @@ from vllm.model_executor.kernels.linear.scaled_mm.xpu import (
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.zentorch import (
|
||||
ZentorchInt8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
prefers_humming,
|
||||
prioritize_humming,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.platforms import PlatformEnum, current_platform
|
||||
|
||||
@@ -376,8 +380,9 @@ def _resolve_backend_kernels(
|
||||
layer_desc: str,
|
||||
*,
|
||||
quantization: str,
|
||||
compute_capability: int | None = None,
|
||||
) -> list[type]:
|
||||
"""Apply --linear-backend filtering to one layer type's kernel list.
|
||||
"""Apply device priorities and --linear-backend filtering to a kernel list.
|
||||
|
||||
When the requested backend has no kernel for this layer type, fall back
|
||||
to the unfiltered list (with a WARNING log) instead of failing engine
|
||||
@@ -386,6 +391,7 @@ def _resolve_backend_kernels(
|
||||
layer types (e.g. NVFP4 MoE projections next to FP8 attention
|
||||
projections).
|
||||
"""
|
||||
kernels = prioritize_humming(kernels, compute_capability)
|
||||
linear_backend = _get_linear_backend(quantization=quantization)
|
||||
if linear_backend == "auto":
|
||||
return kernels
|
||||
@@ -682,6 +688,7 @@ def choose_scaled_mm_linear_kernel(
|
||||
platform_kernels,
|
||||
"scaled-mm",
|
||||
quantization=quantization,
|
||||
compute_capability=compute_capability,
|
||||
)
|
||||
|
||||
for kernel in platform_kernels:
|
||||
@@ -845,6 +852,7 @@ def choose_mp_linear_kernel(
|
||||
platform_kernels,
|
||||
"mixed-precision",
|
||||
quantization="mixed_precision",
|
||||
compute_capability=compute_capability,
|
||||
)
|
||||
|
||||
failure_reasons = []
|
||||
@@ -1121,13 +1129,13 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
|
||||
_cc = current_platform.get_device_capability()
|
||||
compute_capability = _cc.to_int() if _cc is not None else None
|
||||
# Weight-only: prefer FlashInfer CuTe-DSL W4A16 on SM100/103,
|
||||
# otherwise Marlin.
|
||||
# Humming then Marlin where Humming is preferred, and Marlin elsewhere.
|
||||
cutedsl_ok, _ = FlashInferCuteDslNvFp4W4A16LinearKernel.is_supported(
|
||||
compute_capability
|
||||
)
|
||||
if compute_capability in (100, 103) and cutedsl_ok:
|
||||
force_kernel = FlashInferCuteDslNvFp4W4A16LinearKernel
|
||||
else:
|
||||
elif not prefers_humming(compute_capability):
|
||||
force_kernel = MarlinNvFp4LinearKernel
|
||||
|
||||
if force_kernel is not None:
|
||||
|
||||
@@ -4,11 +4,21 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTokenSym,
|
||||
kInt8DynamicTokenSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
|
||||
|
||||
# Per-token activation quantization requested through `act_type`.
|
||||
_ACT_QUANT_KEYS = {
|
||||
torch.int8: kInt8DynamicTokenSym,
|
||||
torch.float8_e4m3fn: kFp8DynamicTokenSym,
|
||||
}
|
||||
|
||||
|
||||
class HummingLinearKernel(MPLinearKernel):
|
||||
@classmethod
|
||||
@@ -21,6 +31,8 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
return False, "Humming is only supported on CUDA"
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
if c.act_type.itemsize == 1 and c.act_type not in _ACT_QUANT_KEYS:
|
||||
return False, f"Activation type ({c.act_type}) not supported by Humming"
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
@@ -28,6 +40,7 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
convert_linear_layer_to_humming_standard,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
quant_key_to_input_schema,
|
||||
)
|
||||
|
||||
name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}
|
||||
@@ -48,8 +61,14 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
input_quant_config = getattr(layer, "_humming_input_quant_config", None)
|
||||
input_schema = None
|
||||
if (act_key := _ACT_QUANT_KEYS.get(self.config.act_type)) is not None:
|
||||
input_schema = quant_key_to_input_schema(act_key)
|
||||
self.layer_config = prepare_humming_linear_layer_config(
|
||||
layer, quant_config, input_quant_config=input_quant_config
|
||||
layer,
|
||||
quant_config,
|
||||
input_quant_config=input_quant_config,
|
||||
input_schema=input_schema,
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
Reference in New Issue
Block a user