[Quantization] Select linear backends per quantization (#51204)

Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>
This commit is contained in:
Netanel Haber
2026-09-14 16:24:18 -04:00
committed by GitHub
parent 995e8581f4
commit d4ee7fe7a9
6 changed files with 154 additions and 17 deletions
+51 -14
View File
@@ -235,14 +235,19 @@ from vllm.platforms import PlatformEnum, current_platform
logger = init_logger(__name__)
def _get_linear_backend() -> str:
def _get_linear_backend(*, quantization: str) -> str:
"""Get the linear_backend setting from the current vllm config."""
from vllm.config import get_current_vllm_config_or_none
config = get_current_vllm_config_or_none()
if config is not None:
return config.kernel_config.linear_backend
return "auto"
if (config := get_current_vllm_config_or_none()) is None:
return "auto"
overrides = config.kernel_config.linear_backend_per_quant
if overrides is not None and (override := overrides.get(quantization)):
logger.info_once(
"Applied linear backend override for %r: %r", quantization, override
)
return override
return config.kernel_config.linear_backend
# Kernel classes covered by each --linear-backend value.
@@ -370,6 +375,8 @@ def _filter_kernels_by_backend(
def _resolve_backend_kernels(
kernels: list[type],
layer_desc: str,
*,
quantization: str,
) -> list[type]:
"""Apply --linear-backend filtering to one layer type's kernel list.
@@ -380,7 +387,7 @@ def _resolve_backend_kernels(
layer types (e.g. NVFP4 MoE projections next to FP8 attention
projections).
"""
linear_backend = _get_linear_backend()
linear_backend = _get_linear_backend(quantization=quantization)
if linear_backend == "auto":
return kernels
filtered = _filter_kernels_by_backend(linear_backend, kernels)
@@ -625,6 +632,8 @@ def choose_scaled_mm_linear_kernel(
possible_kernels: dict[PlatformEnum, list[type[_KernelT]]],
compute_capability: int | None = None,
force_kernel: type[_KernelT] | None = None,
*,
quantization: str,
) -> type[_KernelT]:
"""
Choose a _KernelT that can implement the given config for the
@@ -642,6 +651,7 @@ def choose_scaled_mm_linear_kernel(
force_kernel (Optional[type[_KernelT]]): An Optional forced kernel to override
the possible_kernels if it can be implemented. If None, it will only try the
possible kernels.
quantization: Quantization scheme used to select a backend override.
Raises:
ValueError: If no kernel can implement the given config.
@@ -668,7 +678,11 @@ def choose_scaled_mm_linear_kernel(
platform_kernels = possible_kernels.get(current_platform._enum, [])
# Apply --linear-backend filtering when set.
platform_kernels = _resolve_backend_kernels(platform_kernels, "scaled-mm")
platform_kernels = _resolve_backend_kernels(
platform_kernels,
"scaled-mm",
quantization=quantization,
)
for kernel in platform_kernels:
is_supported_and_can_implement, failure_reason = (
@@ -705,6 +719,7 @@ def init_fp8_linear_kernel(
kernel_type = choose_scaled_mm_linear_kernel(
config=scaled_mm_linear_kernel_config,
possible_kernels=_POSSIBLE_FP8_BLOCK_KERNELS, # type: ignore[misc]
quantization="fp8_block_w8a8",
force_kernel=force_kernel,
)
if module_name:
@@ -736,6 +751,7 @@ def init_fp8_linear_kernel(
kernel_type = choose_scaled_mm_linear_kernel(
config=scaled_mm_linear_kernel_config,
possible_kernels=_POSSIBLE_FP8_KERNELS, # type: ignore[arg-type]
quantization="fp8_w8a8",
force_kernel=force_kernel,
)
if module_name:
@@ -772,6 +788,7 @@ def init_int8_linear_kernel(
kernel_type = choose_scaled_mm_linear_kernel(
config,
_POSSIBLE_INT8_KERNELS,
quantization="int8_w8a8",
)
logger.info_once(
@@ -824,7 +841,11 @@ def choose_mp_linear_kernel(
platform_kernels = _POSSIBLE_KERNELS.get(current_platform._enum, [])
# Apply --linear-backend filtering when set.
platform_kernels = _resolve_backend_kernels(platform_kernels, "mixed-precision")
platform_kernels = _resolve_backend_kernels(
platform_kernels,
"mixed-precision",
quantization="mixed_precision",
)
failure_reasons = []
for kernel in platform_kernels:
@@ -875,7 +896,11 @@ def init_mxfp8_linear_kernel(*, bmm_batch_size: int | None = None) -> Mxfp8Linea
possible = list(_POSSIBLE_MXFP8_KERNELS.get(platform, []))
# Apply --linear-backend filtering when set.
possible = _resolve_backend_kernels(possible, "MXFP8")
possible = _resolve_backend_kernels(
possible,
"MXFP8",
quantization="mxfp8",
)
failure_reasons = []
for kernel_cls in possible:
@@ -917,7 +942,11 @@ def init_mxfp4_linear_kernel(
possible = list(_POSSIBLE_MXFP4_KERNELS.get(platform, []))
# Apply --linear-backend filtering when set.
possible = _resolve_backend_kernels(possible, "MXFP4")
possible = _resolve_backend_kernels(
possible,
"MXFP4",
quantization="mxfp4",
)
failure_reasons = []
for kernel_cls in possible:
@@ -957,7 +986,7 @@ def init_mxfp6_linear_kernel(
activation_quant_key=activation_quant_key,
)
linear_backend = _get_linear_backend()
linear_backend = _get_linear_backend(quantization="mxfp6")
platform = current_platform._enum
possible = list(_POSSIBLE_MXFP6_KERNELS.get(platform, []))
@@ -1017,7 +1046,10 @@ def init_wfp8_a16_linear_kernel(
)
kernel_type = choose_scaled_mm_linear_kernel(
config, _POSSIBLE_WFP8A16_KERNELS, force_kernel=force_kernel
config,
_POSSIBLE_WFP8A16_KERNELS,
quantization="w8a16_fp8",
force_kernel=force_kernel,
)
if module_name:
@@ -1048,7 +1080,8 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
# batch-invariant CUTLASS implementation when available, otherwise fall
# back to emulation. It overrides --linear-backend.
force_kernel: type[NvFp4LinearKernel] | None = None
linear_backend = _get_linear_backend()
quantization = "nvfp4_w4a16" if use_a16 else "nvfp4_w4a4"
linear_backend = _get_linear_backend(quantization=quantization)
if envs.VLLM_BATCH_INVARIANT:
bi_supported, reason = CutlassNvFp4LinearKernel.is_supported()
if bi_supported:
@@ -1110,7 +1143,11 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
possible = [kernel for kernel in possible if kernel in a16_kernels]
# Apply --linear-backend filtering when set.
possible = _resolve_backend_kernels(possible, "NVFP4")
possible = _resolve_backend_kernels(
possible,
"NVFP4",
quantization=quantization,
)
failure_reasons = []
for kernel_cls in possible:
@@ -158,7 +158,7 @@ class AiterMxfp4LinearKernel(MxFp4LinearKernel):
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.model_executor.kernels.linear import _get_linear_backend
linear_backend = _get_linear_backend()
linear_backend = _get_linear_backend(quantization="mxfp4")
if (
current_platform.is_rocm()