[Quantization] Rework quantization_config to use QuantKey and allow for activation override (#41566)

This commit is contained in:
Michael Goin
2026-05-13 16:58:32 -04:00
committed by GitHub
parent cca32d55a2
commit 8efd508204
16 changed files with 621 additions and 322 deletions
@@ -1,13 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import TYPE_CHECKING, Union
from typing import TYPE_CHECKING, Literal, Union
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm.config import get_current_vllm_config
from vllm.config.kernel import MoEBackend
from vllm.config.quantization import QuantizationConfigArgs
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
@@ -207,25 +209,38 @@ def backend_to_kernel_cls(
raise ValueError(f"Unknown MXFP4 MoE backend: {backend.value}")
def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend:
"""Map user's moe_backend string to Mxfp4MoeBackend."""
mapping: dict[str, Mxfp4MoeBackend] = {
"deep_gemm": Mxfp4MoeBackend.DEEPGEMM_MXFP4,
"flashinfer_trtllm": Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
"flashinfer_trtllm_afp8": Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
"flashinfer_cutlass": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
"flashinfer_cutlass_afp8": Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
"triton": Mxfp4MoeBackend.TRITON,
"triton_unfused": Mxfp4MoeBackend.TRITON_UNFUSED,
"humming": Mxfp4MoeBackend.HUMMING,
"marlin": Mxfp4MoeBackend.MARLIN,
"aiter": Mxfp4MoeBackend.AITER_MXFP4_BF16,
"aiter_mxfp4_fp8": Mxfp4MoeBackend.AITER_MXFP4_FP8,
"xpu": Mxfp4MoeBackend.XPU,
"emulation": Mxfp4MoeBackend.EMULATION,
def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
"""Map a moe_backend string to its candidate Mxfp4MoeBackends.
Vendor families return all activation variants; the caller picks one
via ``activation_key`` and ``is_supported_config``.
"""
mapping: dict[str, list[Mxfp4MoeBackend]] = {
"deep_gemm": [Mxfp4MoeBackend.DEEPGEMM_MXFP4],
"flashinfer_trtllm": [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
],
"flashinfer_trtllm_afp8": [Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8],
"flashinfer_cutlass": [
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
],
"flashinfer_cutlass_afp8": [Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8],
"triton": [Mxfp4MoeBackend.TRITON],
"triton_unfused": [Mxfp4MoeBackend.TRITON_UNFUSED],
"humming": [Mxfp4MoeBackend.HUMMING],
"marlin": [Mxfp4MoeBackend.MARLIN],
"aiter": [
Mxfp4MoeBackend.AITER_MXFP4_BF16,
Mxfp4MoeBackend.AITER_MXFP4_FP8,
],
"aiter_mxfp4_fp8": [Mxfp4MoeBackend.AITER_MXFP4_FP8],
"xpu": [Mxfp4MoeBackend.XPU],
"emulation": [Mxfp4MoeBackend.EMULATION],
}
if backend := mapping.get(runner_backend):
return backend
if backends := mapping.get(runner_backend):
return backends
raise ValueError(
f"moe_backend='{runner_backend}' is not supported for MXFP4 MoE. "
f"Expected one of {list(mapping.keys())}."
@@ -233,16 +248,16 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend:
def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
"""
Get available backends in priority order based on platform and config.
Only includes BF16 backends. MXFP8 backends are selected via env vars.
"""
"""Available backends in priority order, BF16-act variant before
activation-quantized variant within each vendor family."""
_AVAILABLE_BACKENDS = [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
Mxfp4MoeBackend.AITER_MXFP4_BF16,
Mxfp4MoeBackend.AITER_MXFP4_FP8,
Mxfp4MoeBackend.TRITON,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
# TRITON_UNFUSED has bug with MTP support
# TODO re-enable after kernel is fixed
# TRITON_UNFUSED
@@ -288,6 +303,79 @@ def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
return None # BF16 activation
def _user_moe_activation_override() -> QuantKey | None:
"""User's MoE activation override from quantization_config, or None."""
args = get_current_vllm_config().model_config.quantization_config
if not isinstance(args, QuantizationConfigArgs) or args.moe is None:
return None
return args.moe.activation
def _resolve_activation_key(
model_activation_key: QuantKey | None,
) -> QuantKey | None:
"""Combine the model-supplied activation key with the user override.
Raises on conflict (both set and disagreeing)."""
user_override = _user_moe_activation_override()
if user_override is None:
return model_activation_key
if model_activation_key is None or model_activation_key == user_override:
return user_override
raise ValueError(
f"checkpoint declares MoE activation={model_activation_key} but "
f"quantization_config.moe.activation={user_override}; remove the "
f"override or align it with the checkpoint."
)
def _make_log_backend(backend: Mxfp4MoeBackend) -> str:
return f"Using '{backend.value}' Mxfp4 MoE backend."
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
base = (
f"Mxfp4 MoE backend '{backend.value}' does not support the "
f"deployment configuration"
)
return f"{base} since {reason}." if reason else f"{base}."
def _return_or_raise(
backend: Mxfp4MoeBackend,
config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
scope: Literal["process", "global", "local"] = "local",
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
reason: str | None = None
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls, config, weight_key, activation_key, activation_format
)
if supported:
logger.info_once(_make_log_backend(backend), scope=scope)
return backend, k_cls
raise ValueError(_make_log_unsupported(backend, reason))
def _filter_by_activation(
backends: list[Mxfp4MoeBackend],
requested_activation_key: QuantKey | None,
) -> list[Mxfp4MoeBackend]:
"""Pick variants matching ``requested_activation_key``; without one,
prefer BF16 if the list has any, else keep the list as-is so explicit
non-BF16 picks (e.g. the ``_afp8`` aliases) still land."""
if requested_activation_key is not None:
return [
b
for b in backends
if _backend_activation_key(b) == requested_activation_key
]
bf16 = [b for b in backends if _backend_activation_key(b) is None]
return bf16 if bf16 else backends
def select_mxfp4_moe_backend(
config: FusedMoEConfig,
activation_key: QuantKey | None = None,
@@ -303,8 +391,7 @@ def select_mxfp4_moe_backend(
Note: Shape-specific fallbacks may still occur at runtime.
"""
# If activation_key is explicitly provided (e.g., W4A8), use it
requested_activation_key = activation_key
requested_activation_key = _resolve_activation_key(activation_key)
activation_format = (
mk.FusedMoEActivationFormat.BatchedExperts
@@ -312,67 +399,55 @@ def select_mxfp4_moe_backend(
else mk.FusedMoEActivationFormat.Standard
)
def _make_log_backend(backend: Mxfp4MoeBackend):
return f"Using '{backend.value}' Mxfp4 MoE backend."
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
if reason:
return (
f"Mxfp4 MoE backend '{backend.value}' does not support the "
f"deployment configuration since {reason}."
)
return (
f"Mxfp4 MoE backend '{backend.value}' does not support the "
"deployment configuration."
)
def _return_or_raise(
backend: Mxfp4MoeBackend,
config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
reason: str | None = None
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls, config, weight_key, activation_key, activation_format
)
if supported:
logger.info_once(_make_log_backend(backend))
return backend, k_cls
raise ValueError(_make_log_unsupported(backend, reason))
runner_backend = config.moe_backend
if runner_backend != "auto":
requested_backend = map_mxfp4_backend(runner_backend)
if (
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
and requested_backend == Mxfp4MoeBackend.MARLIN
):
requested_backend = Mxfp4MoeBackend.BATCHED_MARLIN
# Use requested_activation_key if provided, otherwise use backend default
act_key = (
requested_activation_key
if requested_activation_key is not None
else _backend_activation_key(requested_backend)
)
return _return_or_raise(
requested_backend,
config,
kMxfp4Static,
act_key,
activation_format,
)
requested_backends = map_mxfp4_backend(runner_backend)
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
requested_backends = [
Mxfp4MoeBackend.BATCHED_MARLIN if b == Mxfp4MoeBackend.MARLIN else b
for b in requested_backends
]
candidates = _filter_by_activation(requested_backends, requested_activation_key)
if not candidates:
raise ValueError(
f"moe_backend={runner_backend!r} does not support "
f"activation={requested_activation_key}; supported variants: "
f"{[b.name for b in requested_backends]}"
)
last_error: Exception | None = None
for requested_backend in candidates:
act_key = (
requested_activation_key
if requested_activation_key is not None
else _backend_activation_key(requested_backend)
)
try:
return _return_or_raise(
requested_backend,
config,
kMxfp4Static,
act_key,
activation_format,
)
except ValueError as e:
last_error = e
assert last_error is not None
raise last_error
# Select kernels in order of backend.
AVAILABLE_BACKENDS = _get_priority_backends_for_gpt_oss()
AVAILABLE_BACKENDS = _filter_by_activation(
_get_priority_backends_for_gpt_oss(), requested_activation_key
)
# Handle explicit FlashInfer MXFP4 BF16 configuration.
if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"):
if not envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16:
AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16)
AVAILABLE_BACKENDS.remove(Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16)
for _b in (
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
):
if _b in AVAILABLE_BACKENDS:
AVAILABLE_BACKENDS.remove(_b)
else:
if current_platform.is_device_capability(90):
return _return_or_raise(
@@ -481,54 +556,30 @@ def select_deepseek_v4_mxfp4_moe_backend(
else mk.FusedMoEActivationFormat.Standard
)
def _make_log_backend(backend: Mxfp4MoeBackend):
return f"Using '{backend.value}' Mxfp4 MoE backend."
def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
if reason:
return (
f"Mxfp4 MoE backend '{backend.value}' does not support the "
f"deployment configuration since {reason}."
)
return (
f"Mxfp4 MoE backend '{backend.value}' does not support the "
"deployment configuration."
)
def _return_or_raise(
backend: Mxfp4MoeBackend,
config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
reason: str | None = None
for k_cls in backend_to_kernel_cls(backend):
supported, reason = k_cls.is_supported_config(
k_cls, config, weight_key, activation_key, activation_format
)
if supported:
logger.info_once(_make_log_backend(backend), scope="local")
return backend, k_cls
raise ValueError(_make_log_unsupported(backend, reason))
# Honor explicit moe_backend (e.g. "marlin", "triton_unfused") before
# falling back to the auto priority list.
runner_backend = config.moe_backend
if runner_backend != "auto":
requested_backend = map_mxfp4_backend(runner_backend)
if (
activation_format == mk.FusedMoEActivationFormat.BatchedExperts
and requested_backend == Mxfp4MoeBackend.MARLIN
):
requested_backend = Mxfp4MoeBackend.BATCHED_MARLIN
return _return_or_raise(
requested_backend,
config,
kMxfp4Static,
_backend_activation_key(requested_backend),
activation_format,
)
requested_backends = map_mxfp4_backend(runner_backend)
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
requested_backends = [
Mxfp4MoeBackend.BATCHED_MARLIN if b == Mxfp4MoeBackend.MARLIN else b
for b in requested_backends
]
last_error: Exception | None = None
for requested_backend in requested_backends:
try:
return _return_or_raise(
requested_backend,
config,
kMxfp4Static,
_backend_activation_key(requested_backend),
activation_format,
)
except ValueError as e:
last_error = e
assert last_error is not None
raise last_error
# DeepSeek-V4 on ROCm is more accurate with the unfused Triton MXFP4 path
# than the default AITER path. Prefer Triton-unfused for this routing mode,
@@ -35,10 +35,9 @@ QuantizationMethods = Literal[
"deepseek_v4_fp8",
"cpu_awq",
"online",
# Below are values of the OnlineQuantScheme enum, specified as strings to
# avoid circular import issues. This is here to provide a shortcut where
# the user can specify "LLM(..., quantization='fp8_per_tensor')" as
# shorthand for creating a more complicated online quant config object
# Below are online quant shorthand names (see vllm.config.quantization).
# Listed here as strings to avoid a circular import; kept in sync with
# _ONLINE_SHORTHANDS by the assertion in get_quantization_config().
"fp8_per_tensor",
"fp8_per_block",
"int8_per_channel_weight_only",
@@ -111,7 +110,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
raise ValueError(f"Invalid quantization method: {quantization}")
# lazy import to avoid triggering `torch.compile` too early
from vllm.config.quantization import OnlineQuantScheme
from vllm.config.quantization import _ONLINE_SHORTHANDS
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.models.deepseek_v4 import DeepseekV4FP8Config
@@ -171,16 +170,15 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
"online": OnlineQuantizationConfig,
}
# Below are values of the OnlineQuantScheme enum. This is here to provide
# a shortcut where the user can specify
# "LLM(..., quantization='fp8_per_tensor')" as shorthand for creating a
# more complicated online quant config object
for scheme in OnlineQuantScheme:
assert scheme.value not in method_to_config, (
f"Online quant scheme {scheme.value!r} conflicts with an "
# Register online shorthands as quantization methods so the user can
# specify "LLM(..., quantization='fp8_per_tensor')" as shorthand for
# creating a more complicated online quant config object.
for shorthand in _ONLINE_SHORTHANDS:
assert shorthand not in method_to_config, (
f"Online quant shorthand {shorthand!r} conflicts with an "
f"existing quantization method"
)
method_to_config[scheme.value] = OnlineQuantizationConfig
method_to_config[shorthand] = OnlineQuantizationConfig
# Update the `method_to_config` with customized quantization methods.
method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG)
@@ -5,10 +5,7 @@ from typing import Any
import torch
from vllm.config.quantization import (
OnlineQuantizationConfigArgs,
OnlineQuantScheme,
)
from vllm.config.quantization import QuantizationConfigArgs, QuantSpec
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
RoutedExperts,
@@ -41,31 +38,50 @@ from vllm.model_executor.layers.quantization.online.mxfp8 import (
Mxfp8OnlineLinearMethod,
Mxfp8OnlineMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Static128BlockSym,
kFp8StaticTensorSym,
kInt8StaticChannelSym,
kMxfp8Dynamic,
)
logger = init_logger(__name__)
# Online dispatch tables, keyed by the QuantSpec.weight QuantKey. The
# corresponding method class handles the activation choice via its
# `supported_activation_quant` set.
_ONLINE_LINEAR_METHODS: dict[QuantKey, type] = {
kFp8StaticTensorSym: Fp8PerTensorOnlineLinearMethod,
kFp8Static128BlockSym: Fp8PerBlockOnlineLinearMethod,
kMxfp8Dynamic: Mxfp8OnlineLinearMethod,
}
_ONLINE_MOE_METHODS: dict[QuantKey, type] = {
kFp8StaticTensorSym: Fp8PerTensorOnlineMoEMethod,
kFp8Static128BlockSym: Fp8PerBlockOnlineMoEMethod,
kMxfp8Dynamic: Mxfp8OnlineMoEMethod,
kInt8StaticChannelSym: Int8OnlineMoEMethod,
}
class OnlineQuantizationConfig(QuantizationConfig):
"""Model-level config class for online quantization (quantize fp16/bf16 weights
"""Model-level config for online quantization (quantize fp16/bf16 weights
during model loading, without requiring a pre-quantized checkpoint)."""
def __init__(
self,
args: OnlineQuantizationConfigArgs,
args: QuantizationConfigArgs,
) -> None:
super().__init__()
if (
args.global_scheme is None
and args.linear_scheme_override is None
and args.moe_scheme_override is None
):
if args.linear is None and args.moe is None:
raise ValueError(
"OnlineQuantizationConfig requires at least one of "
"global_scheme, linear_scheme_override, or "
"moe_scheme_override to be set."
"quantization_config.linear or quantization_config.moe "
"to be set."
)
self.args = args
self.quant_scheme = args.global_scheme
self.ignored_layers: list[str] = args.ignore
@classmethod
@@ -94,6 +110,33 @@ class OnlineQuantizationConfig(QuantizationConfig):
"quantization='fp8_per_tensor'/'fp8_per_block' instead."
)
def _dispatch(
self,
spec: QuantSpec | None,
table: dict[QuantKey, type],
layer: torch.nn.Module,
) -> "QuantizeMethodBase | None":
if spec is None or spec.weight is None:
return None
cls = table.get(spec.weight)
if cls is None:
raise ValueError(
f"online quantization for {type(layer).__name__} with "
f"weight={spec.weight} is not supported; supported weight "
f"keys: {sorted(str(k) for k in table)}"
)
# Online method classes pick their own activation format internally.
# Per-class activation overrides are not yet wired through; reject
# explicit overrides until the relevant method class opts in.
if spec.activation is not None:
raise ValueError(
f"activation override (activation={spec.activation}) is not "
f"yet supported for online {cls.__name__}"
)
if isinstance(layer, RoutedExperts):
return cls(layer=layer)
return cls()
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
@@ -104,20 +147,8 @@ class OnlineQuantizationConfig(QuantizationConfig):
fused_mapping=self.packed_modules_mapping,
):
return UnquantizedLinearMethod()
linear_scheme = self.args.linear_scheme_override or self.args.global_scheme
if linear_scheme == OnlineQuantScheme.INT8_PER_CHANNEL_WEIGHT_ONLY:
logger.warning_once(
"INT8 online quantization only quantizes MoE expert "
"weights. linear layers remain in full precision."
)
return UnquantizedLinearMethod()
elif linear_scheme == OnlineQuantScheme.FP8_PER_BLOCK:
return Fp8PerBlockOnlineLinearMethod()
elif linear_scheme == OnlineQuantScheme.MXFP8:
return Mxfp8OnlineLinearMethod()
else:
return Fp8PerTensorOnlineLinearMethod()
method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer)
return method if method is not None else UnquantizedLinearMethod()
elif isinstance(layer, RoutedExperts):
if should_ignore_layer(
prefix,
@@ -125,14 +156,10 @@ class OnlineQuantizationConfig(QuantizationConfig):
fused_mapping=self.packed_modules_mapping,
):
return UnquantizedFusedMoEMethod(layer.moe_config)
moe_scheme = self.args.moe_scheme_override or self.args.global_scheme
if moe_scheme == OnlineQuantScheme.INT8_PER_CHANNEL_WEIGHT_ONLY:
return Int8OnlineMoEMethod(layer=layer)
elif moe_scheme == OnlineQuantScheme.FP8_PER_BLOCK:
return Fp8PerBlockOnlineMoEMethod(layer=layer)
elif moe_scheme == OnlineQuantScheme.MXFP8:
return Mxfp8OnlineMoEMethod(layer=layer)
else:
return Fp8PerTensorOnlineMoEMethod(layer=layer)
method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer)
return (
method
if method is not None
else UnquantizedFusedMoEMethod(layer.moe_config)
)
return None