[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-09-04 11:46:06 -07:00
committed by GitHub
co-authored by Codex mgoin
parent a11dfcff89
commit 5690b02c03
25 changed files with 846 additions and 175 deletions
@@ -4,7 +4,10 @@
from typing import Literal, get_args
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
resolve_quant_method,
)
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -182,6 +185,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
__all__ = [
"QuantizationConfig",
"resolve_quant_method",
"QuantizationMethods",
"get_quantization_config",
"register_quantization_config",
@@ -12,6 +12,9 @@ from transformers import PretrainedConfig
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization import QuantizationMethods
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
from vllm.model_executor.models.utils import WeightsMapper
else:
QuantizationMethods = str
@@ -105,6 +108,7 @@ class QuantizationConfig(ABC):
"""Suffixes of quantization parameters that may be present in the checkpoint but
not in the model, and should be ignored if unexpected during loading. These are used
after remapping, so should be in vLLM format (e.g. .q_scale, not .q.scale)."""
online_quantization_config: "OnlineQuantizationConfig | None" = None
def __init__(self):
super().__init__()
@@ -187,7 +191,8 @@ class QuantizationConfig(ABC):
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> QuantizeMethodBase | None:
"""Get the quantize method to use for the quantized layer.
"""Get the quantize method to use for the quantized layer, from the
pre-quantized checkpoint quant_method.
Args:
layer: The layer for the quant method.
@@ -265,3 +270,46 @@ class QuantizationConfig(ABC):
# TODO: revision is never passed currently in vllm.py,
# but is used in subclasses, should we remove this parameter?
pass
def resolve_quant_method(
quant_config: QuantizationConfig, layer: torch.nn.Module, prefix: str
) -> QuantizeMethodBase | None:
"""Return the checkpoint method with configured online quantization."""
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
base_quant_method = quant_config.get_quant_method(layer, prefix)
if quant_config.online_quantization_config is None:
return base_quant_method
# Online quantization currently supports only LinearBase and RoutedExperts.
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
if not isinstance(layer, (LinearBase, RoutedExperts)):
return base_quant_method
quant_config.online_quantization_config.packed_modules_mapping = (
quant_config.packed_modules_mapping
)
checkpoint_is_quantized = base_quant_method is not None and not isinstance(
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
)
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
layer, prefix
)
if checkpoint_is_quantized:
if online_target is not None:
raise ValueError(
f"Cannot apply requested online quantization {online_target[3]} to "
f"pre-quantized layer {prefix}: {base_quant_method} was already "
"selected by the checkpoint quantization config."
)
return base_quant_method
if online_target is None:
return base_quant_method
return quant_config.online_quantization_config.get_quant_method(layer, prefix)
@@ -56,5 +56,5 @@ class ExpertsInt8Config(QuantizationConfig):
if isinstance(layer, LinearBase):
return UnquantizedLinearMethod()
elif isinstance(layer, RoutedExperts):
return Int8OnlineMoEMethod(layer=layer)
return Int8OnlineMoEMethod(moe=layer.moe_config)
return None
@@ -219,7 +219,7 @@ class Fp8Config(QuantizationConfig):
Fp8PerTensorOnlineMoEMethod,
)
return Fp8PerTensorOnlineMoEMethod(layer=layer)
return Fp8PerTensorOnlineMoEMethod(moe=layer.moe_config)
elif isinstance(layer, Attention):
return Fp8KVCacheMethod(self)
return None
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from enum import Enum
from types import MappingProxyType
from typing import Any
@@ -14,6 +15,7 @@ from vllm.config.quantization import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoEMethodBase,
RoutedExperts,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
@@ -38,6 +40,7 @@ from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PerTensorOnlineMoEMethod,
Fp8PtpcOnlineLinearMethod,
Fp8PtpcOnlineMoEMethod,
OnlineLinearBase,
)
from vllm.model_executor.layers.quantization.online.int8 import (
Int8OnlineMoEMethod,
@@ -71,6 +74,14 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
logger = init_logger(__name__)
class OnlineQuantizationSource(str, Enum):
"""Supported online quantization configuration sources."""
linear = "linear" # LinearBase
moe = "moe" # RoutedExperts
targets = "targets"
# Online dispatch tables, keyed by the QuantSpec.weight QuantKey. The
# corresponding method class handles the activation choice via its
# `supported_activation_quant` set.
@@ -200,12 +211,23 @@ class OnlineQuantizationConfig(QuantizationConfig):
"quantization='fp8_per_tensor'/'fp8_per_block' instead."
)
def _dispatch(
def _get_method_cls(
self,
spec: QuantSpec | None,
table: dict[QuantKey, type],
layer: torch.nn.Module,
) -> "QuantizeMethodBase | None":
) -> type | None:
"""Resolve the online method class for a layer's quantization spec.
Args:
spec: Quantization specification to resolve.
table: Mapping from weight quantization keys to method classes.
layer: Layer that will use the resolved method.
Returns:
The matching method class, or None when ``spec`` has no weight
quantization.
"""
if spec is None or spec.weight is None:
return None
cls = table.get(spec.weight)
@@ -223,13 +245,75 @@ class OnlineQuantizationConfig(QuantizationConfig):
f"activation override (activation={spec.activation}) is not "
f"yet supported for online {cls.__name__}"
)
if isinstance(layer, RoutedExperts):
return cls(layer=layer)
return cls()
return cls
def _dispatch_target(
def resolve_quant_method_cls(
self, layer: torch.nn.Module, prefix: str
) -> tuple[OnlineQuantizationSource, str, str | None, QuantSpec, type] | None:
"""Resolve quantization metadata and method class without instantiating it.
Args:
layer: Layer for which to resolve online quantization.
prefix: Fully qualified layer name.
Returns:
A tuple of source, quantization key string, target pattern, spec,
and method class. Returns None when online quantization does not
apply to the layer.
"""
quant_spec: QuantSpec | None
if self.args.targets is not None:
resolved_pattern = self._resolve_targets_quant_method_metadata(
prefix, layer
)
if resolved_pattern is None:
return None
source, quant_key_str, target_pattern, quant_spec, table = resolved_pattern
else:
if isinstance(layer, LinearBase):
source = OnlineQuantizationSource.linear
quant_spec = self.args.linear
table = _ONLINE_LINEAR_METHODS
elif isinstance(layer, RoutedExperts):
source = OnlineQuantizationSource.moe
quant_spec = self.args.moe
table = _ONLINE_MOE_METHODS
else:
return None
if should_ignore_layer(
prefix,
ignore=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
use_fnmatch=True,
):
return None
quant_key_str = str(quant_spec)
target_pattern = None
quant_method_cls = self._get_method_cls(quant_spec, table, layer)
if quant_method_cls is None:
return None
assert quant_spec is not None
return source, quant_key_str, target_pattern, quant_spec, quant_method_cls
def _resolve_targets_quant_method_metadata(
self, prefix: str, layer: torch.nn.Module
) -> "QuantizeMethodBase | None":
) -> (
tuple[OnlineQuantizationSource, str, str, QuantSpec, dict[QuantKey, type]]
| None
):
"""Resolve target-pattern quantization metadata and dispatch table.
Args:
prefix: Fully qualified layer name.
layer: Layer matched against configured target patterns.
Returns:
A tuple of source, quantization key string, target pattern, spec,
and dispatch table. Returns None when no pattern applies or the
layer is ignored.
"""
assert self.args.targets is not None
ignored = should_ignore_layer(
prefix,
@@ -254,7 +338,8 @@ class OnlineQuantizationConfig(QuantizationConfig):
f"targets patterns: {matches}. Each layer may match at most "
f"one target."
)
quant_key_str = self.args.targets[matches[0]]
target_pattern = matches[0]
quant_key_str = self.args.targets[target_pattern]
shorthand = _ONLINE_SHORTHANDS[quant_key_str]
if isinstance(layer, LinearBase):
quant_spec = shorthand.linear
@@ -265,63 +350,44 @@ class OnlineQuantizationConfig(QuantizationConfig):
else:
raise ValueError(
f"Layer {prefix} was matched by quantization_config.targets "
f"({matches[0]}), but online quantization is not supported for "
f"({target_pattern}), but online quantization is not supported for "
f"{type(layer).__name__}."
)
if quant_spec is None:
raise ValueError(
f"targets pattern {matches[0]} = {quant_key_str} does "
f"targets pattern {target_pattern} = {quant_key_str} does "
f"not define a QuantSpec for {type(layer).__name__} layers "
f"(matched at {prefix})."
)
method = self._dispatch(quant_spec, table, layer)
if method is not None:
self.quantized_layers[prefix] = ("targets", quant_key_str, matches[0])
return method
return (
OnlineQuantizationSource.targets,
quant_key_str,
target_pattern,
quant_spec,
table,
)
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
# `targets` takes precedence over `moe` and `linear` and is exclusive.
if self.args.targets is not None:
method = self._dispatch_target(prefix, layer)
if method is not None:
return method
if isinstance(layer, LinearBase):
return UnquantizedLinearMethod()
resolved = self.resolve_quant_method_cls(layer, prefix)
if resolved is not None:
source, quant_key_str, target_pattern, _, quant_method_cls = resolved
self.quantized_layers[prefix] = (
source.value,
quant_key_str,
target_pattern,
)
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
return None
assert issubclass(quant_method_cls, FusedMoEMethodBase)
return quant_method_cls(moe=layer.moe_config)
assert issubclass(quant_method_cls, OnlineLinearBase)
return quant_method_cls()
if isinstance(layer, LinearBase):
if should_ignore_layer(
prefix,
ignore=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
use_fnmatch=True,
):
return UnquantizedLinearMethod()
method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer)
if method is not None:
self.quantized_layers[prefix] = (
"linear",
str(self.args.linear),
None,
)
return method
return UnquantizedLinearMethod()
elif isinstance(layer, RoutedExperts):
if should_ignore_layer(
prefix,
ignore=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
use_fnmatch=True,
):
return UnquantizedFusedMoEMethod(layer.moe_config)
method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer)
if method is not None:
self.quantized_layers[prefix] = ("moe", str(self.args.moe), None)
return method
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
return None
@@ -23,6 +23,7 @@ from vllm.model_executor.kernels.linear.scaled_mm import (
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
select_fp8_moe_backend,
)
@@ -111,7 +112,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
return is_row_parallel or (reduces_output_dim and is_column_parallel)
class _Fp8OnlineLinearBase(LinearMethodBase):
class OnlineLinearBase(LinearMethodBase):
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
weights onto meta device and materializes them just-in-time."""
@@ -155,7 +156,7 @@ class _Fp8OnlineLinearBase(LinearMethodBase):
initialize_online_processing(layer)
class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
"""Online tensorwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
@@ -256,7 +257,7 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
return self.fp8_linear.apply_weights(layer, x, bias)
class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
"""Online blockwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-block during loading."""
@@ -336,7 +337,7 @@ class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
)
class Fp8PtpcOnlineLinearMethod(_Fp8OnlineLinearBase):
class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
"""Online PTPC FP8 linear quantization.
Per-output-channel weight scale + dynamic per-token activation scale. The
@@ -443,12 +444,12 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
self,
*,
weight_block_size: list[int] | None,
layer: torch.nn.Module,
moe: FusedMoEConfig,
weight_key: "QuantKey | None" = None,
activation_key: "QuantKey | None" = None,
allow_vllm_cutlass: bool = False,
):
super().__init__(layer.moe_config)
super().__init__(moe)
self.weight_block_size = weight_block_size
self.block_quant: bool = self.weight_block_size is not None
self.weight_scale_name = (
@@ -555,11 +556,11 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
super().__init__(
weight_block_size=None,
layer=layer,
moe=moe,
)
def process_weights_after_loading(self, layer: Module) -> None:
@@ -612,11 +613,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
super().__init__(
weight_block_size=[128, 128],
layer=layer,
moe=moe,
)
def maybe_roundup_sizes(
@@ -716,13 +717,13 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
super().__init__(
weight_block_size=None,
layer=layer,
moe=moe,
weight_key=kFp8StaticChannelSym,
activation_key=kFp8DynamicTokenSym,
allow_vllm_cutlass=True,
@@ -12,6 +12,7 @@ if TYPE_CHECKING:
)
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
convert_to_int8_moe_kernel_format,
make_int8_moe_kernel,
@@ -38,9 +39,9 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
super().__init__(layer.moe_config)
super().__init__(moe)
self.int8_backend, self.experts_cls = select_int8_moe_backend(
config=self.moe,
weight_key=kInt8StaticChannelSym,
@@ -17,6 +17,7 @@ if TYPE_CHECKING:
)
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
@@ -27,7 +28,7 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_mxfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.online.fp8 import (
_Fp8OnlineLinearBase,
OnlineLinearBase,
)
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
@@ -73,7 +74,7 @@ def _quantize_mxfp4_moe_weight(
return w_quant, w_scales
class Mxfp4OnlineLinearMethod(_Fp8OnlineLinearBase):
class Mxfp4OnlineLinearMethod(OnlineLinearBase):
"""Online MXFP4 linear method.
Loads bf16/fp16 checkpoints and quantizes weights to MXFP4 (microscaling
FP4 with block-32 scales) during weight loading.
@@ -144,8 +145,8 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
experts_cls: "type[mk.FusedMoEExperts] | None"
activation_quant_key = kMxfp4Dynamic
def __init__(self, *, layer: torch.nn.Module):
super().__init__(layer.moe_config)
def __init__(self, *, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_block_size: list[int] = [1, MXFP4_BLOCK_SIZE]
self.weight_scale_name = "weight_scale"
@@ -8,6 +8,8 @@ from typing import TYPE_CHECKING
import torch
from torch.nn import Module
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
if TYPE_CHECKING:
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe import (
@@ -21,7 +23,7 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
select_mxfp8_moe_backend,
)
from vllm.model_executor.layers.quantization.online.fp8 import (
_Fp8OnlineLinearBase,
OnlineLinearBase,
)
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
@@ -34,7 +36,7 @@ from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
class Mxfp8OnlineLinearMethod(_Fp8OnlineLinearBase):
class Mxfp8OnlineLinearMethod(OnlineLinearBase):
"""Online MXFP8 linear method.
Loads bf16/fp16 checkpoints and quantizes weights to MXFP8 (microscaling
FP8 with block-32 scales) during weight loading.
@@ -100,8 +102,8 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
fp8_backend: "Fp8MoeBackend"
experts_cls: "type[mk.FusedMoEExperts] | None"
def __init__(self, *, layer: torch.nn.Module):
super().__init__(layer.moe_config)
def __init__(self, *, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_block_size: list[int] = [1, MXFP8_BLOCK_SIZE]
self.weight_scale_name = "weight_scale"
@@ -6,7 +6,10 @@ from torch.nn import Module
from vllm._custom_ops import scaled_fp4_quant
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
convert_to_nvfp4_moe_kernel_format,
make_nvfp4_moe_kernel,
@@ -87,13 +90,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
if not current_platform.is_device_capability_family(100):
raise ValueError(
"nvfp4_per_token online quantization requires a Blackwell (SM100) GPU."
)
super().__init__(layer.moe_config)
super().__init__(moe)
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
config=self.moe,
weight_key=kNvfp4Static,