[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-09-04 11:46:06 -07:00
committed by GitHub
co-authored by Codex mgoin
parent a11dfcff89
commit 5690b02c03
25 changed files with 846 additions and 175 deletions
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from enum import Enum
from types import MappingProxyType
from typing import Any
@@ -14,6 +15,7 @@ from vllm.config.quantization import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoEMethodBase,
RoutedExperts,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
@@ -38,6 +40,7 @@ from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PerTensorOnlineMoEMethod,
Fp8PtpcOnlineLinearMethod,
Fp8PtpcOnlineMoEMethod,
OnlineLinearBase,
)
from vllm.model_executor.layers.quantization.online.int8 import (
Int8OnlineMoEMethod,
@@ -71,6 +74,14 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
logger = init_logger(__name__)
class OnlineQuantizationSource(str, Enum):
"""Supported online quantization configuration sources."""
linear = "linear" # LinearBase
moe = "moe" # RoutedExperts
targets = "targets"
# Online dispatch tables, keyed by the QuantSpec.weight QuantKey. The
# corresponding method class handles the activation choice via its
# `supported_activation_quant` set.
@@ -200,12 +211,23 @@ class OnlineQuantizationConfig(QuantizationConfig):
"quantization='fp8_per_tensor'/'fp8_per_block' instead."
)
def _dispatch(
def _get_method_cls(
self,
spec: QuantSpec | None,
table: dict[QuantKey, type],
layer: torch.nn.Module,
) -> "QuantizeMethodBase | None":
) -> type | None:
"""Resolve the online method class for a layer's quantization spec.
Args:
spec: Quantization specification to resolve.
table: Mapping from weight quantization keys to method classes.
layer: Layer that will use the resolved method.
Returns:
The matching method class, or None when ``spec`` has no weight
quantization.
"""
if spec is None or spec.weight is None:
return None
cls = table.get(spec.weight)
@@ -223,13 +245,75 @@ class OnlineQuantizationConfig(QuantizationConfig):
f"activation override (activation={spec.activation}) is not "
f"yet supported for online {cls.__name__}"
)
if isinstance(layer, RoutedExperts):
return cls(layer=layer)
return cls()
return cls
def _dispatch_target(
def resolve_quant_method_cls(
self, layer: torch.nn.Module, prefix: str
) -> tuple[OnlineQuantizationSource, str, str | None, QuantSpec, type] | None:
"""Resolve quantization metadata and method class without instantiating it.
Args:
layer: Layer for which to resolve online quantization.
prefix: Fully qualified layer name.
Returns:
A tuple of source, quantization key string, target pattern, spec,
and method class. Returns None when online quantization does not
apply to the layer.
"""
quant_spec: QuantSpec | None
if self.args.targets is not None:
resolved_pattern = self._resolve_targets_quant_method_metadata(
prefix, layer
)
if resolved_pattern is None:
return None
source, quant_key_str, target_pattern, quant_spec, table = resolved_pattern
else:
if isinstance(layer, LinearBase):
source = OnlineQuantizationSource.linear
quant_spec = self.args.linear
table = _ONLINE_LINEAR_METHODS
elif isinstance(layer, RoutedExperts):
source = OnlineQuantizationSource.moe
quant_spec = self.args.moe
table = _ONLINE_MOE_METHODS
else:
return None
if should_ignore_layer(
prefix,
ignore=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
use_fnmatch=True,
):
return None
quant_key_str = str(quant_spec)
target_pattern = None
quant_method_cls = self._get_method_cls(quant_spec, table, layer)
if quant_method_cls is None:
return None
assert quant_spec is not None
return source, quant_key_str, target_pattern, quant_spec, quant_method_cls
def _resolve_targets_quant_method_metadata(
self, prefix: str, layer: torch.nn.Module
) -> "QuantizeMethodBase | None":
) -> (
tuple[OnlineQuantizationSource, str, str, QuantSpec, dict[QuantKey, type]]
| None
):
"""Resolve target-pattern quantization metadata and dispatch table.
Args:
prefix: Fully qualified layer name.
layer: Layer matched against configured target patterns.
Returns:
A tuple of source, quantization key string, target pattern, spec,
and dispatch table. Returns None when no pattern applies or the
layer is ignored.
"""
assert self.args.targets is not None
ignored = should_ignore_layer(
prefix,
@@ -254,7 +338,8 @@ class OnlineQuantizationConfig(QuantizationConfig):
f"targets patterns: {matches}. Each layer may match at most "
f"one target."
)
quant_key_str = self.args.targets[matches[0]]
target_pattern = matches[0]
quant_key_str = self.args.targets[target_pattern]
shorthand = _ONLINE_SHORTHANDS[quant_key_str]
if isinstance(layer, LinearBase):
quant_spec = shorthand.linear
@@ -265,63 +350,44 @@ class OnlineQuantizationConfig(QuantizationConfig):
else:
raise ValueError(
f"Layer {prefix} was matched by quantization_config.targets "
f"({matches[0]}), but online quantization is not supported for "
f"({target_pattern}), but online quantization is not supported for "
f"{type(layer).__name__}."
)
if quant_spec is None:
raise ValueError(
f"targets pattern {matches[0]} = {quant_key_str} does "
f"targets pattern {target_pattern} = {quant_key_str} does "
f"not define a QuantSpec for {type(layer).__name__} layers "
f"(matched at {prefix})."
)
method = self._dispatch(quant_spec, table, layer)
if method is not None:
self.quantized_layers[prefix] = ("targets", quant_key_str, matches[0])
return method
return (
OnlineQuantizationSource.targets,
quant_key_str,
target_pattern,
quant_spec,
table,
)
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
# `targets` takes precedence over `moe` and `linear` and is exclusive.
if self.args.targets is not None:
method = self._dispatch_target(prefix, layer)
if method is not None:
return method
if isinstance(layer, LinearBase):
return UnquantizedLinearMethod()
resolved = self.resolve_quant_method_cls(layer, prefix)
if resolved is not None:
source, quant_key_str, target_pattern, _, quant_method_cls = resolved
self.quantized_layers[prefix] = (
source.value,
quant_key_str,
target_pattern,
)
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
return None
assert issubclass(quant_method_cls, FusedMoEMethodBase)
return quant_method_cls(moe=layer.moe_config)
assert issubclass(quant_method_cls, OnlineLinearBase)
return quant_method_cls()
if isinstance(layer, LinearBase):
if should_ignore_layer(
prefix,
ignore=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
use_fnmatch=True,
):
return UnquantizedLinearMethod()
method = self._dispatch(self.args.linear, _ONLINE_LINEAR_METHODS, layer)
if method is not None:
self.quantized_layers[prefix] = (
"linear",
str(self.args.linear),
None,
)
return method
return UnquantizedLinearMethod()
elif isinstance(layer, RoutedExperts):
if should_ignore_layer(
prefix,
ignore=self.ignored_layers,
fused_mapping=self.packed_modules_mapping,
use_fnmatch=True,
):
return UnquantizedFusedMoEMethod(layer.moe_config)
method = self._dispatch(self.args.moe, _ONLINE_MOE_METHODS, layer)
if method is not None:
self.quantized_layers[prefix] = ("moe", str(self.args.moe), None)
return method
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
return None
@@ -23,6 +23,7 @@ from vllm.model_executor.kernels.linear.scaled_mm import (
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
select_fp8_moe_backend,
)
@@ -111,7 +112,7 @@ def _is_tp_sharded(layer: Module, *, reduces_output_dim: bool = True) -> bool:
return is_row_parallel or (reduces_output_dim and is_column_parallel)
class _Fp8OnlineLinearBase(LinearMethodBase):
class OnlineLinearBase(LinearMethodBase):
"""Shared base for online FP8 linear methods. Loads fp16/bf16 checkpoint
weights onto meta device and materializes them just-in-time."""
@@ -155,7 +156,7 @@ class _Fp8OnlineLinearBase(LinearMethodBase):
initialize_online_processing(layer)
class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
"""Online tensorwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-tensor during loading."""
@@ -256,7 +257,7 @@ class Fp8PerTensorOnlineLinearMethod(_Fp8OnlineLinearBase):
return self.fp8_linear.apply_weights(layer, x, bias)
class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
"""Online blockwise FP8 linear quantization.
Loads fp16/bf16 weights and quantizes them per-block during loading."""
@@ -336,7 +337,7 @@ class Fp8PerBlockOnlineLinearMethod(_Fp8OnlineLinearBase):
)
class Fp8PtpcOnlineLinearMethod(_Fp8OnlineLinearBase):
class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
"""Online PTPC FP8 linear quantization.
Per-output-channel weight scale + dynamic per-token activation scale. The
@@ -443,12 +444,12 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
self,
*,
weight_block_size: list[int] | None,
layer: torch.nn.Module,
moe: FusedMoEConfig,
weight_key: "QuantKey | None" = None,
activation_key: "QuantKey | None" = None,
allow_vllm_cutlass: bool = False,
):
super().__init__(layer.moe_config)
super().__init__(moe)
self.weight_block_size = weight_block_size
self.block_quant: bool = self.weight_block_size is not None
self.weight_scale_name = (
@@ -555,11 +556,11 @@ class Fp8PerTensorOnlineMoEMethod(_Fp8OnlineMoEBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
super().__init__(
weight_block_size=None,
layer=layer,
moe=moe,
)
def process_weights_after_loading(self, layer: Module) -> None:
@@ -612,11 +613,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
super().__init__(
weight_block_size=[128, 128],
layer=layer,
moe=moe,
)
def maybe_roundup_sizes(
@@ -716,13 +717,13 @@ class Fp8PtpcOnlineMoEMethod(_Fp8OnlineMoEBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
super().__init__(
weight_block_size=None,
layer=layer,
moe=moe,
weight_key=kFp8StaticChannelSym,
activation_key=kFp8DynamicTokenSym,
allow_vllm_cutlass=True,
@@ -12,6 +12,7 @@ if TYPE_CHECKING:
)
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
convert_to_int8_moe_kernel_format,
make_int8_moe_kernel,
@@ -38,9 +39,9 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
super().__init__(layer.moe_config)
super().__init__(moe)
self.int8_backend, self.experts_cls = select_int8_moe_backend(
config=self.moe,
weight_key=kInt8StaticChannelSym,
@@ -17,6 +17,7 @@ if TYPE_CHECKING:
)
from vllm.model_executor.kernels.linear import init_mxfp4_linear_kernel
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
@@ -27,7 +28,7 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_mxfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.online.fp8 import (
_Fp8OnlineLinearBase,
OnlineLinearBase,
)
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
@@ -73,7 +74,7 @@ def _quantize_mxfp4_moe_weight(
return w_quant, w_scales
class Mxfp4OnlineLinearMethod(_Fp8OnlineLinearBase):
class Mxfp4OnlineLinearMethod(OnlineLinearBase):
"""Online MXFP4 linear method.
Loads bf16/fp16 checkpoints and quantizes weights to MXFP4 (microscaling
FP4 with block-32 scales) during weight loading.
@@ -144,8 +145,8 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
experts_cls: "type[mk.FusedMoEExperts] | None"
activation_quant_key = kMxfp4Dynamic
def __init__(self, *, layer: torch.nn.Module):
super().__init__(layer.moe_config)
def __init__(self, *, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_block_size: list[int] = [1, MXFP4_BLOCK_SIZE]
self.weight_scale_name = "weight_scale"
@@ -8,6 +8,8 @@ from typing import TYPE_CHECKING
import torch
from torch.nn import Module
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
if TYPE_CHECKING:
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe import (
@@ -21,7 +23,7 @@ from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
select_mxfp8_moe_backend,
)
from vllm.model_executor.layers.quantization.online.fp8 import (
_Fp8OnlineLinearBase,
OnlineLinearBase,
)
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
@@ -34,7 +36,7 @@ from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
class Mxfp8OnlineLinearMethod(_Fp8OnlineLinearBase):
class Mxfp8OnlineLinearMethod(OnlineLinearBase):
"""Online MXFP8 linear method.
Loads bf16/fp16 checkpoints and quantizes weights to MXFP8 (microscaling
FP8 with block-32 scales) during weight loading.
@@ -100,8 +102,8 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
fp8_backend: "Fp8MoeBackend"
experts_cls: "type[mk.FusedMoEExperts] | None"
def __init__(self, *, layer: torch.nn.Module):
super().__init__(layer.moe_config)
def __init__(self, *, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_block_size: list[int] = [1, MXFP8_BLOCK_SIZE]
self.weight_scale_name = "weight_scale"
@@ -6,7 +6,10 @@ from torch.nn import Module
from vllm._custom_ops import scaled_fp4_quant
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
convert_to_nvfp4_moe_kernel_format,
make_nvfp4_moe_kernel,
@@ -87,13 +90,13 @@ class Nvfp4OnlineMoEMethod(OnlineMoEMethodBase):
def __init__(
self,
*,
layer: torch.nn.Module,
moe: FusedMoEConfig,
):
if not current_platform.is_device_capability_family(100):
raise ValueError(
"nvfp4_per_token online quantization requires a Blackwell (SM100) GPU."
)
super().__init__(layer.moe_config)
super().__init__(moe)
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
config=self.moe,
weight_key=kNvfp4Static,