[Quantization] Layer re-quantization for linear layers through online quantization API (MXFP8 -> FP8 PTPC showcase) (#55684)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-06 11:38:18 -05:00
committed by GitHub
co-authored by vllmellm Tan Pin Siang Misha Goin
parent 29f955cce0
commit f8d92cae41
12 changed files with 422 additions and 45 deletions
@@ -84,6 +84,13 @@ class QuantizeMethodBase(ABC):
"""
return
def dequantize_weight(self, layer: nn.Module) -> torch.Tensor:
"""Materialize a serialized quantized weight for requantization."""
raise NotImplementedError(
f"The quantization method {type(self)} does not implement "
"dequantize_weight. Please open an issue."
)
def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) -> bool:
"""Not all quant methods have embedding implemented, so we need to check that
@@ -294,13 +301,16 @@ def resolve_quant_method(
LinearBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
base_quant_method = quant_config.get_quant_method(layer, prefix)
if quant_config.online_quantization_config is None:
# No online configuration: retain the checkpoint method.
return base_quant_method
# Online quantization currently supports only LinearBase and RoutedExperts.
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
if not isinstance(layer, (LinearBase, RoutedExperts)):
# Online quantization only supports linear and routed MoE layers.
return base_quant_method
quant_config.online_quantization_config.packed_modules_mapping = (
@@ -312,14 +322,34 @@ def resolve_quant_method(
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
layer, prefix
)
if checkpoint_is_quantized:
if online_target is not None:
raise ValueError(
f"Cannot apply requested online quantization {online_target[3]} to "
f"pre-quantized layer {prefix}: {base_quant_method} was already "
"selected by the checkpoint quantization config."
if online_target is None:
# The checkpoint quant method is applied as there is no online override.
return base_quant_method
if isinstance(layer, RoutedExperts):
raise NotImplementedError(
"Requantizing checkpoint-quantized MoE layers is not supported."
)
return base_quant_method
online_quant_method = quant_config.online_quantization_config.get_quant_method(
layer, prefix
)
assert base_quant_method is not None and not isinstance(
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
)
assert isinstance(online_quant_method, OnlineLinearBase)
online_quant_method.set_requantization_source(base_quant_method)
# The online method dequantizes the checkpoint method before requantizing.
return online_quant_method
if online_target is None:
# The layer is unquantized and no online target applies.
return base_quant_method
# Quantize an unquantized layer with the online method.
return quant_config.online_quantization_config.get_quant_method(layer, prefix)