[Quantization] Layer re-quantization for linear layers through online quantization API (MXFP8 -> FP8 PTPC showcase) (#55684)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
vllmellm
Tan Pin Siang
Misha Goin
parent
29f955cce0
commit
f8d92cae41
@@ -84,6 +84,13 @@ class QuantizeMethodBase(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
def dequantize_weight(self, layer: nn.Module) -> torch.Tensor:
|
||||
"""Materialize a serialized quantized weight for requantization."""
|
||||
raise NotImplementedError(
|
||||
f"The quantization method {type(self)} does not implement "
|
||||
"dequantize_weight. Please open an issue."
|
||||
)
|
||||
|
||||
|
||||
def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) -> bool:
|
||||
"""Not all quant methods have embedding implemented, so we need to check that
|
||||
@@ -294,13 +301,16 @@ def resolve_quant_method(
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
|
||||
base_quant_method = quant_config.get_quant_method(layer, prefix)
|
||||
if quant_config.online_quantization_config is None:
|
||||
# No online configuration: retain the checkpoint method.
|
||||
return base_quant_method
|
||||
# Online quantization currently supports only LinearBase and RoutedExperts.
|
||||
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
|
||||
if not isinstance(layer, (LinearBase, RoutedExperts)):
|
||||
# Online quantization only supports linear and routed MoE layers.
|
||||
return base_quant_method
|
||||
|
||||
quant_config.online_quantization_config.packed_modules_mapping = (
|
||||
@@ -312,14 +322,34 @@ def resolve_quant_method(
|
||||
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
|
||||
layer, prefix
|
||||
)
|
||||
|
||||
if checkpoint_is_quantized:
|
||||
if online_target is not None:
|
||||
raise ValueError(
|
||||
f"Cannot apply requested online quantization {online_target[3]} to "
|
||||
f"pre-quantized layer {prefix}: {base_quant_method} was already "
|
||||
"selected by the checkpoint quantization config."
|
||||
if online_target is None:
|
||||
# The checkpoint quant method is applied as there is no online override.
|
||||
return base_quant_method
|
||||
|
||||
if isinstance(layer, RoutedExperts):
|
||||
raise NotImplementedError(
|
||||
"Requantizing checkpoint-quantized MoE layers is not supported."
|
||||
)
|
||||
return base_quant_method
|
||||
|
||||
online_quant_method = quant_config.online_quantization_config.get_quant_method(
|
||||
layer, prefix
|
||||
)
|
||||
|
||||
assert base_quant_method is not None and not isinstance(
|
||||
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
|
||||
)
|
||||
|
||||
assert isinstance(online_quant_method, OnlineLinearBase)
|
||||
online_quant_method.set_requantization_source(base_quant_method)
|
||||
|
||||
# The online method dequantizes the checkpoint method before requantizing.
|
||||
return online_quant_method
|
||||
|
||||
if online_target is None:
|
||||
# The layer is unquantized and no online target applies.
|
||||
return base_quant_method
|
||||
|
||||
# Quantize an unquantized layer with the online method.
|
||||
return quant_config.online_quantization_config.get_quant_method(layer, prefix)
|
||||
|
||||
Reference in New Issue
Block a user