[Quantization] Layer re-quantization for linear layers through online quantization API (MXFP8 -> FP8 PTPC showcase) (#55684)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
vllmellm
Tan Pin Siang
Misha Goin
parent
29f955cce0
commit
f8d92cae41
@@ -84,6 +84,13 @@ class QuantizeMethodBase(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
def dequantize_weight(self, layer: nn.Module) -> torch.Tensor:
|
||||
"""Materialize a serialized quantized weight for requantization."""
|
||||
raise NotImplementedError(
|
||||
f"The quantization method {type(self)} does not implement "
|
||||
"dequantize_weight. Please open an issue."
|
||||
)
|
||||
|
||||
|
||||
def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) -> bool:
|
||||
"""Not all quant methods have embedding implemented, so we need to check that
|
||||
@@ -294,13 +301,16 @@ def resolve_quant_method(
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
|
||||
base_quant_method = quant_config.get_quant_method(layer, prefix)
|
||||
if quant_config.online_quantization_config is None:
|
||||
# No online configuration: retain the checkpoint method.
|
||||
return base_quant_method
|
||||
# Online quantization currently supports only LinearBase and RoutedExperts.
|
||||
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
|
||||
if not isinstance(layer, (LinearBase, RoutedExperts)):
|
||||
# Online quantization only supports linear and routed MoE layers.
|
||||
return base_quant_method
|
||||
|
||||
quant_config.online_quantization_config.packed_modules_mapping = (
|
||||
@@ -312,14 +322,34 @@ def resolve_quant_method(
|
||||
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
|
||||
layer, prefix
|
||||
)
|
||||
|
||||
if checkpoint_is_quantized:
|
||||
if online_target is not None:
|
||||
raise ValueError(
|
||||
f"Cannot apply requested online quantization {online_target[3]} to "
|
||||
f"pre-quantized layer {prefix}: {base_quant_method} was already "
|
||||
"selected by the checkpoint quantization config."
|
||||
if online_target is None:
|
||||
# The checkpoint quant method is applied as there is no online override.
|
||||
return base_quant_method
|
||||
|
||||
if isinstance(layer, RoutedExperts):
|
||||
raise NotImplementedError(
|
||||
"Requantizing checkpoint-quantized MoE layers is not supported."
|
||||
)
|
||||
return base_quant_method
|
||||
|
||||
online_quant_method = quant_config.online_quantization_config.get_quant_method(
|
||||
layer, prefix
|
||||
)
|
||||
|
||||
assert base_quant_method is not None and not isinstance(
|
||||
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
|
||||
)
|
||||
|
||||
assert isinstance(online_quant_method, OnlineLinearBase)
|
||||
online_quant_method.set_requantization_source(base_quant_method)
|
||||
|
||||
# The online method dequantizes the checkpoint method before requantizing.
|
||||
return online_quant_method
|
||||
|
||||
if online_target is None:
|
||||
# The layer is unquantized and no online target applies.
|
||||
return base_quant_method
|
||||
|
||||
# Quantize an unquantized layer with the online method.
|
||||
return quant_config.online_quantization_config.get_quant_method(layer, prefix)
|
||||
|
||||
@@ -76,6 +76,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
FP4_DTYPE,
|
||||
@@ -2572,6 +2573,17 @@ class ModelOptLinearMethod(LinearMethodBase):
|
||||
layer._nvfp4_group_size_for_gather = self.ctx.group_size
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def dequantize_weight(self, layer: torch.nn.Module) -> torch.Tensor:
|
||||
"""Reconstruct serialized weights for online requantization."""
|
||||
if self.wkey.key is kMxfp8Static:
|
||||
return dequant_mxfp8_to_bf16(
|
||||
layer.weight.contiguous(), layer.weight_scale.contiguous()
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"ModelOpt weight dequantization is only supported for MXFP8."
|
||||
)
|
||||
|
||||
def apply(self, layer, x, bias=None):
|
||||
return self.fmt.apply(
|
||||
layer,
|
||||
|
||||
@@ -156,6 +156,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
self.args = args
|
||||
self.ignored_layers: list[str] = args.ignore
|
||||
self.quantized_layers: dict[str, tuple[str, str, str | None]] = {}
|
||||
self.online_quantization_time = 0.0
|
||||
|
||||
@property
|
||||
def quantized_layer_summaries(self) -> list[str]:
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
@@ -125,6 +126,13 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
def __init__(self):
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
self.input_dtype = get_current_vllm_config().model_config.dtype
|
||||
self.requantization_source: QuantizeMethodBase | None = None
|
||||
self.requantization_source_parameters: dict[str, torch.nn.Parameter] = {}
|
||||
|
||||
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
|
||||
"""Configure serialized-weight conversion before online quantization."""
|
||||
self.requantization_source = source_method
|
||||
self.uses_meta_device = False
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -136,6 +144,24 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
if self.requantization_source is not None:
|
||||
existing_parameter_names = set(layer._parameters)
|
||||
self.requantization_source.create_weights(
|
||||
layer,
|
||||
input_size_per_partition,
|
||||
output_partition_sizes,
|
||||
input_size,
|
||||
output_size,
|
||||
params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
self.requantization_source_parameters = {
|
||||
name: parameter
|
||||
for name, parameter in layer._parameters.items()
|
||||
if name not in existing_parameter_names and parameter is not None
|
||||
}
|
||||
return
|
||||
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
layer.logical_widths = output_partition_sizes
|
||||
@@ -159,6 +185,19 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
|
||||
initialize_online_processing(layer)
|
||||
|
||||
def release_requantization_source_weights(self, layer: torch.nn.Module) -> None:
|
||||
"""Release checkpoint parameters after successful requantization."""
|
||||
for name, source_parameter in self.requantization_source_parameters.items():
|
||||
if layer._parameters.get(name) is source_parameter:
|
||||
delattr(layer, name)
|
||||
self.requantization_source_parameters.clear()
|
||||
|
||||
def get_weight_for_quantization(self, layer: Module) -> torch.Tensor:
|
||||
"""Return checkpoint weights materialized for online quantization."""
|
||||
if self.requantization_source is None:
|
||||
return layer.weight
|
||||
return self.requantization_source.dequantize_weight(layer)
|
||||
|
||||
|
||||
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online tensorwise FP8 linear quantization.
|
||||
@@ -213,10 +252,11 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
return
|
||||
|
||||
layer.input_scale = None
|
||||
amax = weight_amax(layer.weight).reshape(1)
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
amax = weight_amax(weight).reshape(1)
|
||||
amax = amax_for_tp_weight_quant(amax, _is_tp_sharded(layer))
|
||||
weight_scale = _fp8_scale(amax)
|
||||
qweight, _ = ops.scaled_fp8_quant(layer.weight, scale=weight_scale)
|
||||
qweight, _ = ops.scaled_fp8_quant(weight, scale=weight_scale)
|
||||
|
||||
# Update layer with new values.
|
||||
replace_parameter(layer, "weight", qweight.t().data)
|
||||
@@ -312,14 +352,18 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
|
||||
layer.input_scale = None
|
||||
block_size = self.weight_block_size
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
|
||||
qweight, weight_scale_inv = per_block_cast_to_fp8(
|
||||
layer.weight, block_size=block_size, use_ue8m0=False
|
||||
weight, block_size=block_size, use_ue8m0=False
|
||||
)
|
||||
|
||||
replace_parameter(layer, "weight", qweight.data)
|
||||
replace_parameter(layer, "weight_scale_inv", weight_scale_inv.data)
|
||||
|
||||
if self.requantization_source is not None and hasattr(layer, "weight_scale"):
|
||||
del layer.weight_scale
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
# Prevent duplicate processing (e.g., during weight reload)
|
||||
@@ -395,18 +439,20 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
return
|
||||
|
||||
layer.input_scale = None
|
||||
amax = weight_amax(layer.weight, dim=-1, keepdim=True)
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
amax = weight_amax(weight, dim=-1, keepdim=True)
|
||||
amax = amax_for_tp_weight_quant(
|
||||
amax, _is_tp_sharded(layer, reduces_output_dim=False)
|
||||
)
|
||||
weight_scale = _fp8_channel_scale(amax)
|
||||
qweight = _fp8_quant_per_channel(layer.weight, weight_scale)
|
||||
qweight = _fp8_quant_per_channel(weight, weight_scale)
|
||||
|
||||
replace_parameter(layer, "weight", qweight.t())
|
||||
replace_parameter(layer, "weight_scale", weight_scale)
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
expose_input_quant_key(layer, self.fp8_linear)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.model_loader.reload.layerwise import (
|
||||
initialize_online_processing,
|
||||
)
|
||||
@@ -24,6 +25,12 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
|
||||
uses_meta_device: bool = True
|
||||
|
||||
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
|
||||
"""Reject requantization from a checkpoint-quantized MoE method."""
|
||||
raise NotImplementedError(
|
||||
"Requantizing checkpoint-quantized MoE layers is not supported."
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
|
||||
@@ -119,7 +119,8 @@ class Mxfp4OnlineLinearMethod(OnlineLinearBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
weight_fp4, weight_scale = mxfp4_quantize(layer.weight.contiguous())
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
weight_fp4, weight_scale = mxfp4_quantize(weight.contiguous())
|
||||
|
||||
layer.input_scale = None
|
||||
replace_parameter(layer, "weight", weight_fp4.data)
|
||||
|
||||
@@ -75,7 +75,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
weight_fp8, weight_scale = mxfp8_e4m3_quantize(layer.weight.contiguous())
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
weight_fp8, weight_scale = mxfp8_e4m3_quantize(weight.contiguous())
|
||||
|
||||
layer.input_scale = None
|
||||
replace_parameter(layer, "weight", weight_fp8.data)
|
||||
|
||||
Reference in New Issue
Block a user