[Quantization] Layer re-quantization for linear layers through online quantization API (MXFP8 -> FP8 PTPC showcase) (#55684)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-06 11:38:18 -05:00
committed by GitHub
co-authored by vllmellm Tan Pin Siang Misha Goin
parent 29f955cce0
commit f8d92cae41
12 changed files with 422 additions and 45 deletions
@@ -84,6 +84,13 @@ class QuantizeMethodBase(ABC):
"""
return
def dequantize_weight(self, layer: nn.Module) -> torch.Tensor:
"""Materialize a serialized quantized weight for requantization."""
raise NotImplementedError(
f"The quantization method {type(self)} does not implement "
"dequantize_weight. Please open an issue."
)
def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) -> bool:
"""Not all quant methods have embedding implemented, so we need to check that
@@ -294,13 +301,16 @@ def resolve_quant_method(
LinearBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
base_quant_method = quant_config.get_quant_method(layer, prefix)
if quant_config.online_quantization_config is None:
# No online configuration: retain the checkpoint method.
return base_quant_method
# Online quantization currently supports only LinearBase and RoutedExperts.
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
if not isinstance(layer, (LinearBase, RoutedExperts)):
# Online quantization only supports linear and routed MoE layers.
return base_quant_method
quant_config.online_quantization_config.packed_modules_mapping = (
@@ -312,14 +322,34 @@ def resolve_quant_method(
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
layer, prefix
)
if checkpoint_is_quantized:
if online_target is not None:
raise ValueError(
f"Cannot apply requested online quantization {online_target[3]} to "
f"pre-quantized layer {prefix}: {base_quant_method} was already "
"selected by the checkpoint quantization config."
if online_target is None:
# The checkpoint quant method is applied as there is no online override.
return base_quant_method
if isinstance(layer, RoutedExperts):
raise NotImplementedError(
"Requantizing checkpoint-quantized MoE layers is not supported."
)
return base_quant_method
online_quant_method = quant_config.online_quantization_config.get_quant_method(
layer, prefix
)
assert base_quant_method is not None and not isinstance(
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
)
assert isinstance(online_quant_method, OnlineLinearBase)
online_quant_method.set_requantization_source(base_quant_method)
# The online method dequantizes the checkpoint method before requantizing.
return online_quant_method
if online_target is None:
# The layer is unquantized and no online target applies.
return base_quant_method
# Quantize an unquantized layer with the online method.
return quant_config.online_quantization_config.get_quant_method(layer, prefix)
@@ -76,6 +76,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
dequant_mxfp8_to_bf16,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP4_DTYPE,
@@ -2572,6 +2573,17 @@ class ModelOptLinearMethod(LinearMethodBase):
layer._nvfp4_group_size_for_gather = self.ctx.group_size
self.kernel.process_weights_after_loading(layer)
def dequantize_weight(self, layer: torch.nn.Module) -> torch.Tensor:
"""Reconstruct serialized weights for online requantization."""
if self.wkey.key is kMxfp8Static:
return dequant_mxfp8_to_bf16(
layer.weight.contiguous(), layer.weight_scale.contiguous()
)
else:
raise NotImplementedError(
"ModelOpt weight dequantization is only supported for MXFP8."
)
def apply(self, layer, x, bias=None):
return self.fmt.apply(
layer,
@@ -156,6 +156,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
self.args = args
self.ignored_layers: list[str] = args.ignore
self.quantized_layers: dict[str, tuple[str, str, str | None]] = {}
self.online_quantization_time = 0.0
@property
def quantized_layer_summaries(self) -> list[str]:
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.fusion.quant_activation import (
from vllm.model_executor.layers.linear import (
LinearMethodBase,
)
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
@@ -125,6 +126,13 @@ class OnlineLinearBase(LinearMethodBase):
def __init__(self):
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
self.requantization_source: QuantizeMethodBase | None = None
self.requantization_source_parameters: dict[str, torch.nn.Parameter] = {}
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
"""Configure serialized-weight conversion before online quantization."""
self.requantization_source = source_method
self.uses_meta_device = False
def create_weights(
self,
@@ -136,6 +144,24 @@ class OnlineLinearBase(LinearMethodBase):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
if self.requantization_source is not None:
existing_parameter_names = set(layer._parameters)
self.requantization_source.create_weights(
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
**extra_weight_attrs,
)
self.requantization_source_parameters = {
name: parameter
for name, parameter in layer._parameters.items()
if name not in existing_parameter_names and parameter is not None
}
return
output_size_per_partition = sum(output_partition_sizes)
weight_loader = extra_weight_attrs.get("weight_loader")
layer.logical_widths = output_partition_sizes
@@ -159,6 +185,19 @@ class OnlineLinearBase(LinearMethodBase):
initialize_online_processing(layer)
def release_requantization_source_weights(self, layer: torch.nn.Module) -> None:
"""Release checkpoint parameters after successful requantization."""
for name, source_parameter in self.requantization_source_parameters.items():
if layer._parameters.get(name) is source_parameter:
delattr(layer, name)
self.requantization_source_parameters.clear()
def get_weight_for_quantization(self, layer: Module) -> torch.Tensor:
"""Return checkpoint weights materialized for online quantization."""
if self.requantization_source is None:
return layer.weight
return self.requantization_source.dequantize_weight(layer)
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
"""Online tensorwise FP8 linear quantization.
@@ -213,10 +252,11 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
return
layer.input_scale = None
amax = weight_amax(layer.weight).reshape(1)
weight = self.get_weight_for_quantization(layer)
amax = weight_amax(weight).reshape(1)
amax = amax_for_tp_weight_quant(amax, _is_tp_sharded(layer))
weight_scale = _fp8_scale(amax)
qweight, _ = ops.scaled_fp8_quant(layer.weight, scale=weight_scale)
qweight, _ = ops.scaled_fp8_quant(weight, scale=weight_scale)
# Update layer with new values.
replace_parameter(layer, "weight", qweight.t().data)
@@ -312,14 +352,18 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
layer.input_scale = None
block_size = self.weight_block_size
weight = self.get_weight_for_quantization(layer)
qweight, weight_scale_inv = per_block_cast_to_fp8(
layer.weight, block_size=block_size, use_ue8m0=False
weight, block_size=block_size, use_ue8m0=False
)
replace_parameter(layer, "weight", qweight.data)
replace_parameter(layer, "weight_scale_inv", weight_scale_inv.data)
if self.requantization_source is not None and hasattr(layer, "weight_scale"):
del layer.weight_scale
self.fp8_linear.process_weights_after_loading(layer)
# Prevent duplicate processing (e.g., during weight reload)
@@ -395,18 +439,20 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
return
layer.input_scale = None
amax = weight_amax(layer.weight, dim=-1, keepdim=True)
weight = self.get_weight_for_quantization(layer)
amax = weight_amax(weight, dim=-1, keepdim=True)
amax = amax_for_tp_weight_quant(
amax, _is_tp_sharded(layer, reduces_output_dim=False)
)
weight_scale = _fp8_channel_scale(amax)
qweight = _fp8_quant_per_channel(layer.weight, weight_scale)
qweight = _fp8_quant_per_channel(weight, weight_scale)
replace_parameter(layer, "weight", qweight.t())
replace_parameter(layer, "weight_scale", weight_scale)
self.fp8_linear.process_weights_after_loading(layer)
expose_input_quant_key(layer, self.fp8_linear)
self.release_requantization_source_weights(layer)
layer._already_called_process_weights_after_loading = True
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.fused_moe import (
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.model_loader.reload.layerwise import (
initialize_online_processing,
)
@@ -24,6 +25,12 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
uses_meta_device: bool = True
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
"""Reject requantization from a checkpoint-quantized MoE method."""
raise NotImplementedError(
"Requantizing checkpoint-quantized MoE layers is not supported."
)
def create_weights(
self,
layer: torch.nn.Module,
@@ -119,7 +119,8 @@ class Mxfp4OnlineLinearMethod(OnlineLinearBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
weight_fp4, weight_scale = mxfp4_quantize(layer.weight.contiguous())
weight = self.get_weight_for_quantization(layer)
weight_fp4, weight_scale = mxfp4_quantize(weight.contiguous())
layer.input_scale = None
replace_parameter(layer, "weight", weight_fp4.data)
@@ -75,7 +75,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
weight_fp8, weight_scale = mxfp8_e4m3_quantize(layer.weight.contiguous())
weight = self.get_weight_for_quantization(layer)
weight_fp8, weight_scale = mxfp8_e4m3_quantize(weight.contiguous())
layer.input_scale = None
replace_parameter(layer, "weight", weight_fp8.data)
@@ -101,6 +101,7 @@ class BaseModelLoader(ABC):
finalize_layerwise_processing(model, model_config)
process_weights_after_loading(model, model_config, target_device)
log_online_quantization_time(vllm_config)
return model.eval()
@@ -115,8 +116,8 @@ def log_model_inspection(model: nn.Module) -> None:
logger.info("vLLM model structure:\n%s", format_model_inspection(model))
def log_online_quantization(vllm_config: VllmConfig) -> None:
"""Log the online-quantized layer count and types, when applicable."""
def _get_online_quantization_config(vllm_config: VllmConfig):
"""Return the online quantization config when one is configured."""
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
@@ -128,15 +129,35 @@ def log_online_quantization(vllm_config: VllmConfig) -> None:
if isinstance(online_quantization_config, OnlineQuantizationConfig):
quant_config = online_quantization_config
if not isinstance(quant_config, OnlineQuantizationConfig):
return None
return quant_config
def log_online_quantization(vllm_config: VllmConfig) -> None:
"""Log the online-quantized layer count and types, when applicable."""
quant_config = _get_online_quantization_config(vllm_config)
if quant_config is None:
return
logger.info(
"Quantized %d layers of types: %s",
"Quantizing %d layers of types: %s",
len(quant_config.quantized_layers),
"; ".join(quant_config.quantized_layer_summaries),
)
def log_online_quantization_time(vllm_config: VllmConfig) -> None:
"""Log online quantization processing time, when applicable."""
quant_config = _get_online_quantization_config(vllm_config)
if quant_config is None:
return
logger.info(
"Online quantization as part of model loading took %.2f seconds",
quant_config.online_quantization_time,
)
def _has_online_quant(model: nn.Module):
for module in model.modules():
quant_method = getattr(module, "quant_method", None)
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import inspect
from collections.abc import Callable
from contextlib import nullcontext
from functools import wraps
from weakref import WeakKeyDictionary, WeakSet
@@ -321,7 +322,21 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) -
_get_weight_loader(param)(*args.args, **args.kwargs)
if quant_method is not None:
quant_method.process_weights_after_loading(layer)
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
from vllm.model_executor.model_loader.utils import (
online_quantization_timing_context,
)
timing_context = (
online_quantization_timing_context(layer, quant_method)
if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
else nullcontext()
)
with timing_context:
quant_method.process_weights_after_loading(layer)
_copy_and_restore_kernel_tensors(layer, info)
@@ -356,7 +371,21 @@ def _layerwise_process(layer: torch.nn.Module, info: LayerReloadingInfo):
# Process weights (quantization, repacking, etc.)
quant_method = getattr(layer, "quant_method", None)
if isinstance(quant_method, QuantizeMethodBase):
quant_method.process_weights_after_loading(layer)
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
from vllm.model_executor.model_loader.utils import (
online_quantization_timing_context,
)
timing_context = (
online_quantization_timing_context(layer, quant_method)
if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
else nullcontext()
)
with timing_context:
quant_method.process_weights_after_loading(layer)
# Re-reconcile parameter TP state: process_weights_after_loading may
# have re-created Parameters (stamped with the global rank), which would
# otherwise break replicated (disable_tp) weights on a subsequent reload.
+49 -1
View File
@@ -3,6 +3,7 @@
"""Utilities for selecting and loading models."""
import inspect
import time
import warnings
from contextlib import contextmanager, nullcontext
from typing import Any
@@ -141,6 +142,11 @@ def process_weights_after_loading(
"""
# Reclaim memory when an explicit lm_head has been
# loaded, but it is identical to the input embeddings.
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
maybe_retie_word_embeddings(model, model_config)
for name, module in model.named_modules():
@@ -166,7 +172,12 @@ def process_weights_after_loading(
if quant_method.requires_device_loading
else nullcontext()
)
with loading_context:
timing_context = (
online_quantization_timing_context(module, quant_method)
if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
else nullcontext()
)
with loading_context, timing_context:
quant_method.process_weights_after_loading(module)
# process_weights_after_loading may swap in freshly-created
# Parameters (e.g. FP8 requantization), which are stamped with the
@@ -245,6 +256,43 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device)
p._vllm_is_uva_offloaded = True
@contextmanager
def online_quantization_timing_context(
layer: nn.Module, quant_method: QuantizeMethodBase
):
"""Accumulate online quantization processing time for ``quant_method``."""
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
assert isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
quant_config = layer.quant_config
assert quant_config is not None
online_quantization_config = (
quant_config
if isinstance(quant_config, OnlineQuantizationConfig)
else quant_config.online_quantization_config
)
# reload/layerwise.py's _layerwise_process calls this for experts_int8,
# whose online MoE method has no OnlineQuantizationConfig.
if online_quantization_config is None:
yield
return
assert isinstance(online_quantization_config, OnlineQuantizationConfig)
start_time = time.perf_counter()
try:
yield
finally:
online_quantization_config.online_quantization_time += (
time.perf_counter() - start_time
)
_MODEL_ARCH_BY_HASH = dict[int, tuple[type[nn.Module], str]]()
"""Caches the outputs of `_get_model_architecture`."""