[Quantization] Layer re-quantization for linear layers through online quantization API (MXFP8 -> FP8 PTPC showcase) (#55684)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
vllmellm
Tan Pin Siang
Misha Goin
parent
29f955cce0
commit
f8d92cae41
@@ -101,6 +101,7 @@ class BaseModelLoader(ABC):
|
||||
finalize_layerwise_processing(model, model_config)
|
||||
|
||||
process_weights_after_loading(model, model_config, target_device)
|
||||
log_online_quantization_time(vllm_config)
|
||||
|
||||
return model.eval()
|
||||
|
||||
@@ -115,8 +116,8 @@ def log_model_inspection(model: nn.Module) -> None:
|
||||
logger.info("vLLM model structure:\n%s", format_model_inspection(model))
|
||||
|
||||
|
||||
def log_online_quantization(vllm_config: VllmConfig) -> None:
|
||||
"""Log the online-quantized layer count and types, when applicable."""
|
||||
def _get_online_quantization_config(vllm_config: VllmConfig):
|
||||
"""Return the online quantization config when one is configured."""
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
@@ -128,15 +129,35 @@ def log_online_quantization(vllm_config: VllmConfig) -> None:
|
||||
if isinstance(online_quantization_config, OnlineQuantizationConfig):
|
||||
quant_config = online_quantization_config
|
||||
if not isinstance(quant_config, OnlineQuantizationConfig):
|
||||
return None
|
||||
return quant_config
|
||||
|
||||
|
||||
def log_online_quantization(vllm_config: VllmConfig) -> None:
|
||||
"""Log the online-quantized layer count and types, when applicable."""
|
||||
quant_config = _get_online_quantization_config(vllm_config)
|
||||
if quant_config is None:
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Quantized %d layers of types: %s",
|
||||
"Quantizing %d layers of types: %s",
|
||||
len(quant_config.quantized_layers),
|
||||
"; ".join(quant_config.quantized_layer_summaries),
|
||||
)
|
||||
|
||||
|
||||
def log_online_quantization_time(vllm_config: VllmConfig) -> None:
|
||||
"""Log online quantization processing time, when applicable."""
|
||||
quant_config = _get_online_quantization_config(vllm_config)
|
||||
if quant_config is None:
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Online quantization as part of model loading took %.2f seconds",
|
||||
quant_config.online_quantization_time,
|
||||
)
|
||||
|
||||
|
||||
def _has_online_quant(model: nn.Module):
|
||||
for module in model.modules():
|
||||
quant_method = getattr(module, "quant_method", None)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import inspect
|
||||
from collections.abc import Callable
|
||||
from contextlib import nullcontext
|
||||
from functools import wraps
|
||||
from weakref import WeakKeyDictionary, WeakSet
|
||||
|
||||
@@ -321,7 +322,21 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) -
|
||||
_get_weight_loader(param)(*args.args, **args.kwargs)
|
||||
|
||||
if quant_method is not None:
|
||||
quant_method.process_weights_after_loading(layer)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.model_loader.utils import (
|
||||
online_quantization_timing_context,
|
||||
)
|
||||
|
||||
timing_context = (
|
||||
online_quantization_timing_context(layer, quant_method)
|
||||
if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
|
||||
else nullcontext()
|
||||
)
|
||||
with timing_context:
|
||||
quant_method.process_weights_after_loading(layer)
|
||||
|
||||
_copy_and_restore_kernel_tensors(layer, info)
|
||||
|
||||
@@ -356,7 +371,21 @@ def _layerwise_process(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
# Process weights (quantization, repacking, etc.)
|
||||
quant_method = getattr(layer, "quant_method", None)
|
||||
if isinstance(quant_method, QuantizeMethodBase):
|
||||
quant_method.process_weights_after_loading(layer)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.model_loader.utils import (
|
||||
online_quantization_timing_context,
|
||||
)
|
||||
|
||||
timing_context = (
|
||||
online_quantization_timing_context(layer, quant_method)
|
||||
if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
|
||||
else nullcontext()
|
||||
)
|
||||
with timing_context:
|
||||
quant_method.process_weights_after_loading(layer)
|
||||
# Re-reconcile parameter TP state: process_weights_after_loading may
|
||||
# have re-created Parameters (stamped with the global rank), which would
|
||||
# otherwise break replicated (disable_tp) weights on a subsequent reload.
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"""Utilities for selecting and loading models."""
|
||||
|
||||
import inspect
|
||||
import time
|
||||
import warnings
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from typing import Any
|
||||
@@ -141,6 +142,11 @@ def process_weights_after_loading(
|
||||
"""
|
||||
# Reclaim memory when an explicit lm_head has been
|
||||
# loaded, but it is identical to the input embeddings.
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
|
||||
maybe_retie_word_embeddings(model, model_config)
|
||||
|
||||
for name, module in model.named_modules():
|
||||
@@ -166,7 +172,12 @@ def process_weights_after_loading(
|
||||
if quant_method.requires_device_loading
|
||||
else nullcontext()
|
||||
)
|
||||
with loading_context:
|
||||
timing_context = (
|
||||
online_quantization_timing_context(module, quant_method)
|
||||
if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
|
||||
else nullcontext()
|
||||
)
|
||||
with loading_context, timing_context:
|
||||
quant_method.process_weights_after_loading(module)
|
||||
# process_weights_after_loading may swap in freshly-created
|
||||
# Parameters (e.g. FP8 requantization), which are stamped with the
|
||||
@@ -245,6 +256,43 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device)
|
||||
p._vllm_is_uva_offloaded = True
|
||||
|
||||
|
||||
@contextmanager
|
||||
def online_quantization_timing_context(
|
||||
layer: nn.Module, quant_method: QuantizeMethodBase
|
||||
):
|
||||
"""Accumulate online quantization processing time for ``quant_method``."""
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
|
||||
assert isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
|
||||
quant_config = layer.quant_config
|
||||
assert quant_config is not None
|
||||
online_quantization_config = (
|
||||
quant_config
|
||||
if isinstance(quant_config, OnlineQuantizationConfig)
|
||||
else quant_config.online_quantization_config
|
||||
)
|
||||
# reload/layerwise.py's _layerwise_process calls this for experts_int8,
|
||||
# whose online MoE method has no OnlineQuantizationConfig.
|
||||
if online_quantization_config is None:
|
||||
yield
|
||||
return
|
||||
assert isinstance(online_quantization_config, OnlineQuantizationConfig)
|
||||
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
online_quantization_config.online_quantization_time += (
|
||||
time.perf_counter() - start_time
|
||||
)
|
||||
|
||||
|
||||
_MODEL_ARCH_BY_HASH = dict[int, tuple[type[nn.Module], str]]()
|
||||
"""Caches the outputs of `_get_model_architecture`."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user