[Quantization] Layer re-quantization for linear layers through online quantization API (MXFP8 -> FP8 PTPC showcase) (#55684)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-06 11:38:18 -05:00
committed by GitHub
co-authored by vllmellm Tan Pin Siang Misha Goin
parent 29f955cce0
commit f8d92cae41
12 changed files with 422 additions and 45 deletions
+49 -1
View File
@@ -3,6 +3,7 @@
"""Utilities for selecting and loading models."""
import inspect
import time
import warnings
from contextlib import contextmanager, nullcontext
from typing import Any
@@ -141,6 +142,11 @@ def process_weights_after_loading(
"""
# Reclaim memory when an explicit lm_head has been
# loaded, but it is identical to the input embeddings.
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
maybe_retie_word_embeddings(model, model_config)
for name, module in model.named_modules():
@@ -166,7 +172,12 @@ def process_weights_after_loading(
if quant_method.requires_device_loading
else nullcontext()
)
with loading_context:
timing_context = (
online_quantization_timing_context(module, quant_method)
if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
else nullcontext()
)
with loading_context, timing_context:
quant_method.process_weights_after_loading(module)
# process_weights_after_loading may swap in freshly-created
# Parameters (e.g. FP8 requantization), which are stamped with the
@@ -245,6 +256,43 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device)
p._vllm_is_uva_offloaded = True
@contextmanager
def online_quantization_timing_context(
layer: nn.Module, quant_method: QuantizeMethodBase
):
"""Accumulate online quantization processing time for ``quant_method``."""
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
assert isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
quant_config = layer.quant_config
assert quant_config is not None
online_quantization_config = (
quant_config
if isinstance(quant_config, OnlineQuantizationConfig)
else quant_config.online_quantization_config
)
# reload/layerwise.py's _layerwise_process calls this for experts_int8,
# whose online MoE method has no OnlineQuantizationConfig.
if online_quantization_config is None:
yield
return
assert isinstance(online_quantization_config, OnlineQuantizationConfig)
start_time = time.perf_counter()
try:
yield
finally:
online_quantization_config.online_quantization_time += (
time.perf_counter() - start_time
)
_MODEL_ARCH_BY_HASH = dict[int, tuple[type[nn.Module], str]]()
"""Caches the outputs of `_get_model_architecture`."""