[Quantization] Layer re-quantization for linear layers through online quantization API (MXFP8 -> FP8 PTPC showcase) (#55684)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-06 11:38:18 -05:00
committed by GitHub
co-authored by vllmellm Tan Pin Siang Misha Goin
parent 29f955cce0
commit f8d92cae41
12 changed files with 422 additions and 45 deletions
@@ -101,6 +101,7 @@ class BaseModelLoader(ABC):
finalize_layerwise_processing(model, model_config)
process_weights_after_loading(model, model_config, target_device)
log_online_quantization_time(vllm_config)
return model.eval()
@@ -115,8 +116,8 @@ def log_model_inspection(model: nn.Module) -> None:
logger.info("vLLM model structure:\n%s", format_model_inspection(model))
def log_online_quantization(vllm_config: VllmConfig) -> None:
"""Log the online-quantized layer count and types, when applicable."""
def _get_online_quantization_config(vllm_config: VllmConfig):
"""Return the online quantization config when one is configured."""
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
@@ -128,15 +129,35 @@ def log_online_quantization(vllm_config: VllmConfig) -> None:
if isinstance(online_quantization_config, OnlineQuantizationConfig):
quant_config = online_quantization_config
if not isinstance(quant_config, OnlineQuantizationConfig):
return None
return quant_config
def log_online_quantization(vllm_config: VllmConfig) -> None:
"""Log the online-quantized layer count and types, when applicable."""
quant_config = _get_online_quantization_config(vllm_config)
if quant_config is None:
return
logger.info(
"Quantized %d layers of types: %s",
"Quantizing %d layers of types: %s",
len(quant_config.quantized_layers),
"; ".join(quant_config.quantized_layer_summaries),
)
def log_online_quantization_time(vllm_config: VllmConfig) -> None:
"""Log online quantization processing time, when applicable."""
quant_config = _get_online_quantization_config(vllm_config)
if quant_config is None:
return
logger.info(
"Online quantization as part of model loading took %.2f seconds",
quant_config.online_quantization_time,
)
def _has_online_quant(model: nn.Module):
for module in model.modules():
quant_method = getattr(module, "quant_method", None)