[Online quantization] Add targeted online quantization configuration based on user patterns (#51285)

This commit is contained in:
fxmarty-amd
2026-09-02 08:55:15 -04:00
committed by GitHub
parent 2a4e3cc3db
commit 7894394b07
11 changed files with 892 additions and 172 deletions
@@ -58,6 +58,7 @@ class BaseModelLoader(ABC):
prefix=prefix,
)
log_online_quantization(vllm_config)
log_model_inspection(model)
logger.debug("Loading weights on %s ...", load_device)
@@ -92,6 +93,23 @@ def log_model_inspection(model: nn.Module) -> None:
logger.info("vLLM model structure:\n%s", format_model_inspection(model))
def log_online_quantization(vllm_config: VllmConfig) -> None:
"""Log the online-quantized layer count and types, when applicable."""
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
quant_config = vllm_config.quant_config
if not isinstance(quant_config, OnlineQuantizationConfig):
return
logger.info(
"Quantized %d layers of types: %s",
len(quant_config.quantized_layers),
"; ".join(quant_config.quantized_layer_summaries),
)
def _has_online_quant(model: nn.Module):
for module in model.modules():
quant_method = getattr(module, "quant_method", None)