[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-09-04 11:46:06 -07:00
committed by GitHub
co-authored by Codex mgoin
parent a11dfcff89
commit 5690b02c03
25 changed files with 846 additions and 175 deletions
@@ -12,6 +12,9 @@ from transformers import PretrainedConfig
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization import QuantizationMethods
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
from vllm.model_executor.models.utils import WeightsMapper
else:
QuantizationMethods = str
@@ -105,6 +108,7 @@ class QuantizationConfig(ABC):
"""Suffixes of quantization parameters that may be present in the checkpoint but
not in the model, and should be ignored if unexpected during loading. These are used
after remapping, so should be in vLLM format (e.g. .q_scale, not .q.scale)."""
online_quantization_config: "OnlineQuantizationConfig | None" = None
def __init__(self):
super().__init__()
@@ -187,7 +191,8 @@ class QuantizationConfig(ABC):
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> QuantizeMethodBase | None:
"""Get the quantize method to use for the quantized layer.
"""Get the quantize method to use for the quantized layer, from the
pre-quantized checkpoint quant_method.
Args:
layer: The layer for the quant method.
@@ -265,3 +270,46 @@ class QuantizationConfig(ABC):
# TODO: revision is never passed currently in vllm.py,
# but is used in subclasses, should we remove this parameter?
pass
def resolve_quant_method(
quant_config: QuantizationConfig, layer: torch.nn.Module, prefix: str
) -> QuantizeMethodBase | None:
"""Return the checkpoint method with configured online quantization."""
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
base_quant_method = quant_config.get_quant_method(layer, prefix)
if quant_config.online_quantization_config is None:
return base_quant_method
# Online quantization currently supports only LinearBase and RoutedExperts.
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
if not isinstance(layer, (LinearBase, RoutedExperts)):
return base_quant_method
quant_config.online_quantization_config.packed_modules_mapping = (
quant_config.packed_modules_mapping
)
checkpoint_is_quantized = base_quant_method is not None and not isinstance(
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
)
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
layer, prefix
)
if checkpoint_is_quantized:
if online_target is not None:
raise ValueError(
f"Cannot apply requested online quantization {online_target[3]} to "
f"pre-quantized layer {prefix}: {base_quant_method} was already "
"selected by the checkpoint quantization config."
)
return base_quant_method
if online_target is None:
return base_quant_method
return quant_config.online_quantization_config.get_quant_method(layer, prefix)