[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
a11dfcff89
commit
5690b02c03
@@ -12,6 +12,9 @@ from transformers import PretrainedConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.quantization import QuantizationMethods
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
else:
|
||||
QuantizationMethods = str
|
||||
@@ -105,6 +108,7 @@ class QuantizationConfig(ABC):
|
||||
"""Suffixes of quantization parameters that may be present in the checkpoint but
|
||||
not in the model, and should be ignored if unexpected during loading. These are used
|
||||
after remapping, so should be in vLLM format (e.g. .q_scale, not .q.scale)."""
|
||||
online_quantization_config: "OnlineQuantizationConfig | None" = None
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -187,7 +191,8 @@ class QuantizationConfig(ABC):
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> QuantizeMethodBase | None:
|
||||
"""Get the quantize method to use for the quantized layer.
|
||||
"""Get the quantize method to use for the quantized layer, from the
|
||||
pre-quantized checkpoint quant_method.
|
||||
|
||||
Args:
|
||||
layer: The layer for the quant method.
|
||||
@@ -265,3 +270,46 @@ class QuantizationConfig(ABC):
|
||||
# TODO: revision is never passed currently in vllm.py,
|
||||
# but is used in subclasses, should we remove this parameter?
|
||||
pass
|
||||
|
||||
|
||||
def resolve_quant_method(
|
||||
quant_config: QuantizationConfig, layer: torch.nn.Module, prefix: str
|
||||
) -> QuantizeMethodBase | None:
|
||||
"""Return the checkpoint method with configured online quantization."""
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
|
||||
base_quant_method = quant_config.get_quant_method(layer, prefix)
|
||||
if quant_config.online_quantization_config is None:
|
||||
return base_quant_method
|
||||
# Online quantization currently supports only LinearBase and RoutedExperts.
|
||||
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
|
||||
if not isinstance(layer, (LinearBase, RoutedExperts)):
|
||||
return base_quant_method
|
||||
|
||||
quant_config.online_quantization_config.packed_modules_mapping = (
|
||||
quant_config.packed_modules_mapping
|
||||
)
|
||||
checkpoint_is_quantized = base_quant_method is not None and not isinstance(
|
||||
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
|
||||
)
|
||||
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
|
||||
layer, prefix
|
||||
)
|
||||
if checkpoint_is_quantized:
|
||||
if online_target is not None:
|
||||
raise ValueError(
|
||||
f"Cannot apply requested online quantization {online_target[3]} to "
|
||||
f"pre-quantized layer {prefix}: {base_quant_method} was already "
|
||||
"selected by the checkpoint quantization config."
|
||||
)
|
||||
return base_quant_method
|
||||
if online_target is None:
|
||||
return base_quant_method
|
||||
return quant_config.online_quantization_config.get_quant_method(layer, prefix)
|
||||
|
||||
Reference in New Issue
Block a user