[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
a11dfcff89
commit
5690b02c03
@@ -225,6 +225,10 @@ ONLINE_QUANT_SHORTHAND_NAMES: tuple[str, ...] = (
|
||||
"online",
|
||||
)
|
||||
|
||||
# These names are also checkpoint quantization methods. Their online configs
|
||||
# are resolved only when checkpoint quantization metadata is absent.
|
||||
_DEFERRED_ONLINE_SHORTHANDS = frozenset(("mxfp4", "mxfp8"))
|
||||
|
||||
|
||||
def resolve_quantization_config(
|
||||
quantization: str | None,
|
||||
@@ -239,17 +243,25 @@ def resolve_quantization_config(
|
||||
take precedence over the shorthand.
|
||||
"""
|
||||
if quantization is not None and quantization not in ONLINE_QUANT_SHORTHAND_NAMES:
|
||||
if quantization_config is not None:
|
||||
raise ValueError(
|
||||
f"quantization_config is only supported when quantization is "
|
||||
f"one of {sorted(ONLINE_QUANT_SHORTHAND_NAMES)}, "
|
||||
f"got quantization={quantization!r}"
|
||||
)
|
||||
return None
|
||||
# Pre-quantized checkpoints can be composed with online quantization
|
||||
# for layers that the base quant_method leaves unquantized. The
|
||||
# checkpoint quant_method remains the primary quantization method; composition
|
||||
# is performed after its config has been loaded.
|
||||
if quantization_config is None:
|
||||
return None
|
||||
|
||||
# `quantization_config` may hold both:
|
||||
# 1. Base quantization method activation key override,
|
||||
# 2. online quantization config to apply on top of the base quant_method.
|
||||
if isinstance(quantization_config, dict):
|
||||
return QuantizationConfigArgs(**quantization_config)
|
||||
return quantization_config
|
||||
|
||||
base = _ONLINE_SHORTHANDS.get(quantization) if quantization else None
|
||||
|
||||
if quantization_config is None:
|
||||
if quantization in _DEFERRED_ONLINE_SHORTHANDS:
|
||||
return None
|
||||
return base
|
||||
|
||||
if isinstance(quantization_config, dict):
|
||||
|
||||
Reference in New Issue
Block a user