[Quantization] Support online quantization with partially pre-quantized checkpoints (#51392)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
a11dfcff89
commit
5690b02c03
@@ -100,6 +100,11 @@ def log_online_quantization(vllm_config: VllmConfig) -> None:
|
||||
)
|
||||
|
||||
quant_config = vllm_config.quant_config
|
||||
online_quantization_config = getattr(
|
||||
quant_config, "online_quantization_config", None
|
||||
)
|
||||
if isinstance(online_quantization_config, OnlineQuantizationConfig):
|
||||
quant_config = online_quantization_config
|
||||
if not isinstance(quant_config, OnlineQuantizationConfig):
|
||||
return
|
||||
|
||||
|
||||
@@ -243,6 +243,26 @@ def get_quant_config(
|
||||
if model_config.quantization is None:
|
||||
raise ValueError("Model quantization method is not specified in the config.")
|
||||
quant_cls = get_quantization_config(model_config.quantization)
|
||||
from vllm.config.quantization import _ONLINE_SHORTHANDS, QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
|
||||
# resolve_quantization_config does not resolve `quantization_config` from
|
||||
# ambiguous shorthands as `"mxfp4"`, `"mxfp8"`.
|
||||
online_args = model_config.quantization_config
|
||||
|
||||
def maybe_compose_online_quantization(
|
||||
checkpoint_config: QuantizationConfig,
|
||||
) -> QuantizationConfig:
|
||||
if online_args is None:
|
||||
return checkpoint_config
|
||||
assert isinstance(online_args, QuantizationConfigArgs)
|
||||
|
||||
checkpoint_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
online_args
|
||||
)
|
||||
return checkpoint_config
|
||||
|
||||
# Read the quantization config from the HF model config, if available.
|
||||
hf_quant_config = getattr(model_config.hf_config, "quantization_config", None)
|
||||
@@ -284,7 +304,9 @@ def get_quant_config(
|
||||
):
|
||||
pass # fall through to file-based loading below
|
||||
else:
|
||||
return quant_cls.from_config(hf_quant_config)
|
||||
return maybe_compose_online_quantization(
|
||||
quant_cls.from_config(hf_quant_config)
|
||||
)
|
||||
|
||||
# if hf_quant_config is None, we will try to get config from
|
||||
# hf_overrides
|
||||
@@ -302,7 +324,9 @@ def get_quant_config(
|
||||
quantization_config_file = hf_overrides.get("quantization_config_file")
|
||||
if quantization_config_file is not None:
|
||||
if hasattr(quant_cls, "from_config_file"):
|
||||
return quant_cls.from_config_file(quantization_config_file)
|
||||
return maybe_compose_online_quantization(
|
||||
quant_cls.from_config_file(quantization_config_file)
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"from_config_file is specified in hf_override config, "
|
||||
@@ -312,7 +336,9 @@ def get_quant_config(
|
||||
quantization_config_json = hf_overrides.get("quantization_config_dict_json")
|
||||
if quantization_config_json is not None:
|
||||
if hasattr(quant_cls, "from_config_dict_json"):
|
||||
return quant_cls.from_config_dict_json(quantization_config_json)
|
||||
return maybe_compose_online_quantization(
|
||||
quant_cls.from_config_dict_json(quantization_config_json)
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"from_config_dict_json is specified in hf_override config, "
|
||||
@@ -320,16 +346,14 @@ def get_quant_config(
|
||||
f"{quant_cls}"
|
||||
)
|
||||
|
||||
# Online quantization doesn't read from checkpoint configs - it quantizes
|
||||
# fp16/bf16 weights on the fly during loading.
|
||||
if model_config.quantization_config is not None:
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
|
||||
assert isinstance(model_config.quantization_config, QuantizationConfigArgs)
|
||||
return OnlineQuantizationConfig(args=model_config.quantization_config)
|
||||
# Raw unambiguous online quantization doesn't read from checkpoint configs
|
||||
# We must continue to load/read the config below in two cases:
|
||||
# 1. composed online quantization, before `maybe_compose_online_quantization`,
|
||||
# 2. Ambiguous online shorthands as `"mxpf4"`, `"mxfp8"`, for which `online_args`
|
||||
# is not set yet.
|
||||
if quant_cls is OnlineQuantizationConfig and online_args is not None:
|
||||
assert isinstance(online_args, QuantizationConfigArgs)
|
||||
return OnlineQuantizationConfig(args=online_args)
|
||||
|
||||
model_name_or_path = (
|
||||
maybe_download_from_modelscope(
|
||||
@@ -359,7 +383,11 @@ def get_quant_config(
|
||||
|
||||
# If the quantization config is not found, use the default config.
|
||||
if not possible_config_filenames:
|
||||
return quant_cls()
|
||||
if model_config.quantization in _ONLINE_SHORTHANDS:
|
||||
args = online_args or _ONLINE_SHORTHANDS[model_config.quantization]
|
||||
assert isinstance(args, QuantizationConfigArgs)
|
||||
return OnlineQuantizationConfig(args=args)
|
||||
return maybe_compose_online_quantization(quant_cls())
|
||||
|
||||
config_files = glob.glob(os.path.join(hf_folder, "*.json"))
|
||||
|
||||
@@ -367,6 +395,10 @@ def get_quant_config(
|
||||
f for f in config_files if any(f.endswith(x) for x in possible_config_filenames)
|
||||
]
|
||||
if len(quant_config_files) == 0:
|
||||
if model_config.quantization in _ONLINE_SHORTHANDS:
|
||||
args = online_args or _ONLINE_SHORTHANDS[model_config.quantization]
|
||||
assert isinstance(args, QuantizationConfigArgs)
|
||||
return OnlineQuantizationConfig(args=args)
|
||||
raise ValueError(f"Cannot find the config file for {model_config.quantization}")
|
||||
if len(quant_config_files) > 1:
|
||||
raise ValueError(
|
||||
@@ -380,14 +412,14 @@ def get_quant_config(
|
||||
|
||||
if model_config.quantization in ("modelopt", "modelopt_mixed"):
|
||||
if config.get("producer", {}).get("name") == "modelopt":
|
||||
return quant_cls.from_config(config)
|
||||
return maybe_compose_online_quantization(quant_cls.from_config(config))
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported quantization config"
|
||||
f" found for {model_config.quantization} in {f}."
|
||||
)
|
||||
|
||||
return quant_cls.from_config(config)
|
||||
return maybe_compose_online_quantization(quant_cls.from_config(config))
|
||||
|
||||
|
||||
def get_sparse_attention_config(
|
||||
|
||||
Reference in New Issue
Block a user