add mixed precision support for modelopt (#35047)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
@@ -287,7 +287,17 @@ def get_quant_config(
|
||||
)
|
||||
|
||||
if hf_quant_config is not None:
|
||||
return quant_cls.from_config(hf_quant_config)
|
||||
# For modelopt_mixed, config.json's quantization_config may or may
|
||||
# not contain the per-layer quantized_layers map. Newer checkpoints
|
||||
# embed it directly; older ones keep it only in hf_quant_config.json.
|
||||
# If it is missing, fall through to the file-based loading path.
|
||||
if (
|
||||
model_config.quantization == "modelopt_mixed"
|
||||
and "quantized_layers" not in hf_quant_config
|
||||
):
|
||||
pass # fall through to file-based loading below
|
||||
else:
|
||||
return quant_cls.from_config(hf_quant_config)
|
||||
|
||||
# if hf_quant_config is None, we will try to get config from
|
||||
# hf_overrides
|
||||
@@ -365,8 +375,8 @@ def get_quant_config(
|
||||
|
||||
if model_config.quantization == "bitsandbytes":
|
||||
config["adapter_name_or_path"] = model_config.model
|
||||
elif model_config.quantization == "modelopt":
|
||||
if config["producer"]["name"] == "modelopt":
|
||||
elif model_config.quantization in ("modelopt", "modelopt_mixed"):
|
||||
if config.get("producer", {}).get("name") == "modelopt":
|
||||
return quant_cls.from_config(config)
|
||||
else:
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user