add mixed precision support for modelopt (#35047)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
sychen52
2026-02-26 21:56:24 +00:00
committed by GitHub
parent 832a780f3a
commit d0105b84f0
4 changed files with 250 additions and 83 deletions
@@ -287,7 +287,17 @@ def get_quant_config(
)
if hf_quant_config is not None:
return quant_cls.from_config(hf_quant_config)
# For modelopt_mixed, config.json's quantization_config may or may
# not contain the per-layer quantized_layers map. Newer checkpoints
# embed it directly; older ones keep it only in hf_quant_config.json.
# If it is missing, fall through to the file-based loading path.
if (
model_config.quantization == "modelopt_mixed"
and "quantized_layers" not in hf_quant_config
):
pass # fall through to file-based loading below
else:
return quant_cls.from_config(hf_quant_config)
# if hf_quant_config is None, we will try to get config from
# hf_overrides
@@ -365,8 +375,8 @@ def get_quant_config(
if model_config.quantization == "bitsandbytes":
config["adapter_name_or_path"] = model_config.model
elif model_config.quantization == "modelopt":
if config["producer"]["name"] == "modelopt":
elif model_config.quantization in ("modelopt", "modelopt_mixed"):
if config.get("producer", {}).get("name") == "modelopt":
return quant_cls.from_config(config)
else:
raise ValueError(