Support RL online quantization with torchao (#23014)

Signed-off-by: Jerry Zhang <jerryzh168@gmail.com>
This commit is contained in:
Jerry Zhang
2025-10-01 16:39:29 -07:00
committed by GitHub
parent 4134312b35
commit c31246800c
6 changed files with 465 additions and 16 deletions
@@ -246,8 +246,34 @@ def get_quant_config(model_config: ModelConfig,
# compressed-tensors uses a compressions_config
hf_quant_config = getattr(model_config.hf_config, "compression_config",
None)
if hf_quant_config is not None:
return quant_cls.from_config(hf_quant_config)
# if hf_quant_config is None, we will try to get config from
# hf_overrides
hf_overrides = model_config.hf_overrides
quantization_config_file = hf_overrides.get("quantization_config_file",
None)
if quantization_config_file is not None:
if hasattr(quant_cls, "from_config_file"):
return quant_cls.from_config_file(quantization_config_file)
else:
raise NotImplementedError(
"from_config_file is specified in hf_override config, "
"but quant_cls.from_config_file is not implemented in "
f"{quant_cls}")
quantization_config_json = hf_overrides.get(
"quantization_config_dict_json", None)
if quantization_config_json is not None:
if hasattr(quant_cls, "from_config_dict_json"):
return quant_cls.from_config_dict_json(quantization_config_json)
else:
raise NotImplementedError(
"from_config_dict_json is specified in hf_override config, "
"but quant_cls.from_config_dict_json is not implemented in "
f"{quant_cls}")
# Inflight BNB quantization
if model_config.quantization == "bitsandbytes":
return quant_cls.from_config({})