Support RL online quantization with torchao (#23014)
Signed-off-by: Jerry Zhang <jerryzh168@gmail.com>
This commit is contained in:
@@ -246,8 +246,34 @@ def get_quant_config(model_config: ModelConfig,
|
||||
# compressed-tensors uses a compressions_config
|
||||
hf_quant_config = getattr(model_config.hf_config, "compression_config",
|
||||
None)
|
||||
|
||||
if hf_quant_config is not None:
|
||||
return quant_cls.from_config(hf_quant_config)
|
||||
|
||||
# if hf_quant_config is None, we will try to get config from
|
||||
# hf_overrides
|
||||
hf_overrides = model_config.hf_overrides
|
||||
quantization_config_file = hf_overrides.get("quantization_config_file",
|
||||
None)
|
||||
if quantization_config_file is not None:
|
||||
if hasattr(quant_cls, "from_config_file"):
|
||||
return quant_cls.from_config_file(quantization_config_file)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"from_config_file is specified in hf_override config, "
|
||||
"but quant_cls.from_config_file is not implemented in "
|
||||
f"{quant_cls}")
|
||||
quantization_config_json = hf_overrides.get(
|
||||
"quantization_config_dict_json", None)
|
||||
if quantization_config_json is not None:
|
||||
if hasattr(quant_cls, "from_config_dict_json"):
|
||||
return quant_cls.from_config_dict_json(quantization_config_json)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"from_config_dict_json is specified in hf_override config, "
|
||||
"but quant_cls.from_config_dict_json is not implemented in "
|
||||
f"{quant_cls}")
|
||||
|
||||
# Inflight BNB quantization
|
||||
if model_config.quantization == "bitsandbytes":
|
||||
return quant_cls.from_config({})
|
||||
|
||||
Reference in New Issue
Block a user