[Feature][Kernel] Support bitsandbytes quantization and QLoRA (#4776)

This commit is contained in:
chenqianfzh
2024-06-01 14:51:10 -06:00
committed by GitHub
parent 37464a0f74
commit b9c0605a8e
11 changed files with 752 additions and 8 deletions
@@ -130,7 +130,17 @@ def get_quant_config(model_config: ModelConfig,
if hf_quant_config is not None:
return quant_cls.from_config(hf_quant_config)
model_name_or_path = model_config.model
# In case of bitsandbytes/QLoRA, get quant config from the adapter model.
if model_config.quantization == "bitsandbytes":
if (not load_config.model_loader_extra_config
or "qlora_adapter_name_or_path"
not in load_config.model_loader_extra_config):
return quant_cls.from_config({"adapter_name_or_path": ""})
model_name_or_path = load_config.model_loader_extra_config[
"qlora_adapter_name_or_path"]
else:
model_name_or_path = model_config.model
is_local = os.path.isdir(model_name_or_path)
if not is_local:
# Download the config files.
@@ -169,6 +179,10 @@ def get_quant_config(model_config: ModelConfig,
quant_config_file = quant_config_files[0]
with open(quant_config_file, "r") as f:
config = json.load(f)
if model_config.quantization == "bitsandbytes":
config["adapter_name_or_path"] = model_name_or_path
return quant_cls.from_config(config)