support bitsandbytes 8-bit and FP4 quantized models (#7445)
This commit is contained in:
@@ -771,7 +771,11 @@ class BitsAndBytesModelLoader(BaseModelLoader):
|
||||
return pt_weights_iterator(hf_weights_files)
|
||||
|
||||
def _get_quantized_weights_iterator(
|
||||
self, model_name_or_path: str, revision: Optional[str], pre_quant: bool
|
||||
self,
|
||||
model_name_or_path: str,
|
||||
revision: Optional[str],
|
||||
pre_quant: bool,
|
||||
load_8bit: bool,
|
||||
) -> Tuple[Generator[Tuple[str, torch.Tensor], None, None], Dict[str,
|
||||
Any]]:
|
||||
"""Get an iterator to the model weights with bitsandbytes quantization,
|
||||
@@ -780,11 +784,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
|
||||
# only load the bitsandbytes module when needed
|
||||
try:
|
||||
import bitsandbytes
|
||||
from bitsandbytes.functional import QuantState
|
||||
if bitsandbytes.__version__ < "0.42.0":
|
||||
raise ImportError("bitsandbytes version is wrong. Please "
|
||||
"install bitsandbytes>=0.42.0.")
|
||||
from bitsandbytes.functional import quantize_4bit
|
||||
except ImportError as err:
|
||||
raise ImportError("Please install bitsandbytes>=0.42.0 via "
|
||||
"`pip install bitsandbytes>=0.42.0` to use "
|
||||
@@ -793,80 +795,111 @@ class BitsAndBytesModelLoader(BaseModelLoader):
|
||||
hf_weights_files, use_safetensors = self._prepare_weights(
|
||||
model_name_or_path, revision)
|
||||
|
||||
quant_state_dict = {}
|
||||
|
||||
def quantized_checkpoint() -> Generator:
|
||||
# First iterate over all quant state weights
|
||||
weight_iterator = self._hf_weight_iter(hf_weights_files,
|
||||
use_safetensors)
|
||||
temp_state_dict = {}
|
||||
for weight_name, weight_tensor in weight_iterator:
|
||||
if weight_name.endswith(".weight"):
|
||||
continue
|
||||
# TODO: only nf4 quantization is supported for now
|
||||
if weight_name.endswith(".quant_state.bitsandbytes__fp4"):
|
||||
raise NotImplementedError(
|
||||
"Only bitsandbytes_nf4 quantization"
|
||||
f"is supported for now. {weight_name} is fp4 quantized"
|
||||
)
|
||||
temp_state_dict[weight_name] = weight_tensor
|
||||
|
||||
# Closure to parse quant_state for each prequant weight
|
||||
def _parse_quant_state(param_name: str,
|
||||
temp_state_dict: Dict) -> QuantState:
|
||||
quant_state = {}
|
||||
for k in temp_state_dict:
|
||||
if param_name + "." in k:
|
||||
quant_state[k] = temp_state_dict[k]
|
||||
# bitsandbytes library requires
|
||||
# weight.quant_state.bitsandbytes__nf4 in CPU
|
||||
quant_state[param_name +
|
||||
".quant_state.bitsandbytes__nf4"] = quant_state[
|
||||
param_name +
|
||||
".quant_state.bitsandbytes__nf4"].cpu().data
|
||||
return QuantState.from_dict(quant_state, device="cuda")
|
||||
|
||||
# Second iterate over all prequant and normal weights
|
||||
# pre quantized weights would have a quant_state
|
||||
for weight_name, weight_tensor in self._hf_weight_iter(
|
||||
hf_weights_files, use_safetensors):
|
||||
# Filter out all weights whose suffix is not ".weight"
|
||||
if not weight_name.endswith(".weight"):
|
||||
continue
|
||||
if weight_name + ".quant_state.bitsandbytes__nf4" \
|
||||
in temp_state_dict:
|
||||
quant_state = _parse_quant_state(weight_name,
|
||||
temp_state_dict)
|
||||
weight_name = weight_name.replace(".weight", ".qweight")
|
||||
quant_state_dict[weight_name] = quant_state
|
||||
yield weight_name.replace(".weight",
|
||||
".qweight"), weight_tensor
|
||||
else:
|
||||
yield weight_name, weight_tensor
|
||||
|
||||
def generator() -> Generator:
|
||||
for weight_name, weight_tensor in self._hf_weight_iter(
|
||||
hf_weights_files, use_safetensors):
|
||||
if any(target_module in weight_name
|
||||
for target_module in self.target_modules):
|
||||
weight_name = weight_name.replace(".weight", ".qweight")
|
||||
# bitsandbytes requires data in GPU
|
||||
loaded_weight = weight_tensor.cuda().data
|
||||
with set_default_torch_dtype(torch.float32):
|
||||
processed_weight, quant_state = quantize_4bit(
|
||||
loaded_weight,
|
||||
compress_statistics=True,
|
||||
quant_type="nf4")
|
||||
|
||||
quant_state_dict[weight_name] = quant_state
|
||||
else:
|
||||
processed_weight = weight_tensor
|
||||
|
||||
yield weight_name, processed_weight
|
||||
quant_state_dict: Dict[str, Any] = {}
|
||||
|
||||
if pre_quant:
|
||||
return quantized_checkpoint(), quant_state_dict
|
||||
return generator(), quant_state_dict
|
||||
if load_8bit:
|
||||
return self._quantized_8bit_generator(
|
||||
hf_weights_files, use_safetensors,
|
||||
quant_state_dict), quant_state_dict
|
||||
else:
|
||||
return self._quantized_4bit_generator(
|
||||
hf_weights_files, use_safetensors,
|
||||
quant_state_dict), quant_state_dict
|
||||
|
||||
return self._unquantized_generator(hf_weights_files, use_safetensors,
|
||||
quant_state_dict), quant_state_dict
|
||||
|
||||
def _quantized_8bit_generator(self, hf_weights_files, use_safetensors,
|
||||
quant_state_dict) -> Generator:
|
||||
for weight_name, weight_tensor in self._hf_weight_iter(
|
||||
hf_weights_files, use_safetensors):
|
||||
if not weight_name.lower().endswith(".scb"):
|
||||
continue
|
||||
|
||||
weight_key = weight_name.lower().replace(".scb", ".qweight")
|
||||
quant_state_dict[weight_key] = weight_tensor
|
||||
|
||||
for weight_name, weight_tensor in self._hf_weight_iter(
|
||||
hf_weights_files, use_safetensors):
|
||||
|
||||
if not weight_name.endswith(".weight"):
|
||||
continue
|
||||
|
||||
qweight_name = weight_name.replace(".weight", ".qweight")
|
||||
if qweight_name in quant_state_dict:
|
||||
set_weight_attrs(weight_tensor, {"load_in_8bit": True})
|
||||
yield qweight_name, weight_tensor
|
||||
else:
|
||||
yield weight_name, weight_tensor
|
||||
|
||||
def _quantized_4bit_generator(self, hf_weights_files, use_safetensors,
|
||||
quant_state_dict) -> Generator:
|
||||
from bitsandbytes.functional import QuantState
|
||||
|
||||
# First iterate over all quant state weights
|
||||
weight_iterator = self._hf_weight_iter(hf_weights_files,
|
||||
use_safetensors)
|
||||
temp_state_dict = {}
|
||||
for weight_name, weight_tensor in weight_iterator:
|
||||
if weight_name.endswith(".weight"):
|
||||
continue
|
||||
# bitsandbytes library requires
|
||||
# weight.quant_state.bitsandbytes__* in CPU
|
||||
if "quant_state.bitsandbytes" in weight_name:
|
||||
temp_state_dict[weight_name] = weight_tensor.cpu().data
|
||||
else:
|
||||
temp_state_dict[weight_name] = weight_tensor
|
||||
|
||||
# Closure to parse quant_state for each prequant weight
|
||||
def _parse_quant_state(param_name: str,
|
||||
temp_state_dict: Dict) -> QuantState:
|
||||
quant_state = {}
|
||||
for k in temp_state_dict:
|
||||
if param_name + "." in k:
|
||||
quant_state[k] = temp_state_dict[k]
|
||||
|
||||
return QuantState.from_dict(quant_state, device="cuda")
|
||||
|
||||
# Second iterate over all prequant and normal weights
|
||||
# pre quantized weights would have a quant_state
|
||||
for weight_name, weight_tensor in self._hf_weight_iter(
|
||||
hf_weights_files, use_safetensors):
|
||||
# Filter out all weights whose suffix is not ".weight"
|
||||
if not weight_name.endswith(".weight"):
|
||||
continue
|
||||
if (f"{weight_name}.quant_state.bitsandbytes__nf4" \
|
||||
in temp_state_dict) or \
|
||||
(f"{weight_name}.quant_state.bitsandbytes__fp4" \
|
||||
in temp_state_dict):
|
||||
quant_state = _parse_quant_state(weight_name, temp_state_dict)
|
||||
weight_name = weight_name.replace(".weight", ".qweight")
|
||||
quant_state_dict[weight_name] = quant_state
|
||||
yield weight_name.replace(".weight", ".qweight"), weight_tensor
|
||||
else:
|
||||
yield weight_name, weight_tensor
|
||||
|
||||
def _unquantized_generator(self, hf_weights_files, use_safetensors,
|
||||
quant_state_dict) -> Generator:
|
||||
from bitsandbytes.functional import quantize_4bit
|
||||
for weight_name, weight_tensor in self._hf_weight_iter(
|
||||
hf_weights_files, use_safetensors):
|
||||
if any(target_module in weight_name
|
||||
for target_module in self.target_modules):
|
||||
weight_name = weight_name.replace(".weight", ".qweight")
|
||||
# bitsandbytes requires data in GPU
|
||||
loaded_weight = weight_tensor.cuda().data
|
||||
with set_default_torch_dtype(torch.float32):
|
||||
processed_weight, quant_state = quantize_4bit(
|
||||
loaded_weight,
|
||||
compress_statistics=True,
|
||||
quant_type="nf4")
|
||||
|
||||
quant_state_dict[weight_name] = quant_state
|
||||
else:
|
||||
processed_weight = weight_tensor
|
||||
|
||||
yield weight_name, processed_weight
|
||||
|
||||
def _load_weights(self, model_config: ModelConfig,
|
||||
model: nn.Module) -> None:
|
||||
@@ -883,16 +916,26 @@ class BitsAndBytesModelLoader(BaseModelLoader):
|
||||
logger.info("Loading weights with BitsAndBytes quantization. "
|
||||
" May take a while ...")
|
||||
|
||||
is_quantized_checkpoint = False
|
||||
quant_config = getattr(model_config.hf_config, "quantization_config",
|
||||
None)
|
||||
if quant_config is not None and quant_config.get(
|
||||
'quant_method') == "bitsandbytes":
|
||||
is_quantized_checkpoint = True
|
||||
|
||||
pre_quant = False
|
||||
if quant_config is not None:
|
||||
quant_method = quant_config.get('quant_method')
|
||||
if quant_method == "bitsandbytes":
|
||||
pre_quant = True
|
||||
else:
|
||||
raise ValueError(
|
||||
f"BitsAndBytes loader does not support {quant_method} "
|
||||
"quantization")
|
||||
|
||||
load_8bit = False
|
||||
if pre_quant:
|
||||
load_8bit = quant_config.get('load_in_8bit', False)
|
||||
|
||||
qweight_iterator, quant_state_dict = \
|
||||
self._get_quantized_weights_iterator(
|
||||
model_config.model, model_config.revision, is_quantized_checkpoint)
|
||||
model_config.model, model_config.revision, pre_quant, load_8bit)
|
||||
|
||||
model.load_weights(qweight_iterator)
|
||||
|
||||
@@ -942,6 +985,10 @@ class BitsAndBytesModelLoader(BaseModelLoader):
|
||||
offsets = np.concatenate(([0], np.cumsum(num_elements)))
|
||||
set_weight_attrs(param, {"bnb_shard_offsets": offsets})
|
||||
|
||||
if load_8bit:
|
||||
set_weight_attrs(
|
||||
param, {"matmul_state": [None] * len(quant_states)})
|
||||
|
||||
def load_model(self, *, model_config: ModelConfig,
|
||||
device_config: DeviceConfig,
|
||||
lora_config: Optional[LoRAConfig],
|
||||
|
||||
Reference in New Issue
Block a user