[Feature][Kernel] Support bitsandbytes quantization and QLoRA (#4776)

This commit is contained in:
chenqianfzh
2024-06-01 14:51:10 -06:00
committed by GitHub
parent 37464a0f74
commit b9c0605a8e
11 changed files with 752 additions and 8 deletions
+8
View File
@@ -319,6 +319,14 @@ class LlamaForCausalLM(nn.Module):
"lm_head": "output_embeddings",
}
embedding_padding_modules = ["lm_head"]
bitsandbytes_stacked_params_mapping = {
# shard_name, weight_name, index
"q_proj": ("qkv_proj", 0),
"k_proj": ("qkv_proj", 1),
"v_proj": ("qkv_proj", 2),
"gate_proj": ("gate_up_proj", 0),
"up_proj": ("gate_up_proj", 1),
}
def __init__(
self,