[Model] add a bunch of supported lora modules for mixtral (#9008)

Signed-off-by: Prashant Gupta <prashantgupta@us.ibm.com>
This commit is contained in:
Prashant Gupta
2024-10-04 16:24:40 +00:00
committed by GitHub
parent 22482e495e
commit 9ade8bbc8d
3 changed files with 69 additions and 20 deletions
+2 -4
View File
@@ -322,10 +322,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
# LoRA specific attributes
supported_lora_modules = [
"qkv_proj",
"o_proj",
"embed_tokens",
"lm_head",
"qkv_proj", "o_proj", "embed_tokens", "lm_head", "w1", "w2", "w3",
"gate"
]
embedding_modules = {
"embed_tokens": "input_embeddings",