[Model] add a bunch of supported lora modules for mixtral (#9008)
Signed-off-by: Prashant Gupta <prashantgupta@us.ibm.com>
This commit is contained in:
@@ -322,10 +322,8 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
|
||||
# LoRA specific attributes
|
||||
supported_lora_modules = [
|
||||
"qkv_proj",
|
||||
"o_proj",
|
||||
"embed_tokens",
|
||||
"lm_head",
|
||||
"qkv_proj", "o_proj", "embed_tokens", "lm_head", "w1", "w2", "w3",
|
||||
"gate"
|
||||
]
|
||||
embedding_modules = {
|
||||
"embed_tokens": "input_embeddings",
|
||||
|
||||
Reference in New Issue
Block a user