Add quantized mixtral support (#2673)

This commit is contained in:
Woosuk Kwon
2024-01-30 16:34:10 -08:00
committed by GitHub
parent 105a40f53a
commit 3dad944485
3 changed files with 422 additions and 4 deletions
+1
View File
@@ -30,6 +30,7 @@ _MODELS = {
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
"MistralForCausalLM": ("mistral", "MistralForCausalLM"),
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
"QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
# transformers's mpt class has lower case
"MptForCausalLM": ("mpt", "MPTForCausalLM"),
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),