Optimize Mixtral with expert parallelism (#2090)
This commit is contained in:
+9
-7
@@ -120,14 +120,16 @@ class ModelConfig:
|
||||
if load_format == "auto":
|
||||
load_format = "pt"
|
||||
|
||||
# FIXME(woosuk): This is a temporary hack. Support safetensor weights.
|
||||
# TODO: Remove this check once HF updates the pt weights of Mixtral.
|
||||
architectures = getattr(self.hf_config, "architectures", [])
|
||||
if "MixtralForCausalLM" in architectures and load_format != "pt":
|
||||
logger.info(
|
||||
"Currently, only 'pt' format is supported for Mixtral. "
|
||||
"Changing the format to 'pt'. This may re-download the "
|
||||
"weights if you have downloaded the safetensor weights.")
|
||||
load_format = "pt"
|
||||
if "MixtralForCausalLM" in architectures:
|
||||
if load_format == "pt":
|
||||
raise ValueError(
|
||||
"Currently, the 'pt' format is not supported for Mixtral. "
|
||||
"Please use the 'safetensors' format instead. ")
|
||||
elif load_format == "auto":
|
||||
# Do not fall back to pt weights.
|
||||
load_format = "safetensors"
|
||||
|
||||
self.load_format = load_format
|
||||
|
||||
|
||||
Reference in New Issue
Block a user