Add Mistral Large 3 and Ministral 3 (#29757)
Signed-off-by: Julien Denize <julien.denize@mistral.ai> Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> Signed-off-by: Mickael Seznec <mickael@mistral.ai> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Mickael Seznec <mickael@mistral.ai>
This commit is contained in:
co-authored by
Roger Wang
Mickael Seznec
parent
8bbcf8b6e7
commit
d8c6210eea
@@ -82,3 +82,9 @@ class EAGLEConfig(PretrainedConfig):
|
||||
pretrained_model_name_or_path, **kwargs
|
||||
)
|
||||
return cls.from_dict(config_dict, **kwargs)
|
||||
|
||||
def to_json_string(self, use_diff: bool = True) -> str:
|
||||
# we override use_diff to False as initializing
|
||||
# EAGLEConfig with default arguments is not supported
|
||||
del use_diff
|
||||
return super().to_json_string(use_diff=False)
|
||||
|
||||
@@ -18,9 +18,31 @@ def adapt_config_dict(
|
||||
if bool(config_dict.get("quantization")):
|
||||
config_dict = _remap_mistral_quantization_args(config_dict)
|
||||
|
||||
is_moe = bool(config_dict.get("moe"))
|
||||
is_mistral_large_3 = (
|
||||
is_moe and (config_dict["moe"].get("num_shared_experts") or 0) > 0
|
||||
)
|
||||
if config_dict.get("model_type") == "mamba":
|
||||
config_dict["architectures"] = ["Mamba2ForCausalLM"]
|
||||
elif bool(config_dict.get("moe")):
|
||||
elif is_moe and is_mistral_large_3:
|
||||
config_dict = _remap_moe_args(config_dict)
|
||||
config_dict["model_type"] = "deepseek_v3"
|
||||
config_dict["architectures"] = ["MistralLarge3ForCausalLM"]
|
||||
|
||||
assert "llama_4_scaling" in config_dict, (
|
||||
"MistralLarge3 expect llama4 scaling config."
|
||||
)
|
||||
llama_4_scaling_config_keys = ["original_max_position_embeddings", "beta"]
|
||||
assert all(
|
||||
[
|
||||
key in config_dict["llama_4_scaling"]
|
||||
for key in llama_4_scaling_config_keys
|
||||
]
|
||||
), (
|
||||
"llama_4_scaling config should define the keys: "
|
||||
f"{','.join(llama_4_scaling_config_keys)}"
|
||||
)
|
||||
elif is_moe:
|
||||
config_dict["architectures"] = ["MixtralForCausalLM"]
|
||||
else:
|
||||
config_dict["architectures"] = ["MistralForCausalLM"]
|
||||
@@ -140,17 +162,20 @@ def _remap_general_mistral_args(config: dict) -> dict:
|
||||
|
||||
|
||||
def _remap_mistral_quantization_args(config: dict) -> dict:
|
||||
quantization = config.get("quantization", {})
|
||||
if quantization.get("qformat_weight") == "fp8_e4m3":
|
||||
# This maps to the FP8 static per-tensor quantization scheme
|
||||
quantization_config = {"quant_method": "fp8", "activation_scheme": "static"}
|
||||
elif quantization.get("quant_method") == "compressed-tensors":
|
||||
# Pass through the quantization config to compressed-tensors
|
||||
quantization_config = quantization
|
||||
else:
|
||||
raise ValueError(f"Found unknown quantization='{quantization}' in config")
|
||||
|
||||
config["quantization_config"] = quantization_config
|
||||
if config.get("quantization"):
|
||||
quantization = config.pop("quantization", {})
|
||||
if quantization.get("qformat_weight") == "fp8_e4m3":
|
||||
qscheme_act = quantization.get("qscheme_act")
|
||||
assert qscheme_act in ("NO_SCALES", "TENSOR", None), (
|
||||
"Only NO_SCALES and TENSOR (default) are supported for qscheme_act"
|
||||
)
|
||||
is_dynamic = qscheme_act == "NO_SCALES"
|
||||
config["quantization_config"] = {
|
||||
"quant_method": "fp8",
|
||||
"activation_scheme": "dynamic" if is_dynamic else "static",
|
||||
}
|
||||
else:
|
||||
raise ValueError(f"Found unknown quantization='{quantization}' in config")
|
||||
|
||||
return config
|
||||
|
||||
@@ -183,3 +208,28 @@ def _remap_mistral_audio_args(config: dict) -> dict:
|
||||
if quant_config:
|
||||
config["quantization_config"] = quant_config
|
||||
return config
|
||||
|
||||
|
||||
def _remap_moe_args(config: dict) -> dict:
|
||||
moe_config_map = {
|
||||
"route_every_n": "moe_layer_freq",
|
||||
"first_k_dense_replace": "first_k_dense_replace",
|
||||
"num_experts_per_tok": "num_experts_per_tok",
|
||||
"num_experts": "n_routed_experts",
|
||||
"expert_hidden_dim": "moe_intermediate_size",
|
||||
"routed_scale": "routed_scaling_factor",
|
||||
"num_shared_experts": "n_shared_experts",
|
||||
"num_expert_groups": "n_group",
|
||||
"num_expert_groups_per_tok": "topk_group",
|
||||
}
|
||||
moe_config = config.get("moe", {})
|
||||
for old_name, new_name in moe_config_map.items():
|
||||
if old_name in moe_config:
|
||||
value = moe_config.pop(old_name)
|
||||
config[new_name] = value
|
||||
|
||||
config["topk_method"] = None
|
||||
config["norm_topk_prob"] = True
|
||||
config["scoring_func"] = "softmax"
|
||||
|
||||
return config
|
||||
|
||||
Reference in New Issue
Block a user