[fix] Mistral-Large-3 accuracy regression on main (#57563)
Signed-off-by: jdebache <jdebache@nvidia.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
76d517fd12
commit
2bdae2a5a8
@@ -324,9 +324,9 @@ class AXK1Attention(nn.Module):
|
||||
assert config.rope_parameters is not None
|
||||
if config.rope_parameters["rope_type"] != "default":
|
||||
config.rope_parameters["rope_type"] = (
|
||||
"deepseek_yarn"
|
||||
if config.rope_parameters.get("apply_yarn_scaling", True)
|
||||
else "deepseek_llama_scaling"
|
||||
"deepseek_llama_scaling"
|
||||
if config.rope_parameters.get("attention_factor") == 1.0
|
||||
else "deepseek_yarn"
|
||||
)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
@@ -496,9 +496,9 @@ class AXK1MLAAttention(nn.Module):
|
||||
assert config.rope_parameters is not None
|
||||
if config.rope_parameters["rope_type"] != "default":
|
||||
config.rope_parameters["rope_type"] = (
|
||||
"deepseek_yarn"
|
||||
if config.rope_parameters.get("apply_yarn_scaling", True)
|
||||
else "deepseek_llama_scaling"
|
||||
"deepseek_llama_scaling"
|
||||
if config.rope_parameters.get("attention_factor") == 1.0
|
||||
else "deepseek_yarn"
|
||||
)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
|
||||
@@ -540,9 +540,9 @@ class DeepseekV2Attention(nn.Module):
|
||||
)
|
||||
if config.rope_parameters["rope_type"] != "default":
|
||||
config.rope_parameters["rope_type"] = (
|
||||
"deepseek_yarn"
|
||||
if config.rope_parameters.get("apply_yarn_scaling", True)
|
||||
else "deepseek_llama_scaling"
|
||||
"deepseek_llama_scaling"
|
||||
if config.rope_parameters.get("attention_factor") == 1.0
|
||||
else "deepseek_yarn"
|
||||
)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
@@ -1107,9 +1107,9 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
|
||||
if config.rope_parameters["rope_type"] != "default":
|
||||
config.rope_parameters["rope_type"] = (
|
||||
"deepseek_yarn"
|
||||
if config.rope_parameters.get("apply_yarn_scaling", True)
|
||||
else "deepseek_llama_scaling"
|
||||
"deepseek_llama_scaling"
|
||||
if config.rope_parameters.get("attention_factor") == 1.0
|
||||
else "deepseek_yarn"
|
||||
)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
|
||||
Reference in New Issue
Block a user