[fix] Mistral-Large-3 accuracy regression on main (#57563)

Signed-off-by: jdebache <jdebache@nvidia.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Julien Debache
2026-09-18 14:19:16 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 76d517fd12
commit 2bdae2a5a8
8 changed files with 79 additions and 27 deletions
+3 -3
View File
@@ -153,9 +153,9 @@ class DeepseekV32Attention(MLAAttention):
# DSA checkpoints may use plain ("default") or yarn-scaled RoPE.
if config.rope_parameters["rope_type"] != "default":
config.rope_parameters["rope_type"] = (
"deepseek_yarn"
if config.rope_parameters.get("apply_yarn_scaling", True)
else "deepseek_llama_scaling"
"deepseek_llama_scaling"
if config.rope_parameters.get("attention_factor") == 1.0
else "deepseek_yarn"
)
if config.rope_parameters["rope_type"] == "deepseek_yarn":
mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False)
+3 -3
View File
@@ -31,9 +31,9 @@ def build_deepseek_v4_rope(
if compress_ratio > 1 and rope_parameters["rope_type"] != "default":
# YaRN applies only to compressor (CSA/HCA) layers.
rope_parameters["rope_type"] = (
"deepseek_yarn"
if rope_parameters.get("apply_yarn_scaling", True)
else "deepseek_llama_scaling"
"deepseek_llama_scaling"
if rope_parameters.get("attention_factor") == 1.0
else "deepseek_yarn"
)
else:
# Sliding-window layers use plain RoPE (theta=rope_theta, no YaRN).
+3 -3
View File
@@ -32,9 +32,9 @@ def build_deepseek_v4_rope(
# YaRN applies to all compressed-KV layers (compress_ratio > 0); only
# pure sliding-window layers (compress_ratio == 0) use plain RoPE.
rope_parameters["rope_type"] = (
"deepseek_yarn"
if rope_parameters.get("apply_yarn_scaling", True)
else "deepseek_llama_scaling"
"deepseek_llama_scaling"
if rope_parameters.get("attention_factor") == 1.0
else "deepseek_yarn"
)
else:
# Sliding-window layers use plain RoPE (theta=rope_theta, no YaRN).
+3 -3
View File
@@ -496,9 +496,9 @@ class Glm5NextMLAAttention(nn.Module):
assert config.rope_parameters is not None
if config.rope_parameters["rope_type"] != "default":
config.rope_parameters["rope_type"] = (
"deepseek_yarn"
if config.rope_parameters.get("apply_yarn_scaling", True)
else "deepseek_llama_scaling"
"deepseek_llama_scaling"
if config.rope_parameters.get("attention_factor") == 1.0
else "deepseek_yarn"
)
self.rotary_emb: RotaryEmbedding | None = get_rope(
+3 -3
View File
@@ -165,9 +165,9 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
rope_parameters = dict(config.rope_parameters)
if rope_parameters["rope_type"] != "default":
rope_parameters["rope_type"] = (
"deepseek_yarn"
if rope_parameters.get("apply_yarn_scaling", True)
else "deepseek_llama_scaling"
"deepseek_llama_scaling"
if rope_parameters.get("attention_factor") == 1.0
else "deepseek_yarn"
)
self.rotary_emb = get_rope(
qk_rope_head_dim,