[Cohere] Fix Cohere2MoE weight loading when using Transformers ≥5.10 (#44747)

Signed-off-by: Terrencezzj <terrence@cohere.ai>
This commit is contained in:
Terrence Zhao
2026-06-09 06:27:40 -07:00
committed by GitHub
parent 6690a0c4de
commit 3e8afdf785
+27 -9
View File
@@ -51,6 +51,13 @@ from .utils import (
)
def is_prefix_dense_layer(config: CohereConfig, layer_idx: int) -> bool:
"""True when layer_idx lies in the contiguous dense MLP prefix."""
if layer_idx >= len(config.mlp_layer_types):
return False
return all(t == "dense" for t in config.mlp_layer_types[: layer_idx + 1])
@torch.compile(backend=current_platform.simple_compile_backend)
def token_choice_with_bias(
hidden_states: torch.Tensor,
@@ -207,17 +214,15 @@ class Cohere2MoeAttention(nn.Module):
):
self.sliding_window = config.sliding_window
# Prefix-dense layers (layer_idx < first_k_dense_replace) have full
# attention (no sliding window). When prefix_dense_sliding_window_pattern
# == 1, they keep RoPE even though they are not sliding-window layers.
first_k_dense_replace = getattr(config, "first_k_dense_replace", 0)
# Prefix-dense layers have full attention (no sliding window). When
# prefix_dense_sliding_window_pattern == 1, they keep RoPE even though
# they are not sliding-window layers.
prefix_dense_sliding_window_pattern = getattr(
config, "prefix_dense_sliding_window_pattern", 1
)
self.force_rope = bool(
first_k_dense_replace
is_prefix_dense_layer(config, self.layer_idx)
and prefix_dense_sliding_window_pattern == 1
and self.layer_idx < first_k_dense_replace
)
self.attn = Attention(
@@ -346,9 +351,7 @@ class Cohere2MoeDecoderLayer(nn.Module):
prefix=f"{prefix}.self_attn",
)
# Layers before first_k_dense_replace use a dense MLP instead of MoE.
first_k_dense_replace = getattr(config, "first_k_dense_replace", 0)
if self.layer_idx < first_k_dense_replace:
if config.mlp_layer_types[self.layer_idx] == "dense":
self.mlp = Cohere2MoeMLP(
config=config,
intermediate_size=getattr(
@@ -402,6 +405,21 @@ class Cohere2MoeModel(nn.Module):
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size, config.hidden_size
)
# Decoder layers read per-layer MLP layout from config.mlp_layer_types
# (dense MLP vs MoE) and use it for weight loading. Transformers >=5.10
# populates this field; older versions only expose first_k_dense_replace.
# Normalize here so layer construction below sees a consistent layout.
if getattr(config, "mlp_layer_types", None) is None:
first_k_dense_replace = getattr(config, "first_k_dense_replace", None)
n = config.num_hidden_layers
if first_k_dense_replace is not None:
config.mlp_layer_types = ["dense"] * first_k_dense_replace + [
"sparse"
] * (n - first_k_dense_replace)
else:
config.mlp_layer_types = ["sparse"] * n
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: Cohere2MoeDecoderLayer(