[Cohere] Fix Cohere2MoE weight loading when using Transformers ≥5.10 (#44747)
Signed-off-by: Terrencezzj <terrence@cohere.ai>
This commit is contained in:
@@ -51,6 +51,13 @@ from .utils import (
|
||||
)
|
||||
|
||||
|
||||
def is_prefix_dense_layer(config: CohereConfig, layer_idx: int) -> bool:
|
||||
"""True when layer_idx lies in the contiguous dense MLP prefix."""
|
||||
if layer_idx >= len(config.mlp_layer_types):
|
||||
return False
|
||||
return all(t == "dense" for t in config.mlp_layer_types[: layer_idx + 1])
|
||||
|
||||
|
||||
@torch.compile(backend=current_platform.simple_compile_backend)
|
||||
def token_choice_with_bias(
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -207,17 +214,15 @@ class Cohere2MoeAttention(nn.Module):
|
||||
):
|
||||
self.sliding_window = config.sliding_window
|
||||
|
||||
# Prefix-dense layers (layer_idx < first_k_dense_replace) have full
|
||||
# attention (no sliding window). When prefix_dense_sliding_window_pattern
|
||||
# == 1, they keep RoPE even though they are not sliding-window layers.
|
||||
first_k_dense_replace = getattr(config, "first_k_dense_replace", 0)
|
||||
# Prefix-dense layers have full attention (no sliding window). When
|
||||
# prefix_dense_sliding_window_pattern == 1, they keep RoPE even though
|
||||
# they are not sliding-window layers.
|
||||
prefix_dense_sliding_window_pattern = getattr(
|
||||
config, "prefix_dense_sliding_window_pattern", 1
|
||||
)
|
||||
self.force_rope = bool(
|
||||
first_k_dense_replace
|
||||
is_prefix_dense_layer(config, self.layer_idx)
|
||||
and prefix_dense_sliding_window_pattern == 1
|
||||
and self.layer_idx < first_k_dense_replace
|
||||
)
|
||||
|
||||
self.attn = Attention(
|
||||
@@ -346,9 +351,7 @@ class Cohere2MoeDecoderLayer(nn.Module):
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
|
||||
# Layers before first_k_dense_replace use a dense MLP instead of MoE.
|
||||
first_k_dense_replace = getattr(config, "first_k_dense_replace", 0)
|
||||
if self.layer_idx < first_k_dense_replace:
|
||||
if config.mlp_layer_types[self.layer_idx] == "dense":
|
||||
self.mlp = Cohere2MoeMLP(
|
||||
config=config,
|
||||
intermediate_size=getattr(
|
||||
@@ -402,6 +405,21 @@ class Cohere2MoeModel(nn.Module):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size, config.hidden_size
|
||||
)
|
||||
|
||||
# Decoder layers read per-layer MLP layout from config.mlp_layer_types
|
||||
# (dense MLP vs MoE) and use it for weight loading. Transformers >=5.10
|
||||
# populates this field; older versions only expose first_k_dense_replace.
|
||||
# Normalize here so layer construction below sees a consistent layout.
|
||||
if getattr(config, "mlp_layer_types", None) is None:
|
||||
first_k_dense_replace = getattr(config, "first_k_dense_replace", None)
|
||||
n = config.num_hidden_layers
|
||||
if first_k_dense_replace is not None:
|
||||
config.mlp_layer_types = ["dense"] * first_k_dense_replace + [
|
||||
"sparse"
|
||||
] * (n - first_k_dense_replace)
|
||||
else:
|
||||
config.mlp_layer_types = ["sparse"] * n
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: Cohere2MoeDecoderLayer(
|
||||
|
||||
Reference in New Issue
Block a user