[Cohere] Misc changes to cohere model definitions (#50156)

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Komal Kumar Teru
2026-08-18 04:59:57 +00:00
committed by GitHub
co-authored by Cursor
parent 101c4477dd
commit aa9903490c
2 changed files with 87 additions and 26 deletions
+1 -1
View File
@@ -208,7 +208,7 @@ class Cohere2MoeAttention(nn.Module):
layer_types is not None
and layer_types[self.layer_idx] == "sliding_attention"
):
self.sliding_window = config.sliding_window
self.sliding_window = config.sliding_window + 1
# Prefix-dense layers have full attention (no sliding window). When
# prefix_dense_sliding_window_pattern == 1, they keep RoPE even though
+86 -25
View File
@@ -32,7 +32,11 @@ from transformers import Cohere2Config, CohereConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_reduce,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import (
@@ -44,9 +48,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.model_loader.weight_utils import (
row_parallel_weight_loader,
)
from vllm.model_executor.model_loader.weight_utils import row_parallel_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
@@ -87,18 +89,61 @@ class LayerNorm(nn.Module):
return hidden_states, residuals
@torch.compile(backend=current_platform.simple_compile_backend)
def rms_norm_func(hidden_states, weight, variance_epsilon):
input_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float32)
variance = hidden_states.pow(2).mean(-1, keepdim=True)
hidden_states = hidden_states * torch.rsqrt(variance + variance_epsilon)
hidden_states = weight.to(torch.float32) * hidden_states
return hidden_states.to(input_dtype)
class RMSNorm(nn.Module):
def __init__(self, param_shape=None, eps=1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(param_shape))
self.variance_epsilon = eps
set_weight_attrs(self.weight, {"weight_loader": row_parallel_weight_loader})
def forward(self, hidden_states, residuals=None):
hidden_states = rms_norm_func(hidden_states, self.weight, self.variance_epsilon)
return hidden_states, residuals
def select_norm_impl(config: CohereConfig) -> tuple[type[nn.Module], float]:
"""
Returns the normalization layer class and epsilon value to use.
If `config.rms_norm_eps` is present, use RMSNorm.
Otherwise default to LayerNorm with `config.layer_norm_eps`.
"""
rms_eps = getattr(config, "rms_norm_eps", None)
if rms_eps is not None:
return RMSNorm, rms_eps
return LayerNorm, config.layer_norm_eps
# Copied from transformers.models.llama.modeling_llama.LlamaMLP Llama->Cohere
class CohereMLP(nn.Module):
def __init__(
self,
config: CohereConfig | Cohere2Config,
config: CohereConfig,
intermediate_size: int | None = None,
quant_config: QuantizationConfig | None = None,
# so we can reduce the attention and MLP outputs together
reduce_results: bool = False,
prefix: str = "",
):
super().__init__()
self.config = config
self.hidden_size = config.hidden_size
self.intermediate_size = config.intermediate_size
if intermediate_size is None:
self.intermediate_size = config.intermediate_size
else:
self.intermediate_size = intermediate_size
self.gate_up_proj = MergedColumnParallelLinear(
self.hidden_size,
[self.intermediate_size] * 2,
@@ -111,6 +156,7 @@ class CohereMLP(nn.Module):
self.hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=f"{prefix}.down_proj",
)
self.act_fn = SiluAndMul()
@@ -133,11 +179,15 @@ class CohereAttention(nn.Module):
super().__init__()
tp_size = get_tensor_model_parallel_world_size()
self.config = config
self.layer_idx = extract_layer_index(prefix)
self.attention_dropout = config.attention_dropout
self.hidden_size = config.hidden_size
self.total_num_heads = config.num_attention_heads
self.num_heads = self.total_num_heads // tp_size
self.head_dim = self.hidden_size // self.total_num_heads
if hasattr(config, "head_dim"):
self.head_dim = config.head_dim
else:
self.head_dim = self.hidden_size // self.total_num_heads
self.total_num_kv_heads = config.num_key_value_heads
if self.total_num_kv_heads >= tp_size:
# Number of KV heads is greater than TP size, so we partition
@@ -171,6 +221,9 @@ class CohereAttention(nn.Module):
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
# NOTE: reduction will happen in the decoder layer forward
# so we can combine the attention and MLP outputs together
reduce_results=False,
)
self.rotary_emb = get_rope(
self.head_dim,
@@ -182,11 +235,14 @@ class CohereAttention(nn.Module):
# Model v2 has interleaved sliding windows, v1 does not
self.v1 = isinstance(config, CohereConfig)
# cohere swa layer sees [pos - sliding_window, pos], i.e. sliding_window + 1
# tokens. vLLM's FlashAttention backend does (value - 1, 0), so we pass
# sliding_window + 1 here to match the training convention. The same +1
# propagates into the KV-cache eviction formula
# (single_type_kv_cache_manager.py), keeping both paths consistent.
self.sliding_window = None
if not self.v1:
layer_idx = extract_layer_index(prefix)
if config.layer_types[layer_idx] == "sliding_attention":
self.sliding_window = config.sliding_window
if not self.v1 and config.layer_types[self.layer_idx] == "sliding_attention":
self.sliding_window = config.sliding_window + 1
self.attn = Attention(
self.num_heads,
@@ -198,13 +254,13 @@ class CohereAttention(nn.Module):
per_layer_sliding_window=self.sliding_window,
prefix=f"{prefix}.attn",
)
norm_cls, norm_eps = select_norm_impl(config)
if self.use_qk_norm:
self.q_norm = LayerNorm(
param_shape=(self.num_heads, self.head_dim), eps=config.layer_norm_eps
self.q_norm = norm_cls(
param_shape=(self.num_heads, self.head_dim), eps=norm_eps
)
self.k_norm = LayerNorm(
param_shape=(self.num_kv_heads, self.head_dim),
eps=config.layer_norm_eps,
self.k_norm = norm_cls(
param_shape=(self.num_kv_heads, self.head_dim), eps=norm_eps
)
def _apply_qk_norm(self, q, k):
@@ -242,18 +298,16 @@ class CohereDecoderLayer(nn.Module):
):
super().__init__()
self.hidden_size = config.hidden_size
self.tp_size = get_tensor_model_parallel_world_size()
self.self_attn = CohereAttention(
config,
cache_config,
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
)
self.mlp = CohereMLP(config, quant_config=quant_config, prefix=f"{prefix}.mlp")
self.input_layernorm = LayerNorm(
param_shape=(config.hidden_size), eps=config.layer_norm_eps
)
norm_cls, norm_eps = select_norm_impl(config)
self.input_layernorm = norm_cls(param_shape=(config.hidden_size), eps=norm_eps)
def forward(
self,
@@ -269,8 +323,14 @@ class CohereDecoderLayer(nn.Module):
hidden_states=hidden_states,
)
hidden_states_mlp = self.mlp(hidden_states)
parallel_block_output = hidden_states_attention + hidden_states_mlp
# Add everything together
hidden_states = residual + hidden_states_attention + hidden_states_mlp
if self.tp_size > 1:
# do the reduction in 1 shot instead of 2 separate all reduce
parallel_block_output = tensor_model_parallel_all_reduce(
parallel_block_output
)
hidden_states = residual + parallel_block_output
return hidden_states, residual
@@ -299,9 +359,10 @@ class CohereModel(nn.Module):
),
prefix=f"{prefix}.layers",
)
self.norm = LayerNorm(
param_shape=(config.hidden_size), eps=config.layer_norm_eps
)
norm_cls, norm_eps = select_norm_impl(config)
self.norm = norm_cls(param_shape=(config.hidden_size), eps=norm_eps)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)