From aa9903490c616dc6871e5acc62cec7bb1e5e9434 Mon Sep 17 00:00:00 2001 From: Komal Kumar Teru <162363718+kkt-cohere@users.noreply.github.com> Date: Tue, 18 Aug 2026 00:59:57 -0400 Subject: [PATCH] [Cohere] Misc changes to cohere model definitions (#50156) Co-authored-by: Cursor --- vllm/model_executor/models/cohere2_moe.py | 2 +- vllm/model_executor/models/commandr.py | 111 +++++++++++++++++----- 2 files changed, 87 insertions(+), 26 deletions(-) diff --git a/vllm/model_executor/models/cohere2_moe.py b/vllm/model_executor/models/cohere2_moe.py index 12291fb142..3873b6ee00 100644 --- a/vllm/model_executor/models/cohere2_moe.py +++ b/vllm/model_executor/models/cohere2_moe.py @@ -208,7 +208,7 @@ class Cohere2MoeAttention(nn.Module): layer_types is not None and layer_types[self.layer_idx] == "sliding_attention" ): - self.sliding_window = config.sliding_window + self.sliding_window = config.sliding_window + 1 # Prefix-dense layers have full attention (no sliding window). When # prefix_dense_sliding_window_pattern == 1, they keep RoPE even though diff --git a/vllm/model_executor/models/commandr.py b/vllm/model_executor/models/commandr.py index 3c755c65e1..5481fb0abc 100644 --- a/vllm/model_executor/models/commandr.py +++ b/vllm/model_executor/models/commandr.py @@ -32,7 +32,11 @@ from transformers import Cohere2Config, CohereConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig -from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.distributed import ( + get_pp_group, + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce, +) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.linear import ( @@ -44,9 +48,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding -from vllm.model_executor.model_loader.weight_utils import ( - row_parallel_weight_loader, -) +from vllm.model_executor.model_loader.weight_utils import row_parallel_weight_loader from vllm.model_executor.utils import set_weight_attrs from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors @@ -87,18 +89,61 @@ class LayerNorm(nn.Module): return hidden_states, residuals +@torch.compile(backend=current_platform.simple_compile_backend) +def rms_norm_func(hidden_states, weight, variance_epsilon): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + variance_epsilon) + + hidden_states = weight.to(torch.float32) * hidden_states + return hidden_states.to(input_dtype) + + +class RMSNorm(nn.Module): + def __init__(self, param_shape=None, eps=1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(param_shape)) + self.variance_epsilon = eps + set_weight_attrs(self.weight, {"weight_loader": row_parallel_weight_loader}) + + def forward(self, hidden_states, residuals=None): + hidden_states = rms_norm_func(hidden_states, self.weight, self.variance_epsilon) + return hidden_states, residuals + + +def select_norm_impl(config: CohereConfig) -> tuple[type[nn.Module], float]: + """ + Returns the normalization layer class and epsilon value to use. + If `config.rms_norm_eps` is present, use RMSNorm. + Otherwise default to LayerNorm with `config.layer_norm_eps`. + """ + rms_eps = getattr(config, "rms_norm_eps", None) + if rms_eps is not None: + return RMSNorm, rms_eps + + return LayerNorm, config.layer_norm_eps + + # Copied from transformers.models.llama.modeling_llama.LlamaMLP Llama->Cohere class CohereMLP(nn.Module): def __init__( self, - config: CohereConfig | Cohere2Config, + config: CohereConfig, + intermediate_size: int | None = None, quant_config: QuantizationConfig | None = None, + # so we can reduce the attention and MLP outputs together + reduce_results: bool = False, prefix: str = "", ): super().__init__() self.config = config self.hidden_size = config.hidden_size - self.intermediate_size = config.intermediate_size + if intermediate_size is None: + self.intermediate_size = config.intermediate_size + else: + self.intermediate_size = intermediate_size self.gate_up_proj = MergedColumnParallelLinear( self.hidden_size, [self.intermediate_size] * 2, @@ -111,6 +156,7 @@ class CohereMLP(nn.Module): self.hidden_size, bias=False, quant_config=quant_config, + reduce_results=reduce_results, prefix=f"{prefix}.down_proj", ) self.act_fn = SiluAndMul() @@ -133,11 +179,15 @@ class CohereAttention(nn.Module): super().__init__() tp_size = get_tensor_model_parallel_world_size() self.config = config + self.layer_idx = extract_layer_index(prefix) self.attention_dropout = config.attention_dropout self.hidden_size = config.hidden_size self.total_num_heads = config.num_attention_heads self.num_heads = self.total_num_heads // tp_size - self.head_dim = self.hidden_size // self.total_num_heads + if hasattr(config, "head_dim"): + self.head_dim = config.head_dim + else: + self.head_dim = self.hidden_size // self.total_num_heads self.total_num_kv_heads = config.num_key_value_heads if self.total_num_kv_heads >= tp_size: # Number of KV heads is greater than TP size, so we partition @@ -171,6 +221,9 @@ class CohereAttention(nn.Module): bias=False, quant_config=quant_config, prefix=f"{prefix}.o_proj", + # NOTE: reduction will happen in the decoder layer forward + # so we can combine the attention and MLP outputs together + reduce_results=False, ) self.rotary_emb = get_rope( self.head_dim, @@ -182,11 +235,14 @@ class CohereAttention(nn.Module): # Model v2 has interleaved sliding windows, v1 does not self.v1 = isinstance(config, CohereConfig) + # cohere swa layer sees [pos - sliding_window, pos], i.e. sliding_window + 1 + # tokens. vLLM's FlashAttention backend does (value - 1, 0), so we pass + # sliding_window + 1 here to match the training convention. The same +1 + # propagates into the KV-cache eviction formula + # (single_type_kv_cache_manager.py), keeping both paths consistent. self.sliding_window = None - if not self.v1: - layer_idx = extract_layer_index(prefix) - if config.layer_types[layer_idx] == "sliding_attention": - self.sliding_window = config.sliding_window + if not self.v1 and config.layer_types[self.layer_idx] == "sliding_attention": + self.sliding_window = config.sliding_window + 1 self.attn = Attention( self.num_heads, @@ -198,13 +254,13 @@ class CohereAttention(nn.Module): per_layer_sliding_window=self.sliding_window, prefix=f"{prefix}.attn", ) + norm_cls, norm_eps = select_norm_impl(config) if self.use_qk_norm: - self.q_norm = LayerNorm( - param_shape=(self.num_heads, self.head_dim), eps=config.layer_norm_eps + self.q_norm = norm_cls( + param_shape=(self.num_heads, self.head_dim), eps=norm_eps ) - self.k_norm = LayerNorm( - param_shape=(self.num_kv_heads, self.head_dim), - eps=config.layer_norm_eps, + self.k_norm = norm_cls( + param_shape=(self.num_kv_heads, self.head_dim), eps=norm_eps ) def _apply_qk_norm(self, q, k): @@ -242,18 +298,16 @@ class CohereDecoderLayer(nn.Module): ): super().__init__() self.hidden_size = config.hidden_size - + self.tp_size = get_tensor_model_parallel_world_size() self.self_attn = CohereAttention( config, cache_config, quant_config=quant_config, prefix=f"{prefix}.self_attn", ) - self.mlp = CohereMLP(config, quant_config=quant_config, prefix=f"{prefix}.mlp") - self.input_layernorm = LayerNorm( - param_shape=(config.hidden_size), eps=config.layer_norm_eps - ) + norm_cls, norm_eps = select_norm_impl(config) + self.input_layernorm = norm_cls(param_shape=(config.hidden_size), eps=norm_eps) def forward( self, @@ -269,8 +323,14 @@ class CohereDecoderLayer(nn.Module): hidden_states=hidden_states, ) hidden_states_mlp = self.mlp(hidden_states) + parallel_block_output = hidden_states_attention + hidden_states_mlp # Add everything together - hidden_states = residual + hidden_states_attention + hidden_states_mlp + if self.tp_size > 1: + # do the reduction in 1 shot instead of 2 separate all reduce + parallel_block_output = tensor_model_parallel_all_reduce( + parallel_block_output + ) + hidden_states = residual + parallel_block_output return hidden_states, residual @@ -299,9 +359,10 @@ class CohereModel(nn.Module): ), prefix=f"{prefix}.layers", ) - self.norm = LayerNorm( - param_shape=(config.hidden_size), eps=config.layer_norm_eps - ) + + norm_cls, norm_eps = select_norm_impl(config) + self.norm = norm_cls(param_shape=(config.hidden_size), eps=norm_eps) + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( ["hidden_states", "residual"], config.hidden_size )