[Cohere] Misc changes to cohere model definitions (#50156)
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
101c4477dd
commit
aa9903490c
@@ -208,7 +208,7 @@ class Cohere2MoeAttention(nn.Module):
|
||||
layer_types is not None
|
||||
and layer_types[self.layer_idx] == "sliding_attention"
|
||||
):
|
||||
self.sliding_window = config.sliding_window
|
||||
self.sliding_window = config.sliding_window + 1
|
||||
|
||||
# Prefix-dense layers have full attention (no sliding window). When
|
||||
# prefix_dense_sliding_window_pattern == 1, they keep RoPE even though
|
||||
|
||||
@@ -32,7 +32,11 @@ from transformers import Cohere2Config, CohereConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.linear import (
|
||||
@@ -44,9 +48,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
row_parallel_weight_loader,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import row_parallel_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -87,18 +89,61 @@ class LayerNorm(nn.Module):
|
||||
return hidden_states, residuals
|
||||
|
||||
|
||||
@torch.compile(backend=current_platform.simple_compile_backend)
|
||||
def rms_norm_func(hidden_states, weight, variance_epsilon):
|
||||
input_dtype = hidden_states.dtype
|
||||
hidden_states = hidden_states.to(torch.float32)
|
||||
|
||||
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
||||
hidden_states = hidden_states * torch.rsqrt(variance + variance_epsilon)
|
||||
|
||||
hidden_states = weight.to(torch.float32) * hidden_states
|
||||
return hidden_states.to(input_dtype)
|
||||
|
||||
|
||||
class RMSNorm(nn.Module):
|
||||
def __init__(self, param_shape=None, eps=1e-6):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(param_shape))
|
||||
self.variance_epsilon = eps
|
||||
set_weight_attrs(self.weight, {"weight_loader": row_parallel_weight_loader})
|
||||
|
||||
def forward(self, hidden_states, residuals=None):
|
||||
hidden_states = rms_norm_func(hidden_states, self.weight, self.variance_epsilon)
|
||||
return hidden_states, residuals
|
||||
|
||||
|
||||
def select_norm_impl(config: CohereConfig) -> tuple[type[nn.Module], float]:
|
||||
"""
|
||||
Returns the normalization layer class and epsilon value to use.
|
||||
If `config.rms_norm_eps` is present, use RMSNorm.
|
||||
Otherwise default to LayerNorm with `config.layer_norm_eps`.
|
||||
"""
|
||||
rms_eps = getattr(config, "rms_norm_eps", None)
|
||||
if rms_eps is not None:
|
||||
return RMSNorm, rms_eps
|
||||
|
||||
return LayerNorm, config.layer_norm_eps
|
||||
|
||||
|
||||
# Copied from transformers.models.llama.modeling_llama.LlamaMLP Llama->Cohere
|
||||
class CohereMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: CohereConfig | Cohere2Config,
|
||||
config: CohereConfig,
|
||||
intermediate_size: int | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
# so we can reduce the attention and MLP outputs together
|
||||
reduce_results: bool = False,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.hidden_size = config.hidden_size
|
||||
self.intermediate_size = config.intermediate_size
|
||||
if intermediate_size is None:
|
||||
self.intermediate_size = config.intermediate_size
|
||||
else:
|
||||
self.intermediate_size = intermediate_size
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
[self.intermediate_size] * 2,
|
||||
@@ -111,6 +156,7 @@ class CohereMLP(nn.Module):
|
||||
self.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
@@ -133,11 +179,15 @@ class CohereAttention(nn.Module):
|
||||
super().__init__()
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.config = config
|
||||
self.layer_idx = extract_layer_index(prefix)
|
||||
self.attention_dropout = config.attention_dropout
|
||||
self.hidden_size = config.hidden_size
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.head_dim = self.hidden_size // self.total_num_heads
|
||||
if hasattr(config, "head_dim"):
|
||||
self.head_dim = config.head_dim
|
||||
else:
|
||||
self.head_dim = self.hidden_size // self.total_num_heads
|
||||
self.total_num_kv_heads = config.num_key_value_heads
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
# Number of KV heads is greater than TP size, so we partition
|
||||
@@ -171,6 +221,9 @@ class CohereAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
# NOTE: reduction will happen in the decoder layer forward
|
||||
# so we can combine the attention and MLP outputs together
|
||||
reduce_results=False,
|
||||
)
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
@@ -182,11 +235,14 @@ class CohereAttention(nn.Module):
|
||||
# Model v2 has interleaved sliding windows, v1 does not
|
||||
self.v1 = isinstance(config, CohereConfig)
|
||||
|
||||
# cohere swa layer sees [pos - sliding_window, pos], i.e. sliding_window + 1
|
||||
# tokens. vLLM's FlashAttention backend does (value - 1, 0), so we pass
|
||||
# sliding_window + 1 here to match the training convention. The same +1
|
||||
# propagates into the KV-cache eviction formula
|
||||
# (single_type_kv_cache_manager.py), keeping both paths consistent.
|
||||
self.sliding_window = None
|
||||
if not self.v1:
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
if config.layer_types[layer_idx] == "sliding_attention":
|
||||
self.sliding_window = config.sliding_window
|
||||
if not self.v1 and config.layer_types[self.layer_idx] == "sliding_attention":
|
||||
self.sliding_window = config.sliding_window + 1
|
||||
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
@@ -198,13 +254,13 @@ class CohereAttention(nn.Module):
|
||||
per_layer_sliding_window=self.sliding_window,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
norm_cls, norm_eps = select_norm_impl(config)
|
||||
if self.use_qk_norm:
|
||||
self.q_norm = LayerNorm(
|
||||
param_shape=(self.num_heads, self.head_dim), eps=config.layer_norm_eps
|
||||
self.q_norm = norm_cls(
|
||||
param_shape=(self.num_heads, self.head_dim), eps=norm_eps
|
||||
)
|
||||
self.k_norm = LayerNorm(
|
||||
param_shape=(self.num_kv_heads, self.head_dim),
|
||||
eps=config.layer_norm_eps,
|
||||
self.k_norm = norm_cls(
|
||||
param_shape=(self.num_kv_heads, self.head_dim), eps=norm_eps
|
||||
)
|
||||
|
||||
def _apply_qk_norm(self, q, k):
|
||||
@@ -242,18 +298,16 @@ class CohereDecoderLayer(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.self_attn = CohereAttention(
|
||||
config,
|
||||
cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
|
||||
self.mlp = CohereMLP(config, quant_config=quant_config, prefix=f"{prefix}.mlp")
|
||||
self.input_layernorm = LayerNorm(
|
||||
param_shape=(config.hidden_size), eps=config.layer_norm_eps
|
||||
)
|
||||
norm_cls, norm_eps = select_norm_impl(config)
|
||||
self.input_layernorm = norm_cls(param_shape=(config.hidden_size), eps=norm_eps)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -269,8 +323,14 @@ class CohereDecoderLayer(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
)
|
||||
hidden_states_mlp = self.mlp(hidden_states)
|
||||
parallel_block_output = hidden_states_attention + hidden_states_mlp
|
||||
# Add everything together
|
||||
hidden_states = residual + hidden_states_attention + hidden_states_mlp
|
||||
if self.tp_size > 1:
|
||||
# do the reduction in 1 shot instead of 2 separate all reduce
|
||||
parallel_block_output = tensor_model_parallel_all_reduce(
|
||||
parallel_block_output
|
||||
)
|
||||
hidden_states = residual + parallel_block_output
|
||||
|
||||
return hidden_states, residual
|
||||
|
||||
@@ -299,9 +359,10 @@ class CohereModel(nn.Module):
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self.norm = LayerNorm(
|
||||
param_shape=(config.hidden_size), eps=config.layer_norm_eps
|
||||
)
|
||||
|
||||
norm_cls, norm_eps = select_norm_impl(config)
|
||||
self.norm = norm_cls(param_shape=(config.hidden_size), eps=norm_eps)
|
||||
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user