diff --git a/vllm/model_executor/models/kimi_k25_vit.py b/vllm/model_executor/models/kimi_k25_vit.py index 20f5b105b9..5e858a129a 100644 --- a/vllm/model_executor/models/kimi_k25_vit.py +++ b/vllm/model_executor/models/kimi_k25_vit.py @@ -29,6 +29,7 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb from vllm.model_executor.models.utils import maybe_prefix from vllm.model_executor.models.vision import ( is_vit_use_data_parallel, @@ -79,29 +80,6 @@ def get_rope_shape(org, interpolation_mode, shape): ) -def apply_rope( - xq: torch.Tensor, xk: torch.Tensor, freqs_cis: torch.Tensor -) -> tuple[torch.Tensor, torch.Tensor]: - """ - Args: (The leading dimensions of all inputs should be the same) - xq: query, tensor of shape (..., num_heads, head_dim) - xk: key, tensor of shape (..., num_heads, head_dim) - freqs_cis: tensor of shape (..., head_dim/2), dtype=torch.complex64. - Returns: - xq_out, xk_out: tensors of shape (..., num_heads, head_dim) - """ - _apply_rope_input_validation(xq, freqs_cis) - _apply_rope_input_validation(xk, freqs_cis) - - freqs_cis = freqs_cis.unsqueeze(-2) # ..., 1, head_dim/2 - # ..., num_heads, head_dim/2 - xq_ = torch.view_as_complex(xq.float().view(*xq.shape[:-1], -1, 2)) - xk_ = torch.view_as_complex(xk.float().view(*xq.shape[:-1], -1, 2)) - xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(-2) # ..., num_heads, head_dim - xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(-2) # ..., num_heads, head_dim - return xq_out.type_as(xq), xk_out.type_as(xk) - - def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): """Generate 1D sincos positional embedding from grid positions.""" assert embed_dim % 2 == 0 @@ -451,12 +429,17 @@ class MoonViTEncoderLayer(nn.Module): scale=self.hidden_size_per_attention_head**-0.5, prefix=f"{prefix}.attn", ) + self.apply_rotary_emb = ApplyRotaryEmb( + enforce_enable=True, + is_neox_style=False, + enable_fp32_compute=True, + ) def attention_qkvpacked( self, x: torch.Tensor, cu_seqlens: torch.Tensor, - rope_freqs_cis: torch.Tensor | None = None, + rope_freqs_cis: torch.Tensor, max_seqlen: torch.Tensor | None = None, sequence_lengths: torch.Tensor | None = None, ): @@ -478,7 +461,12 @@ class MoonViTEncoderLayer(nn.Module): xqkv = xqkv.view(*qkv_shape) xq, xk, xv = torch.unbind(xqkv, dim=-3) - xq, xk = apply_rope(xq, xk, rope_freqs_cis) + _apply_rope_input_validation(xq, rope_freqs_cis) + _apply_rope_input_validation(xk, rope_freqs_cis) + rope_cos = rope_freqs_cis.real.contiguous() + rope_sin = rope_freqs_cis.imag.contiguous() + xq = self.apply_rotary_emb(xq, rope_cos, rope_sin) + xk = self.apply_rotary_emb(xk, rope_cos, rope_sin) if max_seqlen is None: max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() @@ -502,7 +490,7 @@ class MoonViTEncoderLayer(nn.Module): self, hidden_states: torch.Tensor, cu_seqlens: torch.Tensor, - rope_freqs_cis: torch.Tensor | None = None, + rope_freqs_cis: torch.Tensor, max_seqlen: torch.Tensor | None = None, sequence_lengths: torch.Tensor | None = None, ):