[Kernel][Kimi] fused vision q/k roper kernel (#50400)
Signed-off-by: rongfu.leng <lenronfu@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -29,6 +29,7 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.model_executor.models.vision import (
|
||||
is_vit_use_data_parallel,
|
||||
@@ -79,29 +80,6 @@ def get_rope_shape(org, interpolation_mode, shape):
|
||||
)
|
||||
|
||||
|
||||
def apply_rope(
|
||||
xq: torch.Tensor, xk: torch.Tensor, freqs_cis: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Args: (The leading dimensions of all inputs should be the same)
|
||||
xq: query, tensor of shape (..., num_heads, head_dim)
|
||||
xk: key, tensor of shape (..., num_heads, head_dim)
|
||||
freqs_cis: tensor of shape (..., head_dim/2), dtype=torch.complex64.
|
||||
Returns:
|
||||
xq_out, xk_out: tensors of shape (..., num_heads, head_dim)
|
||||
"""
|
||||
_apply_rope_input_validation(xq, freqs_cis)
|
||||
_apply_rope_input_validation(xk, freqs_cis)
|
||||
|
||||
freqs_cis = freqs_cis.unsqueeze(-2) # ..., 1, head_dim/2
|
||||
# ..., num_heads, head_dim/2
|
||||
xq_ = torch.view_as_complex(xq.float().view(*xq.shape[:-1], -1, 2))
|
||||
xk_ = torch.view_as_complex(xk.float().view(*xq.shape[:-1], -1, 2))
|
||||
xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(-2) # ..., num_heads, head_dim
|
||||
xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(-2) # ..., num_heads, head_dim
|
||||
return xq_out.type_as(xq), xk_out.type_as(xk)
|
||||
|
||||
|
||||
def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
|
||||
"""Generate 1D sincos positional embedding from grid positions."""
|
||||
assert embed_dim % 2 == 0
|
||||
@@ -451,12 +429,17 @@ class MoonViTEncoderLayer(nn.Module):
|
||||
scale=self.hidden_size_per_attention_head**-0.5,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
self.apply_rotary_emb = ApplyRotaryEmb(
|
||||
enforce_enable=True,
|
||||
is_neox_style=False,
|
||||
enable_fp32_compute=True,
|
||||
)
|
||||
|
||||
def attention_qkvpacked(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
rope_freqs_cis: torch.Tensor | None = None,
|
||||
rope_freqs_cis: torch.Tensor,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
):
|
||||
@@ -478,7 +461,12 @@ class MoonViTEncoderLayer(nn.Module):
|
||||
xqkv = xqkv.view(*qkv_shape)
|
||||
xq, xk, xv = torch.unbind(xqkv, dim=-3)
|
||||
|
||||
xq, xk = apply_rope(xq, xk, rope_freqs_cis)
|
||||
_apply_rope_input_validation(xq, rope_freqs_cis)
|
||||
_apply_rope_input_validation(xk, rope_freqs_cis)
|
||||
rope_cos = rope_freqs_cis.real.contiguous()
|
||||
rope_sin = rope_freqs_cis.imag.contiguous()
|
||||
xq = self.apply_rotary_emb(xq, rope_cos, rope_sin)
|
||||
xk = self.apply_rotary_emb(xk, rope_cos, rope_sin)
|
||||
|
||||
if max_seqlen is None:
|
||||
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
|
||||
@@ -502,7 +490,7 @@ class MoonViTEncoderLayer(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
rope_freqs_cis: torch.Tensor | None = None,
|
||||
rope_freqs_cis: torch.Tensor,
|
||||
max_seqlen: torch.Tensor | None = None,
|
||||
sequence_lengths: torch.Tensor | None = None,
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user