[Kernel][Kimi] fused vision q/k roper kernel (#50400)

Signed-off-by: rongfu.leng <lenronfu@gmail.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
rongfu.leng
2026-08-20 09:15:47 +00:00
committed by GitHub
co-authored by Isotr0py
parent 5d4d470470
commit c8de519917
+14 -26
View File
@@ -29,6 +29,7 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
from vllm.model_executor.models.utils import maybe_prefix
from vllm.model_executor.models.vision import (
is_vit_use_data_parallel,
@@ -79,29 +80,6 @@ def get_rope_shape(org, interpolation_mode, shape):
)
def apply_rope(
xq: torch.Tensor, xk: torch.Tensor, freqs_cis: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Args: (The leading dimensions of all inputs should be the same)
xq: query, tensor of shape (..., num_heads, head_dim)
xk: key, tensor of shape (..., num_heads, head_dim)
freqs_cis: tensor of shape (..., head_dim/2), dtype=torch.complex64.
Returns:
xq_out, xk_out: tensors of shape (..., num_heads, head_dim)
"""
_apply_rope_input_validation(xq, freqs_cis)
_apply_rope_input_validation(xk, freqs_cis)
freqs_cis = freqs_cis.unsqueeze(-2) # ..., 1, head_dim/2
# ..., num_heads, head_dim/2
xq_ = torch.view_as_complex(xq.float().view(*xq.shape[:-1], -1, 2))
xk_ = torch.view_as_complex(xk.float().view(*xq.shape[:-1], -1, 2))
xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(-2) # ..., num_heads, head_dim
xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(-2) # ..., num_heads, head_dim
return xq_out.type_as(xq), xk_out.type_as(xk)
def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
"""Generate 1D sincos positional embedding from grid positions."""
assert embed_dim % 2 == 0
@@ -451,12 +429,17 @@ class MoonViTEncoderLayer(nn.Module):
scale=self.hidden_size_per_attention_head**-0.5,
prefix=f"{prefix}.attn",
)
self.apply_rotary_emb = ApplyRotaryEmb(
enforce_enable=True,
is_neox_style=False,
enable_fp32_compute=True,
)
def attention_qkvpacked(
self,
x: torch.Tensor,
cu_seqlens: torch.Tensor,
rope_freqs_cis: torch.Tensor | None = None,
rope_freqs_cis: torch.Tensor,
max_seqlen: torch.Tensor | None = None,
sequence_lengths: torch.Tensor | None = None,
):
@@ -478,7 +461,12 @@ class MoonViTEncoderLayer(nn.Module):
xqkv = xqkv.view(*qkv_shape)
xq, xk, xv = torch.unbind(xqkv, dim=-3)
xq, xk = apply_rope(xq, xk, rope_freqs_cis)
_apply_rope_input_validation(xq, rope_freqs_cis)
_apply_rope_input_validation(xk, rope_freqs_cis)
rope_cos = rope_freqs_cis.real.contiguous()
rope_sin = rope_freqs_cis.imag.contiguous()
xq = self.apply_rotary_emb(xq, rope_cos, rope_sin)
xk = self.apply_rotary_emb(xk, rope_cos, rope_sin)
if max_seqlen is None:
max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
@@ -502,7 +490,7 @@ class MoonViTEncoderLayer(nn.Module):
self,
hidden_states: torch.Tensor,
cu_seqlens: torch.Tensor,
rope_freqs_cis: torch.Tensor | None = None,
rope_freqs_cis: torch.Tensor,
max_seqlen: torch.Tensor | None = None,
sequence_lengths: torch.Tensor | None = None,
):