diff --git a/tests/kernels/core/test_apply_rotary_emb.py b/tests/kernels/core/test_apply_rotary_emb.py index a576199663..7d63c3499c 100644 --- a/tests/kernels/core/test_apply_rotary_emb.py +++ b/tests/kernels/core/test_apply_rotary_emb.py @@ -225,3 +225,46 @@ def test_apply_rotary_emb_xpu_matches_native( expected = op.forward_native(x, cos, sin) actual = op.forward_xpu(x, cos, sin) torch.testing.assert_close(actual, expected, atol=0, rtol=0) + + +@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm only test.") +@pytest.mark.parametrize("num_tokens", [257, 2304]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +def test_packed_qk_rope_correctness( + num_tokens: int, dtype: torch.dtype, default_vllm_config +): + """packed_qk_rope_ must be bitwise identical to the per-tensor + ApplyRotaryEmb path with enable_fp32_compute=True, and must leave the V + slice untouched.""" + from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb + from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import ( + packed_qk_rope_, + ) + + num_heads, head_dim = 12, 128 + rng = torch.Generator(device="cuda").manual_seed(0) + + angles = ( + torch.rand(num_tokens, head_dim // 2, device="cuda", generator=rng) * torch.pi + ) + freqs_cis = torch.polar(torch.ones_like(angles), angles) + cos = freqs_cis.real.contiguous() + sin = freqs_cis.imag.contiguous() + + xqkv = torch.randn( + num_tokens, 3, num_heads, head_dim, dtype=dtype, device="cuda", generator=rng + ) + xqkv_ref = xqkv.clone() + xq, xk, xv = torch.unbind(xqkv_ref, dim=-3) + + op = ApplyRotaryEmb( + enforce_enable=True, is_neox_style=False, enable_fp32_compute=True + ) + # op() dispatches to forward_cuda / forward_hip per platform. + xq_ref = op(xq, cos, sin) + xk_ref = op(xk, cos, sin) + + packed_qk_rope_(xqkv, freqs_cis) + torch.testing.assert_close(xqkv[:, 0], xq_ref, atol=0, rtol=0) + torch.testing.assert_close(xqkv[:, 1], xk_ref, atol=0, rtol=0) + torch.testing.assert_close(xqkv[:, 2], xv, atol=0, rtol=0) diff --git a/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py b/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py new file mode 100644 index 0000000000..e7d916e48f --- /dev/null +++ b/vllm/model_executor/layers/rotary_embedding/packed_qk_rope.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""In-place interleaved RoPE on the Q/K slices of a packed QKV buffer.""" + +import torch + +from vllm.triton_utils import tl, triton + + +# seqlen is not specialized: image sizes vary per request. +@triton.jit(do_not_specialize=["seqlen"]) +def _packed_qk_rope_kernel( + qk_ptr, # (seqlen, 2 * nheads, headdim) strided view of the packed QKV + freqs_ptr, # (seqlen, rotary_dim) fp32 re/im view of the complex freqs + seqlen, + qk_row_stride, + qk_head_stride, + rotary_dim: tl.constexpr, + BLOCK_M: tl.constexpr, +): + BLOCK_K: tl.constexpr = triton.next_power_of_2(rotary_dim) + pid_m = tl.program_id(0) + pid_h = tl.program_id(1) + rm = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + rk = tl.arange(0, BLOCK_K) + mask_m = rm < seqlen + + base = qk_ptr + pid_h * qk_head_stride + rm[:, None] * qk_row_stride + mask_x = mask_m[:, None] & (rk < rotary_dim)[None, :] + x = tl.load(base + rk[None, :], mask=mask_x, other=0.0).to(tl.float32) + f = tl.load(freqs_ptr + rm[:, None] * rotary_dim + rk[None, :], mask=mask_x) + cos, sin = tl.split(tl.reshape(f, (BLOCK_M, BLOCK_K // 2, 2))) + + x0, x1 = tl.split(tl.reshape(x, (BLOCK_M, BLOCK_K // 2, 2))) + o0 = x0 * cos - x1 * sin + o1 = x1 * cos + x0 * sin + tl.store(base + rk[None, :], tl.interleave(o0, o1), mask=mask_x) + + +def packed_qk_rope_(xqkv: torch.Tensor, freqs_cis: torch.Tensor) -> None: + """Rotate Q and K in place inside a packed QKV buffer. + + Args: + xqkv: contiguous (seqlen, 3, nheads, headdim); only the Q and K + slices are rotated. + freqs_cis: contiguous (seqlen, headdim // 2) complex64 rotary freqs, + read through its interleaved re/im fp32 view. + + Bitwise identical to ``ApplyRotaryEmb(enable_fp32_compute=True)`` applied + to Q and K separately, but in one kernel with ~5x less memory traffic. + Requires triton: callers must check HAS_TRITON and use the unfused path + otherwise. Contract violations raise. + + """ + assert xqkv.ndim == 4 and xqkv.size(1) == 3, xqkv.shape + assert xqkv.is_contiguous(), xqkv.shape + assert freqs_cis.ndim == 2 and freqs_cis.is_contiguous(), freqs_cis.shape + assert freqs_cis.dtype == torch.complex64, freqs_cis.dtype + + seq_length, _, num_heads, head_dim = xqkv.shape + rotary_dim = freqs_cis.size(-1) * 2 + assert freqs_cis.size(0) == seq_length, (freqs_cis.shape, xqkv.shape) + assert rotary_dim == head_dim, (freqs_cis.shape, xqkv.shape) # full rotation + qk = xqkv.as_strided( + (seq_length, 2 * num_heads, head_dim), + (3 * num_heads * head_dim, head_dim, 1), + ) + + BLOCK_M = 4 + # seqlen on grid.x: HIP caps gridDim.y at 65535. + grid = (triton.cdiv(seq_length, BLOCK_M), 2 * num_heads) + _packed_qk_rope_kernel[grid]( + qk, + freqs_cis.view(torch.float32), + seq_length, + qk.stride(0), + qk.stride(1), + rotary_dim=rotary_dim, + BLOCK_M=BLOCK_M, + num_warps=2 if rotary_dim <= 64 else 4, + ) diff --git a/vllm/model_executor/models/kimi_k25_vit.py b/vllm/model_executor/models/kimi_k25_vit.py index 1b05f1f77b..72e45d82e6 100644 --- a/vllm/model_executor/models/kimi_k25_vit.py +++ b/vllm/model_executor/models/kimi_k25_vit.py @@ -30,6 +30,9 @@ from vllm.model_executor.layers.linear import ( ) from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb +from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import ( + packed_qk_rope_, +) from vllm.model_executor.models.utils import maybe_prefix from vllm.model_executor.models.vision import ( is_vit_use_data_parallel, @@ -37,6 +40,7 @@ from vllm.model_executor.models.vision import ( ) from vllm.platforms import current_platform from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig +from vllm.triton_utils import HAS_TRITON from vllm.utils.torch_utils import async_tensor_h2d logger = init_logger(__name__) @@ -451,14 +455,19 @@ class MoonViTEncoderLayer(nn.Module): ) # xqkv: (seqlen, 3, nheads, headdim) xqkv = xqkv.view(*qkv_shape) - xq, xk, xv = torch.unbind(xqkv, dim=-3) - _apply_rope_input_validation(xq, rope_freqs_cis) - _apply_rope_input_validation(xk, rope_freqs_cis) - rope_cos = rope_freqs_cis.real.contiguous() - rope_sin = rope_freqs_cis.imag.contiguous() - xq = self.apply_rotary_emb(xq, rope_cos, rope_sin) - xk = self.apply_rotary_emb(xk, rope_cos, rope_sin) + # xq/xk alias xqkv, so the in-place rotation below is visible through + # them; only the fallback needs to rebind. + xq, xk, xv = torch.unbind(xqkv, dim=-3) + if HAS_TRITON: + packed_qk_rope_(xqkv, rope_freqs_cis) + else: + _apply_rope_input_validation(xq, rope_freqs_cis) + _apply_rope_input_validation(xk, rope_freqs_cis) + rope_cos = rope_freqs_cis.real.contiguous() + rope_sin = rope_freqs_cis.imag.contiguous() + xq = self.apply_rotary_emb(xq, rope_cos, rope_sin) + xk = self.apply_rotary_emb(xk, rope_cos, rope_sin) if max_seqlen is None: max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()