From 73c59d365b9e1e722fbea5e897e8efb646e3e513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ganczarenko?= Date: Wed, 23 Sep 2026 05:43:24 +0200 Subject: [PATCH] [XPU] Wire up SYCL apply_rotary_emb kernel in ApplyRotaryEmb (#55721) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Michal Ganczarenko Signed-off-by: MichaƂ Ganczarenko Co-authored-by: Claude --- tests/kernels/core/test_apply_rotary_emb.py | 41 +++++++++++++++++++ vllm/_xpu_ops.py | 25 +++++++++++ .../layers/rotary_embedding/common.py | 12 ++++++ 3 files changed, 78 insertions(+) diff --git a/tests/kernels/core/test_apply_rotary_emb.py b/tests/kernels/core/test_apply_rotary_emb.py index fd92de16bb..a576199663 100644 --- a/tests/kernels/core/test_apply_rotary_emb.py +++ b/tests/kernels/core/test_apply_rotary_emb.py @@ -184,3 +184,44 @@ def test_rotary_embedding_dispatch( - forward_cuda/forward methods should call ApplyRotaryEmb.forward() """ run_dispatch_test(test_case, device) + + +@pytest.mark.skipif(not current_platform.is_xpu(), reason="XPU only test.") +@pytest.mark.parametrize( + "num_tokens,num_heads,head_size,rot_dim,is_neox_style", + [ + (128, 32, 128, 128, True), # typical text shape + (4096, 16, 80, 80, False), # GPT-J style (interleaved) + (4096, 32, 128, 64, True), # partial rotary (rot_dim < head_size) + ], + ids=["text", "gptj_interleaved", "partial_rotary"], +) +def test_apply_rotary_emb_xpu_matches_native( + num_tokens: int, + num_heads: int, + head_size: int, + rot_dim: int, + is_neox_style: bool, +): + """ApplyRotaryEmb.forward_xpu (vllm_xpu_kernels SYCL op) must match + forward_native (the CustomOp default fallback) exactly.""" + from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb + + vllm_config = VllmConfig( + compilation_config=CompilationConfig(custom_ops=["all", "+apply_rotary_emb"]) + ) + get_cached_compilation_config.cache_clear() + + with set_current_vllm_config(vllm_config): + op = ApplyRotaryEmb(enforce_enable=True, is_neox_style=is_neox_style) + + x = torch.randn( + num_tokens, num_heads, head_size, device="xpu", dtype=torch.bfloat16 + ) + x = x[..., :rot_dim].contiguous() + cos = torch.randn(num_tokens, rot_dim // 2, device="xpu", dtype=torch.bfloat16) + sin = torch.randn(num_tokens, rot_dim // 2, device="xpu", dtype=torch.bfloat16) + + expected = op.forward_native(x, cos, sin) + actual = op.forward_xpu(x, cos, sin) + torch.testing.assert_close(actual, expected, atol=0, rtol=0) diff --git a/vllm/_xpu_ops.py b/vllm/_xpu_ops.py index 5c19b01408..d25ef6b258 100644 --- a/vllm/_xpu_ops.py +++ b/vllm/_xpu_ops.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING import torch from vllm_xpu_kernels.flash_attn_interface import flash_attn_varlen_func +from vllm_xpu_kernels.rotary import apply_rotary_emb from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.logger import init_logger @@ -249,6 +250,24 @@ def _xpu_ops_deepseek_scaling_rope_fake( return query, key +def _xpu_apply_rotary_emb_impl( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + is_neox_style: bool, +) -> torch.Tensor: + return apply_rotary_emb(x, cos, sin, is_neox_style) + + +def _xpu_apply_rotary_emb_fake( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + is_neox_style: bool, +) -> torch.Tensor: + return torch.empty_like(x) + + def _xpu_fp8_bmm_impl( a: torch.Tensor, b: torch.Tensor, @@ -1257,6 +1276,12 @@ class xpu_ops: dispatch_key=current_platform.dispatch_key, ) + direct_register_custom_op( + op_name="xpu_apply_rotary_emb", + op_func=_xpu_apply_rotary_emb_impl, + fake_impl=_xpu_apply_rotary_emb_fake, + ) + direct_register_custom_op( op_name="xpu_mxfp8_quantize", op_func=_xpu_mxfp8_quantize_impl, diff --git a/vllm/model_executor/layers/rotary_embedding/common.py b/vllm/model_executor/layers/rotary_embedding/common.py index 24c96c2521..b96ab9cb66 100644 --- a/vllm/model_executor/layers/rotary_embedding/common.py +++ b/vllm/model_executor/layers/rotary_embedding/common.py @@ -237,6 +237,18 @@ class ApplyRotaryEmb(CustomOp): output = self._post_process(output, origin_shape, origin_dtype) return output + def forward_xpu( + self, + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + ) -> torch.Tensor: + import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_apply_rotary_emb + + x, cos, sin, origin_shape, origin_dtype = self._pre_process(x, cos, sin) + output = torch.ops.vllm.xpu_apply_rotary_emb(x, cos, sin, self.is_neox_style) + return self._post_process(output, origin_shape, origin_dtype) + def forward_hip( self, x: torch.Tensor,