[XPU] Wire up SYCL apply_rotary_emb kernel in ApplyRotaryEmb (#55721)

Signed-off-by: Michal Ganczarenko <michal.ganczarenko@intel.com>
Signed-off-by: Michał Ganczarenko <michal.ganczarenko@intel.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Michał Ganczarenko
2026-09-22 20:43:24 -07:00
committed by GitHub
co-authored by Claude
parent 94f4170df3
commit 73c59d365b
3 changed files with 78 additions and 0 deletions
@@ -184,3 +184,44 @@ def test_rotary_embedding_dispatch(
- forward_cuda/forward methods should call ApplyRotaryEmb.forward()
"""
run_dispatch_test(test_case, device)
@pytest.mark.skipif(not current_platform.is_xpu(), reason="XPU only test.")
@pytest.mark.parametrize(
"num_tokens,num_heads,head_size,rot_dim,is_neox_style",
[
(128, 32, 128, 128, True), # typical text shape
(4096, 16, 80, 80, False), # GPT-J style (interleaved)
(4096, 32, 128, 64, True), # partial rotary (rot_dim < head_size)
],
ids=["text", "gptj_interleaved", "partial_rotary"],
)
def test_apply_rotary_emb_xpu_matches_native(
num_tokens: int,
num_heads: int,
head_size: int,
rot_dim: int,
is_neox_style: bool,
):
"""ApplyRotaryEmb.forward_xpu (vllm_xpu_kernels SYCL op) must match
forward_native (the CustomOp default fallback) exactly."""
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
vllm_config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["all", "+apply_rotary_emb"])
)
get_cached_compilation_config.cache_clear()
with set_current_vllm_config(vllm_config):
op = ApplyRotaryEmb(enforce_enable=True, is_neox_style=is_neox_style)
x = torch.randn(
num_tokens, num_heads, head_size, device="xpu", dtype=torch.bfloat16
)
x = x[..., :rot_dim].contiguous()
cos = torch.randn(num_tokens, rot_dim // 2, device="xpu", dtype=torch.bfloat16)
sin = torch.randn(num_tokens, rot_dim // 2, device="xpu", dtype=torch.bfloat16)
expected = op.forward_native(x, cos, sin)
actual = op.forward_xpu(x, cos, sin)
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
+25
View File
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING
import torch
from vllm_xpu_kernels.flash_attn_interface import flash_attn_varlen_func
from vllm_xpu_kernels.rotary import apply_rotary_emb
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.logger import init_logger
@@ -249,6 +250,24 @@ def _xpu_ops_deepseek_scaling_rope_fake(
return query, key
def _xpu_apply_rotary_emb_impl(
x: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
is_neox_style: bool,
) -> torch.Tensor:
return apply_rotary_emb(x, cos, sin, is_neox_style)
def _xpu_apply_rotary_emb_fake(
x: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
is_neox_style: bool,
) -> torch.Tensor:
return torch.empty_like(x)
def _xpu_fp8_bmm_impl(
a: torch.Tensor,
b: torch.Tensor,
@@ -1257,6 +1276,12 @@ class xpu_ops:
dispatch_key=current_platform.dispatch_key,
)
direct_register_custom_op(
op_name="xpu_apply_rotary_emb",
op_func=_xpu_apply_rotary_emb_impl,
fake_impl=_xpu_apply_rotary_emb_fake,
)
direct_register_custom_op(
op_name="xpu_mxfp8_quantize",
op_func=_xpu_mxfp8_quantize_impl,
@@ -237,6 +237,18 @@ class ApplyRotaryEmb(CustomOp):
output = self._post_process(output, origin_shape, origin_dtype)
return output
def forward_xpu(
self,
x: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
) -> torch.Tensor:
import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_apply_rotary_emb
x, cos, sin, origin_shape, origin_dtype = self._pre_process(x, cos, sin)
output = torch.ops.vllm.xpu_apply_rotary_emb(x, cos, sin, self.is_neox_style)
return self._post_process(output, origin_shape, origin_dtype)
def forward_hip(
self,
x: torch.Tensor,