[XPU] Enable multiple key kernels for sparse attention (#37888)

Signed-off-by: Xiaochang Wu <xiaochang.wu@intel.com>
Signed-off-by: Wu, Xiaochang <xiaochang.wu@intel.com>
Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
Xiaochang Wu
2026-05-22 12:02:51 +08:00
committed by GitHub
co-authored by Kunshang Ji
parent 18a27cc9a3
commit 8c8b1825eb
3 changed files with 202 additions and 312 deletions
@@ -5,6 +5,7 @@
import torch
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.forward_context import get_forward_context
@@ -28,11 +29,6 @@ from vllm.v1.attention.backends.mla.indexer import (
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
from vllm.v1.worker.workspace import current_workspace_manager
if current_platform.is_cuda_alike():
from vllm import _custom_ops as ops
elif current_platform.is_xpu():
from vllm._xpu_ops import xpu_ops
logger = init_logger(__name__)
RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
@@ -222,42 +218,42 @@ def sparse_attn_indexer(
q_slice_cast = q_slice
k_quant_cast = k_quant
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
clean_logits=False,
)
if current_platform.is_xpu():
if q_scale_slice is not None:
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
logits = torch.ops.vllm.xpu_fp8_mqa_logits(
q_slice_cast,
k_quant_cast,
k_scale_cast,
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
)
else:
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
clean_logits=False,
)
num_rows = logits.shape[0]
topk_indices = topk_indices_buffer[
chunk.token_start : chunk.token_end, :topk_tokens
]
if current_platform.is_xpu():
xpu_ops.top_k_per_row_prefill( # type: ignore[attr-defined]
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
)
else:
torch.ops._C.top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
)
ops.top_k_per_row_prefill(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
)
if has_decode:
decode_metadata = attn_metadata_narrowed.decode
@@ -309,16 +305,32 @@ def sparse_attn_indexer(
if use_fp4_cache
else padded_q_quant_decode_tokens
)
logits = fp8_fp4_paged_mqa_logits(
(padded_q_quant_cast, padded_q_scale),
kv_cache,
weights[:num_padded_tokens],
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_model_len,
clean_logits=False,
)
if current_platform.is_xpu():
if padded_q_scale is not None:
raise RuntimeError("XPU fp8_paged_mqa_logits does not support FP4 Q")
seq_lens_xpu = (
seq_lens[:, -1].contiguous() if seq_lens.ndim == 2 else seq_lens
)
logits = torch.ops.vllm.xpu_fp8_paged_mqa_logits(
padded_q_quant_cast,
kv_cache,
weights[:num_padded_tokens],
seq_lens_xpu,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len,
)
else:
logits = fp8_fp4_paged_mqa_logits(
(padded_q_quant_cast, padded_q_scale),
kv_cache,
weights[:num_padded_tokens],
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_model_len,
clean_logits=False,
)
num_rows = logits.shape[0]
topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
@@ -336,28 +348,16 @@ def sparse_attn_indexer(
attn_metadata_narrowed.max_seq_len,
)
else:
if current_platform.is_xpu():
xpu_ops.top_k_per_row_decode( # type: ignore[attr-defined]
logits,
next_n,
seq_lens,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
)
else:
torch.ops._C.top_k_per_row_decode(
logits,
next_n,
seq_lens,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
)
ops.top_k_per_row_decode(
logits,
next_n,
seq_lens,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
)
if decode_metadata.requires_padding:
# if padded, we need to unpack
@@ -494,6 +494,15 @@ class SparseAttnIndexer(CustomOp):
self.use_fp4_cache,
)
def forward_xpu(
self,
hidden_states: torch.Tensor,
q_fp8: torch.Tensor,
k: torch.Tensor,
weights: torch.Tensor,
):
return self.forward_cuda(hidden_states, q_fp8, k, weights)
def forward_hip(
self,
hidden_states: torch.Tensor,