[Attention][DSA] support dcp for FLASHINFER_MLA_SPARSE (#46076)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Jingyi Yang <girasoleyang@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: GirasoleY <girasoleyang@gmail.com>
Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jiangyun Zhu
2026-07-01 00:32:20 -04:00
committed by GitHub
co-authored by Jingyi Yang Lucas Wilkinson Claude Opus 4.8
parent 3c1396bab6
commit c5200d3565
12 changed files with 1995 additions and 97 deletions
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,420 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import cache
import cutlass
import cutlass.cute as cute
import torch
from cuda.bindings.driver import CUstream
from cutlass import Float32, Int32, Uint32, Uint64
from quack.compile_utils import make_fake_tensor
from vllm.cute_utils import recast_val
from vllm.triton_utils import tl, triton
def stable_topk_from_gathered_candidates_cutedsl(
gathered: torch.Tensor,
topk: int,
out: torch.Tensor | None = None,
) -> torch.Tensor:
if out is None:
out = torch.empty(
(gathered.shape[0], topk),
dtype=torch.int32,
device=gathered.device,
)
StableTopKFromGatheredCandidatesKernel.compile(topk, gathered.shape[1])(
gathered, out
)
return out
def pack_dcp_topk_candidates_cutedsl(
logits: torch.Tensor,
topk_indices: torch.Tensor,
packed: torch.Tensor,
dcp_rank: int,
dcp_world_size: int,
cp_interleave: int,
row_starts: torch.Tensor | None,
) -> None:
topk = topk_indices.shape[1]
grid = (topk_indices.shape[0], triton.cdiv(topk, 512))
row_starts_arg = row_starts if row_starts is not None else topk_indices
_pack_dcp_topk_candidates_triton_kernel[grid](
logits,
topk_indices,
packed,
row_starts_arg,
logits.stride(0),
logits.stride(1),
topk_indices.stride(0),
topk_indices.stride(1),
packed.stride(0),
packed.stride(1),
packed.stride(2),
logits.shape[1],
DCP_RANK=dcp_rank,
DCP_WORLD_SIZE=dcp_world_size,
CP_INTERLEAVE=cp_interleave,
HAS_ROW_STARTS=row_starts is not None,
TOPK=topk,
BLOCK_SIZE=512,
num_warps=8,
)
@triton.jit
def _pack_dcp_topk_candidates_triton_kernel(
logits,
topk_indices,
packed,
row_starts,
logits_stride0: tl.constexpr,
logits_stride1: tl.constexpr,
topk_stride0: tl.constexpr,
topk_stride1: tl.constexpr,
packed_stride0: tl.constexpr,
packed_stride1: tl.constexpr,
packed_stride2: tl.constexpr,
num_cols,
DCP_RANK: tl.constexpr,
DCP_WORLD_SIZE: tl.constexpr,
CP_INTERLEAVE: tl.constexpr,
HAS_ROW_STARTS: tl.constexpr,
TOPK: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
row = tl.program_id(0)
tile = tl.program_id(1)
cols = tile * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = cols < TOPK
local_idx = tl.load(
topk_indices + row * topk_stride0 + cols * topk_stride1,
mask=mask,
other=-1,
)
valid = local_idx >= 0
safe_local_idx = tl.maximum(local_idx, 0)
row_start = 0
if HAS_ROW_STARTS:
row_start = tl.load(row_starts + row)
score_col = safe_local_idx + row_start
score_col = tl.minimum(score_col, tl.maximum(num_cols - 1, 0))
score = tl.load(
logits + row * logits_stride0 + score_col * logits_stride1,
mask=mask & valid,
other=-float("inf"),
)
global_id = (
(safe_local_idx // CP_INTERLEAVE) * (DCP_WORLD_SIZE * CP_INTERLEAVE)
+ DCP_RANK * CP_INTERLEAVE
+ safe_local_idx % CP_INTERLEAVE
)
global_id = tl.where(valid, global_id, -1)
packed_base = packed + row * packed_stride0 + cols * packed_stride1
tl.store(packed_base, score, mask=mask)
tl.store(packed_base + packed_stride2, global_id.to(tl.float32), mask=mask)
@cute.jit
def _warp_scan_inclusive_i32(val: Int32, lane: Int32) -> Int32:
for i in cutlass.range_constexpr(cute.arch.WARP_SIZE.bit_length() - 1):
offset = 1 << i
partial = cute.arch.shuffle_sync_up(val, offset=offset, mask_and_clamp=0)
if lane >= offset:
val += partial
return val
@cute.jit
def _block_scan_inclusive_i32(
val: Int32,
lane: Int32,
warp_id: Int32,
warp_scratch: cute.Tensor,
warps_per_block: int,
) -> Int32:
prefix = _warp_scan_inclusive_i32(val, lane)
if lane == Int32(cute.arch.WARP_SIZE - 1):
warp_scratch[0, warp_id] = prefix
cute.arch.sync_threads()
if warp_id == Int32(0):
warp_total = Int32(0)
if lane < Int32(warps_per_block):
warp_total = warp_scratch[0, lane]
warp_prefix = _warp_scan_inclusive_i32(warp_total, lane)
if lane < Int32(warps_per_block):
warp_scratch[0, lane] = warp_prefix - warp_total
cute.arch.sync_threads()
return prefix + warp_scratch[0, warp_id]
class StableTopKFromGatheredCandidatesKernel:
tb_size = 512
hist_bins = 2048
radix_bits = (hist_bins - 1).bit_length()
assert hist_bins == 1 << radix_bits
key_bits = Uint64.width
radix_passes = (key_bits + radix_bits - 1) // radix_bits
final_radix_bits = key_bits - radix_bits * (radix_passes - 1)
hist_chunks = (hist_bins + tb_size - 1) // tb_size
warps_per_block = tb_size // cute.arch.WARP_SIZE
def __init__(self, topk: int, num_candidates: int):
assert num_candidates % self.tb_size == 0, (
"StableTopKFromGatheredCandidatesKernel requires candidate count "
f"to be a multiple of {self.tb_size}, got {num_candidates}"
)
self.topk = topk
self.keys_per_thread = num_candidates // self.tb_size
@cute.struct
class SharedStorage:
hist: cute.struct.MemRange[Int32, self.hist_bins]
committed_count: cute.struct.MemRange[Int32, 1]
running_count: cute.struct.MemRange[Int32, 1]
threshold_bin: cute.struct.MemRange[Int32, 1]
threshold_found: cute.struct.MemRange[Int32, 1]
include_threshold_bin: cute.struct.MemRange[Int32, 1]
prefix_s: cute.struct.Align[cute.struct.MemRange[Uint64, 1], 8]
warp_totals: cute.struct.MemRange[Int32, self.warps_per_block]
self.shared_storage = SharedStorage
@cute.jit
def __call__(
self,
gathered: cute.Tensor,
out: cute.Tensor,
stream: CUstream,
):
grid = (gathered.shape[0], 1, 1)
self.kernel(gathered, out).launch(
grid=grid,
block=(self.tb_size, 1, 1),
stream=stream,
)
@cute.jit
def _stable_key(self, score: Float32, token_id: Int32) -> Uint64:
bits = recast_val(score, Uint32)
mask = Uint32(0x80000000)
if (bits & Uint32(0x80000000)) != Uint32(0):
mask = Uint32(0xFFFFFFFF)
score_key = Uint64(bits ^ mask) << Uint64(32)
id_key = Uint64(~Uint32(token_id))
key = score_key | id_key
if token_id < Int32(0):
key = Uint64(0)
return key
@cute.jit
def _prefix_matches(
self,
key: Uint64,
prefix: Uint64,
prefix_bits: Int32,
):
matches = prefix_bits == Int32(0)
if prefix_bits != Int32(0):
shift = Int32(self.key_bits) - prefix_bits
matches = (key >> Uint64(shift)) == (prefix >> Uint64(shift))
return matches
@cute.jit
def _radix_pass(
self,
keys: cute.Tensor,
output: cute.Tensor,
storage,
tid: Int32,
step: Int32,
bits: int,
is_final_pass: bool,
):
hist_smem = storage.hist.get_tensor(cute.make_layout((self.hist_bins,)))
committed_count_smem = storage.committed_count.data_ptr()
running_count_smem = storage.running_count.data_ptr()
threshold_bin_smem = storage.threshold_bin.data_ptr()
threshold_found_smem = storage.threshold_found.data_ptr()
include_threshold_bin_smem = storage.include_threshold_bin.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
warp_totals_smem = storage.warp_totals.get_tensor(
cute.make_layout((1, self.warps_per_block))
)
prefix_bits = step * Int32(self.radix_bits)
num_bins = 1 << bits
block_scan_iterations = (num_bins + self.tb_size - 1) // self.tb_size
shift = Int32(self.key_bits) - prefix_bits - Int32(bits)
bin_mask = Uint64(num_bins - 1)
prefix = prefix_smem.load()
for chunk in cutlass.range_constexpr(self.hist_chunks):
hist_smem[tid + Int32(chunk * self.tb_size)] = Int32(0)
if tid == Int32(0):
running_count_smem.store(committed_count_smem.load())
include_threshold_bin_smem.store(Int32(0))
threshold_found_smem.store(Int32(0))
cute.arch.sync_threads()
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
key = keys[key_idx]
if self._prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
cute.arch.atomic_add(
hist_smem.iterator + bin_idx,
Int32(1),
sem="relaxed",
scope="cta",
)
cute.arch.sync_threads()
lane = cute.arch.lane_idx()
warp_id = cute.arch.warp_idx()
# Each iteration scans one tb_size-wide slice of bins, high to low.
iter = Int32(0)
threshold_found = threshold_found_smem.load()
while threshold_found == Int32(0) and iter < Int32(block_scan_iterations):
bin_idx = Int32(num_bins - 1) - (iter * Int32(self.tb_size) + tid)
count = hist_smem[bin_idx]
chunk_inclusive = _block_scan_inclusive_i32(
count,
lane,
warp_id,
warp_totals_smem,
self.warps_per_block,
)
running_count = running_count_smem.load()
prior_in_scan_slice = chunk_inclusive - count
remaining = Int32(self.topk) - running_count - prior_in_scan_slice
if count > Int32(0) and remaining > Int32(0) and remaining <= count:
threshold_bin_smem.store(bin_idx)
if count <= remaining or cutlass.const_expr(is_final_pass):
include_threshold_bin_smem.store(Int32(1))
threshold_found_smem.store(Int32(1))
# Barrier: every thread must finish reading running_count for this
# slice before tb_size-1 advances it, else a warp racing ahead to
# the store makes a lagging thread double-count the slice total
# (-> remaining too small -> threshold too high -> under-fill).
cute.arch.sync_threads()
if tid == Int32(self.tb_size - 1):
running_count_smem.store(running_count + chunk_inclusive)
cute.arch.sync_threads()
threshold_found = threshold_found_smem.load()
iter += Int32(1)
threshold = threshold_bin_smem.load()
should_include_threshold = include_threshold_bin_smem.load() != Int32(0)
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
key = keys[key_idx]
if self._prefix_matches(key, prefix, prefix_bits):
bin_idx = Int32((key >> Uint64(shift)) & bin_mask)
selected = bin_idx > threshold
if should_include_threshold:
selected = selected or bin_idx == threshold
if selected:
dst = cute.arch.atomic_add(
committed_count_smem,
Int32(1),
sem="relaxed",
scope="cta",
)
if dst < Int32(self.topk):
output[dst] = recast_val(~Uint32(key), Int32)
cute.arch.sync_threads()
pass_finished = include_threshold_bin_smem.load()
if tid == Int32(0) and pass_finished == Int32(0):
prefix_smem.store(prefix | (Uint64(threshold) << Uint64(shift)))
cute.arch.sync_threads()
return pass_finished
@cute.kernel
def kernel(
self,
input: cute.Tensor,
out: cute.Tensor,
):
row, _, _ = cute.arch.block_idx()
tid, _, _ = cute.arch.thread_idx()
input_row = input[row, None, None]
output_row = out[row, None]
keys = cute.make_rmem_tensor((self.keys_per_thread,), Uint64)
smem = cutlass.utils.SmemAllocator()
storage = smem.allocate(self.shared_storage, 8)
committed_count_smem = storage.committed_count.data_ptr()
prefix_smem = storage.prefix_s.data_ptr()
for i in range(tid, self.topk, self.tb_size):
output_row[i] = Int32(-1)
for key_idx in cutlass.range_constexpr(self.keys_per_thread):
col = tid + Int32(key_idx * self.tb_size)
score = Float32(input_row[col, 0])
token_id = Int32(input_row[col, 1])
keys[key_idx] = self._stable_key(score, token_id)
if tid == Int32(0):
committed_count_smem.store(Int32(0))
prefix_smem.store(Uint64(0))
cute.arch.sync_threads()
step = Int32(0)
finished = Int32(0)
while finished == Int32(0) and step < Int32(self.radix_passes - 1):
finished = self._radix_pass(
keys,
output_row,
storage,
tid,
step,
self.radix_bits,
False,
)
step += Int32(1)
if finished == Int32(0):
self._radix_pass(
keys,
output_row,
storage,
tid,
Int32(self.radix_passes - 1),
self.final_radix_bits,
True,
)
@cache
@staticmethod
def compile(topk: int, num_candidates: int):
num_rows = cute.sym_int()
gathered = cute.runtime.make_fake_tensor(
Float32,
(num_rows, num_candidates, 2),
stride=(cute.sym_int64(divisibility=2), 2, 1),
assumed_align=8,
)
out = make_fake_tensor(Int32, (num_rows, topk), divisibility=1)
kernel = StableTopKFromGatheredCandidatesKernel(topk, num_candidates)
stream = cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=True)
return cute.compile(
kernel,
gathered,
out,
stream,
options="--enable-tvm-ffi",
)
+167 -44
View File
@@ -8,6 +8,8 @@ import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config
from vllm.distributed import get_dcp_group
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
@@ -21,6 +23,7 @@ from vllm.utils.deep_gemm import (
fp8_fp4_paged_mqa_logits,
has_deep_gemm,
)
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
@@ -41,6 +44,85 @@ RADIX_TOPK_WORKSPACE_SIZE = 1024 * 1024
MXFP4_BLOCK_SIZE = 32
def _assert_cutedsl_dcp_merge_supported(
logits: torch.Tensor,
topk_indices: torch.Tensor,
k: int,
) -> None:
# The DCP merge only supports the CuteDSL path (Triton pack kernel + CuteDSL
# stable-topk selector); there is no PyTorch fallback. The first cut targets
# Blackwell/Hopper with index_topk in (512, 1024, 2048) (the selector's radix
# sizing); the Triton pack itself has no shape/topk constraints.
if not has_cutedsl():
raise RuntimeError(
"DCP sparse-indexer merge requires CuteDSL; install it or disable DCP."
)
if logits.device.type != "cuda":
raise RuntimeError("DCP sparse-indexer merge requires CUDA tensors.")
if logits.dtype != torch.float32 or topk_indices.dtype != torch.int32:
raise RuntimeError(
"DCP sparse-indexer merge requires fp32 logits and int32 indices."
)
if k not in (512, 1024, 2048):
raise RuntimeError(
f"DCP sparse-indexer merge requires index_topk in (512, 1024, 2048); "
f"got {k}."
)
def _merge_dcp_topk_global(
logits: torch.Tensor,
topk_indices: torch.Tensor,
topk_tokens: int,
dcp_rank: int,
dcp_world_size: int,
cp_interleave: int,
row_starts: torch.Tensor | None = None,
) -> None:
"""Merge each DCP rank's local top-K into the global top-K.
``topk_indices`` are this rank's local top-K positions into its 1/N KV
shard. A token in the global top-K must also be in its owning rank's local
top-K (at most ``topk_tokens - 1`` tokens rank globally above it, hence at
most that many on its own rank), so exchanging only the per-rank local
candidates is exact -- equivalent to all-gathering the full logit matrix,
but it ships ``dcp_world_size * topk_tokens`` candidates instead of the whole
score row. Overwrites ``topk_indices`` with global token ids (``-1`` for
padding); the attention backend localizes them back to physical slots per
rank.
"""
if dcp_world_size <= 1:
return
# CuteDSL-only path (no PyTorch fallback): Triton-pack each rank's
# (score, global_id) candidates on-device, all-gather, then the CuteDSL
# stable-topk selector.
_assert_cutedsl_dcp_merge_supported(logits, topk_indices, topk_tokens)
from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import (
pack_dcp_topk_candidates_cutedsl,
stable_topk_from_gathered_candidates_cutedsl,
)
packed = torch.empty(
(*topk_indices.shape, 2),
dtype=torch.float32,
device=topk_indices.device,
)
pack_dcp_topk_candidates_cutedsl(
logits,
topk_indices,
packed,
dcp_rank,
dcp_world_size,
cp_interleave,
row_starts,
)
gathered = get_dcp_group().all_gather(packed, dim=1)
stable_topk_from_gathered_candidates_cutedsl(
gathered, topk_tokens, out=topk_indices
)
@triton.jit
def _fused_indexer_q_rope_quant_kernel(
positions,
@@ -227,6 +309,9 @@ def sparse_attn_indexer(
topk_indices_buffer: torch.Tensor,
skip_k_cache_insert: bool,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
dcp_world_size: int = 1,
cp_kv_cache_interleave_size: int = 1,
skip_topk_buffer_clear: bool = False,
) -> torch.Tensor:
# careful! this will be None in dummy run
@@ -328,16 +413,18 @@ def sparse_attn_indexer(
scales_spec,
)
for chunk in prefill_metadata.chunks:
k_quant = k_quant_full[: chunk.total_seq_lens]
k_scale = k_scale_full[: chunk.total_seq_lens]
if not chunk.skip_kv_gather:
cu_seqlen_ks = chunk.cu_seqlen_ks
cu_seqlen_ke = chunk.cu_seqlen_ke
assert chunk.local_cu_seq_lens is not None
k_quant = k_quant_full[: chunk.max_local_total_seq_lens]
k_scale = k_scale_full[: chunk.max_local_total_seq_lens]
if not chunk.skip_kv_gather and chunk.local_total_seq_lens > 0:
ops.cp_gather_indexer_k_quant_cache(
kv_cache,
k_quant,
k_scale,
chunk.block_table,
chunk.cu_seq_lens,
chunk.local_cu_seq_lens,
)
q_slice = q_quant[chunk.token_start : chunk.token_end]
@@ -346,51 +433,64 @@ def sparse_attn_indexer(
if q_scale is not None
else None
)
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
if use_fp4_cache:
q_slice_cast = q_slice.view(torch.int8)
k_quant_cast = k_quant.view(torch.int8)
k_scale_cast = k_scale.view(torch.int32).squeeze(-1)
else:
q_slice_cast = q_slice
k_quant_cast = k_quant
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
if current_platform.is_xpu():
if q_scale_slice is not None:
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
logits = torch.ops.vllm.xpu_fp8_mqa_logits(
q_slice_cast,
k_quant_cast,
k_scale_cast,
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
)
else:
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
clean_logits=False,
)
num_rows = logits.shape[0]
topk_indices = topk_indices_buffer[
chunk.token_start : chunk.token_end, :topk_tokens
]
ops.top_k_per_row_prefill(
if chunk.local_total_seq_lens == 0:
logits = q_slice.new_empty((q_slice.shape[0], 0), dtype=torch.float32)
topk_indices.fill_(-1)
else:
# DeepGEMM scalar-type tags (zero-copy): MXFP4 values → int8
# (kPackedFP4), scales → int32 squeezed to 1-D kv_sf / 2-D q_sf.
if use_fp4_cache:
q_slice_cast = q_slice.view(torch.int8)
k_quant_cast = k_quant.view(torch.int8)
k_scale_cast = k_scale.view(torch.int32).squeeze(-1)
else:
q_slice_cast = q_slice
k_quant_cast = k_quant
k_scale_cast = k_scale.view(torch.float32).squeeze(-1)
if current_platform.is_xpu():
if q_scale_slice is not None:
raise RuntimeError("XPU fp8_mqa_logits does not support FP4 Q")
logits = torch.ops.vllm.xpu_fp8_mqa_logits(
q_slice_cast,
k_quant_cast,
k_scale_cast,
weights[chunk.token_start : chunk.token_end],
cu_seqlen_ks,
cu_seqlen_ke,
)
else:
logits = fp8_fp4_mqa_logits(
(q_slice_cast, q_scale_slice),
(k_quant_cast, k_scale_cast),
weights[chunk.token_start : chunk.token_end],
cu_seqlen_ks,
cu_seqlen_ke,
clean_logits=False,
)
num_rows = logits.shape[0]
ops.top_k_per_row_prefill(
logits,
cu_seqlen_ks,
cu_seqlen_ke,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
)
_merge_dcp_topk_global(
logits,
chunk.cu_seqlen_ks,
chunk.cu_seqlen_ke,
topk_indices,
num_rows,
logits.stride(0),
logits.stride(1),
topk_tokens,
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
row_starts=chunk.cu_seqlen_ks,
)
if has_decode:
@@ -508,7 +608,7 @@ def sparse_attn_indexer(
topk_indices,
topk_workspace,
topk_tokens,
attn_metadata_narrowed.max_seq_len,
logits.shape[1],
)
else:
ops.top_k_per_row_decode(
@@ -522,6 +622,16 @@ def sparse_attn_indexer(
topk_tokens,
)
if decode_metadata.global_seq_lens is not None:
_merge_dcp_topk_global(
logits,
topk_indices,
topk_tokens,
dcp_rank,
dcp_world_size,
cp_kv_cache_interleave_size,
)
if decode_metadata.requires_padding:
# if padded, we need to unpack
# the topk indices removing padded tokens
@@ -553,6 +663,9 @@ def sparse_attn_indexer_fake(
topk_indices_buffer: torch.Tensor | None,
skip_k_cache_insert: bool,
use_fp4_cache: bool = False,
dcp_rank: int = 0,
dcp_world_size: int = 1,
cp_kv_cache_interleave_size: int = 1,
skip_topk_buffer_clear: bool = False,
) -> torch.Tensor:
return topk_indices_buffer
@@ -604,6 +717,13 @@ class SparseAttnIndexer(CustomOp):
self.topk_indices_buffer = topk_indices_buffer
self.skip_k_cache_insert = skip_k_cache_insert
self.use_fp4_cache = use_fp4_cache
# DCP scalars are constant for the run; resolve them here (config is set
# during model construction) and pass them into the custom op, rather
# than threading them through per-step metadata.
parallel_config = get_current_vllm_config().parallel_config
self.dcp_world_size = parallel_config.decode_context_parallel_size
self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0
self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size
if current_platform.is_cuda() and not has_deep_gemm():
raise RuntimeError(
"Sparse Attention Indexer CUDA op requires DeepGEMM support in "
@@ -657,6 +777,9 @@ class SparseAttnIndexer(CustomOp):
self.topk_indices_buffer,
self.skip_k_cache_insert,
self.use_fp4_cache,
self.dcp_rank,
self.dcp_world_size,
self.cp_kv_cache_interleave_size,
)
def forward_xpu(