[Perf][DSv4] Fuse inverse RoPE + FP8 quant into FlashInfer sparse MLA (#58621)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-24 23:26:50 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 29468dde8b
commit 5ff3bbfb08
5 changed files with 587 additions and 108 deletions
@@ -1099,6 +1099,7 @@ def build_flashinfer_mixed_sparse_indices(
prefill_left_visible: torch.Tensor | None = None,
prefill_right_visible: torch.Tensor | None = None,
max_image_tokens: int = 0,
num_rows: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Build the FlashInfer DSV4 sparse-index matrix for decode-first batches.
@@ -1109,6 +1110,9 @@ def build_flashinfer_mixed_sparse_indices(
tokens derive their SWA window from the position and translate local
compressed indices to global slots via the block tables.
``num_rows`` (>= ``num_tokens``) sizes both outputs for a kernel call that
spans a CUDA-graph-padded batch; the rows past ``num_tokens`` stay unset.
When ``prefill_left_visible``/``prefill_right_visible`` are given (vision
variant), the SWA column region widens by ``max_image_tokens`` and prefill
tokens inside an image span get a bidirectionally widened window; decode
@@ -1170,13 +1174,15 @@ def build_flashinfer_mixed_sparse_indices(
# by ``sparse_topk_lens``, so padding never changes the attention result.
padded_topk = max(topk, decode_compressed_topk)
padded_topk = (padded_topk + 3) // 4 * 4
num_rows = num_tokens if num_rows is None else num_rows
assert num_rows >= num_tokens
sparse_indices = torch.empty(
(num_tokens, swa_total_width + padded_topk),
(num_rows, swa_total_width + padded_topk),
dtype=torch.int32,
device=decode_swa_indices.device,
)
sparse_topk_lens = torch.empty(
num_tokens, dtype=torch.int32, device=decode_swa_indices.device
num_rows, dtype=torch.int32, device=decode_swa_indices.device
)
if num_tokens == 0:
return sparse_indices, sparse_topk_lens
@@ -1193,9 +1199,10 @@ def build_flashinfer_mixed_sparse_indices(
if compressed_block_span is None
else compressed_block_span
)
# The launch grid follows the output rows, so hand over only the real ones.
_BUILD_FLASHINFER_MIXED_SPARSE_INDICES_KERNEL(
sparse_indices,
sparse_topk_lens,
sparse_indices[:num_tokens],
sparse_topk_lens[:num_tokens],
decode_swa_indices,
decode_compressed_indices,
decode_compressed_topk_lens,
@@ -1322,12 +1329,52 @@ class BuildFlashinferMixedSparseIndicesKernel(
SWA_TOTAL_WIDTH: tl.constexpr = SWA_INDEX_WIDTH + IMAGE_WIDTH
if token_idx < NUM_DECODE_TOKENS:
# A decode row wider than the window is DSpark's non-causal block:
# the block-anchored window of context followed by the whole block,
# contiguous from column 0. The kernel treats every entry of its
# active ranges as a key -- the first min(WINDOW_SIZE, pos + 1)
# columns and [WINDOW_SIZE, sparse_topk_lens) -- so -1 must not
# appear in either. Fill the causal window with the first visible
# entries and put the rest from column WINDOW_SIZE on, followed
# directly by the compressed entries.
NONCAUSAL: tl.constexpr = SWA_INDEX_WIDTH > WINDOW_SIZE
compressed_start = SWA_TOTAL_WIDTH
if NONCAUSAL:
req_idx = tl.load(token_to_req_indices_ptr + token_idx)
query_start = tl.load(query_start_loc_ptr + req_idx)
query_len = tl.load(query_start_loc_ptr + req_idx + 1) - query_start
context_len = tl.load(seq_lens_ptr + req_idx) - query_len
token_in_query = token_idx - query_start
window_len = tl.minimum(context_len + token_in_query + 1, WINDOW_SIZE)
spill_len = (
tl.minimum(context_len, WINDOW_SIZE) + query_len - window_len
)
# A CUDA-graph pad token sits outside its empty request, so
# the kernel never reads its row; keep it the shape of a
# causal pad row (all -1, WINDOW_SIZE long) rather than let
# the formulas above go negative.
is_block_token = (token_in_query >= 0) & (token_in_query < query_len)
window_len = tl.where(is_block_token, window_len, 0)
spill_len = tl.where(is_block_token, spill_len, 0)
compressed_start = WINDOW_SIZE + spill_len
for i in range(0, SWA_TOTAL_WIDTH, WINDOW_BLOCK_SIZE):
offset = i + tl.arange(0, WINDOW_BLOCK_SIZE)
mask = offset < SWA_TOTAL_WIDTH
src = offset
src_mask = offset < SWA_INDEX_WIDTH
if NONCAUSAL:
# Columns from compressed_start on belong to the loops below.
mask = mask & (offset < compressed_start)
spill = offset >= WINDOW_SIZE
src = tl.where(spill, offset - WINDOW_SIZE + window_len, offset)
src_mask = tl.where(
spill,
offset < WINDOW_SIZE + spill_len,
offset < window_len,
)
values = tl.load(
decode_swa_indices_ptr + token_idx * decode_swa_stride + offset,
mask=offset < SWA_INDEX_WIDTH,
decode_swa_indices_ptr + token_idx * decode_swa_stride + src,
mask=src_mask,
other=-1,
)
values = _remap_flashinfer_index(values, swa_block_size, swa_block_span)
@@ -1372,12 +1419,27 @@ class BuildFlashinferMixedSparseIndicesKernel(
tl.store(
sparse_indices_ptr
+ token_idx * sparse_indices_stride
+ SWA_TOTAL_WIDTH
+ compressed_start
+ offset,
values,
mask=mask,
)
if NONCAUSAL:
# The row's remaining columns, left over by the shifted
# compressed block.
for i in range(0, SWA_TOTAL_WIDTH - WINDOW_SIZE, WINDOW_BLOCK_SIZE):
offset = i + tl.arange(0, WINDOW_BLOCK_SIZE)
tl.store(
sparse_indices_ptr
+ token_idx * sparse_indices_stride
+ compressed_start
+ PADDED_TOP_K
+ offset,
tl.full((WINDOW_BLOCK_SIZE,), -1, dtype=tl.int32),
mask=offset < SWA_TOTAL_WIDTH - compressed_start,
)
if DECODE_COMPRESSED_TOPK == 0:
compressed_len = tl.zeros((), dtype=tl.int32)
elif not DECODE_COMPRESSED_INDICES_ARE_LOCAL:
@@ -1388,7 +1450,9 @@ class BuildFlashinferMixedSparseIndicesKernel(
else:
compressed_len = tl.full((), DECODE_COMPRESSED_TOPK, dtype=tl.int32)
tl.store(sparse_topk_lens_ptr + token_idx, SWA_TOTAL_WIDTH + compressed_len)
tl.store(
sparse_topk_lens_ptr + token_idx, compressed_start + compressed_len
)
return
prefill_idx = token_idx - NUM_DECODE_TOKENS