[Bugfix] DSV4 TP16 garbage output (#47493)

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Jeff (Junze) Ma
2026-07-07 21:04:33 -07:00
committed by GitHub
co-authored by Lucas Wilkinson
parent d9e57ea82e
commit 80eb01e93d
5 changed files with 61 additions and 6 deletions
@@ -654,6 +654,8 @@ def build_flashinfer_mixed_sparse_indices(
topk: int,
decode_compressed_indices_are_local: bool = False,
decode_is_valid_token: torch.Tensor | None = None,
swa_block_span: int | None = None,
compressed_block_span: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Build the FlashInfer DSV4 sparse-index matrix for decode-first batches.
@@ -730,6 +732,13 @@ def build_flashinfer_mixed_sparse_indices(
max_block_size = max(window_block_size, topk_block_size)
num_warps = 4 if max_block_size >= 256 else 1
# block_span = page_stride / token_stride; == block_size (no-op) for unpacked KV.
swa_span = swa_block_size if swa_block_span is None else swa_block_span
compressed_span = (
compressed_block_size
if compressed_block_span is None
else compressed_block_span
)
_build_flashinfer_mixed_sparse_indices_kernel[(num_tokens,)](
sparse_indices,
sparse_indices.stride(0),
@@ -748,9 +757,11 @@ def build_flashinfer_mixed_sparse_indices(
swa_block_table,
swa_block_table.stride(0),
swa_block_size,
swa_span,
compressed_block_table,
compressed_block_table.stride(0),
compressed_block_size,
compressed_span,
NUM_DECODE_TOKENS=num_decode_tokens,
WINDOW_SIZE=window_size,
COMPRESS_RATIO=compress_ratio,
@@ -767,6 +778,18 @@ def build_flashinfer_mixed_sparse_indices(
return sparse_indices, sparse_topk_lens
@triton.jit
def _remap_flashinfer_index(values, block_size, block_span):
# FlashInfer's DSv4 kernel indexes sparse KV by physical token stride, so
# packed pages (#44577) need block*block_size+off -> block*block_span+off.
# TODO: remove once flashinfer-ai/flashinfer#3856 is fixed.
is_valid = values >= 0
safe_values = tl.where(is_valid, values, 0)
values = (safe_values // block_size) * block_span
values += safe_values % block_size
return tl.where(is_valid, values, -1)
@triton.jit(
do_not_specialize=[
"sparse_indices_stride",
@@ -775,8 +798,10 @@ def build_flashinfer_mixed_sparse_indices(
"prefill_topk_stride",
"swa_block_table_stride",
"swa_block_size",
"swa_block_span",
"compressed_block_table_stride",
"compressed_block_size",
"compressed_block_span",
"NUM_DECODE_TOKENS",
"PREFILL_TOPK_STRIDE",
]
@@ -799,9 +824,11 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
swa_block_table_ptr,
swa_block_table_stride,
swa_block_size,
swa_block_span,
compressed_block_table_ptr,
compressed_block_table_stride,
compressed_block_size,
compressed_block_span,
NUM_DECODE_TOKENS,
WINDOW_SIZE: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
@@ -825,6 +852,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
mask=mask,
other=-1,
)
values = _remap_flashinfer_index(values, swa_block_size, swa_block_span)
tl.store(
sparse_indices_ptr + token_idx * sparse_indices_stride + offset,
values,
@@ -858,6 +886,9 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
values = block_numbers * compressed_block_size + block_offsets
values = tl.where(is_valid, values, -1)
compressed_len += tl.sum((is_valid & token_valid).to(tl.int32), axis=0)
values = _remap_flashinfer_index(
values, compressed_block_size, compressed_block_span
)
tl.store(
sparse_indices_ptr
+ token_idx * sparse_indices_stride
@@ -904,6 +935,7 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
block_offsets = pos_offset % swa_block_size
slot_ids = block_numbers * swa_block_size + block_offsets
slot_ids = tl.where(offset < swa_len, slot_ids, -1)
slot_ids = _remap_flashinfer_index(slot_ids, swa_block_size, swa_block_span)
tl.store(
sparse_indices_ptr + token_idx * sparse_indices_stride + offset,
slot_ids,
@@ -930,6 +962,9 @@ def _build_flashinfer_mixed_sparse_indices_kernel(
block_offsets = local_idx % compressed_block_size
slot_ids = block_numbers * compressed_block_size + block_offsets
slot_ids = tl.where((offset < topk_len) & is_valid, slot_ids, -1)
slot_ids = _remap_flashinfer_index(
slot_ids, compressed_block_size, compressed_block_span
)
tl.store(
sparse_indices_ptr
+ token_idx * sparse_indices_stride