[Perf][DSv4.1] Length-aware candidate block selection in the decode indexer (#61039)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
This commit is contained in:
Juntian Liu
2026-10-14 20:15:58 +00:00
committed by GitHub
parent 2622ee21aa
commit 7c3f7e5500
2 changed files with 69 additions and 33 deletions
@@ -3,7 +3,14 @@
import torch
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import has_deep_select
from vllm.model_executor.layers.indexer_topk import (
deep_select_topk,
get_deep_select_stride_requirement,
is_deep_select_supported,
)
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import round_up
@triton.jit
@@ -17,38 +24,47 @@ def _block_scores_kernel(
starts,
ends,
scores,
valid_blocks,
stride_row,
stride_col,
stride_start,
stride_end,
stride_scores,
width,
nblocks,
BLOCK_SIZE: tl.constexpr,
HAS_STARTS: tl.constexpr,
ROW_REPEAT: tl.constexpr,
TILE: tl.constexpr,
FULL_WIDTH: tl.constexpr,
):
# Writes each row's valid block count; FULL_WIDTH also scores the -inf tail.
row = tl.program_id(0).to(tl.int64)
blocks = tl.program_id(1) * TILE + tl.arange(0, TILE)
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
end = tl.load(ends + row // ROW_REPEAT * stride_end)
col_end = tl.minimum(end, width)
num_valid = tl.cdiv(tl.maximum(col_end - start, 0), BLOCK_SIZE)
if tl.program_id(1) == 0:
tl.store(valid_blocks + row, num_valid)
num_blocks = nblocks if FULL_WIDTH else num_valid
offsets = tl.arange(0, triton.next_power_of_2(BLOCK_SIZE))
cols = start + blocks[:, None] * BLOCK_SIZE + offsets[None, :]
values = tl.load(
logits + row * stride_row + cols * stride_col,
(blocks[:, None] < nblocks)
& (offsets[None, :] < BLOCK_SIZE)
& (cols < end)
& (cols < width),
other=-float("inf"),
)
reduced = tl.reduce(values, 1, _max_with_nan)
reduced = tl.where(
(end > start) & (blocks == (end - start - 1) // BLOCK_SIZE),
float("inf"),
reduced,
)
tl.store(scores + row * nblocks + blocks, reduced, blocks < nblocks)
num_tiles = tl.cdiv(num_blocks, TILE)
for tile_id in range(tl.program_id(1), num_tiles, tl.num_programs(1)):
blocks = tile_id * TILE + tl.arange(0, TILE)
cols = start + blocks[:, None] * BLOCK_SIZE + offsets[None, :]
values = tl.load(
logits + row * stride_row + cols * stride_col,
(offsets[None, :] < BLOCK_SIZE) & (cols < col_end),
other=-float("inf"),
).to(tl.float32)
reduced = tl.reduce(values, 1, _max_with_nan)
# Pin the newest block. DeepSelect leaves NaN rows partially written,
# so also pin NaN blocks (torch.topk ranks NaN first too).
pinned = (end > start) & (blocks == (end - start - 1) // BLOCK_SIZE)
if not FULL_WIDTH:
pinned |= reduced != reduced
reduced = tl.where(pinned, float("inf"), reduced)
tl.store(scores + row * stride_scores + blocks, reduced, blocks < num_blocks)
@triton.jit(do_not_specialize=["k"])
@@ -160,22 +176,42 @@ def select_candidate_blocks(
out.fill_(-1)
return
nblocks = triton.cdiv(width, block_size)
scores = logits.new_empty((rows, nblocks))
_block_scores_kernel[(rows, triton.cdiv(nblocks, 128))](
# Logits are max_model_len wide (CUDA-graph static) but rows are shorter:
# DeepSelect stops at each row's valid block count. Order does not matter.
use_deep_select = has_deep_select() and nblocks >= topk_blocks
if use_deep_select:
in_align, out_align = get_deep_select_stride_requirement()
padded = round_up(nblocks, in_align // logits.element_size())
scores = logits.new_empty((rows, padded))[:, :nblocks]
use_deep_select = (
out.stride(1) == 1
and out.stride(0) * out.element_size() % out_align == 0
and is_deep_select_supported(scores, topk_blocks)
)
else:
scores = logits.new_empty((rows, nblocks))
valid_blocks = torch.empty(rows, dtype=torch.int32, device=logits.device)
_block_scores_kernel[(rows, 32)](
logits,
row_ks,
row_ke,
scores,
valid_blocks,
*logits.stride(),
row_ks.stride(0) if row_ks is not None else 0,
row_ke.stride(0),
scores.stride(0),
width,
nblocks,
block_size,
row_ks is not None,
row_repeat,
128,
TILE=128,
FULL_WIDTH=not use_deep_select,
)
if use_deep_select:
deep_select_topk(scores, topk_blocks, end=valid_blocks, output_idx=out)
return
# Keep the existing top-k tie behavior.
top = scores.topk(min(topk_blocks, nblocks), dim=-1)
_store_candidates_kernel[(rows, triton.cdiv(topk_blocks, 256))](
@@ -877,20 +877,20 @@ def sparse_attn_indexer(
padded_q_scale, decode_weights = mqa_logits_mx_inputs(
padded_q_quant_cast, padded_q_scale, weights[:num_padded_tokens]
)
# SM100 decode is varlen: one row and context length per token.
logits = as_fp32_logits(
fp8_fp4_paged_mqa_logits(
(padded_q_quant_cast, padded_q_scale),
kv_cache,
decode_weights,
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_model_len,
indices=decode_metadata.indices,
),
seq_lens.view(-1),
logits = fp8_fp4_paged_mqa_logits(
(padded_q_quant_cast, padded_q_scale),
kv_cache,
decode_weights,
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_model_len,
indices=decode_metadata.indices,
)
# Keeping bf16 halves top-k/candidate reads; other top-k backends and
# DCP need fp32. SM100 decode is varlen: one context length per row.
if topk_backend != "deep_select" or dcp_world_size != 1:
logits = as_fp32_logits(logits, seq_lens.view(-1))
num_rows = logits.shape[0]
if candidate_blocks is not None:
# Two-level selection (v4.1) on the decode logits; columns are