[Perf][DSv4.1] Length-aware candidate block selection in the decode indexer (#61039)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
This commit is contained in:
@@ -3,7 +3,14 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import has_deep_select
|
||||
from vllm.model_executor.layers.indexer_topk import (
|
||||
deep_select_topk,
|
||||
get_deep_select_stride_requirement,
|
||||
is_deep_select_supported,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -17,38 +24,47 @@ def _block_scores_kernel(
|
||||
starts,
|
||||
ends,
|
||||
scores,
|
||||
valid_blocks,
|
||||
stride_row,
|
||||
stride_col,
|
||||
stride_start,
|
||||
stride_end,
|
||||
stride_scores,
|
||||
width,
|
||||
nblocks,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
HAS_STARTS: tl.constexpr,
|
||||
ROW_REPEAT: tl.constexpr,
|
||||
TILE: tl.constexpr,
|
||||
FULL_WIDTH: tl.constexpr,
|
||||
):
|
||||
# Writes each row's valid block count; FULL_WIDTH also scores the -inf tail.
|
||||
row = tl.program_id(0).to(tl.int64)
|
||||
blocks = tl.program_id(1) * TILE + tl.arange(0, TILE)
|
||||
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
|
||||
end = tl.load(ends + row // ROW_REPEAT * stride_end)
|
||||
col_end = tl.minimum(end, width)
|
||||
num_valid = tl.cdiv(tl.maximum(col_end - start, 0), BLOCK_SIZE)
|
||||
if tl.program_id(1) == 0:
|
||||
tl.store(valid_blocks + row, num_valid)
|
||||
num_blocks = nblocks if FULL_WIDTH else num_valid
|
||||
offsets = tl.arange(0, triton.next_power_of_2(BLOCK_SIZE))
|
||||
cols = start + blocks[:, None] * BLOCK_SIZE + offsets[None, :]
|
||||
values = tl.load(
|
||||
logits + row * stride_row + cols * stride_col,
|
||||
(blocks[:, None] < nblocks)
|
||||
& (offsets[None, :] < BLOCK_SIZE)
|
||||
& (cols < end)
|
||||
& (cols < width),
|
||||
other=-float("inf"),
|
||||
)
|
||||
reduced = tl.reduce(values, 1, _max_with_nan)
|
||||
reduced = tl.where(
|
||||
(end > start) & (blocks == (end - start - 1) // BLOCK_SIZE),
|
||||
float("inf"),
|
||||
reduced,
|
||||
)
|
||||
tl.store(scores + row * nblocks + blocks, reduced, blocks < nblocks)
|
||||
num_tiles = tl.cdiv(num_blocks, TILE)
|
||||
for tile_id in range(tl.program_id(1), num_tiles, tl.num_programs(1)):
|
||||
blocks = tile_id * TILE + tl.arange(0, TILE)
|
||||
cols = start + blocks[:, None] * BLOCK_SIZE + offsets[None, :]
|
||||
values = tl.load(
|
||||
logits + row * stride_row + cols * stride_col,
|
||||
(offsets[None, :] < BLOCK_SIZE) & (cols < col_end),
|
||||
other=-float("inf"),
|
||||
).to(tl.float32)
|
||||
reduced = tl.reduce(values, 1, _max_with_nan)
|
||||
# Pin the newest block. DeepSelect leaves NaN rows partially written,
|
||||
# so also pin NaN blocks (torch.topk ranks NaN first too).
|
||||
pinned = (end > start) & (blocks == (end - start - 1) // BLOCK_SIZE)
|
||||
if not FULL_WIDTH:
|
||||
pinned |= reduced != reduced
|
||||
reduced = tl.where(pinned, float("inf"), reduced)
|
||||
tl.store(scores + row * stride_scores + blocks, reduced, blocks < num_blocks)
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["k"])
|
||||
@@ -160,22 +176,42 @@ def select_candidate_blocks(
|
||||
out.fill_(-1)
|
||||
return
|
||||
nblocks = triton.cdiv(width, block_size)
|
||||
scores = logits.new_empty((rows, nblocks))
|
||||
_block_scores_kernel[(rows, triton.cdiv(nblocks, 128))](
|
||||
# Logits are max_model_len wide (CUDA-graph static) but rows are shorter:
|
||||
# DeepSelect stops at each row's valid block count. Order does not matter.
|
||||
use_deep_select = has_deep_select() and nblocks >= topk_blocks
|
||||
if use_deep_select:
|
||||
in_align, out_align = get_deep_select_stride_requirement()
|
||||
padded = round_up(nblocks, in_align // logits.element_size())
|
||||
scores = logits.new_empty((rows, padded))[:, :nblocks]
|
||||
use_deep_select = (
|
||||
out.stride(1) == 1
|
||||
and out.stride(0) * out.element_size() % out_align == 0
|
||||
and is_deep_select_supported(scores, topk_blocks)
|
||||
)
|
||||
else:
|
||||
scores = logits.new_empty((rows, nblocks))
|
||||
valid_blocks = torch.empty(rows, dtype=torch.int32, device=logits.device)
|
||||
_block_scores_kernel[(rows, 32)](
|
||||
logits,
|
||||
row_ks,
|
||||
row_ke,
|
||||
scores,
|
||||
valid_blocks,
|
||||
*logits.stride(),
|
||||
row_ks.stride(0) if row_ks is not None else 0,
|
||||
row_ke.stride(0),
|
||||
scores.stride(0),
|
||||
width,
|
||||
nblocks,
|
||||
block_size,
|
||||
row_ks is not None,
|
||||
row_repeat,
|
||||
128,
|
||||
TILE=128,
|
||||
FULL_WIDTH=not use_deep_select,
|
||||
)
|
||||
if use_deep_select:
|
||||
deep_select_topk(scores, topk_blocks, end=valid_blocks, output_idx=out)
|
||||
return
|
||||
# Keep the existing top-k tie behavior.
|
||||
top = scores.topk(min(topk_blocks, nblocks), dim=-1)
|
||||
_store_candidates_kernel[(rows, triton.cdiv(topk_blocks, 256))](
|
||||
|
||||
@@ -877,20 +877,20 @@ def sparse_attn_indexer(
|
||||
padded_q_scale, decode_weights = mqa_logits_mx_inputs(
|
||||
padded_q_quant_cast, padded_q_scale, weights[:num_padded_tokens]
|
||||
)
|
||||
# SM100 decode is varlen: one row and context length per token.
|
||||
logits = as_fp32_logits(
|
||||
fp8_fp4_paged_mqa_logits(
|
||||
(padded_q_quant_cast, padded_q_scale),
|
||||
kv_cache,
|
||||
decode_weights,
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_model_len,
|
||||
indices=decode_metadata.indices,
|
||||
),
|
||||
seq_lens.view(-1),
|
||||
logits = fp8_fp4_paged_mqa_logits(
|
||||
(padded_q_quant_cast, padded_q_scale),
|
||||
kv_cache,
|
||||
decode_weights,
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_model_len,
|
||||
indices=decode_metadata.indices,
|
||||
)
|
||||
# Keeping bf16 halves top-k/candidate reads; other top-k backends and
|
||||
# DCP need fp32. SM100 decode is varlen: one context length per row.
|
||||
if topk_backend != "deep_select" or dcp_world_size != 1:
|
||||
logits = as_fp32_logits(logits, seq_lens.view(-1))
|
||||
num_rows = logits.shape[0]
|
||||
if candidate_blocks is not None:
|
||||
# Two-level selection (v4.1) on the decode logits; columns are
|
||||
|
||||
Reference in New Issue
Block a user