[Refactor] Improve indexer decode path metadata preparation (#38865)

This commit is contained in:
Yongye Zhu
2026-04-08 20:49:15 -07:00
committed by GitHub
parent ef5a226819
commit 2e98406048
4 changed files with 162 additions and 102 deletions
@@ -183,13 +183,15 @@ def sparse_attn_indexer(
# TODO: move and optimize below logic with triton kernels
batch_size = padded_q_fp8_decode_tokens.shape[0]
next_n = padded_q_fp8_decode_tokens.shape[1]
assert batch_size == decode_metadata.seq_lens.shape[0]
num_padded_tokens = batch_size * next_n
seq_lens = decode_metadata.seq_lens[:batch_size]
# seq_lens is (B, next_n) for native spec decode, (B,) otherwise.
# fp8_paged_mqa_logits and all topk kernels accept both shapes.
logits = fp8_paged_mqa_logits(
padded_q_fp8_decode_tokens,
kv_cache,
weights[:num_padded_tokens],
decode_metadata.seq_lens,
seq_lens,
decode_metadata.block_table,
decode_metadata.schedule_metadata,
max_model_len=max_model_len,
@@ -198,17 +200,6 @@ def sparse_attn_indexer(
num_rows = logits.shape[0]
topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
if next_n == 1:
lengths = decode_metadata.seq_lens
else:
# (bs,) -> (bs, 1) + (next_n,) -> (bs, next_n) -> (bs * next_n,)
lengths = (
decode_metadata.seq_lens.unsqueeze(1)
- next_n
+ 1
+ decode_metadata.offsets
).flatten()
if current_platform.is_cuda():
workspace_manager = current_workspace_manager()
(topk_workspace,) = workspace_manager.get_simultaneous(
@@ -216,7 +207,7 @@ def sparse_attn_indexer(
)
torch.ops._C.persistent_topk(
logits,
lengths,
decode_metadata.seq_lens,
topk_indices,
topk_workspace,
topk_tokens,
@@ -227,7 +218,7 @@ def sparse_attn_indexer(
ops.top_k_per_row_decode(
logits,
next_n,
decode_metadata.seq_lens,
seq_lens,
topk_indices,
num_rows,
logits.stride(0),
@@ -238,7 +229,7 @@ def sparse_attn_indexer(
torch.ops._C.top_k_per_row_decode(
logits,
next_n,
decode_metadata.seq_lens,
seq_lens,
topk_indices,
num_rows,
logits.stride(0),