[Refactor] Improve indexer decode path metadata preparation (#38865)
This commit is contained in:
@@ -183,13 +183,15 @@ def sparse_attn_indexer(
|
||||
# TODO: move and optimize below logic with triton kernels
|
||||
batch_size = padded_q_fp8_decode_tokens.shape[0]
|
||||
next_n = padded_q_fp8_decode_tokens.shape[1]
|
||||
assert batch_size == decode_metadata.seq_lens.shape[0]
|
||||
num_padded_tokens = batch_size * next_n
|
||||
seq_lens = decode_metadata.seq_lens[:batch_size]
|
||||
# seq_lens is (B, next_n) for native spec decode, (B,) otherwise.
|
||||
# fp8_paged_mqa_logits and all topk kernels accept both shapes.
|
||||
logits = fp8_paged_mqa_logits(
|
||||
padded_q_fp8_decode_tokens,
|
||||
kv_cache,
|
||||
weights[:num_padded_tokens],
|
||||
decode_metadata.seq_lens,
|
||||
seq_lens,
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_model_len,
|
||||
@@ -198,17 +200,6 @@ def sparse_attn_indexer(
|
||||
num_rows = logits.shape[0]
|
||||
topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
|
||||
|
||||
if next_n == 1:
|
||||
lengths = decode_metadata.seq_lens
|
||||
else:
|
||||
# (bs,) -> (bs, 1) + (next_n,) -> (bs, next_n) -> (bs * next_n,)
|
||||
lengths = (
|
||||
decode_metadata.seq_lens.unsqueeze(1)
|
||||
- next_n
|
||||
+ 1
|
||||
+ decode_metadata.offsets
|
||||
).flatten()
|
||||
|
||||
if current_platform.is_cuda():
|
||||
workspace_manager = current_workspace_manager()
|
||||
(topk_workspace,) = workspace_manager.get_simultaneous(
|
||||
@@ -216,7 +207,7 @@ def sparse_attn_indexer(
|
||||
)
|
||||
torch.ops._C.persistent_topk(
|
||||
logits,
|
||||
lengths,
|
||||
decode_metadata.seq_lens,
|
||||
topk_indices,
|
||||
topk_workspace,
|
||||
topk_tokens,
|
||||
@@ -227,7 +218,7 @@ def sparse_attn_indexer(
|
||||
ops.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
decode_metadata.seq_lens,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
@@ -238,7 +229,7 @@ def sparse_attn_indexer(
|
||||
torch.ops._C.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
decode_metadata.seq_lens,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
num_rows,
|
||||
logits.stride(0),
|
||||
|
||||
Reference in New Issue
Block a user