[Bugfix] Fix tensor shape mismatch in sparse attention with speculative decoding (#39542)
Signed-off-by: Santino Ramos <santinor@inferact.ai>
This commit is contained in:
@@ -213,7 +213,7 @@ def sparse_attn_indexer(
|
||||
)
|
||||
torch.ops._C.persistent_topk(
|
||||
logits,
|
||||
decode_metadata.seq_lens,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
topk_workspace,
|
||||
topk_tokens,
|
||||
@@ -250,7 +250,7 @@ def sparse_attn_indexer(
|
||||
topk_indices.reshape(batch_size, -1, topk_indices.shape[-1]),
|
||||
decode_lens,
|
||||
)
|
||||
topk_indices_buffer[:num_decode_tokens, : topk_indices.shape[-1]] = (
|
||||
topk_indices_buffer[: topk_indices.shape[0], : topk_indices.shape[-1]] = (
|
||||
topk_indices
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user