[ROCm] Fix cu_seqlens_q off-by-one in AITER FA speculative decode path (#39120)
Signed-off-by: Bortlesboat <bortstheboat@gmail.com>
This commit is contained in:
@@ -1181,7 +1181,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
)
|
||||
|
||||
descale_shape = (
|
||||
attn_metadata.query_start_loc[:num_decodes].shape[0] - 1,
|
||||
num_decodes,
|
||||
key_cache.shape[2],
|
||||
)
|
||||
unified_attention(
|
||||
@@ -1189,7 +1189,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
out=output[:num_decode_tokens],
|
||||
cu_seqlens_q=attn_metadata.query_start_loc[:num_decodes],
|
||||
cu_seqlens_q=attn_metadata.query_start_loc[: num_decodes + 1],
|
||||
max_seqlen_q=decode_max_query_len,
|
||||
seqused_k=attn_metadata.seq_lens[:num_decodes],
|
||||
max_seqlen_k=attn_metadata.max_seq_len,
|
||||
|
||||
Reference in New Issue
Block a user