[Attention] Move context_lens_tensor compute into GDN prefill path (#51913)
Signed-off-by: Xin Yang <xyangx@amazon.com>
This commit is contained in:
@@ -177,7 +177,6 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
|
||||
query_start_loc = m.query_start_loc
|
||||
query_start_loc_cpu = m.query_start_loc_cpu
|
||||
context_lens_tensor = m.compute_num_computed_tokens()
|
||||
nums_dict, batch_ptr, token_chunk_offset_ptr = None, None, None
|
||||
block_table_tensor = mamba_get_block_table_tensor(
|
||||
m.block_table_tensor,
|
||||
@@ -389,6 +388,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata]
|
||||
)
|
||||
|
||||
if num_prefills > 0:
|
||||
context_lens_tensor = m.compute_num_computed_tokens()
|
||||
has_initial_state = context_lens_tensor > 0
|
||||
if spec_sequence_masks_cpu is not None:
|
||||
has_initial_state = has_initial_state[~spec_sequence_masks_cpu]
|
||||
|
||||
Reference in New Issue
Block a user