[ROCm][V2] Fix failed assertion in Llama models when using EAGLE with ROCM_AITER_FA (#44936)
Signed-off-by: Micah Williamson <micah.williamson@amd.com>
This commit is contained in:
@@ -458,15 +458,16 @@ def split_decodes_prefills_and_extends(
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
num_tokens = common_attn_metadata.num_actual_tokens
|
||||
query_start_loc = common_attn_metadata.query_start_loc_cpu
|
||||
|
||||
if max_query_len <= decode_threshold:
|
||||
return num_reqs, 0, 0, num_tokens, 0, 0
|
||||
|
||||
# Upper bound is exact for prefill rows; decode rows still satisfy
|
||||
# seq_len > query_len under the optimistic bound, so `seq_lens ==
|
||||
# query_lens` identifies prefills correctly either way.
|
||||
assert common_attn_metadata.seq_lens_cpu_upper_bound is not None
|
||||
seq_lens = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
|
||||
if max_query_len <= decode_threshold:
|
||||
return num_reqs, 0, 0, num_tokens, 0, 0
|
||||
|
||||
query_lens = query_start_loc[1:] - query_start_loc[:-1]
|
||||
is_prefill_or_extend = query_lens > decode_threshold
|
||||
is_prefill = (seq_lens == query_lens) & is_prefill_or_extend
|
||||
|
||||
Reference in New Issue
Block a user