[Attention] Make split_decodes_and_prefills(..., require_uniform=True) support padding (#29644)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
co-authored by
Benjamin Chislett
parent
e4605d225e
commit
aed846917f
@@ -883,11 +883,15 @@ def split_decodes_and_prefills(
|
||||
return 0, num_reqs, 0, num_tokens
|
||||
|
||||
if require_uniform:
|
||||
# check if we are in a padded uniform batch; this is used for full-CGs, some
|
||||
# requests may have a query length of 0 but since they are padding its fine
|
||||
# to treat them as decodes (ensures num_decodes matches the captured size)
|
||||
if torch.all((query_lens == query_lens[0]) | (query_lens == 0)):
|
||||
assert num_reqs * query_lens[0] == num_tokens, "tokens not padded correctly"
|
||||
return num_reqs, 0, num_tokens, 0 # all decodes
|
||||
is_prefill = query_lens != query_lens[0]
|
||||
else:
|
||||
# 0-query len indicates a padded request; leave this at the back
|
||||
# of the batch with the prefills
|
||||
is_prefill = (query_lens > decode_threshold) | (query_lens == 0)
|
||||
is_prefill = query_lens > decode_threshold
|
||||
|
||||
if not torch.any(is_prefill):
|
||||
return num_reqs, 0, num_tokens, 0
|
||||
|
||||
Reference in New Issue
Block a user