[Core][5/N] Fully working chunked prefill e2e (#3884)
This commit is contained in:
@@ -386,9 +386,8 @@ class EngineArgs:
|
||||
'prompt latency) before scheduling next prompt.')
|
||||
parser.add_argument(
|
||||
'--enable-chunked-prefill',
|
||||
type=bool,
|
||||
default=False,
|
||||
help='If True, the prefill requests can be chunked based on the '
|
||||
action='store_true',
|
||||
help='If set, the prefill requests can be chunked based on the '
|
||||
'max_num_batched_tokens')
|
||||
|
||||
parser.add_argument(
|
||||
|
||||
@@ -633,7 +633,10 @@ class LLMEngine:
|
||||
seq_group = scheduled_seq_group.seq_group
|
||||
seq_group.update_num_computed_tokens(
|
||||
scheduled_seq_group.token_chunk_size)
|
||||
self._process_sequence_group_outputs(seq_group, outputs)
|
||||
# If uncomputed tokens > 0, it means prefill is chunked.
|
||||
# We don't need to process outputs in that case.
|
||||
if seq_group.get_num_uncomputed_tokens() == 0:
|
||||
self._process_sequence_group_outputs(seq_group, outputs)
|
||||
|
||||
# Free the finished sequence groups.
|
||||
self.scheduler.free_finished_seq_groups()
|
||||
|
||||
Reference in New Issue
Block a user