[2/N] Chunked prefill data update (#3538)
This commit is contained in:
@@ -62,6 +62,7 @@ class EngineArgs:
|
||||
image_input_shape: Optional[str] = None
|
||||
image_feature_size: Optional[int] = None
|
||||
scheduler_delay_factor: float = 0.0
|
||||
enable_chunked_prefill: bool = False
|
||||
|
||||
def __post_init__(self):
|
||||
if self.tokenizer is None:
|
||||
@@ -356,6 +357,12 @@ class EngineArgs:
|
||||
default=EngineArgs.scheduler_delay_factor,
|
||||
help='Apply a delay (of delay factor multiplied by previous'
|
||||
'prompt latency) before scheduling next prompt.')
|
||||
parser.add_argument(
|
||||
'--enable-chunked-prefill',
|
||||
type=bool,
|
||||
default=False,
|
||||
help='If True, the prefill requests can be chunked based on the '
|
||||
'max_num_batched_tokens')
|
||||
return parser
|
||||
|
||||
@classmethod
|
||||
@@ -394,11 +401,14 @@ class EngineArgs:
|
||||
self.tokenizer_pool_type,
|
||||
self.tokenizer_pool_extra_config,
|
||||
), self.ray_workers_use_nsight)
|
||||
scheduler_config = SchedulerConfig(self.max_num_batched_tokens,
|
||||
self.max_num_seqs,
|
||||
model_config.max_model_len,
|
||||
self.use_v2_block_manager,
|
||||
self.scheduler_delay_factor)
|
||||
scheduler_config = SchedulerConfig(
|
||||
self.max_num_batched_tokens,
|
||||
self.max_num_seqs,
|
||||
model_config.max_model_len,
|
||||
self.use_v2_block_manager,
|
||||
delay_factor=self.scheduler_delay_factor,
|
||||
enable_chunked_prefill=self.enable_chunked_prefill,
|
||||
)
|
||||
lora_config = LoRAConfig(
|
||||
max_lora_rank=self.max_lora_rank,
|
||||
max_loras=self.max_loras,
|
||||
|
||||
@@ -553,7 +553,10 @@ class LLMEngine:
|
||||
# Update the scheduled sequence groups with the model outputs.
|
||||
scheduled_seq_groups = scheduler_outputs.scheduled_seq_groups
|
||||
|
||||
for seq_group, outputs in zip(scheduled_seq_groups, output):
|
||||
for scheduled_seq_group, outputs in zip(scheduled_seq_groups, output):
|
||||
seq_group = scheduled_seq_group.seq_group
|
||||
token_chunk_size = scheduled_seq_group.token_chunk_size
|
||||
seq_group.update_num_computed_tokens(token_chunk_size)
|
||||
self._process_sequence_group_outputs(seq_group, outputs)
|
||||
|
||||
# Free the finished sequence groups.
|
||||
@@ -561,7 +564,8 @@ class LLMEngine:
|
||||
|
||||
# Create the outputs.
|
||||
request_outputs: List[RequestOutput] = []
|
||||
for seq_group in scheduled_seq_groups:
|
||||
for scheduled_seq_group in scheduled_seq_groups:
|
||||
seq_group = scheduled_seq_group.seq_group
|
||||
seq_group.maybe_set_first_token_time(now)
|
||||
request_output = RequestOutput.from_seq_group(seq_group)
|
||||
request_outputs.append(request_output)
|
||||
@@ -676,17 +680,20 @@ class LLMEngine:
|
||||
# Number of Tokens.
|
||||
if prompt_run:
|
||||
num_prompt_tokens = sum(
|
||||
len(seq_group.prompt_token_ids)
|
||||
for seq_group in scheduler_outputs.scheduled_seq_groups)
|
||||
len(scheduled_seq_group.seq_group.prompt_token_ids)
|
||||
for scheduled_seq_group in
|
||||
scheduler_outputs.scheduled_seq_groups)
|
||||
num_generation_tokens = sum(
|
||||
seq_group.num_seqs()
|
||||
for seq_group in scheduler_outputs.scheduled_seq_groups)
|
||||
scheduled_seq_group.seq_group.num_seqs()
|
||||
for scheduled_seq_group in
|
||||
scheduler_outputs.scheduled_seq_groups)
|
||||
else:
|
||||
num_generation_tokens = scheduler_outputs.num_batched_tokens
|
||||
|
||||
# Latency Timings.
|
||||
time_last_iters = []
|
||||
for seq_group in scheduler_outputs.scheduled_seq_groups:
|
||||
for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
|
||||
seq_group = scheduled_seq_group.seq_group
|
||||
# Time since last token.
|
||||
# (n.b. updates seq_group.metrics.last_token_time)
|
||||
time_last_iters.append(seq_group.get_last_latency(now))
|
||||
|
||||
Reference in New Issue
Block a user