[Core] Multi-Step + Single Step Prefills via Chunked Prefill code path (#8378)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
This commit is contained in:
co-authored by
Varun Sundar Rabindranath
parent
c5d55356f9
commit
c2ec430ab5
@@ -980,9 +980,13 @@ class EngineArgs:
|
||||
if speculative_config is not None:
|
||||
raise ValueError("Speculative decoding is not supported with "
|
||||
"multi-step (--num-scheduler-steps > 1)")
|
||||
if self.enable_chunked_prefill:
|
||||
raise ValueError("Chunked prefill is not supported with "
|
||||
"multi-step (--num-scheduler-steps > 1)")
|
||||
if self.enable_chunked_prefill and self.enable_prefix_caching:
|
||||
raise ValueError("Multi-Step is not supported with "
|
||||
"both Chunked-Prefill and Prefix-Caching "
|
||||
"enabled together.")
|
||||
if self.enable_chunked_prefill and self.pipeline_parallel_size > 1:
|
||||
raise ValueError("Multi-Step Chunked-Prefill is not supported "
|
||||
"for pipeline-parallel-size > 1")
|
||||
|
||||
# make sure num_lookahead_slots is set the higher value depending on
|
||||
# if we are using speculative decoding or multi-step
|
||||
|
||||
@@ -363,11 +363,18 @@ class _AsyncLLMEngine(LLMEngine):
|
||||
self.cached_scheduler_outputs[
|
||||
virtual_engine] = SchedulerOutputState()
|
||||
|
||||
# is_first_step_output is True only when the num_steps of all
|
||||
# the sequences are 1. When the num_steps > 1,
|
||||
# multi_step_model_runner does the first-step output append.
|
||||
is_first_step_output: bool = False if not seq_group_metadata_list \
|
||||
else seq_group_metadata_list[0].state.num_steps == 1
|
||||
|
||||
ctx.append_output(outputs=outputs,
|
||||
seq_group_metadata_list=seq_group_metadata_list,
|
||||
scheduler_outputs=scheduler_outputs,
|
||||
is_async=allow_async_output_proc,
|
||||
is_last_step=True)
|
||||
is_last_step=True,
|
||||
is_first_step_output=is_first_step_output)
|
||||
|
||||
if outputs and allow_async_output_proc:
|
||||
assert len(
|
||||
|
||||
+113
-17
@@ -90,6 +90,12 @@ class OutputData(NamedTuple):
|
||||
scheduler_outputs: SchedulerOutputs
|
||||
is_async: bool
|
||||
is_last_step: bool
|
||||
# Indicates if this output is from the first step of the
|
||||
# multi-step. When multi-step is disabled, this is always
|
||||
# set to True.
|
||||
# is_first_step_output is invalid when `outputs` has
|
||||
# outputs from multiple steps.
|
||||
is_first_step_output: Optional[bool]
|
||||
skip: List[int]
|
||||
|
||||
|
||||
@@ -108,13 +114,15 @@ class SchedulerContext:
|
||||
def append_output(self, outputs: List[SamplerOutput],
|
||||
seq_group_metadata_list: List[SequenceGroupMetadata],
|
||||
scheduler_outputs: SchedulerOutputs, is_async: bool,
|
||||
is_last_step: bool):
|
||||
is_last_step: bool,
|
||||
is_first_step_output: Optional[bool]):
|
||||
self.output_queue.append(
|
||||
OutputData(outputs=outputs,
|
||||
seq_group_metadata_list=seq_group_metadata_list,
|
||||
scheduler_outputs=scheduler_outputs,
|
||||
is_async=is_async,
|
||||
is_last_step=is_last_step,
|
||||
is_first_step_output=is_first_step_output,
|
||||
skip=[]))
|
||||
|
||||
|
||||
@@ -237,9 +245,10 @@ class LLMEngine:
|
||||
"quantization_param_path=%s, device_config=%s, "
|
||||
"decoding_config=%r, observability_config=%r, "
|
||||
"seed=%d, served_model_name=%s, use_v2_block_manager=%s, "
|
||||
"num_scheduler_steps=%d, multi_step_stream_outputs=%s, "
|
||||
"enable_prefix_caching=%s, use_async_output_proc=%s, "
|
||||
"use_cached_outputs=%s, mm_processor_kwargs=%s)",
|
||||
"num_scheduler_steps=%d, chunked_prefill_enabled=%s "
|
||||
"multi_step_stream_outputs=%s, enable_prefix_caching=%s, "
|
||||
"use_async_output_proc=%s, use_cached_outputs=%s, "
|
||||
"mm_processor_kwargs=%s)",
|
||||
VLLM_VERSION,
|
||||
model_config.model,
|
||||
speculative_config,
|
||||
@@ -270,6 +279,7 @@ class LLMEngine:
|
||||
model_config.served_model_name,
|
||||
scheduler_config.use_v2_block_manager,
|
||||
scheduler_config.num_scheduler_steps,
|
||||
scheduler_config.chunked_prefill_enabled,
|
||||
scheduler_config.multi_step_stream_outputs,
|
||||
cache_config.enable_prefix_caching,
|
||||
model_config.use_async_output_proc,
|
||||
@@ -957,8 +967,66 @@ class LLMEngine:
|
||||
|
||||
ctx: The virtual engine context to work on
|
||||
request_id: If provided, then only this request is going to be processed
|
||||
|
||||
"""
|
||||
|
||||
def update_prefill_num_computed_tokens(
|
||||
seq_group: SequenceGroup,
|
||||
seq_group_meta: SequenceGroupMetadata, num_outputs: int,
|
||||
is_first_step_output: Optional[bool]) -> None:
|
||||
"""
|
||||
When multi-step and chunked-prefill are enabled together, the
|
||||
prefill sequence scheduled for multi-step execution turn into
|
||||
decodes in the first step itself. This function accounts
|
||||
for that conversion.
|
||||
|
||||
seq_group: SequenceGroup - A prefill seq_group
|
||||
seq_group_meta: SequenceGroupMetadata - Metadata of the given
|
||||
prefill seq_group
|
||||
num_outputs: int - number of output tokens being processed for the
|
||||
given seq_group
|
||||
is_first_step_output: Optional[bool] -
|
||||
If multi-step is enabled and num_outputs is 1, this value
|
||||
indicates if this outputs belongs to the first step in the
|
||||
multi-step.
|
||||
If multi-step is enabled and num_outputs > 1, this value
|
||||
must be None, as num_outputs > 1 indicates that outputs from
|
||||
all the steps in multi-step are submitted in a single burst.
|
||||
When multi-step is disabled, this value is always True.
|
||||
"""
|
||||
|
||||
assert seq_group_meta.is_prompt
|
||||
|
||||
token_chunk_size = seq_group_meta.token_chunk_size
|
||||
|
||||
if num_outputs == 1:
|
||||
assert is_first_step_output is not None
|
||||
|
||||
if seq_group_meta.state.num_steps == 1:
|
||||
assert is_first_step_output is True
|
||||
seq_group.update_num_computed_tokens(token_chunk_size)
|
||||
return
|
||||
|
||||
# multi-step prefill is only supported when multi-step is
|
||||
# enabled with chunked prefill
|
||||
assert self.scheduler_config.is_multi_step and \
|
||||
self.scheduler_config.chunked_prefill_enabled
|
||||
if is_first_step_output is True:
|
||||
# This sequence is a prompt during the first step only.
|
||||
seq_group.update_num_computed_tokens(token_chunk_size)
|
||||
return
|
||||
|
||||
assert is_first_step_output is None
|
||||
|
||||
# multi-step prefill is only supported when multi-step is
|
||||
# enabled with chunked prefill. Outputs from all the steps are
|
||||
# submitted in a single burst.
|
||||
assert self.scheduler_config.is_multi_step and \
|
||||
self.scheduler_config.chunked_prefill_enabled
|
||||
assert num_outputs == seq_group_meta.state.num_steps, \
|
||||
f"#outputs {len(outputs)} - num steps {seq_group_meta.state.num_steps}" #noqa
|
||||
# This sequence is a prompt during the first step only.
|
||||
seq_group.update_num_computed_tokens(token_chunk_size)
|
||||
|
||||
now = time.time()
|
||||
|
||||
if len(ctx.output_queue) == 0:
|
||||
@@ -969,20 +1037,27 @@ class LLMEngine:
|
||||
# When we process only one request, no pop is required
|
||||
# (since later we will process all of the rest)
|
||||
(outputs, seq_group_metadata_list, scheduler_outputs, is_async,
|
||||
is_last_step, skip) = ctx.output_queue[0]
|
||||
is_last_step, is_first_step_output, skip) = ctx.output_queue[0]
|
||||
else:
|
||||
(outputs, seq_group_metadata_list, scheduler_outputs, is_async,
|
||||
is_last_step, skip) = ctx.output_queue.popleft()
|
||||
is_last_step, is_first_step_output,
|
||||
skip) = ctx.output_queue.popleft()
|
||||
|
||||
# Sanity check
|
||||
assert len(seq_group_metadata_list) == len(
|
||||
scheduler_outputs.scheduled_seq_groups)
|
||||
|
||||
# Organize outputs by [step][sequence group] instead of
|
||||
# [sequence group][step].
|
||||
if len(outputs) > 1:
|
||||
has_multiple_outputs: bool = len(outputs) > 1
|
||||
if has_multiple_outputs:
|
||||
assert self.scheduler_config.is_multi_step or \
|
||||
self.speculative_config
|
||||
# Organize outputs by [step][sequence group] instead of
|
||||
# [sequence group][step].
|
||||
outputs_by_sequence_group = create_output_by_sequence_group(
|
||||
outputs, num_seq_groups=len(seq_group_metadata_list))
|
||||
# We have outputs for multiple steps submitted in a single burst,
|
||||
# so invalidate is_first_step_output.
|
||||
is_first_step_output = None
|
||||
else:
|
||||
outputs_by_sequence_group = outputs
|
||||
|
||||
@@ -1018,14 +1093,17 @@ class LLMEngine:
|
||||
finished_before.append(i)
|
||||
continue
|
||||
|
||||
if len(outputs) > 1:
|
||||
if has_multiple_outputs:
|
||||
output = outputs_by_sequence_group[i]
|
||||
else:
|
||||
output = [outputs_by_sequence_group[0][i]]
|
||||
|
||||
if not is_async:
|
||||
seq_group.update_num_computed_tokens(
|
||||
scheduled_seq_group.token_chunk_size)
|
||||
if not is_async and seq_group_meta.is_prompt:
|
||||
# Updates for all decodes happen when we actually append the
|
||||
# token ids to the seq in process_outputs.
|
||||
update_prefill_num_computed_tokens(seq_group, seq_group_meta,
|
||||
len(output),
|
||||
is_first_step_output)
|
||||
|
||||
if outputs:
|
||||
for o in outputs:
|
||||
@@ -1159,8 +1237,18 @@ class LLMEngine:
|
||||
if seq_group.is_finished():
|
||||
continue
|
||||
|
||||
seq_group.update_num_computed_tokens(
|
||||
seq_group_metadata.token_chunk_size)
|
||||
if seq_group_metadata.is_prompt:
|
||||
if self.scheduler_config.is_multi_step and \
|
||||
self.scheduler_config.chunked_prefill_enabled:
|
||||
# Prompts are scheduled in multi-step only when
|
||||
# chunking is enabled. These prompts turn into
|
||||
# decodes after the very first step. Therefore,
|
||||
# we skip the update to the num_computed_tokens
|
||||
# here.
|
||||
pass
|
||||
else:
|
||||
seq_group.update_num_computed_tokens(
|
||||
seq_group_metadata.token_chunk_size)
|
||||
|
||||
if seq_group_metadata.do_sample:
|
||||
assert len(sequence_group_outputs.samples) == 1, (
|
||||
@@ -1172,6 +1260,7 @@ class LLMEngine:
|
||||
assert len(seq_group.seqs) == 1
|
||||
seq = seq_group.seqs[0]
|
||||
seq.append_token_id(sample.output_token, sample.logprobs)
|
||||
seq_group.update_num_computed_tokens(1)
|
||||
|
||||
def step(self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
|
||||
"""Performs one decoding iteration and returns newly generated results.
|
||||
@@ -1324,12 +1413,19 @@ class LLMEngine:
|
||||
if self.scheduler_config.is_multi_step:
|
||||
self.cached_scheduler_outputs[0] = SchedulerOutputState()
|
||||
|
||||
# is_first_step_output is True only when the num_steps of all
|
||||
# the sequences are 1. When the num_steps > 1,
|
||||
# multi_step_model_runner does the first-step output append.
|
||||
is_first_step_output: bool = False if not seq_group_metadata_list \
|
||||
else seq_group_metadata_list[0].state.num_steps == 1
|
||||
|
||||
# Add results to the output_queue
|
||||
ctx.append_output(outputs=outputs,
|
||||
seq_group_metadata_list=seq_group_metadata_list,
|
||||
scheduler_outputs=scheduler_outputs,
|
||||
is_async=allow_async_output_proc,
|
||||
is_last_step=True)
|
||||
is_last_step=True,
|
||||
is_first_step_output=is_first_step_output)
|
||||
|
||||
if outputs and allow_async_output_proc:
|
||||
assert len(outputs) == 1, (
|
||||
|
||||
@@ -170,6 +170,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
|
||||
token_id=output_token_id,
|
||||
logprobs=output_logprob,
|
||||
)
|
||||
seq.data.update_num_computed_tokens(1)
|
||||
|
||||
self._process_decode_and_stop(seq, sampling_params)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user