[Core] Multi-Step + Single Step Prefills via Chunked Prefill code path (#8378)

Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
This commit is contained in:
Varun Sundar Rabindranath
2024-09-27 13:32:07 -07:00
committed by GitHub
co-authored by Varun Sundar Rabindranath
parent c5d55356f9
commit c2ec430ab5
19 changed files with 513 additions and 108 deletions
+43 -3
View File
@@ -743,10 +743,35 @@ class SequenceGroup:
return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens\
if self.prompt_adapter_request else 0
def init_multi_step(self, num_scheduler_steps: int) -> None:
self.state.num_steps = num_scheduler_steps
def init_multi_step(self, num_steps: int) -> None:
self.state.num_steps = num_steps
self.state.current_step = 0
def init_multi_step_from_lookahead_slots(self, num_lookahead_slots: int,
num_scheduler_steps: int,
is_multi_step: bool,
enable_chunking: bool) -> None:
if not is_multi_step:
self.init_multi_step(num_steps=num_scheduler_steps)
return
# Multi-Step case
is_prefill = self.is_prefill()
# The asserts below reflect the expectations of the current system.
if is_prefill and enable_chunking:
assert num_lookahead_slots == num_scheduler_steps
self.init_multi_step(num_steps=num_lookahead_slots)
else:
is_decode: bool = not is_prefill
# If it is a prefill, num_lookahead_slots must be 0
assert num_lookahead_slots == 0 or is_decode
# If it is a decode, num_lookahead_slots + 1 must match
# the scheduler steps.
assert num_lookahead_slots + 1 == num_scheduler_steps or is_prefill
self.init_multi_step(num_steps=num_lookahead_slots + 1)
def get_last_latency(self, now: float) -> Optional[float]:
"""Sets the last token time for Request level timings."""
# If still in prefill phase, raise Error.
@@ -1010,6 +1035,20 @@ class SequenceGroupMetadata(
return self.prompt_adapter_request.prompt_adapter_num_virtual_tokens \
if self.prompt_adapter_request else 0
# Multi-Step Chunked-Prefill property
@property
def is_single_step_prompt(self) -> bool:
# do_sample is true, only when the token_chunk_size matches the
# num_uncomputed_tokens of the sequence. This indicates that
# the prompt will finish processing in a single `execute_model`
# step.
return self.is_prompt and self.do_sample
def get_first_seq_id(self) -> int:
# This is an efficient way of fetching the seq_id when
# we know this SequenceGroup has only one sequence.
return next(iter(self.seq_data))
def apply_delta(self,
sequence_group_metadata_delta: SequenceGroupMetadataDelta):
for id, delta in sequence_group_metadata_delta.seq_data_delta.items():
@@ -1022,7 +1061,8 @@ class SequenceGroupMetadata(
def finish_step(self) -> None:
assert self.state is not None
assert self.state.current_step < self.state.num_steps
assert self.state.current_step < self.state.num_steps, \
f"current step {self.state.current_step}, num_steps {self.state.num_steps}" # noqa
self.state.current_step += 1