[Speculative decoding 6/9] Integrate speculative decoding with LLMEngine (#3894)
This commit is contained in:
@@ -104,7 +104,6 @@ class BlockTable:
|
||||
token_ids (List[int]): The sequence of token IDs to be appended.
|
||||
"""
|
||||
assert self._is_allocated
|
||||
assert token_ids, "can't append empty token ids"
|
||||
|
||||
self.ensure_num_empty_slots(num_empty_slots=len(token_ids) +
|
||||
num_lookahead_slots)
|
||||
|
||||
@@ -762,9 +762,7 @@ class Scheduler:
|
||||
blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
|
||||
swapped_in.blocks_to_copy),
|
||||
ignored_seq_groups=prefills.ignored_seq_groups,
|
||||
num_lookahead_slots=(prefills.num_lookahead_slots +
|
||||
running_scheduled.num_lookahead_slots +
|
||||
swapped_in.num_lookahead_slots),
|
||||
num_lookahead_slots=running_scheduled.num_lookahead_slots,
|
||||
)
|
||||
|
||||
def _schedule_chunked_prefill(self):
|
||||
@@ -850,9 +848,7 @@ class Scheduler:
|
||||
blocks_to_copy=merge_dicts(running_scheduled.blocks_to_copy,
|
||||
swapped_in.blocks_to_copy),
|
||||
ignored_seq_groups=prefills.ignored_seq_groups,
|
||||
num_lookahead_slots=(prefills.num_lookahead_slots +
|
||||
running_scheduled.num_lookahead_slots +
|
||||
swapped_in.num_lookahead_slots),
|
||||
num_lookahead_slots=running_scheduled.num_lookahead_slots,
|
||||
)
|
||||
|
||||
def _schedule(self) -> SchedulerOutputs:
|
||||
|
||||
Reference in New Issue
Block a user