[Core] [Bugfix] Add Input Embeddings (#15428)
Signed-off-by: Andrew Sansom <andrew@protopia.ai> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: 临景 <linjing.yx@alibaba-inc.com> Co-authored-by: Bryce1010 <bryceyx@gmail.com> Co-authored-by: Nan2018 <nan@protopia.ai> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
临景
Bryce1010
Nan2018
Cyrus Leung
DarkLight1337
parent
9e2de9b9e9
commit
cc2a77d7f1
@@ -1071,6 +1071,7 @@ class Scheduler:
|
||||
)
|
||||
ignored_seq_groups: List[SequenceGroup] = []
|
||||
seq_groups: List[ScheduledSequenceGroup] = []
|
||||
using_prompt_embeds: bool = False
|
||||
|
||||
waiting_queue = self.waiting
|
||||
|
||||
@@ -1138,6 +1139,15 @@ class Scheduler:
|
||||
waiting_queue.popleft()
|
||||
continue
|
||||
|
||||
# We cannot mix sequence groups that use prompt embeds and
|
||||
# those that do not.
|
||||
if len(seq_groups) == 0:
|
||||
using_prompt_embeds = seq_group.uses_prompt_embeds()
|
||||
if using_prompt_embeds != seq_group.uses_prompt_embeds():
|
||||
leftover_waiting_sequences.appendleft(seq_group)
|
||||
waiting_queue.popleft()
|
||||
continue
|
||||
|
||||
lora_int_id = 0
|
||||
if self.lora_enabled:
|
||||
lora_int_id = seq_group.lora_int_id
|
||||
@@ -1295,17 +1305,39 @@ class Scheduler:
|
||||
|
||||
# Merge lists
|
||||
num_prefill_groups = len(prefills.seq_groups)
|
||||
ignored_seq_groups_for_embeds = list[SequenceGroup]()
|
||||
if num_prefill_groups > 0:
|
||||
scheduled_seq_groups = prefills.seq_groups
|
||||
scheduled_seq_groups.extend(running_scheduled.decode_seq_groups)
|
||||
ignored_seq_groups_for_embeds.clear()
|
||||
else:
|
||||
scheduled_seq_groups = running_scheduled.decode_seq_groups
|
||||
if len(scheduled_seq_groups) > 0:
|
||||
using_prompt_embeds = scheduled_seq_groups[
|
||||
0].seq_group.uses_prompt_embeds()
|
||||
ignored_seq_groups_for_embeds.clear()
|
||||
indices_ignored = list[int]()
|
||||
for i, schedule_seq_group in enumerate(scheduled_seq_groups):
|
||||
if using_prompt_embeds !=\
|
||||
schedule_seq_group.seq_group.uses_prompt_embeds():
|
||||
ignored_seq_groups_for_embeds.append(
|
||||
schedule_seq_group.seq_group)
|
||||
indices_ignored.append(i)
|
||||
if len(ignored_seq_groups_for_embeds) > 0:
|
||||
scheduled_seq_groups = [
|
||||
group for i, group in enumerate(scheduled_seq_groups)
|
||||
if i not in indices_ignored
|
||||
]
|
||||
else:
|
||||
ignored_seq_groups_for_embeds.clear()
|
||||
|
||||
scheduled_seq_groups.extend(swapped_in.decode_seq_groups)
|
||||
|
||||
blocks_to_copy = running_scheduled.blocks_to_copy
|
||||
blocks_to_copy.extend(swapped_in.blocks_to_copy)
|
||||
|
||||
ignored_seq_groups = prefills.ignored_seq_groups
|
||||
ignored_seq_groups.extend(ignored_seq_groups_for_embeds)
|
||||
ignored_seq_groups.extend(swapped_in.infeasible_seq_groups)
|
||||
|
||||
return SchedulerOutputs(
|
||||
|
||||
Reference in New Issue
Block a user