[Model] Jamba support (#4115)
Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai> Co-authored-by: Erez Schwartz <erezs@ai21.com> Co-authored-by: Mor Zusman <morz@ai21.com> Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com> Co-authored-by: Tomer Asida <tomera@ai21.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com> Co-authored-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>
This commit is contained in:
co-authored by
Erez Schwartz
Mor Zusman
tomeras91
Tomer Asida
Zhuohan Li
Muralidhar Andoorveedu
parent
ee93f4f92a
commit
9d6a8daa87
+15
-1
@@ -299,7 +299,10 @@ class Scheduler:
|
||||
# Sequence groups in the SWAPPED state.
|
||||
# Contain decode requests that are swapped out.
|
||||
self.swapped: Deque[SequenceGroup] = deque()
|
||||
|
||||
# Sequence groups finished requests ids since last step iteration.
|
||||
# It lets the model know that any state associated with these requests
|
||||
# can and must be released after the current step.
|
||||
self._finished_requests_ids: List[str] = list()
|
||||
# Time at previous scheduling step
|
||||
self.prev_time = 0.0
|
||||
# Did we schedule a prompt at previous step?
|
||||
@@ -373,6 +376,12 @@ class Scheduler:
|
||||
def get_num_unfinished_seq_groups(self) -> int:
|
||||
return len(self.waiting) + len(self.running) + len(self.swapped)
|
||||
|
||||
def get_and_reset_finished_requests_ids(self) -> List[str]:
|
||||
"""Flushes the list of request ids of previously finished seq_groups."""
|
||||
finished_requests_ids = self._finished_requests_ids
|
||||
self._finished_requests_ids = list()
|
||||
return finished_requests_ids
|
||||
|
||||
def _schedule_running(
|
||||
self,
|
||||
running_queue: deque,
|
||||
@@ -1036,6 +1045,11 @@ class Scheduler:
|
||||
self.block_manager.free(seq)
|
||||
|
||||
def free_finished_seq_groups(self) -> None:
|
||||
for queue in [self.running, self.swapped, self.waiting]:
|
||||
self._finished_requests_ids += [
|
||||
seq_group.request_id for seq_group in queue
|
||||
if seq_group.is_finished()
|
||||
]
|
||||
self.running = deque(seq_group for seq_group in self.running
|
||||
if not seq_group.is_finished())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user