[Model] Jamba support (#4115)

Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>
Co-authored-by: Erez Schwartz <erezs@ai21.com>
Co-authored-by: Mor Zusman <morz@ai21.com>
Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com>
Co-authored-by: Tomer Asida <tomera@ai21.com>
Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
Co-authored-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>
This commit is contained in:
Mor Zusman
2024-07-02 23:11:29 +00:00
committed by GitHub
co-authored by Erez Schwartz Mor Zusman tomeras91 Tomer Asida Zhuohan Li Muralidhar Andoorveedu
parent ee93f4f92a
commit 9d6a8daa87
21 changed files with 1192 additions and 34 deletions
+3 -1
View File
@@ -224,6 +224,8 @@ class _AsyncLLMEngine(LLMEngine):
"""
seq_group_metadata_list, scheduler_outputs = self.scheduler[
virtual_engine].schedule()
finished_requests_ids = self.scheduler[
virtual_engine].get_and_reset_finished_requests_ids()
if not scheduler_outputs.is_empty():
# Execute the model.
@@ -235,7 +237,7 @@ class _AsyncLLMEngine(LLMEngine):
virtual_engine=virtual_engine,
num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
running_queue_size=scheduler_outputs.running_queue_size,
)
finished_requests_ids=finished_requests_ids)
output = await self.model_executor.execute_model_async(
execute_model_req)
else:
+3 -1
View File
@@ -846,6 +846,8 @@ class LLMEngine:
"as performance will be severely degraded otherwise.")
seq_group_metadata_list, scheduler_outputs = self.scheduler[
0].schedule()
finished_requests_ids = self.scheduler[
0].get_and_reset_finished_requests_ids()
if not scheduler_outputs.is_empty():
execute_model_req = ExecuteModelRequest(
@@ -855,7 +857,7 @@ class LLMEngine:
blocks_to_copy=scheduler_outputs.blocks_to_copy,
num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
running_queue_size=scheduler_outputs.running_queue_size,
)
finished_requests_ids=finished_requests_ids)
output = self.model_executor.execute_model(
execute_model_req=execute_model_req)
else: