[Elastic EP] Support Model Runner V2 (#53934)
Signed-off-by: almogtavor <almogtavor@gmail.com> Signed-off-by: Almog Tavor <70065337+almogtavor@users.noreply.github.com>
This commit is contained in:
@@ -4833,6 +4833,23 @@ class GPUModelRunner(
|
||||
self.input_batch.num_tokens_no_spec[i] = pos + 1
|
||||
self.input_batch.prev_req_id_to_index = prev_req_id_to_index
|
||||
|
||||
@contextmanager
|
||||
def preserve_serving_state(self):
|
||||
multi_block_table = self.input_batch.block_table
|
||||
saved = [
|
||||
(bt.block_table.gpu.clone(), bt.block_table.cpu.clone())
|
||||
for bt in multi_block_table.block_tables
|
||||
]
|
||||
multi_block_table.clear()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for bt, (saved_gpu, saved_cpu) in zip(
|
||||
multi_block_table.block_tables, saved
|
||||
):
|
||||
bt.block_table.gpu.copy_(saved_gpu)
|
||||
bt.block_table.cpu.copy_(saved_cpu)
|
||||
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
if not self.num_spec_tokens or not self._draft_token_req_ids:
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user