[Elastic EP] Support Model Runner V2 (#53934)

Signed-off-by: almogtavor <almogtavor@gmail.com>
Signed-off-by: Almog Tavor <70065337+almogtavor@users.noreply.github.com>
This commit is contained in:
Almog Tavor
2026-09-29 00:20:40 +00:00
committed by GitHub
parent c68eb98b69
commit 171d1b9ef6
12 changed files with 171 additions and 80 deletions
+17
View File
@@ -4833,6 +4833,23 @@ class GPUModelRunner(
self.input_batch.num_tokens_no_spec[i] = pos + 1
self.input_batch.prev_req_id_to_index = prev_req_id_to_index
@contextmanager
def preserve_serving_state(self):
multi_block_table = self.input_batch.block_table
saved = [
(bt.block_table.gpu.clone(), bt.block_table.cpu.clone())
for bt in multi_block_table.block_tables
]
multi_block_table.clear()
try:
yield
finally:
for bt, (saved_gpu, saved_cpu) in zip(
multi_block_table.block_tables, saved
):
bt.block_table.gpu.copy_(saved_gpu)
bt.block_table.cpu.copy_(saved_cpu)
def take_draft_token_ids(self) -> DraftTokenIds | None:
if not self.num_spec_tokens or not self._draft_token_req_ids:
return None