[Model Runner V2] Enable encoder token embedding (#50574)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-08-01 07:55:36 -04:00
committed by GitHub
parent 3986b967b2
commit 652ba59229
4 changed files with 106 additions and 7 deletions
+6
View File
@@ -743,6 +743,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def reset_encoder_cache(self) -> None:
if self.encoder_cache is not None:
self.encoder_cache.reset_encoder_cache()
if self.pooling_runner is not None:
self.pooling_runner.clear()
def profile_cudagraph_memory(self) -> int:
# NOTE(woosuk): It is TBD whether we keep this API or not.
@@ -813,6 +815,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def finish_requests(self, scheduler_output: SchedulerOutput) -> None:
finished_req_ids = scheduler_output.finished_req_ids
if self.pooling_runner is not None:
# Preempted docs keep their query-use reservation until rescheduled.
self.pooling_runner.on_requests_finished(finished_req_ids)
preempted_req_ids = scheduler_output.preempted_req_ids
if preempted_req_ids:
finished_req_ids = finished_req_ids.union(preempted_req_ids)
@@ -857,6 +862,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.pooling_runner is not None:
assert new_req_data.pooling_params is not None
self.pooling_runner.add_request(
req_id,
req_index,
new_req_data.pooling_params,
new_req_data.prompt_token_ids,