[Model Runner V2] Enable encoder token embedding (#50574)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -743,6 +743,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
def reset_encoder_cache(self) -> None:
|
||||
if self.encoder_cache is not None:
|
||||
self.encoder_cache.reset_encoder_cache()
|
||||
if self.pooling_runner is not None:
|
||||
self.pooling_runner.clear()
|
||||
|
||||
def profile_cudagraph_memory(self) -> int:
|
||||
# NOTE(woosuk): It is TBD whether we keep this API or not.
|
||||
@@ -813,6 +815,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
def finish_requests(self, scheduler_output: SchedulerOutput) -> None:
|
||||
finished_req_ids = scheduler_output.finished_req_ids
|
||||
if self.pooling_runner is not None:
|
||||
# Preempted docs keep their query-use reservation until rescheduled.
|
||||
self.pooling_runner.on_requests_finished(finished_req_ids)
|
||||
preempted_req_ids = scheduler_output.preempted_req_ids
|
||||
if preempted_req_ids:
|
||||
finished_req_ids = finished_req_ids.union(preempted_req_ids)
|
||||
@@ -857,6 +862,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.pooling_runner is not None:
|
||||
assert new_req_data.pooling_params is not None
|
||||
self.pooling_runner.add_request(
|
||||
req_id,
|
||||
req_index,
|
||||
new_req_data.pooling_params,
|
||||
new_req_data.prompt_token_ids,
|
||||
|
||||
Reference in New Issue
Block a user