[ModelRunner V2] Enable sequence pooling for embedding and classification models (#48791)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-07-29 06:30:45 -07:00
committed by GitHub
parent e0cfa52d22
commit 43eaefba5a
10 changed files with 509 additions and 37 deletions
+13 -3
View File
@@ -366,7 +366,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
if self.is_pooling_model and self.is_last_pp_rank:
self.pooling_runner = PoolingRunner(self.model)
self.pooling_runner = PoolingRunner(self.model, self.vllm_config)
eplb_models_added |= self.eplb.maybe_register_model(
self.model,
self.model_config,
@@ -787,6 +787,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
req_idx = self.req_states.remove_request(req_id)
if req_idx is None:
return False
if self.pooling_runner is not None:
self.pooling_runner.remove_request(req_idx)
if self.pp_handler is not None:
self.pp_handler.on_req_idx_freed(req_idx)
if self.encoder_cache is not None:
@@ -839,6 +841,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
req_index = self.req_states.req_id_to_index[req_id]
if self.pooling_runner is not None:
assert new_req_data.pooling_params is not None
self.pooling_runner.add_request(
req_index,
new_req_data.pooling_params,
new_req_data.prompt_token_ids,
)
if self.encoder_cache is not None:
self.encoder_cache.add_request(req_id, new_req_data.mm_features)
@@ -1591,7 +1601,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
assert self.pooling_runner is not None
pooler_output, is_valid = self.pooling_runner.pool(
pooler_output, finished_mask = self.pooling_runner.pool(
hidden_states, input_batch, self.req_states
)
@@ -1604,7 +1614,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
async_output = AsyncPoolingOutput(
model_runner_output=model_runner_output,
pooler_output=pooler_output,
is_valid=is_valid,
finished_mask=finished_mask,
main_stream=self.main_stream,
copy_stream=self.output_copy_stream,
)