[ModelRunner V2] Enable sequence pooling for embedding and classification models (#48791)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -366,7 +366,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
|
||||
if self.is_pooling_model and self.is_last_pp_rank:
|
||||
self.pooling_runner = PoolingRunner(self.model)
|
||||
self.pooling_runner = PoolingRunner(self.model, self.vllm_config)
|
||||
eplb_models_added |= self.eplb.maybe_register_model(
|
||||
self.model,
|
||||
self.model_config,
|
||||
@@ -787,6 +787,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
req_idx = self.req_states.remove_request(req_id)
|
||||
if req_idx is None:
|
||||
return False
|
||||
if self.pooling_runner is not None:
|
||||
self.pooling_runner.remove_request(req_idx)
|
||||
if self.pp_handler is not None:
|
||||
self.pp_handler.on_req_idx_freed(req_idx)
|
||||
if self.encoder_cache is not None:
|
||||
@@ -839,6 +841,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
req_index = self.req_states.req_id_to_index[req_id]
|
||||
|
||||
if self.pooling_runner is not None:
|
||||
assert new_req_data.pooling_params is not None
|
||||
self.pooling_runner.add_request(
|
||||
req_index,
|
||||
new_req_data.pooling_params,
|
||||
new_req_data.prompt_token_ids,
|
||||
)
|
||||
|
||||
if self.encoder_cache is not None:
|
||||
self.encoder_cache.add_request(req_id, new_req_data.mm_features)
|
||||
|
||||
@@ -1591,7 +1601,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
|
||||
|
||||
assert self.pooling_runner is not None
|
||||
pooler_output, is_valid = self.pooling_runner.pool(
|
||||
pooler_output, finished_mask = self.pooling_runner.pool(
|
||||
hidden_states, input_batch, self.req_states
|
||||
)
|
||||
|
||||
@@ -1604,7 +1614,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
async_output = AsyncPoolingOutput(
|
||||
model_runner_output=model_runner_output,
|
||||
pooler_output=pooler_output,
|
||||
is_valid=is_valid,
|
||||
finished_mask=finished_mask,
|
||||
main_stream=self.main_stream,
|
||||
copy_stream=self.output_copy_stream,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user