[Model Runner V2] Enable encoder token embedding (#50574)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -743,6 +743,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
def reset_encoder_cache(self) -> None:
|
||||
if self.encoder_cache is not None:
|
||||
self.encoder_cache.reset_encoder_cache()
|
||||
if self.pooling_runner is not None:
|
||||
self.pooling_runner.clear()
|
||||
|
||||
def profile_cudagraph_memory(self) -> int:
|
||||
# NOTE(woosuk): It is TBD whether we keep this API or not.
|
||||
@@ -813,6 +815,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
def finish_requests(self, scheduler_output: SchedulerOutput) -> None:
|
||||
finished_req_ids = scheduler_output.finished_req_ids
|
||||
if self.pooling_runner is not None:
|
||||
# Preempted docs keep their query-use reservation until rescheduled.
|
||||
self.pooling_runner.on_requests_finished(finished_req_ids)
|
||||
preempted_req_ids = scheduler_output.preempted_req_ids
|
||||
if preempted_req_ids:
|
||||
finished_req_ids = finished_req_ids.union(preempted_req_ids)
|
||||
@@ -857,6 +862,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.pooling_runner is not None:
|
||||
assert new_req_data.pooling_params is not None
|
||||
self.pooling_runner.add_request(
|
||||
req_id,
|
||||
req_index,
|
||||
new_req_data.pooling_params,
|
||||
new_req_data.prompt_token_ids,
|
||||
|
||||
@@ -12,13 +12,15 @@ from vllm.pooling_params import PoolingParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.outputs import PoolerOutput
|
||||
from vllm.v1.pool.late_interaction_runner import LateInteractionRunner
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
_SUPPORTED_TASKS: frozenset[PoolingTask] = frozenset(
|
||||
{"embed", "classify", "token_classify"}
|
||||
{"embed", "classify", "token_embed", "token_classify"}
|
||||
)
|
||||
_TOKEN_TASKS: frozenset[PoolingTask] = frozenset({"token_embed", "token_classify"})
|
||||
|
||||
|
||||
class PoolingRunner:
|
||||
@@ -53,13 +55,14 @@ class PoolingRunner:
|
||||
self.pooling_params: dict[int, PoolingParams] = {}
|
||||
self.pooling_states: dict[int, PoolingStates] = {}
|
||||
self.prompt_token_ids: dict[int, torch.Tensor] = {}
|
||||
self.late_interaction_runner = LateInteractionRunner()
|
||||
|
||||
@staticmethod
|
||||
def _get_enabled_tasks(model_config: ModelConfig) -> frozenset[PoolingTask]:
|
||||
# Token classification is validated only for unchunked encoder-only prefill.
|
||||
# Token-wise pooling is validated only for unchunked encoder-only prefill.
|
||||
if model_config.attn_type == "encoder_only":
|
||||
return _SUPPORTED_TASKS
|
||||
return _SUPPORTED_TASKS - {"token_classify"}
|
||||
return _SUPPORTED_TASKS - _TOKEN_TASKS
|
||||
|
||||
@staticmethod
|
||||
def get_supported_tasks(
|
||||
@@ -72,6 +75,7 @@ class PoolingRunner:
|
||||
|
||||
def add_request(
|
||||
self,
|
||||
req_id: str,
|
||||
req_index: int,
|
||||
pooling_params: PoolingParams,
|
||||
prompt_token_ids: list[int],
|
||||
@@ -83,6 +87,7 @@ class PoolingRunner:
|
||||
f"Supported tasks: {sorted(self.supported_tasks)}"
|
||||
)
|
||||
self.model.pooler.get_pooling_updates(task).apply(pooling_params)
|
||||
self.late_interaction_runner.register_request(req_id, pooling_params)
|
||||
self.pooling_params[req_index] = pooling_params
|
||||
self.pooling_states[req_index] = PoolingStates()
|
||||
if pooling_params.requires_token_ids:
|
||||
@@ -96,6 +101,12 @@ class PoolingRunner:
|
||||
state.clean()
|
||||
self.prompt_token_ids.pop(req_index, None)
|
||||
|
||||
def on_requests_finished(self, req_ids: set[str]) -> None:
|
||||
self.late_interaction_runner.on_requests_finished(req_ids)
|
||||
|
||||
def clear(self) -> None:
|
||||
self.late_interaction_runner.clear()
|
||||
|
||||
def _get_pooling_metadata(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
@@ -153,8 +164,13 @@ class PoolingRunner:
|
||||
query_start_loc_gpu=input_batch.query_start_loc[: num_reqs + 1],
|
||||
)
|
||||
pooler_output = self.model.pooler(hidden_states, pooling_metadata)
|
||||
|
||||
finished_mask = pooling_metadata.get_pooling_cursor().is_finished().tolist()
|
||||
pooler_output = self.late_interaction_runner.postprocess_pooler_output(
|
||||
raw_pooler_output=pooler_output,
|
||||
pooling_params=pooling_metadata.pooling_params,
|
||||
req_ids=input_batch.req_ids,
|
||||
finished_mask=finished_mask,
|
||||
)
|
||||
return pooler_output, finished_mask
|
||||
|
||||
def _dummy_pooler_run_task(
|
||||
|
||||
Reference in New Issue
Block a user