[Model Runner V2] Enable encoder token embedding (#50574)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-08-01 07:55:36 -04:00
committed by GitHub
parent 3986b967b2
commit 652ba59229
4 changed files with 106 additions and 7 deletions
+6
View File
@@ -743,6 +743,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def reset_encoder_cache(self) -> None:
if self.encoder_cache is not None:
self.encoder_cache.reset_encoder_cache()
if self.pooling_runner is not None:
self.pooling_runner.clear()
def profile_cudagraph_memory(self) -> int:
# NOTE(woosuk): It is TBD whether we keep this API or not.
@@ -813,6 +815,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def finish_requests(self, scheduler_output: SchedulerOutput) -> None:
finished_req_ids = scheduler_output.finished_req_ids
if self.pooling_runner is not None:
# Preempted docs keep their query-use reservation until rescheduled.
self.pooling_runner.on_requests_finished(finished_req_ids)
preempted_req_ids = scheduler_output.preempted_req_ids
if preempted_req_ids:
finished_req_ids = finished_req_ids.union(preempted_req_ids)
@@ -857,6 +862,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.pooling_runner is not None:
assert new_req_data.pooling_params is not None
self.pooling_runner.add_request(
req_id,
req_index,
new_req_data.pooling_params,
new_req_data.prompt_token_ids,
+20 -4
View File
@@ -12,13 +12,15 @@ from vllm.pooling_params import PoolingParams
from vllm.tasks import PoolingTask
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.outputs import PoolerOutput
from vllm.v1.pool.late_interaction_runner import LateInteractionRunner
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.states import RequestState
_SUPPORTED_TASKS: frozenset[PoolingTask] = frozenset(
{"embed", "classify", "token_classify"}
{"embed", "classify", "token_embed", "token_classify"}
)
_TOKEN_TASKS: frozenset[PoolingTask] = frozenset({"token_embed", "token_classify"})
class PoolingRunner:
@@ -53,13 +55,14 @@ class PoolingRunner:
self.pooling_params: dict[int, PoolingParams] = {}
self.pooling_states: dict[int, PoolingStates] = {}
self.prompt_token_ids: dict[int, torch.Tensor] = {}
self.late_interaction_runner = LateInteractionRunner()
@staticmethod
def _get_enabled_tasks(model_config: ModelConfig) -> frozenset[PoolingTask]:
# Token classification is validated only for unchunked encoder-only prefill.
# Token-wise pooling is validated only for unchunked encoder-only prefill.
if model_config.attn_type == "encoder_only":
return _SUPPORTED_TASKS
return _SUPPORTED_TASKS - {"token_classify"}
return _SUPPORTED_TASKS - _TOKEN_TASKS
@staticmethod
def get_supported_tasks(
@@ -72,6 +75,7 @@ class PoolingRunner:
def add_request(
self,
req_id: str,
req_index: int,
pooling_params: PoolingParams,
prompt_token_ids: list[int],
@@ -83,6 +87,7 @@ class PoolingRunner:
f"Supported tasks: {sorted(self.supported_tasks)}"
)
self.model.pooler.get_pooling_updates(task).apply(pooling_params)
self.late_interaction_runner.register_request(req_id, pooling_params)
self.pooling_params[req_index] = pooling_params
self.pooling_states[req_index] = PoolingStates()
if pooling_params.requires_token_ids:
@@ -96,6 +101,12 @@ class PoolingRunner:
state.clean()
self.prompt_token_ids.pop(req_index, None)
def on_requests_finished(self, req_ids: set[str]) -> None:
self.late_interaction_runner.on_requests_finished(req_ids)
def clear(self) -> None:
self.late_interaction_runner.clear()
def _get_pooling_metadata(
self,
input_batch: InputBatch,
@@ -153,8 +164,13 @@ class PoolingRunner:
query_start_loc_gpu=input_batch.query_start_loc[: num_reqs + 1],
)
pooler_output = self.model.pooler(hidden_states, pooling_metadata)
finished_mask = pooling_metadata.get_pooling_cursor().is_finished().tolist()
pooler_output = self.late_interaction_runner.postprocess_pooler_output(
raw_pooler_output=pooler_output,
pooling_params=pooling_metadata.pooling_params,
req_ids=input_batch.req_ids,
finished_mask=finished_mask,
)
return pooler_output, finished_mask
def _dummy_pooler_run_task(