[ModelRunner V2] Enable sequence pooling for embedding and classification models (#48791)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-07-29 06:30:45 -07:00
committed by GitHub
parent e0cfa52d22
commit 43eaefba5a
10 changed files with 509 additions and 37 deletions
+27 -14
View File
@@ -6,7 +6,12 @@ import numpy as np
import torch
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
from vllm.v1.outputs import AsyncModelRunnerOutput, LogprobsTensors, ModelRunnerOutput
from vllm.v1.outputs import (
AsyncModelRunnerOutput,
LogprobsTensors,
ModelRunnerOutput,
PoolerOutput,
)
from vllm.v1.worker.gpu.sample.output import SamplerOutput
@@ -89,34 +94,42 @@ class AsyncPoolingOutput(AsyncModelRunnerOutput):
def __init__(
self,
model_runner_output: ModelRunnerOutput,
pooler_output: torch.Tensor,
is_valid: torch.Tensor | None,
pooler_output: PoolerOutput,
finished_mask: list[bool],
main_stream: torch.cuda.Stream,
copy_stream: torch.cuda.Stream,
):
self.model_runner_output = model_runner_output
self.pooler_output = pooler_output
self.is_valid = is_valid
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
self.copy_event = torch.cuda.Event(blocking=True)
with stream(copy_stream, main_stream):
copy_stream.wait_stream(main_stream)
self.pooler_output_cpu = self.pooler_output.to("cpu", non_blocking=True)
if self.is_valid is not None:
self.is_valid_cpu = self.is_valid.to("cpu", non_blocking=True)
if isinstance(self.pooler_output, torch.Tensor) and all(finished_mask):
self.pooler_output_cpu: PoolerOutput = self.pooler_output.to(
"cpu", non_blocking=True
)
else:
self.is_valid_cpu = None
outputs = (
self.pooler_output.unbind()
if isinstance(self.pooler_output, torch.Tensor)
else self.pooler_output
)
self.pooler_output_cpu = [
None
if output is None or not is_finished
else output.to("cpu", non_blocking=True)
for output, is_finished in zip(outputs, finished_mask, strict=True)
]
self.copy_event.record(copy_stream)
def get_output(self) -> ModelRunnerOutput:
pooler_output = list(self.pooler_output_cpu.unbind(dim=0))
if isinstance(self.pooler_output_cpu, torch.Tensor):
pooler_output = list(self.pooler_output_cpu.unbind(dim=0))
else:
pooler_output = self.pooler_output_cpu
self.copy_event.synchronize()
if self.is_valid_cpu is not None:
is_valid_cpu = self.is_valid_cpu.tolist()
for i, is_valid in enumerate(is_valid_cpu):
if not is_valid:
pooler_output[i] = None
self.model_runner_output.pooler_output = pooler_output
return self.model_runner_output
+13 -3
View File
@@ -366,7 +366,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
if self.is_pooling_model and self.is_last_pp_rank:
self.pooling_runner = PoolingRunner(self.model)
self.pooling_runner = PoolingRunner(self.model, self.vllm_config)
eplb_models_added |= self.eplb.maybe_register_model(
self.model,
self.model_config,
@@ -787,6 +787,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
req_idx = self.req_states.remove_request(req_id)
if req_idx is None:
return False
if self.pooling_runner is not None:
self.pooling_runner.remove_request(req_idx)
if self.pp_handler is not None:
self.pp_handler.on_req_idx_freed(req_idx)
if self.encoder_cache is not None:
@@ -839,6 +841,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
req_index = self.req_states.req_id_to_index[req_id]
if self.pooling_runner is not None:
assert new_req_data.pooling_params is not None
self.pooling_runner.add_request(
req_index,
new_req_data.pooling_params,
new_req_data.prompt_token_ids,
)
if self.encoder_cache is not None:
self.encoder_cache.add_request(req_id, new_req_data.mm_features)
@@ -1591,7 +1601,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return ModelRunnerOutput.with_kv_conn_output_only(kv_connector_output)
assert self.pooling_runner is not None
pooler_output, is_valid = self.pooling_runner.pool(
pooler_output, finished_mask = self.pooling_runner.pool(
hidden_states, input_batch, self.req_states
)
@@ -1604,7 +1614,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
async_output = AsyncPoolingOutput(
model_runner_output=model_runner_output,
pooler_output=pooler_output,
is_valid=is_valid,
finished_mask=finished_mask,
main_stream=self.main_stream,
copy_stream=self.output_copy_stream,
)
@@ -8,12 +8,13 @@ import torch.nn as nn
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.config.compilation import CUDAGraphMode
from vllm.model_executor.layers.attention import Attention
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.utils.torch_utils import PIN_MEMORY, STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.attention.backend import (
AttentionCGSupport,
AttentionType,
CommonAttentionMetadata,
)
from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.kv_cache_interface import (
AttentionSpec,
EncoderOnlyAttentionSpec,
@@ -22,6 +23,7 @@ from vllm.v1.kv_cache_interface import (
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
from vllm.v1.worker.gpu.states import RequestState
from vllm.v1.worker.utils import AttentionGroup
@@ -95,6 +97,52 @@ class EncoderOnlyModelState(DefaultModelState):
self._dummy_slot_mapping = torch.zeros(
self.max_num_tokens, dtype=torch.int64, device=device
)
self.token_type_ids: dict[str, torch.Tensor] = {}
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
super().add_request(req_index, new_req_data)
pooling_params = new_req_data.pooling_params
if pooling_params is None or pooling_params.extra_kwargs is None:
return
token_type_start = pooling_params.extra_kwargs.get("compressed_token_type_ids")
if token_type_start is not None:
assert new_req_data.prompt_token_ids is not None
self.token_type_ids[new_req_data.req_id] = (
torch.arange(len(new_req_data.prompt_token_ids), dtype=torch.int32)
>= token_type_start
).to(torch.int32)
def remove_request(self, req_id: str) -> None:
super().remove_request(req_id)
self.token_type_ids.pop(req_id, None)
def prepare_inputs(
self, input_batch: InputBatch, req_states: RequestState
) -> dict[str, torch.Tensor | None]:
model_inputs = super().prepare_inputs(input_batch, req_states)
if not self.token_type_ids:
return model_inputs
token_type_ids_cpu = torch.zeros(
input_batch.num_tokens_after_padding,
dtype=torch.int32,
pin_memory=PIN_MEMORY,
)
offset = 0
for i, req_id in enumerate(input_batch.req_ids):
num_tokens = int(input_batch.num_scheduled_tokens[i])
request_token_type_ids = self.token_type_ids.get(req_id)
if request_token_type_ids is not None:
start = int(input_batch.num_computed_tokens_np[i])
token_type_ids_cpu[offset : offset + num_tokens].copy_(
request_token_type_ids[start : start + num_tokens]
)
offset += num_tokens
model_inputs["token_type_ids"] = token_type_ids_cpu.to(
self.device, non_blocking=True
)
return model_inputs
def get_additional_cg_support(self) -> tuple[AttentionCGSupport, str | None]:
# Encoder groups are built here rather than in init_attn_backend, so
+162 -17
View File
@@ -2,45 +2,190 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import cast
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from vllm.config import VllmConfig
from vllm.model_executor.models import VllmModelForPooling, is_pooling_model
from vllm.pooling_params import PoolingParams
from vllm.tasks import PoolingTask
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.outputs import PoolerOutput
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.states import RequestState
_SUPPORTED_TASKS: frozenset[PoolingTask] = frozenset({"embed", "classify"})
# NOTE(woosuk): Currently, this class only supports the "LAST" pooling task
# on decoder-only models. How to support other pooling tasks and models
# is to be determined.
class PoolingRunner:
def __init__(self, model: nn.Module):
def __init__(self, model: nn.Module, vllm_config: VllmConfig):
self.model = cast(VllmModelForPooling, model)
self.model_config = vllm_config.model_config
self.max_num_reqs = vllm_config.scheduler_config.max_num_seqs
model_tasks = tuple(sorted(self.model.pooler.get_supported_tasks()))
selected_task = self.model_config.get_pooling_task(model_tasks)
if selected_task not in _SUPPORTED_TASKS:
raise ValueError(
"Model Runner V2 supports only sequence-level pooling tasks "
f"{sorted(_SUPPORTED_TASKS)}, but this model selects "
f"{selected_task!r} from {list(model_tasks)}. Set an explicitly "
"supported task or VLLM_USE_V2_MODEL_RUNNER=0."
)
self.supported_tasks = frozenset(self.get_supported_tasks(model))
if not self.supported_tasks:
raise ValueError(
"Model Runner V2 supports only sequence-level pooling tasks "
f"{sorted(_SUPPORTED_TASKS)}, but this model supports "
f"{list(model_tasks)}. "
"Set VLLM_USE_V2_MODEL_RUNNER=0 to use this model."
)
self.pooling_params: dict[int, PoolingParams] = {}
self.pooling_states: dict[int, PoolingStates] = {}
self.prompt_token_ids: dict[int, torch.Tensor] = {}
@staticmethod
def get_supported_tasks(model: nn.Module) -> list[PoolingTask]:
if not is_pooling_model(model):
return []
assert "embed" in model.pooler.get_supported_tasks()
return ["embed"]
return sorted(model.pooler.get_supported_tasks() & _SUPPORTED_TASKS)
def add_request(
self,
req_index: int,
pooling_params: PoolingParams,
prompt_token_ids: list[int],
) -> None:
task = pooling_params.task
if task not in self.supported_tasks:
raise ValueError(
f"Unsupported task: {task!r}. "
f"Supported tasks: {sorted(self.supported_tasks)}"
)
self.model.pooler.get_pooling_updates(task).apply(pooling_params)
self.pooling_params[req_index] = pooling_params
self.pooling_states[req_index] = PoolingStates()
if pooling_params.requires_token_ids:
self.prompt_token_ids[req_index] = torch.tensor(
prompt_token_ids, dtype=torch.int64
)
def remove_request(self, req_index: int) -> None:
self.pooling_params.pop(req_index, None)
if state := self.pooling_states.pop(req_index, None):
state.clean()
self.prompt_token_ids.pop(req_index, None)
def _get_pooling_metadata(
self,
input_batch: InputBatch,
req_states: RequestState,
device: torch.device,
) -> PoolingMetadata:
req_indices = input_batch.idx_mapping_np.tolist()
pooling_params = [self.pooling_params[i] for i in req_indices]
pooling_states = [self.pooling_states[i] for i in req_indices]
prompt_lens = torch.from_numpy(
req_states.prompt_len.np[input_batch.idx_mapping_np].copy()
)
prompt_token_ids_cpu = None
prompt_token_ids = None
if any(params.requires_token_ids for params in pooling_params):
max_prompt_len = int(prompt_lens.max())
prompt_token_ids_cpu = torch.zeros(
(input_batch.num_reqs, max_prompt_len),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
)
for i, (req_index, params) in enumerate(zip(req_indices, pooling_params)):
if not params.requires_token_ids:
continue
token_ids = self.prompt_token_ids[req_index]
prompt_token_ids_cpu[i, : token_ids.numel()] = token_ids
prompt_token_ids = prompt_token_ids_cpu.to(device, non_blocking=True)
return PoolingMetadata(
prompt_lens=prompt_lens,
prompt_token_ids=prompt_token_ids,
prompt_token_ids_cpu=prompt_token_ids_cpu,
pooling_params=pooling_params,
pooling_states=pooling_states,
)
def pool(
self,
hidden_states: torch.Tensor,
input_batch: InputBatch,
req_states: RequestState,
) -> tuple[torch.Tensor, torch.Tensor | None]:
# TODO(woosuk): Support different types of pooling tasks.
last_hidden_states = hidden_states[input_batch.logits_indices]
# TODO(woosuk): Make normalization optional.
last_hidden_states = F.normalize(last_hidden_states, p=2, dim=-1)
) -> tuple[PoolerOutput, list[bool]]:
hidden_states = hidden_states[: input_batch.num_tokens]
pooling_metadata = self._get_pooling_metadata(
input_batch, req_states, hidden_states.device
)
num_reqs = input_batch.num_reqs
# Pooling has no speculative tokens, so this CPU upper bound is exact.
seq_lens_cpu = input_batch.seq_lens_cpu_upper_bound[:num_reqs]
pooling_metadata.build_pooling_cursor(
input_batch.num_scheduled_tokens,
seq_lens_cpu,
device=hidden_states.device,
query_start_loc_gpu=input_batch.query_start_loc[: num_reqs + 1],
)
pooler_output = self.model.pooler(hidden_states, pooling_metadata)
prompt_len = req_states.prompt_len.gpu[input_batch.idx_mapping]
is_valid = input_batch.seq_lens == prompt_len
return last_hidden_states, is_valid
finished_mask = pooling_metadata.get_pooling_cursor().is_finished().tolist()
return pooler_output, finished_mask
def _dummy_pooler_run_task(
self, hidden_states: torch.Tensor, task: PoolingTask
) -> PoolerOutput:
num_tokens = hidden_states.shape[0]
num_reqs = min(num_tokens, self.max_num_reqs)
base_tokens = num_tokens // num_reqs
num_extra = num_tokens % num_reqs
num_scheduled_tokens = np.full(num_reqs, base_tokens, dtype=np.int32)
if num_extra > 0:
num_scheduled_tokens[-num_extra:] += 1
prompt_lens = torch.from_numpy(num_scheduled_tokens)
pooling_params = PoolingParams(task=task)
pooling_params.verify(self.model_config)
self.model.pooler.get_pooling_updates(task).apply(pooling_params)
prompt_token_ids = None
if pooling_params.requires_token_ids:
prompt_token_ids = torch.zeros(
(num_reqs, int(prompt_lens.max())),
dtype=torch.int64,
device=hidden_states.device,
)
pooling_metadata = PoolingMetadata(
prompt_lens=prompt_lens,
prompt_token_ids=prompt_token_ids,
prompt_token_ids_cpu=None
if prompt_token_ids is None
else prompt_token_ids.cpu(),
pooling_params=[pooling_params] * num_reqs,
pooling_states=[PoolingStates() for _ in range(num_reqs)],
)
pooling_metadata.build_pooling_cursor(
num_scheduled_tokens,
seq_lens_cpu=prompt_lens,
device=hidden_states.device,
)
try:
return self.model.pooler(hidden_states, pooling_metadata)
except RuntimeError as e:
if "out of memory" not in str(e):
raise
raise RuntimeError(
"CUDA out of memory occurred when warming up pooler "
f"({task=}) with {num_reqs} dummy requests. Please try "
"lowering `max_num_seqs` or `gpu_memory_utilization` when "
"initializing the engine."
) from e
def dummy_pooler_run(self, hidden_states: torch.Tensor) -> None:
F.normalize(hidden_states, p=2, dim=-1)
return
for task in sorted(self.supported_tasks):
self._dummy_pooler_run_task(hidden_states, task)
+5 -1
View File
@@ -230,7 +230,11 @@ def warmup_kernels(
# SamplingParams exercising all sampling features.
if model_runner.is_pooling_model:
sampling_params = None
pooling_params = PoolingParams()
pooling_task = model_runner.model_config.get_pooling_task(
model_runner.get_supported_tasks()
)
pooling_params = PoolingParams(task=pooling_task)
pooling_params.verify(model_runner.model_config)
else:
sampling_params = SamplingParams.for_sampler_warmup()
pooling_params = None