[ModelRunner V2] Enable sequence pooling for embedding and classification models (#48791)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-07-29 06:30:45 -07:00
committed by GitHub
parent e0cfa52d22
commit 43eaefba5a
10 changed files with 509 additions and 37 deletions
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock, patch
import numpy as np
import pytest
import torch
import torch.nn as nn
@@ -9,8 +12,13 @@ from vllm.model_executor.models.bert import (
BertMLMHead,
SPLADESparsePooler,
)
from vllm.platforms import current_platform
from vllm.pooling_params import PoolingParams
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
from vllm.v1.worker.gpu.states import RequestState
# ---------------------------------------------------------------------
# Functional test: SPLADE formula correctness (no HF download needed)
@@ -91,3 +99,70 @@ def test_splade_pooler_matches_reference_formula(B, T, H, V):
rtol=1e-4,
atol=1e-4,
)
def test_pooling_runner_gathers_required_token_ids() -> None:
runner = PoolingRunner.__new__(PoolingRunner)
pooling_params = PoolingParams(task="embed", requires_token_ids=True)
runner.pooling_params = {1: pooling_params, 3: pooling_params}
runner.pooling_states = {1: PoolingStates(), 3: PoolingStates()}
runner.prompt_token_ids = {
1: torch.tensor([101, 102]),
3: torch.tensor([101, 11, 102]),
}
input_batch = MagicMock(spec=InputBatch)
input_batch.idx_mapping_np = np.array([3, 1], dtype=np.int32)
input_batch.num_reqs = 2
req_states = MagicMock(spec=RequestState)
req_states.prompt_len = MagicMock(np=np.array([0, 2, 0, 3], dtype=np.int32))
metadata = runner._get_pooling_metadata(
input_batch, req_states, torch.device(current_platform.device_type)
)
expected = torch.tensor([[101, 11, 102], [101, 102, 0]])
assert metadata.prompt_token_ids_cpu is not None
assert metadata.prompt_token_ids is not None
assert metadata.prompt_token_ids_cpu.is_pinned() == PIN_MEMORY
torch.testing.assert_close(
metadata.prompt_lens, torch.tensor([3, 2], dtype=torch.int32)
)
torch.testing.assert_close(metadata.prompt_token_ids_cpu, expected)
torch.testing.assert_close(metadata.prompt_token_ids.cpu(), expected)
def test_pooling_runner_stores_only_required_token_ids() -> None:
runner = PoolingRunner.__new__(PoolingRunner)
runner.model = MagicMock()
runner.supported_tasks = frozenset({"embed"})
runner.pooling_params = {}
runner.pooling_states = {}
runner.prompt_token_ids = {}
runner.add_request(1, PoolingParams(task="embed"), [101, 102])
runner.add_request(
2,
PoolingParams(task="embed", requires_token_ids=True),
[101, 11, 102],
)
assert 1 not in runner.prompt_token_ids
torch.testing.assert_close(runner.prompt_token_ids[2], torch.tensor([101, 11, 102]))
def test_pooling_runner_rejects_unsupported_selected_task() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {
"embed",
"embed&token_classify",
"token_classify",
}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.get_pooling_task.return_value = "embed&token_classify"
with (
patch.object(PoolingRunner, "get_supported_tasks", return_value=["embed"]),
pytest.raises(ValueError, match="selects 'embed&token_classify'"),
):
PoolingRunner(model, vllm_config)