[ModelRunner V2] Enable sequence pooling for embedding and classification models (#48791)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
@@ -15,7 +15,9 @@ from vllm.config import PoolerConfig
|
||||
["Qwen/Qwen3-Embedding-0.6B"],
|
||||
)
|
||||
@torch.inference_mode
|
||||
def test_embed_models(hf_runner, vllm_runner, model: str):
|
||||
def test_embed_models(hf_runner, vllm_runner, monkeypatch, model: str):
|
||||
# Keep token_embed on MRV1 when sequence pooling becomes MRV2 by default.
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
chunk_size = 10
|
||||
n_prompt_tokens = [55, 56, 57]
|
||||
token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
|
||||
@@ -104,3 +106,44 @@ def test_last_pool_score_chunked_prefill_matches_unchunked(vllm_runner, model: s
|
||||
assert chunked == pytest.approx(unchunked, abs=5e-2), (
|
||||
f"chunked score {chunked} diverged from unchunked {unchunked}"
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode
|
||||
def test_sequence_embed_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
|
||||
model = "Qwen/Qwen3-Embedding-0.6B"
|
||||
chunk_size = 10
|
||||
token_prompts = [[1024 + i for i in range(n)] for n in (25, 27)]
|
||||
prompts = [TokensPrompt(prompt_token_ids=t) for t in token_prompts]
|
||||
|
||||
with hf_runner(model, auto_cls=AutoModel) as hf_model:
|
||||
hf_outputs = []
|
||||
for token_prompt in token_prompts:
|
||||
inputs = hf_model.wrap_device({"input_ids": torch.tensor([token_prompt])})
|
||||
output = hf_model.model(inputs["input_ids"])
|
||||
embedding = torch.nn.functional.normalize(
|
||||
output.last_hidden_state.float()[0, -1], dim=0
|
||||
)
|
||||
hf_outputs.append(embedding.cpu().tolist())
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
pooler_config=PoolerConfig(task="embed"),
|
||||
max_model_len=64,
|
||||
max_num_batched_tokens=chunk_size,
|
||||
max_num_seqs=2,
|
||||
gpu_memory_utilization=0.25,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=True,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.embed(prompts)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
@@ -49,3 +49,60 @@ def test_models(
|
||||
vllm_output,
|
||||
rtol=2e-3 if dtype == "float" else 1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
def test_bert_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
|
||||
model = "cross-encoder/ms-marco-TinyBERT-L-2-v2"
|
||||
score_inputs = (
|
||||
"What is the capital of France?",
|
||||
[
|
||||
"Paris.",
|
||||
"Paris is the capital and largest city of France.",
|
||||
"William Shakespeare wrote Hamlet in the early seventeenth century.",
|
||||
],
|
||||
)
|
||||
prompt_batches = [
|
||||
["short input"],
|
||||
[
|
||||
"short input",
|
||||
"a longer input that exercises mixed sequence lengths",
|
||||
],
|
||||
]
|
||||
|
||||
with hf_runner(
|
||||
model, dtype="half", auto_cls=AutoModelForSequenceClassification
|
||||
) as hf_model:
|
||||
# HfRunner uses problem_type to preserve the model's
|
||||
# sbert_ce_default_activation_function=Identity raw logits.
|
||||
hf_model.config.problem_type = "regression"
|
||||
hf_outputs = [hf_model.classify(prompts) for prompts in prompt_batches]
|
||||
|
||||
text_1, text_2 = score_inputs
|
||||
text_pairs = [[text_1, document] for document in text_2]
|
||||
with hf_runner(model, dtype="half", is_cross_encoder=True) as hf_model:
|
||||
hf_scores = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype="half",
|
||||
max_model_len=64,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = [vllm_model.classify(prompts) for prompts in prompt_batches]
|
||||
vllm_scores = vllm_model.score(*score_inputs)
|
||||
|
||||
for hf_batch, vllm_batch in zip(hf_outputs, vllm_outputs):
|
||||
hf_tensor = torch.tensor(hf_batch)
|
||||
vllm_tensor = torch.tensor(vllm_batch)
|
||||
assert vllm_tensor.shape == hf_tensor.shape
|
||||
assert torch.allclose(vllm_tensor, hf_tensor, rtol=1e-2, atol=1e-4)
|
||||
|
||||
assert torch.allclose(
|
||||
torch.tensor(vllm_scores),
|
||||
torch.tensor(hf_scores),
|
||||
rtol=1e-2,
|
||||
atol=1e-4,
|
||||
)
|
||||
|
||||
@@ -5,6 +5,7 @@ import pytest
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import PoolerConfig
|
||||
|
||||
from ...utils import check_embeddings_close
|
||||
@@ -132,6 +133,7 @@ def test_encoder_only_model_runner_v2_attention(
|
||||
task="embed", seq_pooling_type="LAST", use_activation=True
|
||||
),
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.embed(prompts)
|
||||
|
||||
check_embeddings_close(
|
||||
@@ -141,3 +143,77 @@ def test_encoder_only_model_runner_v2_attention(
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
def test_encoder_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
|
||||
model = "sentence-transformers/all-MiniLM-L6-v2"
|
||||
prompt_batches = [
|
||||
["short input"],
|
||||
[
|
||||
"short input",
|
||||
"a longer input that exercises mixed sequence lengths",
|
||||
],
|
||||
]
|
||||
|
||||
with hf_runner(model, is_sentence_transformer=True) as hf_model:
|
||||
hf_outputs = [hf_model.encode(prompts) for prompts in prompt_batches]
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
max_model_len=64,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = [vllm_model.embed(prompts) for prompts in prompt_batches]
|
||||
|
||||
for hf_batch, vllm_batch in zip(hf_outputs, vllm_outputs):
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_batch,
|
||||
embeddings_1_lst=vllm_batch,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
def test_matryoshka_dimensions_model_runner_v2(
|
||||
hf_runner, vllm_runner, monkeypatch
|
||||
) -> None:
|
||||
model = "Snowflake/snowflake-arctic-embed-m-v1.5"
|
||||
prompts = ["short input", "a longer input for a different output width"]
|
||||
dimensions = [None, 256]
|
||||
|
||||
with hf_runner(model, is_sentence_transformer=True) as hf_model:
|
||||
hf_outputs = hf_model.encode(prompts)
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
max_model_len=64,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.embed(
|
||||
prompts,
|
||||
pooling_params=[PoolingParams(dimensions=d) for d in dimensions],
|
||||
)
|
||||
|
||||
expected_outputs = []
|
||||
for output, dimension in zip(hf_outputs, dimensions):
|
||||
output = torch.as_tensor(output)
|
||||
if dimension is not None:
|
||||
output = torch.nn.functional.normalize(output[:dimension], dim=0)
|
||||
expected_outputs.append(output.tolist())
|
||||
|
||||
assert [len(output) for output in vllm_outputs] == [768, 256]
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=expected_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -9,8 +12,13 @@ from vllm.model_executor.models.bert import (
|
||||
BertMLMHead,
|
||||
SPLADESparsePooler,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Functional test: SPLADE formula correctness (no HF download needed)
|
||||
@@ -91,3 +99,70 @@ def test_splade_pooler_matches_reference_formula(B, T, H, V):
|
||||
rtol=1e-4,
|
||||
atol=1e-4,
|
||||
)
|
||||
|
||||
|
||||
def test_pooling_runner_gathers_required_token_ids() -> None:
|
||||
runner = PoolingRunner.__new__(PoolingRunner)
|
||||
pooling_params = PoolingParams(task="embed", requires_token_ids=True)
|
||||
runner.pooling_params = {1: pooling_params, 3: pooling_params}
|
||||
runner.pooling_states = {1: PoolingStates(), 3: PoolingStates()}
|
||||
runner.prompt_token_ids = {
|
||||
1: torch.tensor([101, 102]),
|
||||
3: torch.tensor([101, 11, 102]),
|
||||
}
|
||||
|
||||
input_batch = MagicMock(spec=InputBatch)
|
||||
input_batch.idx_mapping_np = np.array([3, 1], dtype=np.int32)
|
||||
input_batch.num_reqs = 2
|
||||
req_states = MagicMock(spec=RequestState)
|
||||
req_states.prompt_len = MagicMock(np=np.array([0, 2, 0, 3], dtype=np.int32))
|
||||
metadata = runner._get_pooling_metadata(
|
||||
input_batch, req_states, torch.device(current_platform.device_type)
|
||||
)
|
||||
|
||||
expected = torch.tensor([[101, 11, 102], [101, 102, 0]])
|
||||
assert metadata.prompt_token_ids_cpu is not None
|
||||
assert metadata.prompt_token_ids is not None
|
||||
assert metadata.prompt_token_ids_cpu.is_pinned() == PIN_MEMORY
|
||||
torch.testing.assert_close(
|
||||
metadata.prompt_lens, torch.tensor([3, 2], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(metadata.prompt_token_ids_cpu, expected)
|
||||
torch.testing.assert_close(metadata.prompt_token_ids.cpu(), expected)
|
||||
|
||||
|
||||
def test_pooling_runner_stores_only_required_token_ids() -> None:
|
||||
runner = PoolingRunner.__new__(PoolingRunner)
|
||||
runner.model = MagicMock()
|
||||
runner.supported_tasks = frozenset({"embed"})
|
||||
runner.pooling_params = {}
|
||||
runner.pooling_states = {}
|
||||
runner.prompt_token_ids = {}
|
||||
|
||||
runner.add_request(1, PoolingParams(task="embed"), [101, 102])
|
||||
runner.add_request(
|
||||
2,
|
||||
PoolingParams(task="embed", requires_token_ids=True),
|
||||
[101, 11, 102],
|
||||
)
|
||||
|
||||
assert 1 not in runner.prompt_token_ids
|
||||
torch.testing.assert_close(runner.prompt_token_ids[2], torch.tensor([101, 11, 102]))
|
||||
|
||||
|
||||
def test_pooling_runner_rejects_unsupported_selected_task() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {
|
||||
"embed",
|
||||
"embed&token_classify",
|
||||
"token_classify",
|
||||
}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.get_pooling_task.return_value = "embed&token_classify"
|
||||
|
||||
with (
|
||||
patch.object(PoolingRunner, "get_supported_tasks", return_value=["embed"]),
|
||||
pytest.raises(ValueError, match="selects 'embed&token_classify'"),
|
||||
):
|
||||
PoolingRunner(model, vllm_config)
|
||||
|
||||
Reference in New Issue
Block a user