[Frontend] Overlap preprocessing and computation for pooling models offline inference (#47699)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-07-16 14:00:20 +00:00
committed by GitHub
parent 7cd1d57b74
commit 3e90d015ba
10 changed files with 676 additions and 153 deletions
+3 -3
View File
@@ -103,10 +103,10 @@ steps:
commands:
- pytest -v -s models/language/generation_ppl_test
- label: Language Models Test (Extended Pooling) # 36min
- label: Language Models Test (Extended Pooling)
device: h200_35gb
key: language-models-test-extended-pooling
timeout_in_minutes: 70
timeout_in_minutes: 120
optional: true
source_file_dependencies:
- vllm/
@@ -116,7 +116,7 @@ steps:
mirror:
amd:
device: mi325_1
timeout_in_minutes: 100
timeout_in_minutes: 120
depends_on:
- image-build-amd
@@ -0,0 +1,108 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import weakref
from unittest import mock
import pytest
from vllm import LLM, PoolingParams
from vllm.distributed import cleanup_dist_env_and_memory
MODEL_NAME = "intfloat/multilingual-e5-small"
@pytest.fixture(scope="module")
def llm():
llm = LLM(
model=MODEL_NAME,
max_num_seqs=2, # small to trigger tiling
tensor_parallel_size=1,
gpu_memory_utilization=0.75,
enforce_eager=True,
seed=0,
)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
@pytest.mark.skip_global_cleanup
def test_tiling_engine_basic(llm):
"""
Basic test with a small number of prompts (less than max_num_seqs).
No tiling should be triggered, but the engine still processes correctly.
"""
prompts = ["Hello", "World"]
outputs = llm.encode(prompts, pooling_task="embed")
assert len(outputs) == len(prompts)
@pytest.mark.skip_global_cleanup
def test_tiling_engine_many_requests(llm):
"""
Test with a large number of prompts that exceeds max_num_seqs.
This verifies that _run_tiling_engine correctly chunks requests,
processes all of them, and returns outputs in the correct order.
"""
num_prompts = 10
prompts = [f"Prompt {i}" for i in range(num_prompts)]
outputs = llm.encode(prompts, pooling_task="embed")
assert len(outputs) == num_prompts
@pytest.mark.skip_global_cleanup
def test_tiling_engine_with_pooling_params(llm):
"""
Test the tiling engine when different PoolingParams are provided.
The engine must handle a list of params that matches the number of prompts.
"""
num_prompts = 10
prompts = [f"Prompt {i}" for i in range(num_prompts)]
pooling_params = [PoolingParams() for _ in range(num_prompts)]
outputs = llm.encode(prompts, pooling_params=pooling_params, pooling_task="embed")
assert len(outputs) == num_prompts
# Single PoolingParams shared across all prompts
single_param = PoolingParams()
outputs = llm.encode(prompts, pooling_params=single_param, pooling_task="embed")
assert len(outputs) == num_prompts
# None PoolingParams should fall back to default
outputs = llm.encode(prompts, pooling_params=None, pooling_task="embed")
assert len(outputs) == num_prompts
@pytest.mark.skip_global_cleanup
def test_tiling_engine_abort_on_exception(llm):
"""
Test that abort_request IS called with the correct arguments when an
exception occurs inside the engine's step() loop.
"""
prompts = ["Prompt 0", "Prompt 1", "Prompt 2"]
# Mock the step method to throw an exception on the second call
original_step = llm.llm_engine.step
call_count = 0
def mocked_step():
nonlocal call_count
call_count += 1
if call_count == 2:
raise RuntimeError("Simulated engine error")
return original_step()
with mock.patch.object(llm.llm_engine, "step", side_effect=mocked_step):
# We expect an exception to be raised from encode
with mock.patch.object(llm.llm_engine, "abort_request") as mock_abort: # noqa: SIM117
with pytest.raises(RuntimeError, match="Simulated engine error"):
llm.encode(prompts, pooling_task="embed")
args, kwargs = mock_abort.call_args
request_ids = args[0]
assert isinstance(request_ids, list)
assert len(request_ids) > 0
@@ -12,7 +12,7 @@ from vllm.config import PoolerConfig
"model",
["BAAI/bge-m3"],
)
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("dtype", ["float"])
@torch.inference_mode
def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype: str):
with vllm_runner(
@@ -20,13 +20,11 @@ def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype
runner="pooling",
pooler_config=PoolerConfig(task="token_embed"),
max_model_len=None,
dtype=dtype,
) as vllm_model:
vllm_outputs = vllm_model.token_embed(example_prompts)
with hf_runner(
model,
auto_cls=AutoModel,
) as hf_model:
with hf_runner(model, auto_cls=AutoModel, dtype=dtype) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in example_prompts:
@@ -34,7 +32,6 @@ def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
embedding = output.last_hidden_state[0].float()
# normal
hf_outputs.append(embedding.cpu())
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
+1 -1
View File
@@ -356,7 +356,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
# path; the synchronous `LLM` entrypoint runs multimodal
# preprocessing serially. Warn so the setting is not a silent
# no-op. See vllm-project/vllm#42901.
if self.model_config.renderer_num_workers > 1:
if self.model_config.renderer_num_workers > 1 and self.runner_type != "pooling":
logger.warning_once(
"`renderer_num_workers=%d` was set, but the offline `LLM` "
"entrypoint uses the synchronous renderer path and runs "
+133 -30
View File
@@ -2,9 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import Any, Final
from typing import Any, Final, cast
from vllm import PoolingParams, PoolingRequestOutput, PromptType
from vllm import (
PoolingParams,
PoolingRequestOutput,
)
from vllm.config import VllmConfig
from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
@@ -14,18 +17,25 @@ from vllm.entrypoints.chat_utils import (
)
from vllm.entrypoints.serve.engine.typing import RendererChatRequest, RendererRequest
from vllm.inputs import EngineInput, SingletonPrompt
from vllm.renderers import BaseRenderer, TokenizeParams, merge_kwargs
from vllm.lora.request import LoRARequest
from vllm.renderers import BaseRenderer, merge_kwargs
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
from vllm.tool_parsers import ToolParser
from vllm.utils.mistral import is_mistral_tokenizer
from ..scoring.typing import ScoringData
from ..typing import (
OfflineInputsContext,
ALLOfflineInputsContext,
EncodeChatRenderParams,
EncodeCMPLRenderParams,
OfflineEncodeInputsContext,
OfflineOutputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
RequestGenerator,
ScoringRenderParams,
)
@@ -98,16 +108,62 @@ class PoolingIOProcessor:
#######################################
# offline APIs
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
assert not isinstance(ctx.prompts, ScoringData) and not (
isinstance(ctx.prompts, dict) and "data" in ctx.prompts
)
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineEncodeInputsContext)
prompts_seq = prompt_to_seq(ctx.prompts)
num_requests = len(prompts_seq)
pooling_task = ctx.pooling_task
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(self.model_config, prompt)
)
for prompt in prompts_seq
]
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
return self._preprocess_cmpl_offline(prompts=prompts_seq, tok_params=tok_params)
pooling_params: PoolingParams | Sequence[PoolingParams]
if ctx.pooling_params is None:
pooling_params = PoolingParams()
else:
pooling_params = ctx.pooling_params
params_seq = self._params_to_seq(pooling_params, num_requests)
for param in params_seq:
if param.task is None:
param.task = pooling_task
elif pooling_task == "plugin":
# `plugin` task uses io_processor.parse_request to verify inputs.
# We actually allow plugin to overwrite pooling_task.
pass
elif param.task != pooling_task:
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
raise ValueError(msg)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
def request_factory() -> RequestGenerator:
for i in range(num_requests):
yield EncodeCMPLRenderParams(
prompts=parsed_prompts[i],
tok_params=tok_params,
prompt_extras=None,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
return request_factory, num_requests
def post_process_offline(
self,
@@ -118,6 +174,41 @@ class PoolingIOProcessor:
#######################################
# helpers
def render(
self,
render_params: EncodeCMPLRenderParams
| EncodeChatRenderParams
| ScoringRenderParams,
) -> PoolingEngineInput:
if "conversations" in render_params:
render_params = cast(EncodeChatRenderParams, render_params)
(_,), engine_input = self.renderer.render_chat(
conversations=[render_params["conversations"]],
chat_params=render_params["chat_params"],
tok_params=render_params["tok_params"],
prompt_extras=render_params["prompt_extras"],
skip_mm_cache=render_params["skip_mm_cache"],
)
elif "prompts" in render_params:
render_params = cast(EncodeCMPLRenderParams, render_params)
engine_input = self.renderer.render_cmpl(
prompts=[render_params["prompts"]],
tok_params=render_params["tok_params"],
prompt_extras=render_params["prompt_extras"],
skip_mm_cache=render_params["skip_mm_cache"],
)
else:
raise ValueError(
f"Unsupported render_params type {render_params.__class__.__name__}"
)
return PoolingEngineInput(
prompts=engine_input[0],
params=render_params["params"],
lora_requests=render_params["lora_requests"],
priorities=render_params["priorities"],
)
def _preprocess_cmpl_online(
self,
request: RendererRequest,
@@ -196,26 +287,6 @@ class PoolingIOProcessor:
return conversation, [engine_input]
def _preprocess_cmpl_offline(
self,
prompts: PromptType | Sequence[PromptType],
tok_params: TokenizeParams,
prompt_extras: dict[str, Any] | None = None,
) -> Sequence[EngineInput]:
prompts = prompt_to_seq(prompts)
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(self.model_config, prompt)
)
for prompt in prompts
]
return self.renderer.render_cmpl(
parsed_prompts, tok_params, prompt_extras=prompt_extras
)
def _validate_chat_template(
self,
request_chat_template: str | None,
@@ -251,3 +322,35 @@ class PoolingIOProcessor:
return params
return [params] * num_requests
def _lora_request_to_seq(
self,
lora_request: LoRARequest | None | Sequence[LoRARequest | None],
num_requests: int,
) -> Sequence[LoRARequest | None]:
if isinstance(lora_request, Sequence):
if len(lora_request) != num_requests:
raise ValueError(
f"The lengths of prompts ({num_requests}) "
f"and lora_request ({len(lora_request)}) must be the same."
)
return lora_request
return [lora_request] * num_requests
def _priority_to_seq(
self,
priority: Sequence[int] | None,
num_requests: int,
) -> Sequence[int]:
if priority is not None:
if len(priority) != num_requests:
raise ValueError(
f"The lengths of prompts ({num_requests}) "
f"and priority ({len(priority)}) must be the same."
)
return priority
return [0] * num_requests
@@ -28,11 +28,13 @@ from vllm.utils.mistral import is_mistral_tokenizer
from ..base.io_processor import PoolingIOProcessor
from ..scoring.io_processor import JinaRankingIOProcessorMixin
from ..typing import (
ALLOfflineInputsContext,
ChunkedEmbeddingMetadata,
OfflineInputsContext,
OfflineEncodeInputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
PoolingServeContext,
RequestFactory,
)
from .protocol import (
CohereEmbedContent,
@@ -665,7 +667,10 @@ class JinaRankingTokenEmbedIOProcessor(
ctx.engine_inputs = engine_inputs
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineEncodeInputsContext)
if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2:
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
@@ -677,4 +682,4 @@ class JinaRankingTokenEmbedIOProcessor(
query=text_prompts[-1], docs=text_prompts[:-1]
)
return super().pre_process_offline(ctx)
return super().get_request_factory_offline(ctx)
+134 -61
View File
@@ -20,10 +20,18 @@ from vllm.outputs import (
from vllm.pooling_params import PoolingParams
from vllm.tasks import SCORE_TYPE_MAP, PoolingTask, SupportedTask
from .base.io_processor import PoolingIOProcessor
from .factories import init_pooling_io_processors
from .scoring.io_processor import ScoringIOProcessor
from .scoring.typing import ScoreInput
from .typing import OfflineInputsContext, OfflineOutputsContext
from .typing import (
ALLOfflineInputsContext,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflinePluginInputsContext,
OfflineScoringInputsContext,
RequestFactory,
)
logger = init_logger(__name__)
@@ -48,6 +56,9 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
chat_template_config=self.chat_template_config,
)
# Use thread pool executor to accelerate preprocessing.
self._executor = self.renderer._executor
def encode(
self,
prompts: PromptType | Sequence[PromptType] | DataPrompt,
@@ -93,43 +104,30 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
io_processor = self.pooling_io_processors[pooling_task]
if pooling_params is None:
pooling_params = PoolingParams()
ctx: ALLOfflineInputsContext
if isinstance(prompts, dict) and "data" in prompts:
ctx = OfflinePluginInputsContext(
pooling_task=pooling_task,
prompts=prompts, # type: ignore[arg-type]
tokenization_kwargs=tokenization_kwargs,
pooling_params=pooling_params,
lora_request=lora_request,
priorities=None,
)
else:
ctx = OfflineEncodeInputsContext(
pooling_task=pooling_task,
prompts=prompts,
tokenization_kwargs=tokenization_kwargs,
pooling_params=pooling_params,
lora_request=lora_request,
priorities=None,
)
ctx = OfflineInputsContext(
prompts=prompts,
pooling_params=pooling_params,
tokenization_kwargs=tokenization_kwargs,
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
outputs = self._run_tiling_engine(
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
)
engine_inputs = io_processor.pre_process_offline(ctx)
n_inputs = len(engine_inputs)
assert ctx.pooling_params is not None
params_seq = self._params_to_seq(ctx.pooling_params, n_inputs)
for param in params_seq:
if param.task is None:
param.task = pooling_task
elif pooling_task == "plugin":
# `plugin` task uses io_processor.parse_request to verify inputs.
# We actually allow plugin to overwrite pooling_task.
pass
elif param.task != pooling_task:
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
raise ValueError(msg)
seq_lora_requests = self._lora_request_to_seq(lora_request, n_inputs)
seq_priority = self._priority_to_seq(None, n_inputs)
self._render_and_add_requests(
prompts=engine_inputs,
params=params_seq,
lora_requests=seq_lora_requests,
priorities=seq_priority,
)
outputs = self._run_engine(use_tqdm=use_tqdm, output_type=PoolingRequestOutput)
outputs = io_processor.post_process_offline(
ctx=OfflineOutputsContext(outputs=outputs)
)
@@ -357,46 +355,121 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
io_processor = self.pooling_io_processors[score_type]
assert isinstance(io_processor, ScoringIOProcessor)
pooling_task = io_processor.pooling_task
scoring_data = io_processor.valid_inputs(data_1, data_2)
n_queries = len(scoring_data.data_1)
if pooling_params is None:
pooling_params = PoolingParams()
ctx = OfflineInputsContext(
prompts=scoring_data,
assert isinstance(pooling_params, PoolingParams)
pooling_task = io_processor.pooling_task
if pooling_params.task is None:
pooling_params.task = pooling_task
elif pooling_params.task != pooling_task:
msg = (
f"You cannot overwrite {pooling_params.task=!r} with {pooling_task=!r}!"
)
raise ValueError(msg)
ctx = OfflineScoringInputsContext(
pooling_task=pooling_task,
scoring_data=scoring_data,
pooling_params=pooling_params,
tokenization_kwargs=tokenization_kwargs,
lora_request=lora_request,
chat_template=chat_template,
n_queries=n_queries,
priorities=None,
)
engine_inputs = io_processor.pre_process_offline(ctx)
n_inputs = len(engine_inputs)
seq_lora_requests = self._lora_request_to_seq(lora_request, n_inputs)
params_seq = self._params_to_seq(ctx.pooling_params, n_inputs)
for param in params_seq:
if param.task is None:
param.task = pooling_task
elif param.task != pooling_task:
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
raise ValueError(msg)
seq_priority = self._priority_to_seq(None, n_inputs)
self._render_and_add_requests(
prompts=engine_inputs,
params=params_seq,
lora_requests=seq_lora_requests,
priorities=seq_priority,
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
outputs = self._run_tiling_engine(
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
)
outputs = self._run_engine(use_tqdm=use_tqdm, output_type=PoolingRequestOutput)
outputs = io_processor.post_process_offline(
ctx=OfflineOutputsContext(outputs=outputs, n_queries=n_queries),
)
return [ScoringRequestOutput.from_base(item) for item in outputs]
def _run_tiling_engine(
self,
io_processor: PoolingIOProcessor,
request_factory: RequestFactory,
num_requests: int,
use_tqdm: bool | Callable[..., tqdm] = True,
):
# Keeping max_num_seqs * 2 requests in the core can already saturate the core.
# Therefore, keep most requests waiting outside the core.
max_requests_in_core = (
self.llm_engine.vllm_config.scheduler_config.max_num_seqs * 2
)
num_requests_in_core = 0
num_waited_requests = num_requests
if use_tqdm:
tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
pbar = tqdm_func(
total=num_requests,
desc="Processed prompts",
dynamic_ncols=True,
postfix=f"est. speed input: {0:.2f} toks/s, output: {0:.2f} toks/s",
)
outputs: list[PoolingRequestOutput] = []
added_request_ids: set[str] = set()
it = self._executor.map(io_processor.render, request_factory())
try:
while num_waited_requests or self.llm_engine.has_unfinished_requests():
requests = []
for _ in range(max_requests_in_core - num_requests_in_core):
if num_waited_requests == 0:
break
try:
request = next(it)
requests.append(request)
except StopIteration:
num_waited_requests = 0
break
num_waited_requests -= 1
num_requests_in_core += 1
if requests:
request_ids = self._render_and_add_requests(
prompts=[x["prompts"] for x in requests],
params=[x["params"] for x in requests],
lora_requests=[x["lora_requests"] for x in requests],
priorities=[x["priorities"] for x in requests],
)
for request_id in request_ids:
# undo assign_request_id
request_id = request_id.split("-", 1)[0]
added_request_ids.add(request_id)
step_outputs = self.llm_engine.step()
for output in step_outputs:
assert isinstance(output, PoolingRequestOutput)
assert output.finished
outputs.append(output)
added_request_ids.discard(output.request_id)
num_requests_in_core -= 1
if use_tqdm:
pbar.update(1)
if pbar.n == num_requests:
pbar.refresh()
except Exception:
if added_request_ids:
self.llm_engine.abort_request(list(added_request_ids))
raise
finally:
if use_tqdm:
pbar.close()
return sorted(outputs, key=lambda x: int(x.request_id))
@@ -4,13 +4,19 @@ from collections.abc import Sequence
from typing import Any
from vllm import PoolingParams, PoolingRequestOutput
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.plugins.io_processors import get_io_processor
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
from ..base.io_processor import PoolingIOProcessor
from ..typing import OfflineInputsContext, OfflineOutputsContext, PoolingServeContext
from ..typing import (
ALLOfflineInputsContext,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflinePluginInputsContext,
PoolingServeContext,
RequestFactory,
)
from .protocol import IOProcessorRequest, IOProcessorResponse
logger = init_logger(__name__)
@@ -107,9 +113,11 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
#######################################
# offline APIs
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflinePluginInputsContext)
assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts
assert ctx.pooling_params is not None
# Validate the request data is valid for the loaded plugin
prompt_data = ctx.prompts.get("data")
@@ -126,20 +134,35 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
prompts = self.io_processor.pre_process(prompt=validated_prompt)
prompts_seq = prompt_to_seq(prompts)
num_requests = len(prompts_seq)
pooling_params: PoolingParams | Sequence[PoolingParams]
if ctx.pooling_params is None:
pooling_params = PoolingParams()
else:
pooling_params = ctx.pooling_params
params_seq: list[PoolingParams] = [
self.io_processor.merge_pooling_params(param)
for param in self._params_to_seq(
ctx.pooling_params,
len(prompts_seq),
pooling_params,
num_requests,
)
]
for p in params_seq:
if p.task is None:
p.task = "plugin"
ctx.pooling_params = params_seq
ctx.prompts = prompts_seq
return super().pre_process_offline(ctx)
return super().get_request_factory_offline(
OfflineEncodeInputsContext(
prompts=prompts_seq,
pooling_params=params_seq,
pooling_task="plugin",
tokenization_kwargs=ctx.tokenization_kwargs,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
)
)
def post_process_offline(
self,
+180 -31
View File
@@ -2,24 +2,36 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Sequence
from typing import Any, TypeAlias
from typing import Any, TypeAlias, cast
import torch.nn.functional as F
from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
from vllm import PoolingParams, PoolingRequestOutput, PromptType, TokensPrompt
from vllm.inputs import EngineInput
from vllm.renderers import TokenizeParams
from vllm.renderers.hf import safe_apply_chat_template
from vllm.renderers.inputs.preprocess import extract_target_prompt
from vllm.renderers.inputs.preprocess import (
extract_target_prompt,
parse_model_prompt,
prompt_to_seq,
)
from vllm.tasks import PoolingTask
from vllm.utils.mistral import is_mistral_tokenizer
from ...chat_utils import ChatTemplateResolutionError
from ..base.io_processor import PoolingIOProcessor
from ..typing import (
OfflineInputsContext,
ALLOfflineInputsContext,
EncodeChatRenderParams,
EncodeCMPLRenderParams,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflineScoringInputsContext,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
RequestGenerator,
ScoringRenderParams,
)
from .protocol import RerankRequest, ScoreRequest, ScoringRequest
from .typing import ScoreData, ScoreInput, ScoringData
@@ -213,25 +225,37 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
#######################################
# offline APIs
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
assert isinstance(ctx.prompts, ScoringData)
assert not isinstance(ctx.pooling_params, Sequence)
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=ctx.pooling_params
)
scoring_data = ctx.prompts
scoring_data = ctx.scoring_data
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
scoring_data = self._truncate_scoring_data(
scoring_data, max_tokens_per_query, max_tokens_per_doc
)
return self._pre_process(scoring_data, tok_params)
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
data_2 = score_data_to_prompts(
scoring_data.data_2, "document", self.model_config
)
prompts = data_1 + data_2
return super().get_request_factory_offline(
OfflineEncodeInputsContext(
pooling_task=self.pooling_task,
prompts=prompts,
tokenization_kwargs=ctx.tokenization_kwargs,
pooling_params=ctx.pooling_params,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
)
)
def post_process_offline(
self,
@@ -258,6 +282,26 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
prompts=data_1 + data_2, tok_params=tok_params, prompt_extras=prompt_extras
)
def _preprocess_cmpl_offline(
self,
prompts: PromptType | Sequence[PromptType],
tok_params: TokenizeParams,
prompt_extras: dict[str, Any] | None = None,
) -> Sequence[EngineInput]:
prompts = prompt_to_seq(prompts)
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(self.model_config, prompt)
)
for prompt in prompts
]
return self.renderer.render_cmpl(
parsed_prompts, tok_params, prompt_extras=prompt_extras
)
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
emb_data_1 = outputs[:n_queries]
emb_data_2 = outputs[n_queries:]
@@ -429,34 +473,96 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
#######################################
# offline APIs
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
assert isinstance(ctx.prompts, ScoringData)
assert not isinstance(ctx.pooling_params, Sequence)
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
data_1 = ctx.scoring_data.data_1
data_2 = ctx.scoring_data.data_2
num_requests = len(data_2)
if len(data_1) == 1:
data_1 = data_1 * num_requests
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=ctx.pooling_params
)
prompt_extras = ctx.pooling_params.extra_kwargs
prompt_extras = ctx.pooling_params.extra_kwargs if ctx.pooling_params else None
engine_inputs, pooling_params_list = self._pre_process(
ctx.prompts,
tok_params,
ctx.pooling_params,
ctx.chat_template,
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
prompt_extras=prompt_extras,
)
ctx.pooling_params = pooling_params_list
return engine_inputs
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
def request_factory() -> RequestGenerator:
for i in range(num_requests):
yield ScoringRenderParams(
data_1=data_1[i],
data_2=data_2[i],
chat_template=ctx.chat_template,
tok_params=tok_params,
prompt_extras=prompt_extras,
skip_mm_cache=False,
params=ctx.pooling_params,
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
return request_factory, num_requests
#######################################
# helpers
def render(
self,
render_params: EncodeCMPLRenderParams
| EncodeChatRenderParams
| ScoringRenderParams,
) -> PoolingEngineInput:
if "data_1" not in render_params:
raise ValueError(
f"Unsupported render_params type {render_params.__class__.__name__}"
)
render_params = cast(ScoringRenderParams, render_params)
arrival_time = time.time()
tok_params = render_params["tok_params"]
params = render_params["params"]
prompt_extras = render_params["prompt_extras"]
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=params
)
_, engine_prompt = self.get_score_prompt(
data_1=render_params["data_1"],
data_2=render_params["data_2"],
encode_kwargs=tok_params.get_encode_kwargs(),
chat_template=render_params["chat_template"],
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
if prompt_extras
else None,
)
tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
if token_type_ids := engine_prompt.pop("token_type_ids", None):
params = params.clone()
compressed = compress_token_type_ids(token_type_ids)
params.extra_kwargs = {"compressed_token_type_ids": compressed}
engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time)
return PoolingEngineInput(
prompts=engine_input,
params=params,
lora_requests=render_params["lora_requests"],
priorities=render_params["priorities"],
)
def _pre_process(
self,
scoring_data: ScoringData,
@@ -731,6 +837,49 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM
name = "jina-reranking-scoring"
pooling_task: PoolingTask = "token_embed"
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
scoring_data = ctx.scoring_data
prompt_extras = ctx.pooling_params.extra_kwargs
queries = self.ensure_str(scoring_data.data_1)
docs = self.ensure_str(scoring_data.data_2)
chat_template_kwargs = (
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
)
instruction = (
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
)
if len(queries) == 1:
prompts = [
self.format_docs_prompts_func(
query=queries[0], docs=docs, instruction=instruction
)
]
else:
prompts = [
self.format_docs_prompts_func(
query=q, docs=[d], instruction=instruction
)
for q, d in zip(queries, docs)
]
return PoolingIOProcessor.get_request_factory_offline(
self,
OfflineEncodeInputsContext(
pooling_task=self.pooling_task,
prompts=prompts,
tokenization_kwargs=ctx.tokenization_kwargs,
pooling_params=ctx.pooling_params,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
),
)
def _pre_process(
self,
scoring_data: ScoringData,
+74 -9
View File
@@ -1,17 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import AsyncGenerator, Sequence
from collections.abc import AsyncGenerator, Callable, Generator, Sequence
from dataclasses import dataclass, field
from typing import Any, Generic, TypeAlias, TypeVar
from typing import Any, Generic, TypeAlias, TypedDict, TypeVar
from fastapi import Request
from pydantic import ConfigDict
from vllm import PoolingParams, PoolingRequestOutput, PromptType
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.inputs import DataPrompt, EngineInput
from vllm.lora.request import LoRARequest
from vllm.renderers import ChatParams, TokenizeParams
from vllm.renderers.inputs import DictPrompt
from ...tasks import PoolingTask
from .classify.protocol import (
ClassificationChatRequest,
ClassificationCompletionRequest,
@@ -34,7 +38,7 @@ from .pooling.protocol import (
PoolingResponse,
)
from .scoring.protocol import ScoringRequest, ScoringResponse
from .scoring.typing import ScoringData
from .scoring.typing import ScoreData, ScoringData
PoolingCompletionLikeRequest: TypeAlias = (
EmbeddingCompletionRequest
@@ -111,13 +115,36 @@ class PoolingServeContext(Generic[PoolingRequestT]):
@dataclass
class OfflineInputsContext:
prompts: PromptType | Sequence[PromptType] | DataPrompt | ScoringData
pooling_params: PoolingParams | Sequence[PoolingParams]
tokenization_kwargs: dict[str, Any] | None = None
chat_template: str | None = None
pooling_task: PoolingTask
tokenization_kwargs: dict[str, Any] | None
lora_request: Sequence[LoRARequest | None] | None
priorities: Sequence[int] | None
## for bi-encoder & late-interaction
n_queries: int | None = None
@dataclass
class OfflineEncodeInputsContext(OfflineInputsContext):
prompts: PromptType | Sequence[PromptType]
pooling_params: PoolingParams | Sequence[PoolingParams] | None
@dataclass
class OfflineScoringInputsContext(OfflineInputsContext):
scoring_data: ScoringData
chat_template: str | None
pooling_params: PoolingParams
@dataclass
class OfflinePluginInputsContext(OfflineInputsContext):
prompts: DataPrompt
pooling_params: PoolingParams | Sequence[PoolingParams] | None
ALLOfflineInputsContext: TypeAlias = (
OfflineEncodeInputsContext
| OfflineScoringInputsContext
| OfflinePluginInputsContext
)
@dataclass
@@ -126,3 +153,41 @@ class OfflineOutputsContext:
## for bi-encoder & late-interaction
n_queries: int | None = None
class RenderParams(TypedDict):
tok_params: TokenizeParams
prompt_extras: dict[str, Any] | None
skip_mm_cache: bool
params: PoolingParams
lora_requests: LoRARequest | None
priorities: int
class EncodeCMPLRenderParams(RenderParams):
prompts: DictPrompt
class EncodeChatRenderParams(RenderParams):
conversations: list["ChatCompletionMessageParam"]
chat_params: ChatParams
class ScoringRenderParams(RenderParams):
data_1: ScoreData
data_2: ScoreData
chat_template: str | None
class PoolingEngineInput(TypedDict):
prompts: EngineInput
params: PoolingParams
lora_requests: LoRARequest | None
priorities: int
RequestGenerator: TypeAlias = Generator[
EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams
]
RequestFactory: TypeAlias = Callable[[], RequestGenerator]