[Frontend] Overlap preprocessing and computation for pooling models offline inference (#47699)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-07-16 14:00:20 +00:00
committed by GitHub
parent 7cd1d57b74
commit 3e90d015ba
10 changed files with 676 additions and 153 deletions
+1 -1
View File
@@ -356,7 +356,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
# path; the synchronous `LLM` entrypoint runs multimodal
# preprocessing serially. Warn so the setting is not a silent
# no-op. See vllm-project/vllm#42901.
if self.model_config.renderer_num_workers > 1:
if self.model_config.renderer_num_workers > 1 and self.runner_type != "pooling":
logger.warning_once(
"`renderer_num_workers=%d` was set, but the offline `LLM` "
"entrypoint uses the synchronous renderer path and runs "
+133 -30
View File
@@ -2,9 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import Any, Final
from typing import Any, Final, cast
from vllm import PoolingParams, PoolingRequestOutput, PromptType
from vllm import (
PoolingParams,
PoolingRequestOutput,
)
from vllm.config import VllmConfig
from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
@@ -14,18 +17,25 @@ from vllm.entrypoints.chat_utils import (
)
from vllm.entrypoints.serve.engine.typing import RendererChatRequest, RendererRequest
from vllm.inputs import EngineInput, SingletonPrompt
from vllm.renderers import BaseRenderer, TokenizeParams, merge_kwargs
from vllm.lora.request import LoRARequest
from vllm.renderers import BaseRenderer, merge_kwargs
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
from vllm.tool_parsers import ToolParser
from vllm.utils.mistral import is_mistral_tokenizer
from ..scoring.typing import ScoringData
from ..typing import (
OfflineInputsContext,
ALLOfflineInputsContext,
EncodeChatRenderParams,
EncodeCMPLRenderParams,
OfflineEncodeInputsContext,
OfflineOutputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
RequestGenerator,
ScoringRenderParams,
)
@@ -98,16 +108,62 @@ class PoolingIOProcessor:
#######################################
# offline APIs
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
assert not isinstance(ctx.prompts, ScoringData) and not (
isinstance(ctx.prompts, dict) and "data" in ctx.prompts
)
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineEncodeInputsContext)
prompts_seq = prompt_to_seq(ctx.prompts)
num_requests = len(prompts_seq)
pooling_task = ctx.pooling_task
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(self.model_config, prompt)
)
for prompt in prompts_seq
]
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
return self._preprocess_cmpl_offline(prompts=prompts_seq, tok_params=tok_params)
pooling_params: PoolingParams | Sequence[PoolingParams]
if ctx.pooling_params is None:
pooling_params = PoolingParams()
else:
pooling_params = ctx.pooling_params
params_seq = self._params_to_seq(pooling_params, num_requests)
for param in params_seq:
if param.task is None:
param.task = pooling_task
elif pooling_task == "plugin":
# `plugin` task uses io_processor.parse_request to verify inputs.
# We actually allow plugin to overwrite pooling_task.
pass
elif param.task != pooling_task:
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
raise ValueError(msg)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
def request_factory() -> RequestGenerator:
for i in range(num_requests):
yield EncodeCMPLRenderParams(
prompts=parsed_prompts[i],
tok_params=tok_params,
prompt_extras=None,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
return request_factory, num_requests
def post_process_offline(
self,
@@ -118,6 +174,41 @@ class PoolingIOProcessor:
#######################################
# helpers
def render(
self,
render_params: EncodeCMPLRenderParams
| EncodeChatRenderParams
| ScoringRenderParams,
) -> PoolingEngineInput:
if "conversations" in render_params:
render_params = cast(EncodeChatRenderParams, render_params)
(_,), engine_input = self.renderer.render_chat(
conversations=[render_params["conversations"]],
chat_params=render_params["chat_params"],
tok_params=render_params["tok_params"],
prompt_extras=render_params["prompt_extras"],
skip_mm_cache=render_params["skip_mm_cache"],
)
elif "prompts" in render_params:
render_params = cast(EncodeCMPLRenderParams, render_params)
engine_input = self.renderer.render_cmpl(
prompts=[render_params["prompts"]],
tok_params=render_params["tok_params"],
prompt_extras=render_params["prompt_extras"],
skip_mm_cache=render_params["skip_mm_cache"],
)
else:
raise ValueError(
f"Unsupported render_params type {render_params.__class__.__name__}"
)
return PoolingEngineInput(
prompts=engine_input[0],
params=render_params["params"],
lora_requests=render_params["lora_requests"],
priorities=render_params["priorities"],
)
def _preprocess_cmpl_online(
self,
request: RendererRequest,
@@ -196,26 +287,6 @@ class PoolingIOProcessor:
return conversation, [engine_input]
def _preprocess_cmpl_offline(
self,
prompts: PromptType | Sequence[PromptType],
tok_params: TokenizeParams,
prompt_extras: dict[str, Any] | None = None,
) -> Sequence[EngineInput]:
prompts = prompt_to_seq(prompts)
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(self.model_config, prompt)
)
for prompt in prompts
]
return self.renderer.render_cmpl(
parsed_prompts, tok_params, prompt_extras=prompt_extras
)
def _validate_chat_template(
self,
request_chat_template: str | None,
@@ -251,3 +322,35 @@ class PoolingIOProcessor:
return params
return [params] * num_requests
def _lora_request_to_seq(
self,
lora_request: LoRARequest | None | Sequence[LoRARequest | None],
num_requests: int,
) -> Sequence[LoRARequest | None]:
if isinstance(lora_request, Sequence):
if len(lora_request) != num_requests:
raise ValueError(
f"The lengths of prompts ({num_requests}) "
f"and lora_request ({len(lora_request)}) must be the same."
)
return lora_request
return [lora_request] * num_requests
def _priority_to_seq(
self,
priority: Sequence[int] | None,
num_requests: int,
) -> Sequence[int]:
if priority is not None:
if len(priority) != num_requests:
raise ValueError(
f"The lengths of prompts ({num_requests}) "
f"and priority ({len(priority)}) must be the same."
)
return priority
return [0] * num_requests
@@ -28,11 +28,13 @@ from vllm.utils.mistral import is_mistral_tokenizer
from ..base.io_processor import PoolingIOProcessor
from ..scoring.io_processor import JinaRankingIOProcessorMixin
from ..typing import (
ALLOfflineInputsContext,
ChunkedEmbeddingMetadata,
OfflineInputsContext,
OfflineEncodeInputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
PoolingServeContext,
RequestFactory,
)
from .protocol import (
CohereEmbedContent,
@@ -665,7 +667,10 @@ class JinaRankingTokenEmbedIOProcessor(
ctx.engine_inputs = engine_inputs
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineEncodeInputsContext)
if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2:
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
@@ -677,4 +682,4 @@ class JinaRankingTokenEmbedIOProcessor(
query=text_prompts[-1], docs=text_prompts[:-1]
)
return super().pre_process_offline(ctx)
return super().get_request_factory_offline(ctx)
+134 -61
View File
@@ -20,10 +20,18 @@ from vllm.outputs import (
from vllm.pooling_params import PoolingParams
from vllm.tasks import SCORE_TYPE_MAP, PoolingTask, SupportedTask
from .base.io_processor import PoolingIOProcessor
from .factories import init_pooling_io_processors
from .scoring.io_processor import ScoringIOProcessor
from .scoring.typing import ScoreInput
from .typing import OfflineInputsContext, OfflineOutputsContext
from .typing import (
ALLOfflineInputsContext,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflinePluginInputsContext,
OfflineScoringInputsContext,
RequestFactory,
)
logger = init_logger(__name__)
@@ -48,6 +56,9 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
chat_template_config=self.chat_template_config,
)
# Use thread pool executor to accelerate preprocessing.
self._executor = self.renderer._executor
def encode(
self,
prompts: PromptType | Sequence[PromptType] | DataPrompt,
@@ -93,43 +104,30 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
io_processor = self.pooling_io_processors[pooling_task]
if pooling_params is None:
pooling_params = PoolingParams()
ctx: ALLOfflineInputsContext
if isinstance(prompts, dict) and "data" in prompts:
ctx = OfflinePluginInputsContext(
pooling_task=pooling_task,
prompts=prompts, # type: ignore[arg-type]
tokenization_kwargs=tokenization_kwargs,
pooling_params=pooling_params,
lora_request=lora_request,
priorities=None,
)
else:
ctx = OfflineEncodeInputsContext(
pooling_task=pooling_task,
prompts=prompts,
tokenization_kwargs=tokenization_kwargs,
pooling_params=pooling_params,
lora_request=lora_request,
priorities=None,
)
ctx = OfflineInputsContext(
prompts=prompts,
pooling_params=pooling_params,
tokenization_kwargs=tokenization_kwargs,
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
outputs = self._run_tiling_engine(
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
)
engine_inputs = io_processor.pre_process_offline(ctx)
n_inputs = len(engine_inputs)
assert ctx.pooling_params is not None
params_seq = self._params_to_seq(ctx.pooling_params, n_inputs)
for param in params_seq:
if param.task is None:
param.task = pooling_task
elif pooling_task == "plugin":
# `plugin` task uses io_processor.parse_request to verify inputs.
# We actually allow plugin to overwrite pooling_task.
pass
elif param.task != pooling_task:
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
raise ValueError(msg)
seq_lora_requests = self._lora_request_to_seq(lora_request, n_inputs)
seq_priority = self._priority_to_seq(None, n_inputs)
self._render_and_add_requests(
prompts=engine_inputs,
params=params_seq,
lora_requests=seq_lora_requests,
priorities=seq_priority,
)
outputs = self._run_engine(use_tqdm=use_tqdm, output_type=PoolingRequestOutput)
outputs = io_processor.post_process_offline(
ctx=OfflineOutputsContext(outputs=outputs)
)
@@ -357,46 +355,121 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
io_processor = self.pooling_io_processors[score_type]
assert isinstance(io_processor, ScoringIOProcessor)
pooling_task = io_processor.pooling_task
scoring_data = io_processor.valid_inputs(data_1, data_2)
n_queries = len(scoring_data.data_1)
if pooling_params is None:
pooling_params = PoolingParams()
ctx = OfflineInputsContext(
prompts=scoring_data,
assert isinstance(pooling_params, PoolingParams)
pooling_task = io_processor.pooling_task
if pooling_params.task is None:
pooling_params.task = pooling_task
elif pooling_params.task != pooling_task:
msg = (
f"You cannot overwrite {pooling_params.task=!r} with {pooling_task=!r}!"
)
raise ValueError(msg)
ctx = OfflineScoringInputsContext(
pooling_task=pooling_task,
scoring_data=scoring_data,
pooling_params=pooling_params,
tokenization_kwargs=tokenization_kwargs,
lora_request=lora_request,
chat_template=chat_template,
n_queries=n_queries,
priorities=None,
)
engine_inputs = io_processor.pre_process_offline(ctx)
n_inputs = len(engine_inputs)
seq_lora_requests = self._lora_request_to_seq(lora_request, n_inputs)
params_seq = self._params_to_seq(ctx.pooling_params, n_inputs)
for param in params_seq:
if param.task is None:
param.task = pooling_task
elif param.task != pooling_task:
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
raise ValueError(msg)
seq_priority = self._priority_to_seq(None, n_inputs)
self._render_and_add_requests(
prompts=engine_inputs,
params=params_seq,
lora_requests=seq_lora_requests,
priorities=seq_priority,
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
outputs = self._run_tiling_engine(
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
)
outputs = self._run_engine(use_tqdm=use_tqdm, output_type=PoolingRequestOutput)
outputs = io_processor.post_process_offline(
ctx=OfflineOutputsContext(outputs=outputs, n_queries=n_queries),
)
return [ScoringRequestOutput.from_base(item) for item in outputs]
def _run_tiling_engine(
self,
io_processor: PoolingIOProcessor,
request_factory: RequestFactory,
num_requests: int,
use_tqdm: bool | Callable[..., tqdm] = True,
):
# Keeping max_num_seqs * 2 requests in the core can already saturate the core.
# Therefore, keep most requests waiting outside the core.
max_requests_in_core = (
self.llm_engine.vllm_config.scheduler_config.max_num_seqs * 2
)
num_requests_in_core = 0
num_waited_requests = num_requests
if use_tqdm:
tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
pbar = tqdm_func(
total=num_requests,
desc="Processed prompts",
dynamic_ncols=True,
postfix=f"est. speed input: {0:.2f} toks/s, output: {0:.2f} toks/s",
)
outputs: list[PoolingRequestOutput] = []
added_request_ids: set[str] = set()
it = self._executor.map(io_processor.render, request_factory())
try:
while num_waited_requests or self.llm_engine.has_unfinished_requests():
requests = []
for _ in range(max_requests_in_core - num_requests_in_core):
if num_waited_requests == 0:
break
try:
request = next(it)
requests.append(request)
except StopIteration:
num_waited_requests = 0
break
num_waited_requests -= 1
num_requests_in_core += 1
if requests:
request_ids = self._render_and_add_requests(
prompts=[x["prompts"] for x in requests],
params=[x["params"] for x in requests],
lora_requests=[x["lora_requests"] for x in requests],
priorities=[x["priorities"] for x in requests],
)
for request_id in request_ids:
# undo assign_request_id
request_id = request_id.split("-", 1)[0]
added_request_ids.add(request_id)
step_outputs = self.llm_engine.step()
for output in step_outputs:
assert isinstance(output, PoolingRequestOutput)
assert output.finished
outputs.append(output)
added_request_ids.discard(output.request_id)
num_requests_in_core -= 1
if use_tqdm:
pbar.update(1)
if pbar.n == num_requests:
pbar.refresh()
except Exception:
if added_request_ids:
self.llm_engine.abort_request(list(added_request_ids))
raise
finally:
if use_tqdm:
pbar.close()
return sorted(outputs, key=lambda x: int(x.request_id))
@@ -4,13 +4,19 @@ from collections.abc import Sequence
from typing import Any
from vllm import PoolingParams, PoolingRequestOutput
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.plugins.io_processors import get_io_processor
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
from ..base.io_processor import PoolingIOProcessor
from ..typing import OfflineInputsContext, OfflineOutputsContext, PoolingServeContext
from ..typing import (
ALLOfflineInputsContext,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflinePluginInputsContext,
PoolingServeContext,
RequestFactory,
)
from .protocol import IOProcessorRequest, IOProcessorResponse
logger = init_logger(__name__)
@@ -107,9 +113,11 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
#######################################
# offline APIs
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflinePluginInputsContext)
assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts
assert ctx.pooling_params is not None
# Validate the request data is valid for the loaded plugin
prompt_data = ctx.prompts.get("data")
@@ -126,20 +134,35 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
prompts = self.io_processor.pre_process(prompt=validated_prompt)
prompts_seq = prompt_to_seq(prompts)
num_requests = len(prompts_seq)
pooling_params: PoolingParams | Sequence[PoolingParams]
if ctx.pooling_params is None:
pooling_params = PoolingParams()
else:
pooling_params = ctx.pooling_params
params_seq: list[PoolingParams] = [
self.io_processor.merge_pooling_params(param)
for param in self._params_to_seq(
ctx.pooling_params,
len(prompts_seq),
pooling_params,
num_requests,
)
]
for p in params_seq:
if p.task is None:
p.task = "plugin"
ctx.pooling_params = params_seq
ctx.prompts = prompts_seq
return super().pre_process_offline(ctx)
return super().get_request_factory_offline(
OfflineEncodeInputsContext(
prompts=prompts_seq,
pooling_params=params_seq,
pooling_task="plugin",
tokenization_kwargs=ctx.tokenization_kwargs,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
)
)
def post_process_offline(
self,
+180 -31
View File
@@ -2,24 +2,36 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Sequence
from typing import Any, TypeAlias
from typing import Any, TypeAlias, cast
import torch.nn.functional as F
from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
from vllm import PoolingParams, PoolingRequestOutput, PromptType, TokensPrompt
from vllm.inputs import EngineInput
from vllm.renderers import TokenizeParams
from vllm.renderers.hf import safe_apply_chat_template
from vllm.renderers.inputs.preprocess import extract_target_prompt
from vllm.renderers.inputs.preprocess import (
extract_target_prompt,
parse_model_prompt,
prompt_to_seq,
)
from vllm.tasks import PoolingTask
from vllm.utils.mistral import is_mistral_tokenizer
from ...chat_utils import ChatTemplateResolutionError
from ..base.io_processor import PoolingIOProcessor
from ..typing import (
OfflineInputsContext,
ALLOfflineInputsContext,
EncodeChatRenderParams,
EncodeCMPLRenderParams,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflineScoringInputsContext,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
RequestGenerator,
ScoringRenderParams,
)
from .protocol import RerankRequest, ScoreRequest, ScoringRequest
from .typing import ScoreData, ScoreInput, ScoringData
@@ -213,25 +225,37 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
#######################################
# offline APIs
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
assert isinstance(ctx.prompts, ScoringData)
assert not isinstance(ctx.pooling_params, Sequence)
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=ctx.pooling_params
)
scoring_data = ctx.prompts
scoring_data = ctx.scoring_data
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
scoring_data = self._truncate_scoring_data(
scoring_data, max_tokens_per_query, max_tokens_per_doc
)
return self._pre_process(scoring_data, tok_params)
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
data_2 = score_data_to_prompts(
scoring_data.data_2, "document", self.model_config
)
prompts = data_1 + data_2
return super().get_request_factory_offline(
OfflineEncodeInputsContext(
pooling_task=self.pooling_task,
prompts=prompts,
tokenization_kwargs=ctx.tokenization_kwargs,
pooling_params=ctx.pooling_params,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
)
)
def post_process_offline(
self,
@@ -258,6 +282,26 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
prompts=data_1 + data_2, tok_params=tok_params, prompt_extras=prompt_extras
)
def _preprocess_cmpl_offline(
self,
prompts: PromptType | Sequence[PromptType],
tok_params: TokenizeParams,
prompt_extras: dict[str, Any] | None = None,
) -> Sequence[EngineInput]:
prompts = prompt_to_seq(prompts)
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(self.model_config, prompt)
)
for prompt in prompts
]
return self.renderer.render_cmpl(
parsed_prompts, tok_params, prompt_extras=prompt_extras
)
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
emb_data_1 = outputs[:n_queries]
emb_data_2 = outputs[n_queries:]
@@ -429,34 +473,96 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
#######################################
# offline APIs
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
assert isinstance(ctx.prompts, ScoringData)
assert not isinstance(ctx.pooling_params, Sequence)
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
data_1 = ctx.scoring_data.data_1
data_2 = ctx.scoring_data.data_2
num_requests = len(data_2)
if len(data_1) == 1:
data_1 = data_1 * num_requests
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=ctx.pooling_params
)
prompt_extras = ctx.pooling_params.extra_kwargs
prompt_extras = ctx.pooling_params.extra_kwargs if ctx.pooling_params else None
engine_inputs, pooling_params_list = self._pre_process(
ctx.prompts,
tok_params,
ctx.pooling_params,
ctx.chat_template,
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
prompt_extras=prompt_extras,
)
ctx.pooling_params = pooling_params_list
return engine_inputs
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
def request_factory() -> RequestGenerator:
for i in range(num_requests):
yield ScoringRenderParams(
data_1=data_1[i],
data_2=data_2[i],
chat_template=ctx.chat_template,
tok_params=tok_params,
prompt_extras=prompt_extras,
skip_mm_cache=False,
params=ctx.pooling_params,
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
return request_factory, num_requests
#######################################
# helpers
def render(
self,
render_params: EncodeCMPLRenderParams
| EncodeChatRenderParams
| ScoringRenderParams,
) -> PoolingEngineInput:
if "data_1" not in render_params:
raise ValueError(
f"Unsupported render_params type {render_params.__class__.__name__}"
)
render_params = cast(ScoringRenderParams, render_params)
arrival_time = time.time()
tok_params = render_params["tok_params"]
params = render_params["params"]
prompt_extras = render_params["prompt_extras"]
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=params
)
_, engine_prompt = self.get_score_prompt(
data_1=render_params["data_1"],
data_2=render_params["data_2"],
encode_kwargs=tok_params.get_encode_kwargs(),
chat_template=render_params["chat_template"],
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
if prompt_extras
else None,
)
tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
if token_type_ids := engine_prompt.pop("token_type_ids", None):
params = params.clone()
compressed = compress_token_type_ids(token_type_ids)
params.extra_kwargs = {"compressed_token_type_ids": compressed}
engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time)
return PoolingEngineInput(
prompts=engine_input,
params=params,
lora_requests=render_params["lora_requests"],
priorities=render_params["priorities"],
)
def _pre_process(
self,
scoring_data: ScoringData,
@@ -731,6 +837,49 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM
name = "jina-reranking-scoring"
pooling_task: PoolingTask = "token_embed"
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
scoring_data = ctx.scoring_data
prompt_extras = ctx.pooling_params.extra_kwargs
queries = self.ensure_str(scoring_data.data_1)
docs = self.ensure_str(scoring_data.data_2)
chat_template_kwargs = (
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
)
instruction = (
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
)
if len(queries) == 1:
prompts = [
self.format_docs_prompts_func(
query=queries[0], docs=docs, instruction=instruction
)
]
else:
prompts = [
self.format_docs_prompts_func(
query=q, docs=[d], instruction=instruction
)
for q, d in zip(queries, docs)
]
return PoolingIOProcessor.get_request_factory_offline(
self,
OfflineEncodeInputsContext(
pooling_task=self.pooling_task,
prompts=prompts,
tokenization_kwargs=ctx.tokenization_kwargs,
pooling_params=ctx.pooling_params,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
),
)
def _pre_process(
self,
scoring_data: ScoringData,
+74 -9
View File
@@ -1,17 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import AsyncGenerator, Sequence
from collections.abc import AsyncGenerator, Callable, Generator, Sequence
from dataclasses import dataclass, field
from typing import Any, Generic, TypeAlias, TypeVar
from typing import Any, Generic, TypeAlias, TypedDict, TypeVar
from fastapi import Request
from pydantic import ConfigDict
from vllm import PoolingParams, PoolingRequestOutput, PromptType
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
from vllm.inputs import DataPrompt, EngineInput
from vllm.lora.request import LoRARequest
from vllm.renderers import ChatParams, TokenizeParams
from vllm.renderers.inputs import DictPrompt
from ...tasks import PoolingTask
from .classify.protocol import (
ClassificationChatRequest,
ClassificationCompletionRequest,
@@ -34,7 +38,7 @@ from .pooling.protocol import (
PoolingResponse,
)
from .scoring.protocol import ScoringRequest, ScoringResponse
from .scoring.typing import ScoringData
from .scoring.typing import ScoreData, ScoringData
PoolingCompletionLikeRequest: TypeAlias = (
EmbeddingCompletionRequest
@@ -111,13 +115,36 @@ class PoolingServeContext(Generic[PoolingRequestT]):
@dataclass
class OfflineInputsContext:
prompts: PromptType | Sequence[PromptType] | DataPrompt | ScoringData
pooling_params: PoolingParams | Sequence[PoolingParams]
tokenization_kwargs: dict[str, Any] | None = None
chat_template: str | None = None
pooling_task: PoolingTask
tokenization_kwargs: dict[str, Any] | None
lora_request: Sequence[LoRARequest | None] | None
priorities: Sequence[int] | None
## for bi-encoder & late-interaction
n_queries: int | None = None
@dataclass
class OfflineEncodeInputsContext(OfflineInputsContext):
prompts: PromptType | Sequence[PromptType]
pooling_params: PoolingParams | Sequence[PoolingParams] | None
@dataclass
class OfflineScoringInputsContext(OfflineInputsContext):
scoring_data: ScoringData
chat_template: str | None
pooling_params: PoolingParams
@dataclass
class OfflinePluginInputsContext(OfflineInputsContext):
prompts: DataPrompt
pooling_params: PoolingParams | Sequence[PoolingParams] | None
ALLOfflineInputsContext: TypeAlias = (
OfflineEncodeInputsContext
| OfflineScoringInputsContext
| OfflinePluginInputsContext
)
@dataclass
@@ -126,3 +153,41 @@ class OfflineOutputsContext:
## for bi-encoder & late-interaction
n_queries: int | None = None
class RenderParams(TypedDict):
tok_params: TokenizeParams
prompt_extras: dict[str, Any] | None
skip_mm_cache: bool
params: PoolingParams
lora_requests: LoRARequest | None
priorities: int
class EncodeCMPLRenderParams(RenderParams):
prompts: DictPrompt
class EncodeChatRenderParams(RenderParams):
conversations: list["ChatCompletionMessageParam"]
chat_params: ChatParams
class ScoringRenderParams(RenderParams):
data_1: ScoreData
data_2: ScoreData
chat_template: str | None
class PoolingEngineInput(TypedDict):
prompts: EngineInput
params: PoolingParams
lora_requests: LoRARequest | None
priorities: int
RequestGenerator: TypeAlias = Generator[
EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams
]
RequestFactory: TypeAlias = Callable[[], RequestGenerator]