[Frontend] Overlap preprocessing and computation for pooling models offline inference (#47699)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-07-16 14:00:20 +00:00
committed by GitHub
parent 7cd1d57b74
commit 3e90d015ba
10 changed files with 676 additions and 153 deletions
@@ -4,13 +4,19 @@ from collections.abc import Sequence
from typing import Any
from vllm import PoolingParams, PoolingRequestOutput
from vllm.inputs import EngineInput
from vllm.logger import init_logger
from vllm.plugins.io_processors import get_io_processor
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
from ..base.io_processor import PoolingIOProcessor
from ..typing import OfflineInputsContext, OfflineOutputsContext, PoolingServeContext
from ..typing import (
ALLOfflineInputsContext,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflinePluginInputsContext,
PoolingServeContext,
RequestFactory,
)
from .protocol import IOProcessorRequest, IOProcessorResponse
logger = init_logger(__name__)
@@ -107,9 +113,11 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
#######################################
# offline APIs
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
def get_request_factory_offline(
self, ctx: ALLOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflinePluginInputsContext)
assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts
assert ctx.pooling_params is not None
# Validate the request data is valid for the loaded plugin
prompt_data = ctx.prompts.get("data")
@@ -126,20 +134,35 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
prompts = self.io_processor.pre_process(prompt=validated_prompt)
prompts_seq = prompt_to_seq(prompts)
num_requests = len(prompts_seq)
pooling_params: PoolingParams | Sequence[PoolingParams]
if ctx.pooling_params is None:
pooling_params = PoolingParams()
else:
pooling_params = ctx.pooling_params
params_seq: list[PoolingParams] = [
self.io_processor.merge_pooling_params(param)
for param in self._params_to_seq(
ctx.pooling_params,
len(prompts_seq),
pooling_params,
num_requests,
)
]
for p in params_seq:
if p.task is None:
p.task = "plugin"
ctx.pooling_params = params_seq
ctx.prompts = prompts_seq
return super().pre_process_offline(ctx)
return super().get_request_factory_offline(
OfflineEncodeInputsContext(
prompts=prompts_seq,
pooling_params=params_seq,
pooling_task="plugin",
tokenization_kwargs=ctx.tokenization_kwargs,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
)
)
def post_process_offline(
self,