[Frontend] Overlap preprocessing and computation for pooling models offline inference (#47699)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -4,13 +4,19 @@ from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.plugins.io_processors import get_io_processor
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..typing import OfflineInputsContext, OfflineOutputsContext, PoolingServeContext
|
||||
from ..typing import (
|
||||
ALLOfflineInputsContext,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
OfflinePluginInputsContext,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
)
|
||||
from .protocol import IOProcessorRequest, IOProcessorResponse
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -107,9 +113,11 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflinePluginInputsContext)
|
||||
assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts
|
||||
assert ctx.pooling_params is not None
|
||||
|
||||
# Validate the request data is valid for the loaded plugin
|
||||
prompt_data = ctx.prompts.get("data")
|
||||
@@ -126,20 +134,35 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
|
||||
prompts = self.io_processor.pre_process(prompt=validated_prompt)
|
||||
prompts_seq = prompt_to_seq(prompts)
|
||||
|
||||
num_requests = len(prompts_seq)
|
||||
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams]
|
||||
if ctx.pooling_params is None:
|
||||
pooling_params = PoolingParams()
|
||||
else:
|
||||
pooling_params = ctx.pooling_params
|
||||
|
||||
params_seq: list[PoolingParams] = [
|
||||
self.io_processor.merge_pooling_params(param)
|
||||
for param in self._params_to_seq(
|
||||
ctx.pooling_params,
|
||||
len(prompts_seq),
|
||||
pooling_params,
|
||||
num_requests,
|
||||
)
|
||||
]
|
||||
for p in params_seq:
|
||||
if p.task is None:
|
||||
p.task = "plugin"
|
||||
|
||||
ctx.pooling_params = params_seq
|
||||
ctx.prompts = prompts_seq
|
||||
return super().pre_process_offline(ctx)
|
||||
return super().get_request_factory_offline(
|
||||
OfflineEncodeInputsContext(
|
||||
prompts=prompts_seq,
|
||||
pooling_params=params_seq,
|
||||
pooling_task="plugin",
|
||||
tokenization_kwargs=ctx.tokenization_kwargs,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
)
|
||||
)
|
||||
|
||||
def post_process_offline(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user