[Frontend] Offload blocking preprocessing & postprocessing ops to thread pool for pooling entrypoints. (#39763)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
wang.yuqi
2026-04-14 08:29:25 +00:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent 0008729abf
commit c0ecaed950
10 changed files with 68 additions and 83 deletions
+4 -24
View File
@@ -72,7 +72,7 @@ class PoolingIOProcessor:
default_template_kwargs=None,
)
elif isinstance(request, PoolingCompletionLikeRequest):
engine_inputs = self._preprocess_completion_online(
engine_inputs = self._preprocess_cmpl_online(
request,
prompt_input=request.input,
prompt_embeds=None,
@@ -82,21 +82,12 @@ class PoolingIOProcessor:
ctx.engine_inputs = engine_inputs
async def pre_process_online_async(self, ctx: PoolingServeContext):
self.pre_process_online(ctx)
def post_process_online(
self,
ctx: PoolingServeContext,
):
pass
async def post_process_online_async(
self,
ctx: PoolingServeContext,
):
self.post_process_online(ctx)
#######################################
# offline APIs
@@ -109,12 +100,7 @@ class PoolingIOProcessor:
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
return self._preprocess_completion_offline(
prompts=prompts_seq, tok_params=tok_params
)
async def pre_process_offline_async(self, ctx: OfflineInputsContext):
return self.pre_process_offline(ctx)
return self._preprocess_cmpl_offline(prompts=prompts_seq, tok_params=tok_params)
def post_process_offline(
self,
@@ -122,16 +108,10 @@ class PoolingIOProcessor:
) -> list[PoolingRequestOutput]:
return ctx.outputs
async def post_process_offline_async(
self,
ctx: OfflineOutputsContext,
) -> list[PoolingRequestOutput]:
return self.post_process_offline(ctx)
#######################################
# helpers
def _preprocess_completion_online(
def _preprocess_cmpl_online(
self,
request: RendererRequest,
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
@@ -209,7 +189,7 @@ class PoolingIOProcessor:
return conversation, [engine_input]
def _preprocess_completion_offline(
def _preprocess_cmpl_offline(
self,
prompts: PromptType | Sequence[PromptType],
tok_params: TokenizeParams,
+40 -18
View File
@@ -3,9 +3,11 @@
from abc import ABC, abstractmethod
from collections.abc import AsyncGenerator, Mapping
from concurrent.futures import Executor
from http import HTTPStatus
from typing import ClassVar
import torch
from fastapi import Request
from fastapi.responses import Response
from starlette.datastructures import Headers
@@ -32,7 +34,7 @@ from vllm.tracing import (
log_tracing_disabled_warning,
)
from vllm.utils import random_uuid
from vllm.utils.async_utils import merge_async_iterators
from vllm.utils.async_utils import make_async, merge_async_iterators
from .io_processor import PoolingIOProcessor
@@ -67,16 +69,47 @@ class PoolingServingBase(ABC):
trust_request_chat_template=trust_request_chat_template,
)
@abstractmethod
# Shared thread pool executor for preprocessing and postprocessing.
self._executor: Executor = models.renderer._executor
self._preprocessing_async = make_async(
self._preprocessing, executor=self._executor
)
self._postprocessing_async = make_async(
self._postprocessing, executor=self._executor
)
async def __call__(
self,
request: AnyPoolingRequest,
raw_request: Request | None = None,
) -> Response:
io_processor = self.get_io_processor(request)
ctx = await self._init_ctx(io_processor, request, raw_request)
await self._preprocessing_async(io_processor, ctx)
await self._prepare_generators(ctx)
await self._collect_batch(ctx)
return await self._postprocessing_async(io_processor, ctx)
@abstractmethod
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
raise NotImplementedError
@torch.inference_mode()
def _preprocessing(
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
):
return io_processor.pre_process_online(ctx)
@torch.inference_mode()
def _postprocessing(
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
):
io_processor.post_process_online(ctx)
return self._build_response(ctx)
async def _init_ctx(
self,
io_processor: PoolingIOProcessor,
request: AnyPoolingRequest,
raw_request: Request | None = None,
):
@@ -84,10 +117,12 @@ class PoolingServingBase(ABC):
request_id = f"{self.request_id_prefix}-{self._base_request_id(raw_request)}"
await self._check_model(request)
pooling_params = io_processor.create_pooling_params(request)
ctx = PoolingServeContext(
request=request,
raw_request=raw_request,
model_name=model_name,
pooling_params=pooling_params,
request_id=request_id,
)
@@ -175,7 +210,7 @@ class PoolingServingBase(ABC):
ctx.final_res_batch = [res for res in final_res_batch if res is not None]
@abstractmethod
async def _build_response(
def _build_response(
self,
ctx: PoolingServeContext,
) -> Response:
@@ -362,18 +397,5 @@ class PoolingServing(PoolingServingBase, ABC):
) -> PoolingIOProcessor:
raise NotImplementedError
async def __call__(
self,
request: AnyPoolingRequest,
raw_request: Request | None = None,
) -> Response:
ctx = await self._init_ctx(request, raw_request)
await self.io_processor.pre_process_online_async(ctx)
if ctx.pooling_params is None:
ctx.pooling_params = self.io_processor.create_pooling_params(request)
await self._prepare_generators(ctx)
await self._collect_batch(ctx)
await self.io_processor.post_process_online_async(ctx)
return await self._build_response(ctx)
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
return self.io_processor