[Frontend] Offload blocking preprocessing & postprocessing ops to thread pool for pooling entrypoints. (#39763)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
0008729abf
commit
c0ecaed950
@@ -72,7 +72,7 @@ class PoolingIOProcessor:
|
||||
default_template_kwargs=None,
|
||||
)
|
||||
elif isinstance(request, PoolingCompletionLikeRequest):
|
||||
engine_inputs = self._preprocess_completion_online(
|
||||
engine_inputs = self._preprocess_cmpl_online(
|
||||
request,
|
||||
prompt_input=request.input,
|
||||
prompt_embeds=None,
|
||||
@@ -82,21 +82,12 @@ class PoolingIOProcessor:
|
||||
|
||||
ctx.engine_inputs = engine_inputs
|
||||
|
||||
async def pre_process_online_async(self, ctx: PoolingServeContext):
|
||||
self.pre_process_online(ctx)
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
pass
|
||||
|
||||
async def post_process_online_async(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
self.post_process_online(ctx)
|
||||
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
@@ -109,12 +100,7 @@ class PoolingIOProcessor:
|
||||
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
|
||||
**(ctx.tokenization_kwargs or {})
|
||||
)
|
||||
return self._preprocess_completion_offline(
|
||||
prompts=prompts_seq, tok_params=tok_params
|
||||
)
|
||||
|
||||
async def pre_process_offline_async(self, ctx: OfflineInputsContext):
|
||||
return self.pre_process_offline(ctx)
|
||||
return self._preprocess_cmpl_offline(prompts=prompts_seq, tok_params=tok_params)
|
||||
|
||||
def post_process_offline(
|
||||
self,
|
||||
@@ -122,16 +108,10 @@ class PoolingIOProcessor:
|
||||
) -> list[PoolingRequestOutput]:
|
||||
return ctx.outputs
|
||||
|
||||
async def post_process_offline_async(
|
||||
self,
|
||||
ctx: OfflineOutputsContext,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
return self.post_process_offline(ctx)
|
||||
|
||||
#######################################
|
||||
# helpers
|
||||
|
||||
def _preprocess_completion_online(
|
||||
def _preprocess_cmpl_online(
|
||||
self,
|
||||
request: RendererRequest,
|
||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
|
||||
@@ -209,7 +189,7 @@ class PoolingIOProcessor:
|
||||
|
||||
return conversation, [engine_input]
|
||||
|
||||
def _preprocess_completion_offline(
|
||||
def _preprocess_cmpl_offline(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
tok_params: TokenizeParams,
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from concurrent.futures import Executor
|
||||
from http import HTTPStatus
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from fastapi import Request
|
||||
from fastapi.responses import Response
|
||||
from starlette.datastructures import Headers
|
||||
@@ -32,7 +34,7 @@ from vllm.tracing import (
|
||||
log_tracing_disabled_warning,
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
from vllm.utils.async_utils import make_async, merge_async_iterators
|
||||
|
||||
from .io_processor import PoolingIOProcessor
|
||||
|
||||
@@ -67,16 +69,47 @@ class PoolingServingBase(ABC):
|
||||
trust_request_chat_template=trust_request_chat_template,
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
# Shared thread pool executor for preprocessing and postprocessing.
|
||||
self._executor: Executor = models.renderer._executor
|
||||
self._preprocessing_async = make_async(
|
||||
self._preprocessing, executor=self._executor
|
||||
)
|
||||
self._postprocessing_async = make_async(
|
||||
self._postprocessing, executor=self._executor
|
||||
)
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
request: AnyPoolingRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> Response:
|
||||
io_processor = self.get_io_processor(request)
|
||||
ctx = await self._init_ctx(io_processor, request, raw_request)
|
||||
await self._preprocessing_async(io_processor, ctx)
|
||||
await self._prepare_generators(ctx)
|
||||
await self._collect_batch(ctx)
|
||||
return await self._postprocessing_async(io_processor, ctx)
|
||||
|
||||
@abstractmethod
|
||||
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
|
||||
raise NotImplementedError
|
||||
|
||||
@torch.inference_mode()
|
||||
def _preprocessing(
|
||||
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
|
||||
):
|
||||
return io_processor.pre_process_online(ctx)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _postprocessing(
|
||||
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
|
||||
):
|
||||
io_processor.post_process_online(ctx)
|
||||
return self._build_response(ctx)
|
||||
|
||||
async def _init_ctx(
|
||||
self,
|
||||
io_processor: PoolingIOProcessor,
|
||||
request: AnyPoolingRequest,
|
||||
raw_request: Request | None = None,
|
||||
):
|
||||
@@ -84,10 +117,12 @@ class PoolingServingBase(ABC):
|
||||
request_id = f"{self.request_id_prefix}-{self._base_request_id(raw_request)}"
|
||||
await self._check_model(request)
|
||||
|
||||
pooling_params = io_processor.create_pooling_params(request)
|
||||
ctx = PoolingServeContext(
|
||||
request=request,
|
||||
raw_request=raw_request,
|
||||
model_name=model_name,
|
||||
pooling_params=pooling_params,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
@@ -175,7 +210,7 @@ class PoolingServingBase(ABC):
|
||||
ctx.final_res_batch = [res for res in final_res_batch if res is not None]
|
||||
|
||||
@abstractmethod
|
||||
async def _build_response(
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> Response:
|
||||
@@ -362,18 +397,5 @@ class PoolingServing(PoolingServingBase, ABC):
|
||||
) -> PoolingIOProcessor:
|
||||
raise NotImplementedError
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
request: AnyPoolingRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> Response:
|
||||
ctx = await self._init_ctx(request, raw_request)
|
||||
await self.io_processor.pre_process_online_async(ctx)
|
||||
|
||||
if ctx.pooling_params is None:
|
||||
ctx.pooling_params = self.io_processor.create_pooling_params(request)
|
||||
|
||||
await self._prepare_generators(ctx)
|
||||
await self._collect_batch(ctx)
|
||||
await self.io_processor.post_process_online_async(ctx)
|
||||
return await self._build_response(ctx)
|
||||
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
|
||||
return self.io_processor
|
||||
|
||||
Reference in New Issue
Block a user