[Frontend] Parallelize preprocessing within the same request for pooling models online serving. (#49153)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -9,6 +9,7 @@ from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.utils.serial_utils import EmbedDType, Endianness
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..base.serving import PoolingServing
|
||||
from ..typing import PoolingServeContext
|
||||
from ..utils import (
|
||||
@@ -53,6 +54,12 @@ class ServingEmbedding(PoolingServing):
|
||||
def init_io_processor(self, *args, **kwargs) -> EmbedIOProcessor:
|
||||
return EmbedIOProcessor(*args, **kwargs)
|
||||
|
||||
async def _preprocessing(
|
||||
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
|
||||
):
|
||||
await super()._preprocessing(io_processor, ctx)
|
||||
self.io_processor.maybe_pre_process_chunked(ctx)
|
||||
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
|
||||
Reference in New Issue
Block a user