[Frontend] Parallelize preprocessing within the same request for pooling models online serving. (#49153)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
wang.yuqi
2026-07-22 10:56:23 +00:00
committed by GitHub
parent d6dbdb9b0d
commit 61c9ef986a
17 changed files with 538 additions and 820 deletions
@@ -9,6 +9,7 @@ from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput
from vllm.utils.serial_utils import EmbedDType, Endianness
from ..base.io_processor import PoolingIOProcessor
from ..base.serving import PoolingServing
from ..typing import PoolingServeContext
from ..utils import (
@@ -53,6 +54,12 @@ class ServingEmbedding(PoolingServing):
def init_io_processor(self, *args, **kwargs) -> EmbedIOProcessor:
return EmbedIOProcessor(*args, **kwargs)
async def _preprocessing(
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
):
await super()._preprocessing(io_processor, ctx)
self.io_processor.maybe_pre_process_chunked(ctx)
def _build_response(
self,
ctx: PoolingServeContext,