[Frontend] Offload blocking preprocessing & postprocessing ops to thread pool for pooling entrypoints. (#39763)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Signed-off-by: wang.yuqi <noooop@126.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
wang.yuqi
2026-04-14 08:29:25 +00:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent 0008729abf
commit c0ecaed950
10 changed files with 68 additions and 83 deletions
+4 -24
View File
@@ -72,7 +72,7 @@ class PoolingIOProcessor:
default_template_kwargs=None,
)
elif isinstance(request, PoolingCompletionLikeRequest):
engine_inputs = self._preprocess_completion_online(
engine_inputs = self._preprocess_cmpl_online(
request,
prompt_input=request.input,
prompt_embeds=None,
@@ -82,21 +82,12 @@ class PoolingIOProcessor:
ctx.engine_inputs = engine_inputs
async def pre_process_online_async(self, ctx: PoolingServeContext):
self.pre_process_online(ctx)
def post_process_online(
self,
ctx: PoolingServeContext,
):
pass
async def post_process_online_async(
self,
ctx: PoolingServeContext,
):
self.post_process_online(ctx)
#######################################
# offline APIs
@@ -109,12 +100,7 @@ class PoolingIOProcessor:
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
return self._preprocess_completion_offline(
prompts=prompts_seq, tok_params=tok_params
)
async def pre_process_offline_async(self, ctx: OfflineInputsContext):
return self.pre_process_offline(ctx)
return self._preprocess_cmpl_offline(prompts=prompts_seq, tok_params=tok_params)
def post_process_offline(
self,
@@ -122,16 +108,10 @@ class PoolingIOProcessor:
) -> list[PoolingRequestOutput]:
return ctx.outputs
async def post_process_offline_async(
self,
ctx: OfflineOutputsContext,
) -> list[PoolingRequestOutput]:
return self.post_process_offline(ctx)
#######################################
# helpers
def _preprocess_completion_online(
def _preprocess_cmpl_online(
self,
request: RendererRequest,
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
@@ -209,7 +189,7 @@ class PoolingIOProcessor:
return conversation, [engine_input]
def _preprocess_completion_offline(
def _preprocess_cmpl_offline(
self,
prompts: PromptType | Sequence[PromptType],
tok_params: TokenizeParams,