[Frontend] Offload blocking preprocessing & postprocessing ops to thread pool for pooling entrypoints. (#39763)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
0008729abf
commit
c0ecaed950
@@ -220,7 +220,7 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
|
||||
scoring_data.data_2, "document", self.model_config
|
||||
)
|
||||
|
||||
return self._preprocess_completion_offline(
|
||||
return self._preprocess_cmpl_offline(
|
||||
prompts=data_1 + data_2, tok_params=tok_params, prompt_extras=prompt_extras
|
||||
)
|
||||
|
||||
@@ -682,7 +682,7 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM
|
||||
for q, d in zip(queries, docs)
|
||||
]
|
||||
|
||||
return self._preprocess_completion_offline(
|
||||
return self._preprocess_cmpl_offline(
|
||||
prompts=prompts, tok_params=tok_params, prompt_extras=prompt_extras
|
||||
)
|
||||
|
||||
|
||||
@@ -65,7 +65,7 @@ class ServingScores(PoolingServing):
|
||||
|
||||
return await self.flash_late_interaction(*args, **kwargs)
|
||||
|
||||
async def _build_response(
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: ScoringServeContext,
|
||||
) -> JSONResponse:
|
||||
@@ -183,17 +183,15 @@ class ServingScores(PoolingServing):
|
||||
### Can significantly improve late-interaction scoring performance.
|
||||
|
||||
async def flash_late_interaction(self, *args, **kwargs) -> Response:
|
||||
ctx = await self._init_ctx(*args, **kwargs)
|
||||
ctx.pooling_params = self.io_processor.create_pooling_params(ctx.request)
|
||||
await self.io_processor.pre_process_online_async(ctx)
|
||||
ctx = await self._init_ctx(self.io_processor, *args, **kwargs)
|
||||
await self._preprocessing_async(self.io_processor, ctx)
|
||||
|
||||
# stage 1: encode queries and cache token embeddings on workers.
|
||||
await self._flash_late_interaction_encode_queries(ctx)
|
||||
# stage 2: encode docs and return scalar scores from workers.
|
||||
await self._flash_late_interaction_encode_docs(ctx)
|
||||
|
||||
await self.io_processor.post_process_online_async(ctx)
|
||||
return await self._build_response(ctx)
|
||||
return await self._postprocessing_async(self.io_processor, ctx)
|
||||
|
||||
async def _flash_late_interaction_encode_queries(self, ctx: ScoringServeContext):
|
||||
assert ctx.n_queries is not None
|
||||
|
||||
Reference in New Issue
Block a user