[Bugfix][Pooling] Handle multimodal cache misses without crashing (#58975)
Signed-off-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
linchaoying6
OpenAI Codex
parent
443dccd3c9
commit
592c6f3fbc
@@ -195,13 +195,36 @@ class ServingScores(PoolingServing):
|
||||
ctx = await self._init_ctx(self.io_processor, *args, **kwargs)
|
||||
await self._preprocessing(self.io_processor, ctx)
|
||||
|
||||
# stage 1: encode queries and cache token embeddings on workers.
|
||||
await self._flash_late_interaction_encode_queries(ctx)
|
||||
# stage 2: encode docs and return scalar scores from workers.
|
||||
await self._flash_late_interaction_encode_docs(ctx)
|
||||
try:
|
||||
# stage 1: encode queries and cache token embeddings on workers.
|
||||
await self._flash_late_interaction_encode_queries(ctx)
|
||||
# stage 2: encode docs and return scalar scores from workers.
|
||||
await self._flash_late_interaction_encode_docs(ctx)
|
||||
except BaseException:
|
||||
try:
|
||||
await self._cleanup_flash_late_interaction(ctx)
|
||||
except Exception:
|
||||
logger.exception("Failed to clean up late-interaction query cache.")
|
||||
raise
|
||||
|
||||
return await self._postprocessing_async(self.io_processor, ctx)
|
||||
|
||||
async def _cleanup_flash_late_interaction(self, ctx: ScoringServeContext) -> None:
|
||||
query_keys = ctx.late_interaction_query_keys
|
||||
if not query_keys:
|
||||
return
|
||||
|
||||
doc_keys = ctx.late_interaction_doc_keys or []
|
||||
|
||||
try:
|
||||
# Stop documents before removing the query tensors they reference.
|
||||
await self.engine_client.abort([*query_keys, *doc_keys])
|
||||
finally:
|
||||
await self.engine_client.collective_rpc(
|
||||
"release_late_interaction_query_cache",
|
||||
args=(query_keys,),
|
||||
)
|
||||
|
||||
async def _flash_late_interaction_encode_queries(self, ctx: ScoringServeContext):
|
||||
assert ctx.n_queries is not None
|
||||
assert ctx.engine_inputs is not None
|
||||
@@ -259,7 +282,9 @@ class ServingScores(PoolingServing):
|
||||
query_keys = ctx.late_interaction_query_keys
|
||||
if query_keys is None:
|
||||
raise RuntimeError("Late-interaction query keys were not initialized.")
|
||||
doc_keys = [f"{ctx.request_id}-doc-{i}" for i in range(n_docs)]
|
||||
doc_namespace = random_uuid()
|
||||
doc_keys = [f"late-interaction-{doc_namespace}-doc-{i}" for i in range(n_docs)]
|
||||
ctx.late_interaction_doc_keys = doc_keys
|
||||
|
||||
for i in range(n_docs):
|
||||
query_idx = 0 if n_queries == 1 else i
|
||||
|
||||
Reference in New Issue
Block a user