[Bugfix][Pooling] Handle multimodal cache misses without crashing (#58975)
Signed-off-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
linchaoying6
OpenAI Codex
parent
443dccd3c9
commit
592c6f3fbc
@@ -21,6 +21,7 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import GenerationError, RetryableRequestError
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers.base import BaseRenderer
|
||||
from vllm.tracing import (
|
||||
@@ -206,6 +207,10 @@ class PoolingBaseServing(ABC, BaseServing):
|
||||
final_res_batch = [None] * num_inputs
|
||||
|
||||
async for i, res in ctx.result_generator:
|
||||
if res.error is not None:
|
||||
if res.error.retryable:
|
||||
raise RetryableRequestError(res.error.message)
|
||||
raise GenerationError(res.error.message)
|
||||
final_res_batch[i] = res
|
||||
|
||||
if None in final_res_batch:
|
||||
|
||||
@@ -191,6 +191,27 @@ class ScoringIOProcessor(PoolingIOProcessor):
|
||||
scoring_data = self.valid_inputs(data_1, data_2)
|
||||
return scoring_data
|
||||
|
||||
def _pair_error_output(
|
||||
self,
|
||||
output_1: PoolingRequestOutput,
|
||||
output_2: PoolingRequestOutput,
|
||||
) -> PoolingRequestOutput | None:
|
||||
failed = output_1 if output_1.error is not None else output_2
|
||||
if failed.error is None:
|
||||
return None
|
||||
|
||||
padding = [self.pad_token_id] if self.pad_token_id is not None else []
|
||||
return PoolingRequestOutput(
|
||||
request_id=f"{output_1.request_id}_{output_2.request_id}",
|
||||
outputs=failed.outputs,
|
||||
prompt_token_ids=(
|
||||
output_1.prompt_token_ids + padding + output_2.prompt_token_ids
|
||||
),
|
||||
num_cached_tokens=(output_1.num_cached_tokens + output_2.num_cached_tokens),
|
||||
finished=failed.finished,
|
||||
error=failed.error,
|
||||
)
|
||||
|
||||
|
||||
class BiEncoderIOProcessor(ScoringIOProcessor):
|
||||
name = "bi-encoder"
|
||||
@@ -310,6 +331,10 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
|
||||
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
for emb_1, emb_2 in zip(emb_data_1, emb_data_2):
|
||||
if error_output := self._pair_error_output(emb_1, emb_2):
|
||||
final_res_batch.append(error_output)
|
||||
continue
|
||||
|
||||
pair_score = F.cosine_similarity(
|
||||
emb_1.outputs.data.float(), emb_2.outputs.data.float(), dim=0
|
||||
)
|
||||
@@ -352,6 +377,10 @@ class LateInteractionIOProcessor(BiEncoderIOProcessor):
|
||||
|
||||
# Compute MaxSim scores
|
||||
for emb_1, emb_2 in zip(emb_data_1, emb_data_2):
|
||||
if error_output := self._pair_error_output(emb_1, emb_2):
|
||||
final_res_batch.append(error_output)
|
||||
continue
|
||||
|
||||
# emb_1.outputs.data: [query_len, dim]
|
||||
# emb_2.outputs.data: [doc_len, dim]
|
||||
q_emb = emb_1.outputs.data
|
||||
@@ -907,6 +936,10 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
|
||||
for i in range(len(outputs)):
|
||||
if outputs[i].error is not None:
|
||||
final_res_batch.append(outputs[i])
|
||||
continue
|
||||
|
||||
embeds = outputs[i].outputs.data.float()
|
||||
|
||||
# The JinaForRanking model concatenates docs first, then query.
|
||||
|
||||
@@ -195,13 +195,36 @@ class ServingScores(PoolingServing):
|
||||
ctx = await self._init_ctx(self.io_processor, *args, **kwargs)
|
||||
await self._preprocessing(self.io_processor, ctx)
|
||||
|
||||
# stage 1: encode queries and cache token embeddings on workers.
|
||||
await self._flash_late_interaction_encode_queries(ctx)
|
||||
# stage 2: encode docs and return scalar scores from workers.
|
||||
await self._flash_late_interaction_encode_docs(ctx)
|
||||
try:
|
||||
# stage 1: encode queries and cache token embeddings on workers.
|
||||
await self._flash_late_interaction_encode_queries(ctx)
|
||||
# stage 2: encode docs and return scalar scores from workers.
|
||||
await self._flash_late_interaction_encode_docs(ctx)
|
||||
except BaseException:
|
||||
try:
|
||||
await self._cleanup_flash_late_interaction(ctx)
|
||||
except Exception:
|
||||
logger.exception("Failed to clean up late-interaction query cache.")
|
||||
raise
|
||||
|
||||
return await self._postprocessing_async(self.io_processor, ctx)
|
||||
|
||||
async def _cleanup_flash_late_interaction(self, ctx: ScoringServeContext) -> None:
|
||||
query_keys = ctx.late_interaction_query_keys
|
||||
if not query_keys:
|
||||
return
|
||||
|
||||
doc_keys = ctx.late_interaction_doc_keys or []
|
||||
|
||||
try:
|
||||
# Stop documents before removing the query tensors they reference.
|
||||
await self.engine_client.abort([*query_keys, *doc_keys])
|
||||
finally:
|
||||
await self.engine_client.collective_rpc(
|
||||
"release_late_interaction_query_cache",
|
||||
args=(query_keys,),
|
||||
)
|
||||
|
||||
async def _flash_late_interaction_encode_queries(self, ctx: ScoringServeContext):
|
||||
assert ctx.n_queries is not None
|
||||
assert ctx.engine_inputs is not None
|
||||
@@ -259,7 +282,9 @@ class ServingScores(PoolingServing):
|
||||
query_keys = ctx.late_interaction_query_keys
|
||||
if query_keys is None:
|
||||
raise RuntimeError("Late-interaction query keys were not initialized.")
|
||||
doc_keys = [f"{ctx.request_id}-doc-{i}" for i in range(n_docs)]
|
||||
doc_namespace = random_uuid()
|
||||
doc_keys = [f"late-interaction-{doc_namespace}-doc-{i}" for i in range(n_docs)]
|
||||
ctx.late_interaction_doc_keys = doc_keys
|
||||
|
||||
for i in range(n_docs):
|
||||
query_idx = 0 if n_queries == 1 else i
|
||||
|
||||
@@ -115,6 +115,7 @@ class PoolingServeContext(Generic[PoolingRequestT]):
|
||||
## for flash-late-interaction
|
||||
query_final_res_batch: list[PoolingRequestOutput] | None = None
|
||||
late_interaction_query_keys: list[str] | None = None
|
||||
late_interaction_doc_keys: list[str] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from vllm.exceptions import GenerationError
|
||||
from vllm.exceptions import GenerationError, RetryableRequestError
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..engine.protocol import ErrorInfo, ErrorResponse
|
||||
@@ -56,7 +56,11 @@ def create_error_response(
|
||||
status_code = HTTPStatus.BAD_REQUEST
|
||||
param = None
|
||||
elif isinstance(exc, GenerationError):
|
||||
err_type = "InternalServerError"
|
||||
err_type = (
|
||||
"ServiceUnavailableError"
|
||||
if isinstance(exc, RetryableRequestError)
|
||||
else "InternalServerError"
|
||||
)
|
||||
status_code = exc.status_code
|
||||
param = None
|
||||
elif isinstance(exc, VLLMServerError):
|
||||
|
||||
@@ -70,8 +70,8 @@ async def generation_error_handler(req: Request, exc: GenerationError):
|
||||
"""Handle GenerationError without logging stack traces.
|
||||
|
||||
GenerationError is a known, expected error (e.g. KV cache load failure)
|
||||
that should be returned to the client as a 500 response without polluting
|
||||
server logs with stack traces.
|
||||
that should be returned to the client without polluting server logs with
|
||||
stack traces.
|
||||
"""
|
||||
err = create_error_response(exc)
|
||||
return JSONResponse(err.model_dump(), status_code=err.error.code)
|
||||
|
||||
Reference in New Issue
Block a user