[Bugfix][Pooling] Handle multimodal cache misses without crashing (#58975)

Signed-off-by: linchaoying6 <linchaoying6@jd.com>
Co-authored-by: linchaoying6 <linchaoying6@jd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Chaoying
2026-10-02 10:14:33 -04:00
committed by GitHub
co-authored by linchaoying6 OpenAI Codex
parent 443dccd3c9
commit 592c6f3fbc
21 changed files with 525 additions and 17 deletions
+19
View File
@@ -16,6 +16,7 @@ from vllm.outputs import (
CompletionOutput,
PoolingOutput,
PoolingRequestOutput,
RequestError,
RequestOutput,
SamplingMask,
)
@@ -304,6 +305,7 @@ class RequestState:
stop_reason: int | str | None,
kv_transfer_params: dict[str, Any] | None = None,
ec_transfer_params: dict[str, Any] | None = None,
error: RequestError | None = None,
) -> RequestOutput | PoolingRequestOutput | None:
finished = finish_reason is not None
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
@@ -342,6 +344,7 @@ class RequestState:
external_req_id,
[self._new_pooling_output(pooling_output)],
finished,
error=error,
)
output = self._new_completion_output(new_token_ids, finish_reason, stop_reason)
@@ -369,6 +372,7 @@ class RequestState:
finished: bool,
kv_transfer_params: dict[str, Any] | None = None,
ec_transfer_params: dict[str, Any] | None = None,
error: RequestError | None = None,
) -> RequestOutput | PoolingRequestOutput:
# If prompt embeds were used, put placeholder prompt token ids
prompt_token_ids = self.prompt_token_ids
@@ -385,6 +389,7 @@ class RequestState:
num_cached_tokens=self.num_cached_tokens,
prompt_token_ids=prompt_token_ids,
finished=finished,
error=error,
)
assert self.logprobs_processor is not None
if self.output_kind == RequestOutputKind.DELTA:
@@ -715,6 +720,19 @@ class OutputProcessor:
if engine_core_output.spec_decode_metrics is not None:
req_state.spec_decode_metrics = engine_core_output.spec_decode_metrics
request_error = None
if (
req_state.detokenizer is None
and engine_core_output.mm_cache_miss_hashes
and finish_reason == FinishReason.ERROR
):
request_error = RequestError(
code="multimodal_cache_miss",
message="Multi-modal processor cache miss.",
retryable=True,
)
pooling_output = EMPTY_CPU_TENSOR
if pooling_output is None:
assert req_state.detokenizer is not None
assert req_state.logprobs_processor is not None
@@ -742,6 +760,7 @@ class OutputProcessor:
stop_reason,
kv_transfer_params,
ec_transfer_params,
error=request_error,
):
if req_state.streaming_input:
request_output.finished = False