[Bugfix][Pooling] Handle multimodal cache misses without crashing (#58975)
Signed-off-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
linchaoying6
OpenAI Codex
parent
443dccd3c9
commit
592c6f3fbc
@@ -16,6 +16,7 @@ from vllm.outputs import (
|
||||
CompletionOutput,
|
||||
PoolingOutput,
|
||||
PoolingRequestOutput,
|
||||
RequestError,
|
||||
RequestOutput,
|
||||
SamplingMask,
|
||||
)
|
||||
@@ -304,6 +305,7 @@ class RequestState:
|
||||
stop_reason: int | str | None,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
ec_transfer_params: dict[str, Any] | None = None,
|
||||
error: RequestError | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput | None:
|
||||
finished = finish_reason is not None
|
||||
final_only = self.output_kind == RequestOutputKind.FINAL_ONLY
|
||||
@@ -342,6 +344,7 @@ class RequestState:
|
||||
external_req_id,
|
||||
[self._new_pooling_output(pooling_output)],
|
||||
finished,
|
||||
error=error,
|
||||
)
|
||||
|
||||
output = self._new_completion_output(new_token_ids, finish_reason, stop_reason)
|
||||
@@ -369,6 +372,7 @@ class RequestState:
|
||||
finished: bool,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
ec_transfer_params: dict[str, Any] | None = None,
|
||||
error: RequestError | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput:
|
||||
# If prompt embeds were used, put placeholder prompt token ids
|
||||
prompt_token_ids = self.prompt_token_ids
|
||||
@@ -385,6 +389,7 @@ class RequestState:
|
||||
num_cached_tokens=self.num_cached_tokens,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=finished,
|
||||
error=error,
|
||||
)
|
||||
assert self.logprobs_processor is not None
|
||||
if self.output_kind == RequestOutputKind.DELTA:
|
||||
@@ -715,6 +720,19 @@ class OutputProcessor:
|
||||
if engine_core_output.spec_decode_metrics is not None:
|
||||
req_state.spec_decode_metrics = engine_core_output.spec_decode_metrics
|
||||
|
||||
request_error = None
|
||||
if (
|
||||
req_state.detokenizer is None
|
||||
and engine_core_output.mm_cache_miss_hashes
|
||||
and finish_reason == FinishReason.ERROR
|
||||
):
|
||||
request_error = RequestError(
|
||||
code="multimodal_cache_miss",
|
||||
message="Multi-modal processor cache miss.",
|
||||
retryable=True,
|
||||
)
|
||||
pooling_output = EMPTY_CPU_TENSOR
|
||||
|
||||
if pooling_output is None:
|
||||
assert req_state.detokenizer is not None
|
||||
assert req_state.logprobs_processor is not None
|
||||
@@ -742,6 +760,7 @@ class OutputProcessor:
|
||||
stop_reason,
|
||||
kv_transfer_params,
|
||||
ec_transfer_params,
|
||||
error=request_error,
|
||||
):
|
||||
if req_state.streaming_input:
|
||||
request_output.finished = False
|
||||
|
||||
Reference in New Issue
Block a user