[Bugfix][Pooling] Handle multimodal cache misses without crashing (#58975)

Signed-off-by: linchaoying6 <linchaoying6@jd.com>
Co-authored-by: linchaoying6 <linchaoying6@jd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Chaoying
2026-10-02 10:14:33 -04:00
committed by GitHub
co-authored by linchaoying6 OpenAI Codex
parent 443dccd3c9
commit 592c6f3fbc
21 changed files with 525 additions and 17 deletions
+37 -4
View File
@@ -18,6 +18,15 @@ from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats
logger = init_logger(__name__)
@dataclass(frozen=True)
class RequestError:
"""A request-level error that does not mark the engine as unhealthy."""
code: str
message: str
retryable: bool = False
@dataclass
class SamplingMask:
"""Per-token sampling support sets aligned with completion token IDs.
@@ -259,6 +268,7 @@ class PoolingRequestOutput(Generic[_O]):
prompt_token_ids (list[int]): A list of token IDs used in the prompt.
num_cached_tokens: The number of tokens with prefix cache hit.
finished (bool): A flag indicating whether the pooling is completed.
error: Structured request-level error information, if the request failed.
"""
@@ -269,20 +279,25 @@ class PoolingRequestOutput(Generic[_O]):
prompt_token_ids: list[int],
num_cached_tokens: int,
finished: bool,
*,
error: RequestError | None = None,
):
self.request_id = request_id
self.prompt_token_ids = prompt_token_ids
self.num_cached_tokens = num_cached_tokens
self.finished = finished
self.outputs = outputs
self.error = error
def __repr__(self) -> str:
error = f", error={self.error!r}" if self.error is not None else ""
return (
f"{type(self).__name__}(request_id={self.request_id!r}, "
f"outputs={self.outputs!r}, "
f"prompt_token_ids={self.prompt_token_ids}, "
f"num_cached_tokens={self.num_cached_tokens}, "
f"finished={self.finished})"
f"finished={self.finished}"
f"{error})"
)
@@ -319,12 +334,18 @@ class EmbeddingRequestOutput(PoolingRequestOutput[EmbeddingOutput]):
def from_base(
request_output: PoolingRequestOutput,
) -> "EmbeddingRequestOutput":
outputs = (
EmbeddingOutput([])
if request_output.error is not None
else EmbeddingOutput.from_base(request_output.outputs)
)
return EmbeddingRequestOutput(
request_id=request_output.request_id,
outputs=EmbeddingOutput.from_base(request_output.outputs),
outputs=outputs,
prompt_token_ids=request_output.prompt_token_ids,
num_cached_tokens=request_output.num_cached_tokens,
finished=request_output.finished,
error=request_output.error,
)
@@ -362,12 +383,18 @@ class ClassificationRequestOutput(PoolingRequestOutput[ClassificationOutput]):
def from_base(
request_output: PoolingRequestOutput,
) -> "ClassificationRequestOutput":
outputs = (
ClassificationOutput([])
if request_output.error is not None
else ClassificationOutput.from_base(request_output.outputs)
)
return ClassificationRequestOutput(
request_id=request_output.request_id,
outputs=ClassificationOutput.from_base(request_output.outputs),
outputs=outputs,
prompt_token_ids=request_output.prompt_token_ids,
num_cached_tokens=request_output.num_cached_tokens,
finished=request_output.finished,
error=request_output.error,
)
@@ -402,10 +429,16 @@ class ScoringRequestOutput(PoolingRequestOutput[ScoringOutput]):
def from_base(
request_output: PoolingRequestOutput,
) -> "ScoringRequestOutput":
outputs = (
ScoringOutput(0.0)
if request_output.error is not None
else ScoringOutput.from_base(request_output.outputs)
)
return ScoringRequestOutput(
request_id=request_output.request_id,
outputs=ScoringOutput.from_base(request_output.outputs),
outputs=outputs,
prompt_token_ids=request_output.prompt_token_ids,
num_cached_tokens=request_output.num_cached_tokens,
finished=request_output.finished,
error=request_output.error,
)