[Bugfix][Pooling] Handle multimodal cache misses without crashing (#58975)
Signed-off-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
linchaoying6
OpenAI Codex
parent
443dccd3c9
commit
592c6f3fbc
+37
-4
@@ -18,6 +18,15 @@ from vllm.v1.metrics.stats import RequestSpecDecodeMetrics, RequestStateStats
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RequestError:
|
||||
"""A request-level error that does not mark the engine as unhealthy."""
|
||||
|
||||
code: str
|
||||
message: str
|
||||
retryable: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class SamplingMask:
|
||||
"""Per-token sampling support sets aligned with completion token IDs.
|
||||
@@ -259,6 +268,7 @@ class PoolingRequestOutput(Generic[_O]):
|
||||
prompt_token_ids (list[int]): A list of token IDs used in the prompt.
|
||||
num_cached_tokens: The number of tokens with prefix cache hit.
|
||||
finished (bool): A flag indicating whether the pooling is completed.
|
||||
error: Structured request-level error information, if the request failed.
|
||||
|
||||
"""
|
||||
|
||||
@@ -269,20 +279,25 @@ class PoolingRequestOutput(Generic[_O]):
|
||||
prompt_token_ids: list[int],
|
||||
num_cached_tokens: int,
|
||||
finished: bool,
|
||||
*,
|
||||
error: RequestError | None = None,
|
||||
):
|
||||
self.request_id = request_id
|
||||
self.prompt_token_ids = prompt_token_ids
|
||||
self.num_cached_tokens = num_cached_tokens
|
||||
self.finished = finished
|
||||
self.outputs = outputs
|
||||
self.error = error
|
||||
|
||||
def __repr__(self) -> str:
|
||||
error = f", error={self.error!r}" if self.error is not None else ""
|
||||
return (
|
||||
f"{type(self).__name__}(request_id={self.request_id!r}, "
|
||||
f"outputs={self.outputs!r}, "
|
||||
f"prompt_token_ids={self.prompt_token_ids}, "
|
||||
f"num_cached_tokens={self.num_cached_tokens}, "
|
||||
f"finished={self.finished})"
|
||||
f"finished={self.finished}"
|
||||
f"{error})"
|
||||
)
|
||||
|
||||
|
||||
@@ -319,12 +334,18 @@ class EmbeddingRequestOutput(PoolingRequestOutput[EmbeddingOutput]):
|
||||
def from_base(
|
||||
request_output: PoolingRequestOutput,
|
||||
) -> "EmbeddingRequestOutput":
|
||||
outputs = (
|
||||
EmbeddingOutput([])
|
||||
if request_output.error is not None
|
||||
else EmbeddingOutput.from_base(request_output.outputs)
|
||||
)
|
||||
return EmbeddingRequestOutput(
|
||||
request_id=request_output.request_id,
|
||||
outputs=EmbeddingOutput.from_base(request_output.outputs),
|
||||
outputs=outputs,
|
||||
prompt_token_ids=request_output.prompt_token_ids,
|
||||
num_cached_tokens=request_output.num_cached_tokens,
|
||||
finished=request_output.finished,
|
||||
error=request_output.error,
|
||||
)
|
||||
|
||||
|
||||
@@ -362,12 +383,18 @@ class ClassificationRequestOutput(PoolingRequestOutput[ClassificationOutput]):
|
||||
def from_base(
|
||||
request_output: PoolingRequestOutput,
|
||||
) -> "ClassificationRequestOutput":
|
||||
outputs = (
|
||||
ClassificationOutput([])
|
||||
if request_output.error is not None
|
||||
else ClassificationOutput.from_base(request_output.outputs)
|
||||
)
|
||||
return ClassificationRequestOutput(
|
||||
request_id=request_output.request_id,
|
||||
outputs=ClassificationOutput.from_base(request_output.outputs),
|
||||
outputs=outputs,
|
||||
prompt_token_ids=request_output.prompt_token_ids,
|
||||
num_cached_tokens=request_output.num_cached_tokens,
|
||||
finished=request_output.finished,
|
||||
error=request_output.error,
|
||||
)
|
||||
|
||||
|
||||
@@ -402,10 +429,16 @@ class ScoringRequestOutput(PoolingRequestOutput[ScoringOutput]):
|
||||
def from_base(
|
||||
request_output: PoolingRequestOutput,
|
||||
) -> "ScoringRequestOutput":
|
||||
outputs = (
|
||||
ScoringOutput(0.0)
|
||||
if request_output.error is not None
|
||||
else ScoringOutput.from_base(request_output.outputs)
|
||||
)
|
||||
return ScoringRequestOutput(
|
||||
request_id=request_output.request_id,
|
||||
outputs=ScoringOutput.from_base(request_output.outputs),
|
||||
outputs=outputs,
|
||||
prompt_token_ids=request_output.prompt_token_ids,
|
||||
num_cached_tokens=request_output.num_cached_tokens,
|
||||
finished=request_output.finished,
|
||||
error=request_output.error,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user