[Bugfix][Pooling] Handle multimodal cache misses without crashing (#58975)
Signed-off-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
linchaoying6
OpenAI Codex
parent
443dccd3c9
commit
592c6f3fbc
@@ -2,11 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import random
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest.mock import MagicMock, call
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
from vllm.v1.metrics.reader import Counter, Gauge, Histogram, Metric, Vector
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -18,6 +20,38 @@ MODEL = "facebook/opt-125m"
|
||||
DTYPE = "half"
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_step_invalidates_mm_processor_cache_misses():
|
||||
engine = object.__new__(LLMEngine)
|
||||
engine.should_execute_dummy_batch = False
|
||||
engine.log_stats = False
|
||||
engine.logger_manager = None
|
||||
|
||||
mm_processor_cache = MagicMock()
|
||||
engine.renderer = MagicMock(mm_processor_cache=mm_processor_cache)
|
||||
|
||||
engine_core_output = MagicMock(
|
||||
mm_cache_miss_hashes=["missing-hash-1", "missing-hash-2"]
|
||||
)
|
||||
outputs = MagicMock(
|
||||
outputs=[engine_core_output],
|
||||
timestamp=0.0,
|
||||
scheduler_stats=None,
|
||||
)
|
||||
engine.engine_core = MagicMock()
|
||||
engine.engine_core.get_output.return_value = outputs
|
||||
|
||||
processed_outputs = MagicMock(reqs_to_abort=[], request_outputs=[])
|
||||
engine.output_processor = MagicMock()
|
||||
engine.output_processor.process_outputs.return_value = processed_outputs
|
||||
|
||||
assert engine.step() == []
|
||||
assert mm_processor_cache.invalidate.call_args_list == [
|
||||
call("missing-hash-1"),
|
||||
call("missing-hash-2"),
|
||||
]
|
||||
|
||||
|
||||
def _vllm_model(
|
||||
apc: bool,
|
||||
vllm_runner: type[VllmRunner],
|
||||
|
||||
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.engine.utils import (
|
||||
NUM_PROMPT_LOGPROBS_UNDER_TEST,
|
||||
@@ -18,7 +19,12 @@ from tests.v1.engine.utils import (
|
||||
from vllm import PoolingParams
|
||||
from vllm.logprobs import FlatLogprobs, Logprob, PromptLogprobs, SampleLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask
|
||||
from vllm.outputs import (
|
||||
CompletionOutput,
|
||||
PoolingRequestOutput,
|
||||
RequestOutput,
|
||||
SamplingMask,
|
||||
)
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.v1.engine import (
|
||||
@@ -1630,3 +1636,92 @@ def test_request_output_add_merges_delta_sampling_masks():
|
||||
|
||||
assert merged.outputs[0].token_ids == [10, 20]
|
||||
assert merged.outputs[0].sampling_mask.token_ids == [[10, 11], [20]]
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_pooling_cache_miss_is_a_request_error_and_batch_continues():
|
||||
output_processor = OutputProcessor(tokenizer=None, log_stats=False)
|
||||
requests = [
|
||||
EngineCoreRequest(
|
||||
request_id=f"request-{idx}-int",
|
||||
external_req_id=f"request-{idx}",
|
||||
prompt_token_ids=[1, 2],
|
||||
mm_features=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
sampling_params=None,
|
||||
pooling_params=PoolingParams(task="embed"),
|
||||
)
|
||||
for idx in range(2)
|
||||
]
|
||||
for request in requests:
|
||||
output_processor.add_request(request, prompt=None)
|
||||
|
||||
processed = output_processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id=requests[0].request_id,
|
||||
new_token_ids=[],
|
||||
finish_reason=FinishReason.ERROR,
|
||||
mm_cache_miss_hashes=["missing-hash"],
|
||||
),
|
||||
EngineCoreOutput(
|
||||
request_id=requests[1].request_id,
|
||||
new_token_ids=[],
|
||||
pooling_output=torch.tensor([1.0, 2.0]),
|
||||
finish_reason=FinishReason.STOP,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
assert len(processed.request_outputs) == 2
|
||||
failed, succeeded = processed.request_outputs
|
||||
assert isinstance(failed, PoolingRequestOutput)
|
||||
assert failed.error is not None
|
||||
assert failed.error.code == "multimodal_cache_miss"
|
||||
assert failed.error.retryable
|
||||
assert isinstance(succeeded, PoolingRequestOutput)
|
||||
assert succeeded.error is None
|
||||
assert torch.equal(succeeded.outputs.data, torch.tensor([1.0, 2.0]))
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_async_pooling_cache_miss_does_not_fail_output_processor():
|
||||
output_processor = OutputProcessor(tokenizer=None, log_stats=False)
|
||||
request = EngineCoreRequest(
|
||||
request_id="request-int",
|
||||
external_req_id="request",
|
||||
prompt_token_ids=[1, 2],
|
||||
mm_features=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
sampling_params=None,
|
||||
pooling_params=PoolingParams(task="embed"),
|
||||
)
|
||||
queue = RequestOutputCollector(
|
||||
output_kind=RequestOutputKind.FINAL_ONLY,
|
||||
request_id=request.request_id,
|
||||
)
|
||||
output_processor.add_request(request, prompt=None, queue=queue)
|
||||
|
||||
processed = output_processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id=request.request_id,
|
||||
new_token_ids=[],
|
||||
finish_reason=FinishReason.ERROR,
|
||||
mm_cache_miss_hashes=["missing-hash"],
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
assert processed.request_outputs == []
|
||||
output = queue.get_nowait()
|
||||
assert isinstance(output, PoolingRequestOutput)
|
||||
assert output.error is not None
|
||||
assert output.error.retryable
|
||||
assert not output_processor.has_unfinished_requests()
|
||||
|
||||
Reference in New Issue
Block a user