[Bugfix][Pooling] Handle multimodal cache misses without crashing (#58975)
Signed-off-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
linchaoying6
OpenAI Codex
parent
443dccd3c9
commit
592c6f3fbc
@@ -1,20 +1,25 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ScoringIOProcessor post-tokenization helpers."""
|
||||
"""Unit tests for ScoringIOProcessor helpers."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from tokenizers import Tokenizer, models, pre_tokenizers, processors
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm import TokensPrompt
|
||||
from vllm.entrypoints.pooling.scoring.io_processor import (
|
||||
BiEncoderIOProcessor,
|
||||
CrossEncoderIOProcessor,
|
||||
JinaRankingIOProcessor,
|
||||
LateInteractionIOProcessor,
|
||||
_apply_post_tokenization_to_token_type_ids,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.utils import compress_token_type_ids
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput, RequestError
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
@@ -113,3 +118,59 @@ def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt():
|
||||
assert first_doc == 10
|
||||
assert prompt_token_ids[:first_doc] == list(range(10, num_query))
|
||||
assert prompt_token_ids[first_doc:40] == list(range(num_query, 50))
|
||||
|
||||
|
||||
def _pooling_output(
|
||||
request_id: str,
|
||||
data: torch.Tensor,
|
||||
error: RequestError | None = None,
|
||||
) -> PoolingRequestOutput:
|
||||
return PoolingRequestOutput(
|
||||
request_id=request_id,
|
||||
outputs=PoolingOutput(data),
|
||||
prompt_token_ids=[1],
|
||||
num_cached_tokens=0,
|
||||
finished=True,
|
||||
error=error,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"processor_type", [BiEncoderIOProcessor, LateInteractionIOProcessor]
|
||||
)
|
||||
@pytest.mark.parametrize("failed_index", [0, 1])
|
||||
def test_paired_scoring_preserves_request_error(processor_type, failed_index):
|
||||
processor = processor_type.__new__(processor_type)
|
||||
processor.pad_token_id = None
|
||||
error = RequestError(
|
||||
code="multimodal_cache_miss",
|
||||
message="Multi-modal processor cache miss.",
|
||||
retryable=True,
|
||||
)
|
||||
outputs = [
|
||||
_pooling_output("query", torch.tensor([1.0, 2.0])),
|
||||
_pooling_output("document", torch.tensor([3.0, 4.0])),
|
||||
]
|
||||
outputs[failed_index] = _pooling_output(
|
||||
outputs[failed_index].request_id,
|
||||
torch.empty(0),
|
||||
error,
|
||||
)
|
||||
|
||||
(result,) = processor._post_process(outputs, n_queries=1)
|
||||
|
||||
assert result.request_id == "query_document"
|
||||
assert result.error is error
|
||||
assert result.outputs is outputs[failed_index].outputs
|
||||
|
||||
|
||||
def test_jina_scoring_preserves_request_error_before_indexing_output():
|
||||
processor = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor)
|
||||
error = RequestError(
|
||||
code="multimodal_cache_miss",
|
||||
message="Multi-modal processor cache miss.",
|
||||
retryable=True,
|
||||
)
|
||||
failed = _pooling_output("request", torch.empty(0), error)
|
||||
|
||||
assert processor._post_process([failed], n_queries=1) == [failed]
|
||||
|
||||
@@ -49,10 +49,60 @@ def _query_key(context: PoolingServeContext) -> str:
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_colliding_request_ids_use_distinct_query_cache_keys(
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize("failed_stage", ["query", "doc"])
|
||||
async def test_flash_late_interaction_cleans_up_queries_on_failure(
|
||||
failed_stage: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
serving = object.__new__(ServingScores)
|
||||
serving.io_processor = Mock()
|
||||
serving.engine_client = Mock(abort=AsyncMock(), collective_rpc=AsyncMock())
|
||||
|
||||
context = _make_context()
|
||||
monkeypatch.setattr(serving, "_init_ctx", AsyncMock(return_value=context))
|
||||
monkeypatch.setattr(serving, "_preprocessing", AsyncMock())
|
||||
|
||||
async def encode_queries(ctx):
|
||||
ctx.late_interaction_query_keys = ["query-key"]
|
||||
if failed_stage == "query":
|
||||
raise RuntimeError("query failed")
|
||||
|
||||
async def encode_docs(ctx):
|
||||
ctx.late_interaction_doc_keys = ["doc-key"]
|
||||
if failed_stage == "doc":
|
||||
raise RuntimeError("doc failed")
|
||||
|
||||
monkeypatch.setattr(
|
||||
serving,
|
||||
"_flash_late_interaction_encode_queries",
|
||||
AsyncMock(side_effect=encode_queries),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
serving,
|
||||
"_flash_late_interaction_encode_docs",
|
||||
AsyncMock(side_effect=encode_docs),
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match=f"{failed_stage} failed"):
|
||||
await serving.flash_late_interaction()
|
||||
|
||||
expected_doc_keys = [] if failed_stage == "query" else ["doc-key"]
|
||||
serving.engine_client.abort.assert_awaited_once_with(
|
||||
["query-key", *expected_doc_keys]
|
||||
)
|
||||
serving.engine_client.collective_rpc.assert_awaited_once_with(
|
||||
"release_late_interaction_query_cache",
|
||||
args=(["query-key"],),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_colliding_request_ids_use_distinct_late_interaction_keys(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
serving = object.__new__(ServingScores)
|
||||
serving.engine_client = Mock(abort=AsyncMock(), collective_rpc=AsyncMock())
|
||||
prepare_generators = AsyncMock()
|
||||
monkeypatch.setattr(serving, "_prepare_generators", prepare_generators)
|
||||
monkeypatch.setattr(serving, "_collect_batch", AsyncMock())
|
||||
@@ -68,9 +118,24 @@ async def test_colliding_request_ids_use_distinct_query_cache_keys(
|
||||
first_doc_key = _query_key(prepared_contexts[1])
|
||||
second_query_key = _query_key(prepared_contexts[2])
|
||||
second_doc_key = _query_key(prepared_contexts[3])
|
||||
first_doc_request_ids = prepared_contexts[1].prompt_request_ids
|
||||
second_doc_request_ids = prepared_contexts[3].prompt_request_ids
|
||||
|
||||
assert first_query_key == first_doc_key
|
||||
assert second_query_key == second_doc_key
|
||||
assert first_query_key != second_query_key
|
||||
assert "caller-controlled-id" not in first_query_key
|
||||
assert "caller-controlled-id" not in second_query_key
|
||||
assert first_doc_request_ids is not None
|
||||
assert second_doc_request_ids is not None
|
||||
assert first.late_interaction_doc_keys == first_doc_request_ids
|
||||
assert second.late_interaction_doc_keys == second_doc_request_ids
|
||||
assert first_doc_request_ids != second_doc_request_ids
|
||||
assert all("caller-controlled-id" not in key for key in first_doc_request_ids)
|
||||
assert all("caller-controlled-id" not in key for key in second_doc_request_ids)
|
||||
|
||||
assert first.late_interaction_query_keys is not None
|
||||
await serving._cleanup_flash_late_interaction(first)
|
||||
serving.engine_client.abort.assert_awaited_once_with(
|
||||
[*first.late_interaction_query_keys, *first_doc_request_ids]
|
||||
)
|
||||
|
||||
+43
-1
@@ -2,8 +2,17 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.outputs import (
|
||||
ClassificationRequestOutput,
|
||||
EmbeddingRequestOutput,
|
||||
PoolingOutput,
|
||||
PoolingRequestOutput,
|
||||
RequestError,
|
||||
RequestOutput,
|
||||
ScoringRequestOutput,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -19,3 +28,36 @@ def test_request_output_forward_compatible():
|
||||
example_arg_added_in_new_version="some_value",
|
||||
)
|
||||
assert output is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("output_type", "empty_value"),
|
||||
[
|
||||
(EmbeddingRequestOutput, []),
|
||||
(ClassificationRequestOutput, []),
|
||||
(ScoringRequestOutput, 0.0),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_specialized_pooling_outputs_preserve_request_error(output_type, empty_value):
|
||||
error = RequestError(
|
||||
code="multimodal_cache_miss",
|
||||
message="Multi-modal processor cache miss.",
|
||||
retryable=True,
|
||||
)
|
||||
base_output = PoolingRequestOutput(
|
||||
request_id="request",
|
||||
outputs=PoolingOutput(torch.empty(0)),
|
||||
prompt_token_ids=[1, 2],
|
||||
num_cached_tokens=0,
|
||||
finished=True,
|
||||
error=error,
|
||||
)
|
||||
|
||||
output = output_type.from_base(base_output)
|
||||
|
||||
assert output.error is error
|
||||
value = getattr(output.outputs, "embedding", None)
|
||||
value = getattr(output.outputs, "probs", value)
|
||||
value = getattr(output.outputs, "score", value)
|
||||
assert value == empty_value
|
||||
|
||||
@@ -2,11 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import random
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest.mock import MagicMock, call
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
from vllm.v1.metrics.reader import Counter, Gauge, Histogram, Metric, Vector
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -18,6 +20,38 @@ MODEL = "facebook/opt-125m"
|
||||
DTYPE = "half"
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_step_invalidates_mm_processor_cache_misses():
|
||||
engine = object.__new__(LLMEngine)
|
||||
engine.should_execute_dummy_batch = False
|
||||
engine.log_stats = False
|
||||
engine.logger_manager = None
|
||||
|
||||
mm_processor_cache = MagicMock()
|
||||
engine.renderer = MagicMock(mm_processor_cache=mm_processor_cache)
|
||||
|
||||
engine_core_output = MagicMock(
|
||||
mm_cache_miss_hashes=["missing-hash-1", "missing-hash-2"]
|
||||
)
|
||||
outputs = MagicMock(
|
||||
outputs=[engine_core_output],
|
||||
timestamp=0.0,
|
||||
scheduler_stats=None,
|
||||
)
|
||||
engine.engine_core = MagicMock()
|
||||
engine.engine_core.get_output.return_value = outputs
|
||||
|
||||
processed_outputs = MagicMock(reqs_to_abort=[], request_outputs=[])
|
||||
engine.output_processor = MagicMock()
|
||||
engine.output_processor.process_outputs.return_value = processed_outputs
|
||||
|
||||
assert engine.step() == []
|
||||
assert mm_processor_cache.invalidate.call_args_list == [
|
||||
call("missing-hash-1"),
|
||||
call("missing-hash-2"),
|
||||
]
|
||||
|
||||
|
||||
def _vllm_model(
|
||||
apc: bool,
|
||||
vllm_runner: type[VllmRunner],
|
||||
|
||||
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.engine.utils import (
|
||||
NUM_PROMPT_LOGPROBS_UNDER_TEST,
|
||||
@@ -18,7 +19,12 @@ from tests.v1.engine.utils import (
|
||||
from vllm import PoolingParams
|
||||
from vllm.logprobs import FlatLogprobs, Logprob, PromptLogprobs, SampleLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask
|
||||
from vllm.outputs import (
|
||||
CompletionOutput,
|
||||
PoolingRequestOutput,
|
||||
RequestOutput,
|
||||
SamplingMask,
|
||||
)
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.v1.engine import (
|
||||
@@ -1630,3 +1636,92 @@ def test_request_output_add_merges_delta_sampling_masks():
|
||||
|
||||
assert merged.outputs[0].token_ids == [10, 20]
|
||||
assert merged.outputs[0].sampling_mask.token_ids == [[10, 11], [20]]
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_pooling_cache_miss_is_a_request_error_and_batch_continues():
|
||||
output_processor = OutputProcessor(tokenizer=None, log_stats=False)
|
||||
requests = [
|
||||
EngineCoreRequest(
|
||||
request_id=f"request-{idx}-int",
|
||||
external_req_id=f"request-{idx}",
|
||||
prompt_token_ids=[1, 2],
|
||||
mm_features=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
sampling_params=None,
|
||||
pooling_params=PoolingParams(task="embed"),
|
||||
)
|
||||
for idx in range(2)
|
||||
]
|
||||
for request in requests:
|
||||
output_processor.add_request(request, prompt=None)
|
||||
|
||||
processed = output_processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id=requests[0].request_id,
|
||||
new_token_ids=[],
|
||||
finish_reason=FinishReason.ERROR,
|
||||
mm_cache_miss_hashes=["missing-hash"],
|
||||
),
|
||||
EngineCoreOutput(
|
||||
request_id=requests[1].request_id,
|
||||
new_token_ids=[],
|
||||
pooling_output=torch.tensor([1.0, 2.0]),
|
||||
finish_reason=FinishReason.STOP,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
assert len(processed.request_outputs) == 2
|
||||
failed, succeeded = processed.request_outputs
|
||||
assert isinstance(failed, PoolingRequestOutput)
|
||||
assert failed.error is not None
|
||||
assert failed.error.code == "multimodal_cache_miss"
|
||||
assert failed.error.retryable
|
||||
assert isinstance(succeeded, PoolingRequestOutput)
|
||||
assert succeeded.error is None
|
||||
assert torch.equal(succeeded.outputs.data, torch.tensor([1.0, 2.0]))
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_async_pooling_cache_miss_does_not_fail_output_processor():
|
||||
output_processor = OutputProcessor(tokenizer=None, log_stats=False)
|
||||
request = EngineCoreRequest(
|
||||
request_id="request-int",
|
||||
external_req_id="request",
|
||||
prompt_token_ids=[1, 2],
|
||||
mm_features=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
sampling_params=None,
|
||||
pooling_params=PoolingParams(task="embed"),
|
||||
)
|
||||
queue = RequestOutputCollector(
|
||||
output_kind=RequestOutputKind.FINAL_ONLY,
|
||||
request_id=request.request_id,
|
||||
)
|
||||
output_processor.add_request(request, prompt=None, queue=queue)
|
||||
|
||||
processed = output_processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id=request.request_id,
|
||||
new_token_ids=[],
|
||||
finish_reason=FinishReason.ERROR,
|
||||
mm_cache_miss_hashes=["missing-hash"],
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
assert processed.request_outputs == []
|
||||
output = queue.get_nowait()
|
||||
assert isinstance(output, PoolingRequestOutput)
|
||||
assert output.error is not None
|
||||
assert output.error.retryable
|
||||
assert not output_processor.has_unfinished_requests()
|
||||
|
||||
@@ -137,6 +137,52 @@ def test_finished_request_releases_unscored_doc_use():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_release_queries_is_targeted_and_idempotent():
|
||||
runner = LateInteractionRunner()
|
||||
query_emb = torch.tensor([[1.0, 0.0], [0.0, 1.0]], dtype=torch.float32)
|
||||
|
||||
for query_key in ("query-release", "query-keep"):
|
||||
query_params = _make_pooling_params(
|
||||
build_late_interaction_query_params(query_key=query_key, query_uses=1)
|
||||
)
|
||||
runner.postprocess_pooler_output(
|
||||
raw_pooler_output=[query_emb],
|
||||
pooling_params=[query_params],
|
||||
req_ids=[f"{query_key}-req"],
|
||||
finished_mask=[True],
|
||||
)
|
||||
|
||||
released_doc_params = _make_pooling_params(
|
||||
build_late_interaction_doc_params(query_key="query-release")
|
||||
)
|
||||
kept_doc_params = _make_pooling_params(
|
||||
build_late_interaction_doc_params(query_key="query-keep")
|
||||
)
|
||||
runner.register_request("released-doc", released_doc_params)
|
||||
runner.register_request("kept-doc", kept_doc_params)
|
||||
|
||||
runner.release_queries(["query-release"])
|
||||
runner.release_queries(["query-release"])
|
||||
|
||||
with pytest.raises(ValueError, match="query cache miss"):
|
||||
runner.postprocess_pooler_output(
|
||||
raw_pooler_output=[query_emb],
|
||||
pooling_params=[released_doc_params],
|
||||
req_ids=["released-doc"],
|
||||
finished_mask=[True],
|
||||
)
|
||||
|
||||
kept_output = runner.postprocess_pooler_output(
|
||||
raw_pooler_output=[query_emb],
|
||||
pooling_params=[kept_doc_params],
|
||||
req_ids=["kept-doc"],
|
||||
finished_mask=[True],
|
||||
)
|
||||
assert isinstance(kept_output, list)
|
||||
assert kept_output[0] is not None
|
||||
|
||||
|
||||
def test_invalid_query_uses_raises():
|
||||
runner = LateInteractionRunner()
|
||||
bad_meta = LateInteractionParams(
|
||||
|
||||
Reference in New Issue
Block a user