diff --git a/tests/entrypoints/pooling/scoring/test_late_interaction_serving.py b/tests/entrypoints/pooling/scoring/test_late_interaction_serving.py new file mode 100644 index 0000000000..464baf89e7 --- /dev/null +++ b/tests/entrypoints/pooling/scoring/test_late_interaction_serving.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from unittest.mock import AsyncMock, Mock + +import pytest + +from vllm.entrypoints.pooling.scoring.serving import ServingScores +from vllm.entrypoints.pooling.typing import PoolingServeContext +from vllm.inputs import tokens_input +from vllm.pooling_params import PoolingParams + + +def _make_context() -> PoolingServeContext: + return PoolingServeContext( + request=Mock(), + model_name="model", + request_id="score-caller-controlled-id", + pooling_params=PoolingParams(task="token_embed"), + lora_request=None, + priorities=None, + prompt_extras=None, + engine_inputs=[ + tokens_input(prompt_token_ids=[1]), + tokens_input(prompt_token_ids=[2]), + ], + n_queries=1, + ) + + +def _query_key(context: PoolingServeContext) -> str: + assert context.engine_inputs is not None + pooling_params = context.engine_inputs[0]["params"] + late_interaction_params = pooling_params.late_interaction_params + assert late_interaction_params is not None + query_key = late_interaction_params.query_key + assert query_key is not None + return query_key + + +@pytest.mark.asyncio +async def test_colliding_request_ids_use_distinct_query_cache_keys(): + serving = object.__new__(ServingScores) + serving._prepare_generators = AsyncMock() + serving._collect_batch = AsyncMock() + + first = _make_context() + second = _make_context() + for context in (first, second): + await serving._flash_late_interaction_encode_queries(context) + await serving._flash_late_interaction_encode_docs(context) + + prepared_contexts = [ + call.args[0] for call in serving._prepare_generators.await_args_list + ] + first_query_key = _query_key(prepared_contexts[0]) + first_doc_key = _query_key(prepared_contexts[1]) + second_query_key = _query_key(prepared_contexts[2]) + second_doc_key = _query_key(prepared_contexts[3]) + + assert first_query_key == first_doc_key + assert second_query_key == second_doc_key + assert first_query_key != second_query_key + assert "caller-controlled-id" not in first_query_key + assert "caller-controlled-id" not in second_query_key diff --git a/vllm/entrypoints/pooling/scoring/serving.py b/vllm/entrypoints/pooling/scoring/serving.py index d9a067dbab..3b1e138115 100644 --- a/vllm/entrypoints/pooling/scoring/serving.py +++ b/vllm/entrypoints/pooling/scoring/serving.py @@ -9,6 +9,7 @@ from vllm.entrypoints.serve.engine.protocol import UsageInfo from vllm.logger import init_logger from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput from vllm.tasks import SCORE_TYPE_MAP, SupportedTask +from vllm.utils import random_uuid from vllm.v1.pool.late_interaction import ( build_late_interaction_doc_params, build_late_interaction_query_params, @@ -208,7 +209,11 @@ class ServingScores(PoolingServing): n_docs = len(ctx.engine_inputs) - n_queries query_engine_inputs = ctx.engine_inputs[:n_queries] - query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)] + query_namespace = random_uuid() + query_keys = [ + f"late-interaction-{query_namespace}-query-{i}" for i in range(n_queries) + ] + ctx.late_interaction_query_keys = query_keys query_uses = [n_docs if n_queries == 1 else 1] * n_queries for i in range(n_queries): @@ -249,7 +254,9 @@ class ServingScores(PoolingServing): n_docs = len(ctx.engine_inputs) - n_queries doc_engine_inputs = ctx.engine_inputs[n_queries:] - query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)] + query_keys = ctx.late_interaction_query_keys + if query_keys is None: + raise RuntimeError("Late-interaction query keys were not initialized.") doc_keys = [f"{ctx.request_id}-doc-{i}" for i in range(n_docs)] for i in range(n_docs): diff --git a/vllm/entrypoints/pooling/typing.py b/vllm/entrypoints/pooling/typing.py index 288f329d09..7ce26dd6cf 100644 --- a/vllm/entrypoints/pooling/typing.py +++ b/vllm/entrypoints/pooling/typing.py @@ -114,6 +114,7 @@ class PoolingServeContext(Generic[PoolingRequestT]): ## for flash-late-interaction query_final_res_batch: list[PoolingRequestOutput] | None = None + late_interaction_query_keys: list[str] | None = None @dataclass