Use server-generated keys for late-interaction query caches (#51445)

Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
This commit is contained in:
Clinton Thomas
2026-09-03 07:15:58 +00:00
committed by GitHub
co-authored by Lucas Bourtoule
parent 3f1af35e01
commit ee17d0d869
3 changed files with 75 additions and 2 deletions
@@ -0,0 +1,65 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import AsyncMock, Mock
import pytest
from vllm.entrypoints.pooling.scoring.serving import ServingScores
from vllm.entrypoints.pooling.typing import PoolingServeContext
from vllm.inputs import tokens_input
from vllm.pooling_params import PoolingParams
def _make_context() -> PoolingServeContext:
return PoolingServeContext(
request=Mock(),
model_name="model",
request_id="score-caller-controlled-id",
pooling_params=PoolingParams(task="token_embed"),
lora_request=None,
priorities=None,
prompt_extras=None,
engine_inputs=[
tokens_input(prompt_token_ids=[1]),
tokens_input(prompt_token_ids=[2]),
],
n_queries=1,
)
def _query_key(context: PoolingServeContext) -> str:
assert context.engine_inputs is not None
pooling_params = context.engine_inputs[0]["params"]
late_interaction_params = pooling_params.late_interaction_params
assert late_interaction_params is not None
query_key = late_interaction_params.query_key
assert query_key is not None
return query_key
@pytest.mark.asyncio
async def test_colliding_request_ids_use_distinct_query_cache_keys():
serving = object.__new__(ServingScores)
serving._prepare_generators = AsyncMock()
serving._collect_batch = AsyncMock()
first = _make_context()
second = _make_context()
for context in (first, second):
await serving._flash_late_interaction_encode_queries(context)
await serving._flash_late_interaction_encode_docs(context)
prepared_contexts = [
call.args[0] for call in serving._prepare_generators.await_args_list
]
first_query_key = _query_key(prepared_contexts[0])
first_doc_key = _query_key(prepared_contexts[1])
second_query_key = _query_key(prepared_contexts[2])
second_doc_key = _query_key(prepared_contexts[3])
assert first_query_key == first_doc_key
assert second_query_key == second_doc_key
assert first_query_key != second_query_key
assert "caller-controlled-id" not in first_query_key
assert "caller-controlled-id" not in second_query_key