Use server-generated keys for late-interaction query caches (#51445)
Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Bourtoule
parent
3f1af35e01
commit
ee17d0d869
@@ -0,0 +1,65 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.serving import ServingScores
|
||||
from vllm.entrypoints.pooling.typing import PoolingServeContext
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.pooling_params import PoolingParams
|
||||
|
||||
|
||||
def _make_context() -> PoolingServeContext:
|
||||
return PoolingServeContext(
|
||||
request=Mock(),
|
||||
model_name="model",
|
||||
request_id="score-caller-controlled-id",
|
||||
pooling_params=PoolingParams(task="token_embed"),
|
||||
lora_request=None,
|
||||
priorities=None,
|
||||
prompt_extras=None,
|
||||
engine_inputs=[
|
||||
tokens_input(prompt_token_ids=[1]),
|
||||
tokens_input(prompt_token_ids=[2]),
|
||||
],
|
||||
n_queries=1,
|
||||
)
|
||||
|
||||
|
||||
def _query_key(context: PoolingServeContext) -> str:
|
||||
assert context.engine_inputs is not None
|
||||
pooling_params = context.engine_inputs[0]["params"]
|
||||
late_interaction_params = pooling_params.late_interaction_params
|
||||
assert late_interaction_params is not None
|
||||
query_key = late_interaction_params.query_key
|
||||
assert query_key is not None
|
||||
return query_key
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_colliding_request_ids_use_distinct_query_cache_keys():
|
||||
serving = object.__new__(ServingScores)
|
||||
serving._prepare_generators = AsyncMock()
|
||||
serving._collect_batch = AsyncMock()
|
||||
|
||||
first = _make_context()
|
||||
second = _make_context()
|
||||
for context in (first, second):
|
||||
await serving._flash_late_interaction_encode_queries(context)
|
||||
await serving._flash_late_interaction_encode_docs(context)
|
||||
|
||||
prepared_contexts = [
|
||||
call.args[0] for call in serving._prepare_generators.await_args_list
|
||||
]
|
||||
first_query_key = _query_key(prepared_contexts[0])
|
||||
first_doc_key = _query_key(prepared_contexts[1])
|
||||
second_query_key = _query_key(prepared_contexts[2])
|
||||
second_doc_key = _query_key(prepared_contexts[3])
|
||||
|
||||
assert first_query_key == first_doc_key
|
||||
assert second_query_key == second_doc_key
|
||||
assert first_query_key != second_query_key
|
||||
assert "caller-controlled-id" not in first_query_key
|
||||
assert "caller-controlled-id" not in second_query_key
|
||||
@@ -9,6 +9,7 @@ from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
|
||||
from vllm.tasks import SCORE_TYPE_MAP, SupportedTask
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.v1.pool.late_interaction import (
|
||||
build_late_interaction_doc_params,
|
||||
build_late_interaction_query_params,
|
||||
@@ -208,7 +209,11 @@ class ServingScores(PoolingServing):
|
||||
n_docs = len(ctx.engine_inputs) - n_queries
|
||||
query_engine_inputs = ctx.engine_inputs[:n_queries]
|
||||
|
||||
query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)]
|
||||
query_namespace = random_uuid()
|
||||
query_keys = [
|
||||
f"late-interaction-{query_namespace}-query-{i}" for i in range(n_queries)
|
||||
]
|
||||
ctx.late_interaction_query_keys = query_keys
|
||||
query_uses = [n_docs if n_queries == 1 else 1] * n_queries
|
||||
|
||||
for i in range(n_queries):
|
||||
@@ -249,7 +254,9 @@ class ServingScores(PoolingServing):
|
||||
n_docs = len(ctx.engine_inputs) - n_queries
|
||||
doc_engine_inputs = ctx.engine_inputs[n_queries:]
|
||||
|
||||
query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)]
|
||||
query_keys = ctx.late_interaction_query_keys
|
||||
if query_keys is None:
|
||||
raise RuntimeError("Late-interaction query keys were not initialized.")
|
||||
doc_keys = [f"{ctx.request_id}-doc-{i}" for i in range(n_docs)]
|
||||
|
||||
for i in range(n_docs):
|
||||
|
||||
@@ -114,6 +114,7 @@ class PoolingServeContext(Generic[PoolingRequestT]):
|
||||
|
||||
## for flash-late-interaction
|
||||
query_final_res_batch: list[PoolingRequestOutput] | None = None
|
||||
late_interaction_query_keys: list[str] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
Reference in New Issue
Block a user