Use server-generated keys for late-interaction query caches (#51445)

Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
This commit is contained in:
Clinton Thomas
2026-09-03 07:15:58 +00:00
committed by GitHub
co-authored by Lucas Bourtoule
parent 3f1af35e01
commit ee17d0d869
3 changed files with 75 additions and 2 deletions
+9 -2
View File
@@ -9,6 +9,7 @@ from vllm.entrypoints.serve.engine.protocol import UsageInfo
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
from vllm.tasks import SCORE_TYPE_MAP, SupportedTask
from vllm.utils import random_uuid
from vllm.v1.pool.late_interaction import (
build_late_interaction_doc_params,
build_late_interaction_query_params,
@@ -208,7 +209,11 @@ class ServingScores(PoolingServing):
n_docs = len(ctx.engine_inputs) - n_queries
query_engine_inputs = ctx.engine_inputs[:n_queries]
query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)]
query_namespace = random_uuid()
query_keys = [
f"late-interaction-{query_namespace}-query-{i}" for i in range(n_queries)
]
ctx.late_interaction_query_keys = query_keys
query_uses = [n_docs if n_queries == 1 else 1] * n_queries
for i in range(n_queries):
@@ -249,7 +254,9 @@ class ServingScores(PoolingServing):
n_docs = len(ctx.engine_inputs) - n_queries
doc_engine_inputs = ctx.engine_inputs[n_queries:]
query_keys = [f"{ctx.request_id}-query-{i}" for i in range(n_queries)]
query_keys = ctx.late_interaction_query_keys
if query_keys is None:
raise RuntimeError("Late-interaction query keys were not initialized.")
doc_keys = [f"{ctx.request_id}-doc-{i}" for i in range(n_docs)]
for i in range(n_docs):
+1
View File
@@ -114,6 +114,7 @@ class PoolingServeContext(Generic[PoolingRequestT]):
## for flash-late-interaction
query_final_res_batch: list[PoolingRequestOutput] | None = None
late_interaction_query_keys: list[str] | None = None
@dataclass