[Bugfix][Pooling] Handle multimodal cache misses without crashing (#58975)

Signed-off-by: linchaoying6 <linchaoying6@jd.com>
Co-authored-by: linchaoying6 <linchaoying6@jd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Chaoying
2026-10-02 10:14:33 -04:00
committed by GitHub
co-authored by linchaoying6 OpenAI Codex
parent 443dccd3c9
commit 592c6f3fbc
21 changed files with 525 additions and 17 deletions
+12
View File
@@ -42,6 +42,18 @@ class LateInteractionRunner:
self._query_uses.clear()
self._doc_query_keys.clear()
def release_queries(self, query_keys: Iterable[str]) -> None:
query_keys = set(query_keys)
for query_key in query_keys:
self._query_cache.pop(query_key, None)
self._query_uses.pop(query_key, None)
self._doc_query_keys = {
req_id: query_key
for req_id, query_key in self._doc_query_keys.items()
if query_key not in query_keys
}
def register_request(
self, req_id: str, pooling_params: PoolingParams | None
) -> None: