[Bugfix][Pooling] Handle multimodal cache misses without crashing (#58975)
Signed-off-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: linchaoying6 <linchaoying6@jd.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
linchaoying6
OpenAI Codex
parent
443dccd3c9
commit
592c6f3fbc
@@ -42,6 +42,18 @@ class LateInteractionRunner:
|
||||
self._query_uses.clear()
|
||||
self._doc_query_keys.clear()
|
||||
|
||||
def release_queries(self, query_keys: Iterable[str]) -> None:
|
||||
query_keys = set(query_keys)
|
||||
for query_key in query_keys:
|
||||
self._query_cache.pop(query_key, None)
|
||||
self._query_uses.pop(query_key, None)
|
||||
|
||||
self._doc_query_keys = {
|
||||
req_id: query_key
|
||||
for req_id, query_key in self._doc_query_keys.items()
|
||||
if query_key not in query_keys
|
||||
}
|
||||
|
||||
def register_request(
|
||||
self, req_id: str, pooling_params: PoolingParams | None
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user