[Bugfix][KVConnector] Make ExampleHiddenStatesConnector abort-safe (#56841)

Signed-off-by: ys2025-AI <yuanshuaien@163.com>
This commit is contained in:
ys2025-AI
2026-09-18 21:53:02 +00:00
committed by GitHub
parent 62af3df734
commit 6aab78f477
2 changed files with 93 additions and 9 deletions
@@ -103,6 +103,18 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
Must be used in conjunction with the `extract_hidden_states` spec decoding method.
"""
def get_finished_count(self) -> int:
"""Number of ``finished_sending`` notifications expected per request.
Only TP rank 0 writes hidden states to disk, so each request yields a
single notification. The default (``None``) makes ``KVOutputAggregator``
wait for every worker in the world size; since ranks 1..N-1 never submit
a copy, the request would never be reported as finished and its blocks
would be leaked forever (the scheduler then spins on
``has_finished_requests()``).
"""
return 1
@classmethod
def _find_cache_kv_group_id(cls, kv_cache_config: "KVCacheConfig | None") -> int:
"""Index of the KV cache group holding the extracted hidden states.
@@ -517,7 +529,13 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
the hidden states from the KV cache.
"""
req_id = request.request_id
filename = self._request_filenames.pop(req_id)
filename = self._request_filenames.pop(req_id, None)
if filename is None:
# Request was aborted before it was ever scheduled, so
# build_connector_meta never recorded a filename for it. There are
# no hidden states to save and no blocks worth delaying.
self._pending_saves.pop(req_id, None)
return False, None
kv_params = request.kv_transfer_params or {}
if kv_params.get("include_output_tokens", False):
# Exclude the final token — it was the model's output, never an
@@ -566,15 +584,18 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
done_sending: set[str] = set()
for req_id in list(self._accumulated_finished_req_ids):
event = self._req_copy_events.get(req_id)
if event is None or event.query():
self._req_copy_events.pop(req_id, None)
if event is not None and not event.query():
continue # DtoH copy still in flight
self._req_copy_events.pop(req_id, None)
self._accumulated_finished_req_ids.discard(req_id)
# Only report requests we actually tracked for sending.
if event is not None:
done_sending.add(req_id)
self._accumulated_finished_req_ids.discard(req_id)
# Clean up any leftover lock fds (e.g. aborted requests
# that never went through _submit_async_write).
lock_fd = self._lock_fds.pop(req_id, None)
if lock_fd is not None:
os.close(lock_fd)
# Clean up any leftover lock fds (e.g. aborted requests that never
# went through _submit_async_write).
lock_fd = self._lock_fds.pop(req_id, None)
if lock_fd is not None:
os.close(lock_fd)
return done_sending or None, None
@@ -583,6 +604,10 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
if not 0 <= self._cache_kv_group_id < len(block_ids):
# No per-group block table (e.g. a request that was never
# scheduled); treat as a no-op instead of indexing out of bounds.
return False, None
return self.request_finished(request, block_ids[self._cache_kv_group_id])
@classmethod