[Bugfix][KVConnector] Make ExampleHiddenStatesConnector abort-safe (#56841)
Signed-off-by: ys2025-AI <yuanshuaien@163.com>
This commit is contained in:
@@ -103,6 +103,18 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
Must be used in conjunction with the `extract_hidden_states` spec decoding method.
|
||||
"""
|
||||
|
||||
def get_finished_count(self) -> int:
|
||||
"""Number of ``finished_sending`` notifications expected per request.
|
||||
|
||||
Only TP rank 0 writes hidden states to disk, so each request yields a
|
||||
single notification. The default (``None``) makes ``KVOutputAggregator``
|
||||
wait for every worker in the world size; since ranks 1..N-1 never submit
|
||||
a copy, the request would never be reported as finished and its blocks
|
||||
would be leaked forever (the scheduler then spins on
|
||||
``has_finished_requests()``).
|
||||
"""
|
||||
return 1
|
||||
|
||||
@classmethod
|
||||
def _find_cache_kv_group_id(cls, kv_cache_config: "KVCacheConfig | None") -> int:
|
||||
"""Index of the KV cache group holding the extracted hidden states.
|
||||
@@ -517,7 +529,13 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
the hidden states from the KV cache.
|
||||
"""
|
||||
req_id = request.request_id
|
||||
filename = self._request_filenames.pop(req_id)
|
||||
filename = self._request_filenames.pop(req_id, None)
|
||||
if filename is None:
|
||||
# Request was aborted before it was ever scheduled, so
|
||||
# build_connector_meta never recorded a filename for it. There are
|
||||
# no hidden states to save and no blocks worth delaying.
|
||||
self._pending_saves.pop(req_id, None)
|
||||
return False, None
|
||||
kv_params = request.kv_transfer_params or {}
|
||||
if kv_params.get("include_output_tokens", False):
|
||||
# Exclude the final token — it was the model's output, never an
|
||||
@@ -566,15 +584,18 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
done_sending: set[str] = set()
|
||||
for req_id in list(self._accumulated_finished_req_ids):
|
||||
event = self._req_copy_events.get(req_id)
|
||||
if event is None or event.query():
|
||||
self._req_copy_events.pop(req_id, None)
|
||||
if event is not None and not event.query():
|
||||
continue # DtoH copy still in flight
|
||||
self._req_copy_events.pop(req_id, None)
|
||||
self._accumulated_finished_req_ids.discard(req_id)
|
||||
# Only report requests we actually tracked for sending.
|
||||
if event is not None:
|
||||
done_sending.add(req_id)
|
||||
self._accumulated_finished_req_ids.discard(req_id)
|
||||
# Clean up any leftover lock fds (e.g. aborted requests
|
||||
# that never went through _submit_async_write).
|
||||
lock_fd = self._lock_fds.pop(req_id, None)
|
||||
if lock_fd is not None:
|
||||
os.close(lock_fd)
|
||||
# Clean up any leftover lock fds (e.g. aborted requests that never
|
||||
# went through _submit_async_write).
|
||||
lock_fd = self._lock_fds.pop(req_id, None)
|
||||
if lock_fd is not None:
|
||||
os.close(lock_fd)
|
||||
|
||||
return done_sending or None, None
|
||||
|
||||
@@ -583,6 +604,10 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
if not 0 <= self._cache_kv_group_id < len(block_ids):
|
||||
# No per-group block table (e.g. a request that was never
|
||||
# scheduled); treat as a no-op instead of indexing out of bounds.
|
||||
return False, None
|
||||
return self.request_finished(request, block_ids[self._cache_kv_group_id])
|
||||
|
||||
@classmethod
|
||||
|
||||
Reference in New Issue
Block a user