[Bugfix][KVConnector] Make ExampleHiddenStatesConnector abort-safe (#56841)

Signed-off-by: ys2025-AI <yuanshuaien@163.com>
This commit is contained in:
ys2025-AI
2026-09-18 21:53:02 +00:00
committed by GitHub
parent 62af3df734
commit 6aab78f477
2 changed files with 93 additions and 9 deletions
@@ -169,3 +169,62 @@ def test_hidden_state_group_isolated_from_packed_mixed_page_groups():
assert (
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0]
)
# ---- abort-path robustness --------------------------------------------------
def _bare_connector() -> ExampleHiddenStatesConnector:
"""Instance with scheduler-side state but bypassing ``__init__`` (no engine)."""
conn = ExampleHiddenStatesConnector.__new__(ExampleHiddenStatesConnector)
conn._request_filenames = {}
conn._pending_saves = {}
conn._lock_fds = {}
conn._cache_kv_group_id = 1
conn._connector_metadata = None
conn._req_copy_events = {}
conn._accumulated_finished_req_ids = set()
return conn
def test_get_finished_count_is_one():
# Only TP rank 0 writes, so KVOutputAggregator must expect a single
# finished_sending notification per request (not the TP world size).
assert _bare_connector().get_finished_count() == 1
def test_request_finished_is_noop_for_never_scheduled_request():
# A request aborted while still queued never reaches build_connector_meta,
# so no filename was recorded. request_finished must not raise KeyError.
conn = _bare_connector()
request = SimpleNamespace(request_id="cmpl-aborted", kv_transfer_params=None)
assert conn.request_finished(request, []) == (False, None)
def test_request_finished_all_groups_handles_missing_group():
# Guard against indexing a nonexistent per-group block table.
conn = _bare_connector()
conn._cache_kv_group_id = 2
request = SimpleNamespace(request_id="cmpl-aborted", kv_transfer_params=None)
assert conn.request_finished_all_groups(request, ([], [])) == (False, None)
def test_get_finished_does_not_report_untracked_request():
# A never-scheduled aborted request has no copy event. get_finished must not
# report it as done_sending, or the scheduler asserts it is still tracked.
conn = _bare_connector()
assert conn.get_finished({"cmpl-aborted"}) == (None, None)
assert conn._accumulated_finished_req_ids == set()
def test_get_finished_reports_tracked_completed_request():
conn = _bare_connector()
class _DoneEvent:
def query(self) -> bool:
return True
conn._req_copy_events["cmpl-done"] = _DoneEvent()
done_sending, done_recving = conn.get_finished({"cmpl-done"})
assert done_sending == {"cmpl-done"}
assert done_recving is None
@@ -103,6 +103,18 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
Must be used in conjunction with the `extract_hidden_states` spec decoding method.
"""
def get_finished_count(self) -> int:
"""Number of ``finished_sending`` notifications expected per request.
Only TP rank 0 writes hidden states to disk, so each request yields a
single notification. The default (``None``) makes ``KVOutputAggregator``
wait for every worker in the world size; since ranks 1..N-1 never submit
a copy, the request would never be reported as finished and its blocks
would be leaked forever (the scheduler then spins on
``has_finished_requests()``).
"""
return 1
@classmethod
def _find_cache_kv_group_id(cls, kv_cache_config: "KVCacheConfig | None") -> int:
"""Index of the KV cache group holding the extracted hidden states.
@@ -517,7 +529,13 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
the hidden states from the KV cache.
"""
req_id = request.request_id
filename = self._request_filenames.pop(req_id)
filename = self._request_filenames.pop(req_id, None)
if filename is None:
# Request was aborted before it was ever scheduled, so
# build_connector_meta never recorded a filename for it. There are
# no hidden states to save and no blocks worth delaying.
self._pending_saves.pop(req_id, None)
return False, None
kv_params = request.kv_transfer_params or {}
if kv_params.get("include_output_tokens", False):
# Exclude the final token — it was the model's output, never an
@@ -566,15 +584,18 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
done_sending: set[str] = set()
for req_id in list(self._accumulated_finished_req_ids):
event = self._req_copy_events.get(req_id)
if event is None or event.query():
self._req_copy_events.pop(req_id, None)
if event is not None and not event.query():
continue # DtoH copy still in flight
self._req_copy_events.pop(req_id, None)
self._accumulated_finished_req_ids.discard(req_id)
# Only report requests we actually tracked for sending.
if event is not None:
done_sending.add(req_id)
self._accumulated_finished_req_ids.discard(req_id)
# Clean up any leftover lock fds (e.g. aborted requests
# that never went through _submit_async_write).
lock_fd = self._lock_fds.pop(req_id, None)
if lock_fd is not None:
os.close(lock_fd)
# Clean up any leftover lock fds (e.g. aborted requests that never
# went through _submit_async_write).
lock_fd = self._lock_fds.pop(req_id, None)
if lock_fd is not None:
os.close(lock_fd)
return done_sending or None, None
@@ -583,6 +604,10 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
if not 0 <= self._cache_kv_group_id < len(block_ids):
# No per-group block table (e.g. a request that was never
# scheduled); treat as a no-op instead of indexing out of bounds.
return False, None
return self.request_finished(request, block_ids[self._cache_kv_group_id])
@classmethod