[Bugfix][KVConnector] Make ExampleHiddenStatesConnector abort-safe (#56841)
Signed-off-by: ys2025-AI <yuanshuaien@163.com>
This commit is contained in:
@@ -169,3 +169,62 @@ def test_hidden_state_group_isolated_from_packed_mixed_page_groups():
|
||||
assert (
|
||||
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0]
|
||||
)
|
||||
|
||||
|
||||
# ---- abort-path robustness --------------------------------------------------
|
||||
|
||||
|
||||
def _bare_connector() -> ExampleHiddenStatesConnector:
|
||||
"""Instance with scheduler-side state but bypassing ``__init__`` (no engine)."""
|
||||
conn = ExampleHiddenStatesConnector.__new__(ExampleHiddenStatesConnector)
|
||||
conn._request_filenames = {}
|
||||
conn._pending_saves = {}
|
||||
conn._lock_fds = {}
|
||||
conn._cache_kv_group_id = 1
|
||||
conn._connector_metadata = None
|
||||
conn._req_copy_events = {}
|
||||
conn._accumulated_finished_req_ids = set()
|
||||
return conn
|
||||
|
||||
|
||||
def test_get_finished_count_is_one():
|
||||
# Only TP rank 0 writes, so KVOutputAggregator must expect a single
|
||||
# finished_sending notification per request (not the TP world size).
|
||||
assert _bare_connector().get_finished_count() == 1
|
||||
|
||||
|
||||
def test_request_finished_is_noop_for_never_scheduled_request():
|
||||
# A request aborted while still queued never reaches build_connector_meta,
|
||||
# so no filename was recorded. request_finished must not raise KeyError.
|
||||
conn = _bare_connector()
|
||||
request = SimpleNamespace(request_id="cmpl-aborted", kv_transfer_params=None)
|
||||
assert conn.request_finished(request, []) == (False, None)
|
||||
|
||||
|
||||
def test_request_finished_all_groups_handles_missing_group():
|
||||
# Guard against indexing a nonexistent per-group block table.
|
||||
conn = _bare_connector()
|
||||
conn._cache_kv_group_id = 2
|
||||
request = SimpleNamespace(request_id="cmpl-aborted", kv_transfer_params=None)
|
||||
assert conn.request_finished_all_groups(request, ([], [])) == (False, None)
|
||||
|
||||
|
||||
def test_get_finished_does_not_report_untracked_request():
|
||||
# A never-scheduled aborted request has no copy event. get_finished must not
|
||||
# report it as done_sending, or the scheduler asserts it is still tracked.
|
||||
conn = _bare_connector()
|
||||
assert conn.get_finished({"cmpl-aborted"}) == (None, None)
|
||||
assert conn._accumulated_finished_req_ids == set()
|
||||
|
||||
|
||||
def test_get_finished_reports_tracked_completed_request():
|
||||
conn = _bare_connector()
|
||||
|
||||
class _DoneEvent:
|
||||
def query(self) -> bool:
|
||||
return True
|
||||
|
||||
conn._req_copy_events["cmpl-done"] = _DoneEvent()
|
||||
done_sending, done_recving = conn.get_finished({"cmpl-done"})
|
||||
assert done_sending == {"cmpl-done"}
|
||||
assert done_recving is None
|
||||
|
||||
@@ -103,6 +103,18 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
Must be used in conjunction with the `extract_hidden_states` spec decoding method.
|
||||
"""
|
||||
|
||||
def get_finished_count(self) -> int:
|
||||
"""Number of ``finished_sending`` notifications expected per request.
|
||||
|
||||
Only TP rank 0 writes hidden states to disk, so each request yields a
|
||||
single notification. The default (``None``) makes ``KVOutputAggregator``
|
||||
wait for every worker in the world size; since ranks 1..N-1 never submit
|
||||
a copy, the request would never be reported as finished and its blocks
|
||||
would be leaked forever (the scheduler then spins on
|
||||
``has_finished_requests()``).
|
||||
"""
|
||||
return 1
|
||||
|
||||
@classmethod
|
||||
def _find_cache_kv_group_id(cls, kv_cache_config: "KVCacheConfig | None") -> int:
|
||||
"""Index of the KV cache group holding the extracted hidden states.
|
||||
@@ -517,7 +529,13 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
the hidden states from the KV cache.
|
||||
"""
|
||||
req_id = request.request_id
|
||||
filename = self._request_filenames.pop(req_id)
|
||||
filename = self._request_filenames.pop(req_id, None)
|
||||
if filename is None:
|
||||
# Request was aborted before it was ever scheduled, so
|
||||
# build_connector_meta never recorded a filename for it. There are
|
||||
# no hidden states to save and no blocks worth delaying.
|
||||
self._pending_saves.pop(req_id, None)
|
||||
return False, None
|
||||
kv_params = request.kv_transfer_params or {}
|
||||
if kv_params.get("include_output_tokens", False):
|
||||
# Exclude the final token — it was the model's output, never an
|
||||
@@ -566,15 +584,18 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
done_sending: set[str] = set()
|
||||
for req_id in list(self._accumulated_finished_req_ids):
|
||||
event = self._req_copy_events.get(req_id)
|
||||
if event is None or event.query():
|
||||
self._req_copy_events.pop(req_id, None)
|
||||
if event is not None and not event.query():
|
||||
continue # DtoH copy still in flight
|
||||
self._req_copy_events.pop(req_id, None)
|
||||
self._accumulated_finished_req_ids.discard(req_id)
|
||||
# Only report requests we actually tracked for sending.
|
||||
if event is not None:
|
||||
done_sending.add(req_id)
|
||||
self._accumulated_finished_req_ids.discard(req_id)
|
||||
# Clean up any leftover lock fds (e.g. aborted requests
|
||||
# that never went through _submit_async_write).
|
||||
lock_fd = self._lock_fds.pop(req_id, None)
|
||||
if lock_fd is not None:
|
||||
os.close(lock_fd)
|
||||
# Clean up any leftover lock fds (e.g. aborted requests that never
|
||||
# went through _submit_async_write).
|
||||
lock_fd = self._lock_fds.pop(req_id, None)
|
||||
if lock_fd is not None:
|
||||
os.close(lock_fd)
|
||||
|
||||
return done_sending or None, None
|
||||
|
||||
@@ -583,6 +604,10 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
if not 0 <= self._cache_kv_group_id < len(block_ids):
|
||||
# No per-group block table (e.g. a request that was never
|
||||
# scheduled); treat as a no-op instead of indexing out of bounds.
|
||||
return False, None
|
||||
return self.request_finished(request, block_ids[self._cache_kv_group_id])
|
||||
|
||||
@classmethod
|
||||
|
||||
Reference in New Issue
Block a user