[Bugfix][KVConnector] Make ExampleHiddenStatesConnector abort-safe (#56841)

Signed-off-by: ys2025-AI <yuanshuaien@163.com>
This commit is contained in:
ys2025-AI
2026-09-18 21:53:02 +00:00
committed by GitHub
parent 62af3df734
commit 6aab78f477
2 changed files with 93 additions and 9 deletions
@@ -169,3 +169,62 @@ def test_hidden_state_group_isolated_from_packed_mixed_page_groups():
assert (
ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0]
)
# ---- abort-path robustness --------------------------------------------------
def _bare_connector() -> ExampleHiddenStatesConnector:
"""Instance with scheduler-side state but bypassing ``__init__`` (no engine)."""
conn = ExampleHiddenStatesConnector.__new__(ExampleHiddenStatesConnector)
conn._request_filenames = {}
conn._pending_saves = {}
conn._lock_fds = {}
conn._cache_kv_group_id = 1
conn._connector_metadata = None
conn._req_copy_events = {}
conn._accumulated_finished_req_ids = set()
return conn
def test_get_finished_count_is_one():
# Only TP rank 0 writes, so KVOutputAggregator must expect a single
# finished_sending notification per request (not the TP world size).
assert _bare_connector().get_finished_count() == 1
def test_request_finished_is_noop_for_never_scheduled_request():
# A request aborted while still queued never reaches build_connector_meta,
# so no filename was recorded. request_finished must not raise KeyError.
conn = _bare_connector()
request = SimpleNamespace(request_id="cmpl-aborted", kv_transfer_params=None)
assert conn.request_finished(request, []) == (False, None)
def test_request_finished_all_groups_handles_missing_group():
# Guard against indexing a nonexistent per-group block table.
conn = _bare_connector()
conn._cache_kv_group_id = 2
request = SimpleNamespace(request_id="cmpl-aborted", kv_transfer_params=None)
assert conn.request_finished_all_groups(request, ([], [])) == (False, None)
def test_get_finished_does_not_report_untracked_request():
# A never-scheduled aborted request has no copy event. get_finished must not
# report it as done_sending, or the scheduler asserts it is still tracked.
conn = _bare_connector()
assert conn.get_finished({"cmpl-aborted"}) == (None, None)
assert conn._accumulated_finished_req_ids == set()
def test_get_finished_reports_tracked_completed_request():
conn = _bare_connector()
class _DoneEvent:
def query(self) -> bool:
return True
conn._req_copy_events["cmpl-done"] = _DoneEvent()
done_sending, done_recving = conn.get_finished({"cmpl-done"})
assert done_sending == {"cmpl-done"}
assert done_recving is None