diff --git a/tests/v1/kv_connector/unit/test_hidden_states_connector.py b/tests/v1/kv_connector/unit/test_hidden_states_connector.py index 8427833b12..735b042259 100644 --- a/tests/v1/kv_connector/unit/test_hidden_states_connector.py +++ b/tests/v1/kv_connector/unit/test_hidden_states_connector.py @@ -169,3 +169,62 @@ def test_hidden_state_group_isolated_from_packed_mixed_page_groups(): assert ( ExampleHiddenStatesConnector._find_cache_kv_group_id(cfg) == hidden_group_ids[0] ) + + +# ---- abort-path robustness -------------------------------------------------- + + +def _bare_connector() -> ExampleHiddenStatesConnector: + """Instance with scheduler-side state but bypassing ``__init__`` (no engine).""" + conn = ExampleHiddenStatesConnector.__new__(ExampleHiddenStatesConnector) + conn._request_filenames = {} + conn._pending_saves = {} + conn._lock_fds = {} + conn._cache_kv_group_id = 1 + conn._connector_metadata = None + conn._req_copy_events = {} + conn._accumulated_finished_req_ids = set() + return conn + + +def test_get_finished_count_is_one(): + # Only TP rank 0 writes, so KVOutputAggregator must expect a single + # finished_sending notification per request (not the TP world size). + assert _bare_connector().get_finished_count() == 1 + + +def test_request_finished_is_noop_for_never_scheduled_request(): + # A request aborted while still queued never reaches build_connector_meta, + # so no filename was recorded. request_finished must not raise KeyError. + conn = _bare_connector() + request = SimpleNamespace(request_id="cmpl-aborted", kv_transfer_params=None) + assert conn.request_finished(request, []) == (False, None) + + +def test_request_finished_all_groups_handles_missing_group(): + # Guard against indexing a nonexistent per-group block table. + conn = _bare_connector() + conn._cache_kv_group_id = 2 + request = SimpleNamespace(request_id="cmpl-aborted", kv_transfer_params=None) + assert conn.request_finished_all_groups(request, ([], [])) == (False, None) + + +def test_get_finished_does_not_report_untracked_request(): + # A never-scheduled aborted request has no copy event. get_finished must not + # report it as done_sending, or the scheduler asserts it is still tracked. + conn = _bare_connector() + assert conn.get_finished({"cmpl-aborted"}) == (None, None) + assert conn._accumulated_finished_req_ids == set() + + +def test_get_finished_reports_tracked_completed_request(): + conn = _bare_connector() + + class _DoneEvent: + def query(self) -> bool: + return True + + conn._req_copy_events["cmpl-done"] = _DoneEvent() + done_sending, done_recving = conn.get_finished({"cmpl-done"}) + assert done_sending == {"cmpl-done"} + assert done_recving is None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py index e9a3e5d99a..47990d22a5 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py @@ -103,6 +103,18 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): Must be used in conjunction with the `extract_hidden_states` spec decoding method. """ + def get_finished_count(self) -> int: + """Number of ``finished_sending`` notifications expected per request. + + Only TP rank 0 writes hidden states to disk, so each request yields a + single notification. The default (``None``) makes ``KVOutputAggregator`` + wait for every worker in the world size; since ranks 1..N-1 never submit + a copy, the request would never be reported as finished and its blocks + would be leaked forever (the scheduler then spins on + ``has_finished_requests()``). + """ + return 1 + @classmethod def _find_cache_kv_group_id(cls, kv_cache_config: "KVCacheConfig | None") -> int: """Index of the KV cache group holding the extracted hidden states. @@ -517,7 +529,13 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): the hidden states from the KV cache. """ req_id = request.request_id - filename = self._request_filenames.pop(req_id) + filename = self._request_filenames.pop(req_id, None) + if filename is None: + # Request was aborted before it was ever scheduled, so + # build_connector_meta never recorded a filename for it. There are + # no hidden states to save and no blocks worth delaying. + self._pending_saves.pop(req_id, None) + return False, None kv_params = request.kv_transfer_params or {} if kv_params.get("include_output_tokens", False): # Exclude the final token — it was the model's output, never an @@ -566,15 +584,18 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): done_sending: set[str] = set() for req_id in list(self._accumulated_finished_req_ids): event = self._req_copy_events.get(req_id) - if event is None or event.query(): - self._req_copy_events.pop(req_id, None) + if event is not None and not event.query(): + continue # DtoH copy still in flight + self._req_copy_events.pop(req_id, None) + self._accumulated_finished_req_ids.discard(req_id) + # Only report requests we actually tracked for sending. + if event is not None: done_sending.add(req_id) - self._accumulated_finished_req_ids.discard(req_id) - # Clean up any leftover lock fds (e.g. aborted requests - # that never went through _submit_async_write). - lock_fd = self._lock_fds.pop(req_id, None) - if lock_fd is not None: - os.close(lock_fd) + # Clean up any leftover lock fds (e.g. aborted requests that never + # went through _submit_async_write). + lock_fd = self._lock_fds.pop(req_id, None) + if lock_fd is not None: + os.close(lock_fd) return done_sending or None, None @@ -583,6 +604,10 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA): request: "Request", block_ids: tuple[list[int], ...], ) -> tuple[bool, dict[str, Any] | None]: + if not 0 <= self._cache_kv_group_id < len(block_ids): + # No per-group block table (e.g. a request that was never + # scheduled); treat as a no-op instead of indexing out of bounds. + return False, None return self.request_finished(request, block_ids[self._cache_kv_group_id]) @classmethod