diff --git a/docs/features/nixl_connector_usage.md b/docs/features/nixl_connector_usage.md index 0d1f16e2aa..dab0faaf98 100644 --- a/docs/features/nixl_connector_usage.md +++ b/docs/features/nixl_connector_usage.md @@ -454,6 +454,7 @@ exported when NixlConnector is active: | `vllm:nixl_num_failed_transfers` | Counter | Cumulative count of failed NIXL KV-block transfers, including handshake and notification (`send_notif`) failures. These are grouped because all are sporadic lower-transport-layer events. | | `vllm:nixl_num_failed_notifications` | Counter | Cumulative count of failed notifications (`send_notif`), retained for compatibility. These failures are also included in `vllm:nixl_num_failed_transfers`; do not sum the two counters. | | `vllm:nixl_num_kv_expired_reqs` | Counter | Requests whose KV blocks expired on the prefiller before the decoder read them (tracked on the P instance). Kept separate from the failure counters above: KV expiry is an autoscaler/lease-tuning signal, not a transport issue. | +| `vllm:nixl_num_notifications_after_expiry` | Counter | Completion notifications that arrive for a request that is no longer tracked, usually because its KV lease expired first. Pull connector: counted on P, where it means the decoder may have read blocks that were already reused; each matches a "Potentially invalid KV blocks" error log. Push connector: counted on whichever instance tracks the request as neither a send nor a receive (P after the lease expired, or D after the receive completed or was abandoned); each matches an "Unrecognized request" error log. Counted per notification, not per request: each remote rank notifies separately, so with TP > 1 or heterogeneous TP one late request counts several times, and the value is not comparable to `vllm:nixl_num_kv_expired_reqs`. | !!! tip High `vllm:nixl_num_kv_expired_reqs` indicates that the prefiller's lease diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index 9d04a2ea53..e636aabfd9 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -601,6 +601,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } } @@ -626,6 +627,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], }, "MockConnector": {"mock_field": [1, 2, 3]}, } @@ -657,6 +659,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], }, } @@ -678,6 +681,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) mock_stats = MockConnectorStats(data={"mock_field": [1, 2, 3]}) @@ -709,6 +713,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) @@ -742,6 +747,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], }, "ExampleConnector": {"some_field": [1, 2, 3]}, } @@ -781,6 +787,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) } @@ -798,6 +805,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) } @@ -831,6 +839,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) } @@ -859,6 +868,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) } @@ -887,6 +897,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) } @@ -918,6 +929,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) } diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 6be77b7b9d..07ae290219 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -1698,6 +1698,7 @@ def test_kv_connector_stats_failure_grouping(): stats.record_failed_handshake() stats.record_failed_notification() stats.record_kv_expired_req() + stats.record_notification_after_expiry() assert not stats.is_empty() # No successful transfers: latency stats are zero but the failure @@ -1706,6 +1707,7 @@ def test_kv_connector_stats_failure_grouping(): assert reduced["Num successful transfers"] == 0 assert reduced["Num failed transfers"] == 3 assert reduced["Num KV expired reqs"] == 1 + assert reduced["Num notifs after expiry"] == 1 def test_nixl_prom_metrics_group_handshake_with_transfer_failures(): @@ -1750,6 +1752,7 @@ def test_nixl_prom_metrics_group_handshake_with_transfer_failures(): stats.record_failed_handshake() stats.record_failed_notification() stats.record_kv_expired_req() + stats.record_notification_after_expiry() prom.observe(stats.data, engine_idx=0) def counter_value(name: str) -> float: @@ -1761,6 +1764,33 @@ def test_nixl_prom_metrics_group_handshake_with_transfer_failures(): assert counter_value("vllm:nixl_num_failed_transfers_total") == 3.0 assert counter_value("vllm:nixl_num_kv_expired_reqs_total") == 1.0 + assert counter_value("vllm:nixl_num_notifications_after_expiry_total") == 1.0 + + +@patch( + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", + FakeNixlWrapper, +) +def test_notification_after_expiry_is_counted(default_vllm_config, dist_init): + vllm_config = create_vllm_config() + connector = NixlConnector( + vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + ) + connector.connector_worker = FakeNixlConnectorWorker( + vllm_config, connector.engine_id, hand_shake_latency=0 + ) + worker = connector.connector_worker + worker._reqs_to_process.add("known") + worker._reqs_to_send["known"] = time.perf_counter() + 10 + worker.nixl_wrapper.get_new_notifs = MagicMock( + return_value={"decode-agent": [b"known:1", b"unknown:1"]} + ) + + assert worker._get_new_notifs() == {"known"} + + stats = connector.get_kv_connector_stats() + assert isinstance(stats, NixlKVConnectorStats) + assert stats.data["num_notifications_after_expiry"] == [1] def test_multi_kv_connector_stats_aggregation(): diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index de47376117..47779c3611 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -1357,6 +1357,7 @@ class TestPushWriterNegative: assert notified == set() # Did not register anywhere. assert "never-heard-of-you" not in w._recving_transfers + assert w.xfer_stats.data["num_notifications_after_expiry"] == [1] def test_start_load_kv_with_empty_metadata_is_noop(self): """Empty metadata must not wake the writer or enqueue anything.""" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index 11b7215950..4a18632eb4 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -660,6 +660,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): req_id not in self._reqs_to_send and req_id not in self._reqs_to_process ): + self.xfer_stats.record_notification_after_expiry() logger.error( "Potentially invalid KV blocks for " "unrecognized request %s were retrieved by " diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py index ec77f426c5..6d82fbcc0a 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py @@ -786,6 +786,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): else: # Not tracked on either side (lease may have expired # before the notif arrived). Log and skip. + self.xfer_stats.record_notification_after_expiry() logger.error( "Unrecognized request %s notif (may have expired).", req_id, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py index f6d0684b6b..26c4275efb 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py @@ -41,6 +41,7 @@ class NixlKVConnectorStats(KVConnectorStats): "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } def record_transfer(self, res: "nixlXferTelemetry"): @@ -66,6 +67,10 @@ class NixlKVConnectorStats(KVConnectorStats): """Record a request that had its KV blocks expire.""" self.data["num_kv_expired_reqs"].append(1) + def record_notification_after_expiry(self): + """Record a completion notification that arrived after its request expired.""" + self.data["num_notifications_after_expiry"].append(1) + def clone_and_reset(self) -> "NixlKVConnectorStats": old = copy.copy(self) self.reset() @@ -79,6 +84,7 @@ class NixlKVConnectorStats(KVConnectorStats): and len(self.data["num_failed_notifications"]) == 0 and len(self.data["num_failed_handshakes"]) == 0 and len(self.data["num_kv_expired_reqs"]) == 0 + and len(self.data["num_notifications_after_expiry"]) == 0 ) def aggregate(self, other: KVConnectorStats) -> KVConnectorStats: @@ -95,12 +101,14 @@ class NixlKVConnectorStats(KVConnectorStats): # and notification failures are grouped as sporadic # lower-transport-layer events, while KV expiry is reported separately # as it is an actionable autoscaler signal rather than a transport - # issue. + # issue. Notifications arriving after expiry are surfaced too, as the + # reader may have consumed KV blocks that were already reused. failure_counts = { "Num failed transfers": len(self.data["num_failed_transfers"]) + len(self.data["num_failed_handshakes"]) + len(self.data["num_failed_notifications"]), "Num KV expired reqs": len(self.data["num_kv_expired_reqs"]), + "Num notifs after expiry": len(self.data["num_notifications_after_expiry"]), } if self.num_successful_transfers == 0: # Timing / throughput stats only cover successful transfers. If @@ -262,6 +270,18 @@ class NixlPromMetrics(KVConnectorPromMetrics): counter_nixl_num_kv_expired_reqs, self.per_engine_labelvalues ) + counter_nixl_num_notifications_after_expiry = self._counter_cls( + name="vllm:nixl_num_notifications_after_expiry", + documentation="Number of completion notifications for requests that " + "were no longer tracked, usually because their KV lease expired. The " + "KV blocks may have been reused before the transfer finished. Counted " + "per notification (one per remote rank), not per request.", + labelnames=labelnames, + ) + self.counter_nixl_num_notifications_after_expiry = create_metric_per_engine( + counter_nixl_num_notifications_after_expiry, self.per_engine_labelvalues + ) + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): for prom_obj, list_item_key in zip( [ @@ -284,6 +304,7 @@ class NixlPromMetrics(KVConnectorPromMetrics): self.counter_nixl_num_failed_transfers, self.counter_nixl_num_failed_notifications, self.counter_nixl_num_kv_expired_reqs, + self.counter_nixl_num_notifications_after_expiry, ], [ # Transfer, handshake and notification failures are grouped: @@ -297,6 +318,7 @@ class NixlPromMetrics(KVConnectorPromMetrics): ), ("num_failed_notifications",), ("num_kv_expired_reqs",), + ("num_notifications_after_expiry",), ], ): for counter_item_key in counter_item_keys: