diff --git a/docs/features/nixl_connector_usage.md b/docs/features/nixl_connector_usage.md index 9799eb9cf1..0d1f16e2aa 100644 --- a/docs/features/nixl_connector_usage.md +++ b/docs/features/nixl_connector_usage.md @@ -418,12 +418,13 @@ activity for the last reporting interval. Example output: ```text KV Transfer metrics: Num successful transfers=4, Avg xfer time (ms)=1.381, P90 xfer time (ms)=2.601, Avg post time (ms)=0.672, P90 post time (ms)=0.801, -Avg MB per transfer=2.25, Throughput (MB/s)=1629.549, Avg number of descriptors=72.0 +Avg MB per transfer=2.25, Throughput (MB/s)=1629.549, Avg number of descriptors=72.0, +Num failed transfers=0, Num KV expired reqs=0 ``` The table below describes each field. All timing values cover only the -successful transfers recorded in the current interval; failed transfers are -counted separately via Prometheus (see +successful transfers recorded in the current interval; failures are reported +as interval counts in the same line and via Prometheus (see [Prometheus metrics](#prometheus-metrics) below). | Metric | Unit | Description | @@ -436,6 +437,8 @@ counted separately via Prometheus (see | `Avg MB per transfer` | MB | Mean payload size per transfer, computed as `total bytes transferred / number of transfers`. Reflects the average KV cache footprint of a single request (sequence length × layers × head dimension × dtype bytes). | | `Throughput (MB/s)` | MB/s | Effective bandwidth over the interval: `total MB transferred / total xfer time (s)` across all successful transfers. This is aggregate throughput, not per-request bandwidth. | | `Avg number of descriptors` | count | Mean number of NIXL memory descriptors (scatter-gather segments) submitted per transfer. More descriptors indicate more fragmented or larger KV cache allocations; very high counts can increase descriptor-registration overhead. | +| `Num failed transfers` | count | Failed NIXL transfers, handshakes, and completion notifications (`send_notif`) in the interval, grouped because all are sporadic lower-transport-layer events. | +| `Num KV expired reqs` | count | Requests whose KV blocks expired before being read. Reported separately from the failure counts above because it is an autoscaler/lease-tuning signal rather than a transport issue. | ### Prometheus metrics @@ -448,9 +451,9 @@ exported when NixlConnector is active: | `vllm:nixl_post_time_seconds` | Histogram | Time to submit the transfer request to the RDMA backend (seconds). | | `vllm:nixl_bytes_transferred` | Histogram | Bytes moved per transfer. | | `vllm:nixl_num_descriptors` | Histogram | Descriptor count per transfer. | -| `vllm:nixl_num_failed_transfers` | Counter | Cumulative count of failed NIXL KV-block transfers. | -| `vllm:nixl_num_failed_notifications` | Counter | Cumulative count of failed completion notifications (`send_notif`). | -| `vllm:nixl_num_kv_expired_reqs` | Counter | Requests whose KV blocks expired on the prefiller before the decoder read them (tracked on the P instance). | +| `vllm:nixl_num_failed_transfers` | Counter | Cumulative count of failed NIXL KV-block transfers, including handshake and notification (`send_notif`) failures. These are grouped because all are sporadic lower-transport-layer events. | +| `vllm:nixl_num_failed_notifications` | Counter | Cumulative count of failed notifications (`send_notif`), retained for compatibility. These failures are also included in `vllm:nixl_num_failed_transfers`; do not sum the two counters. | +| `vllm:nixl_num_kv_expired_reqs` | Counter | Requests whose KV blocks expired on the prefiller before the decoder read them (tracked on the P instance). Kept separate from the failure counters above: KV expiry is an autoscaler/lease-tuning signal, not a transport issue. | !!! tip High `vllm:nixl_num_kv_expired_reqs` indicates that the prefiller's lease diff --git a/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py b/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py index 78a6d930f9..3d539f97c8 100644 --- a/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py +++ b/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py @@ -969,6 +969,9 @@ def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined): assert worker.xfer_stats.data["num_kv_expired_reqs"] == ( [1] if expect_declined else [] ) + # KV expiry must not pollute the transport failure counters. + assert worker.xfer_stats.data["num_failed_transfers"] == [] + assert worker.xfer_stats.data["num_failed_handshakes"] == [] _, done_recving = connector.get_finished(finished_req_ids=set()) assert "req" in done_recving diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index f6d8ed97d8..1a85865560 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -567,6 +567,8 @@ class TestMultiConnectorStats: "num_descriptors": [10, 20], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } } @@ -590,6 +592,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], }, "MockConnector": {"mock_field": [1, 2, 3]}, } @@ -619,6 +623,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], }, } @@ -638,6 +644,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) mock_stats = MockConnectorStats(data={"mock_field": [1, 2, 3]}) @@ -667,6 +675,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) @@ -698,6 +708,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], }, "ExampleConnector": {"some_field": [1, 2, 3]}, } @@ -735,6 +747,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) } @@ -750,6 +764,8 @@ class TestMultiConnectorStats: "num_descriptors": [20], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) } @@ -781,6 +797,8 @@ class TestMultiConnectorStats: "num_descriptors": [10], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) } @@ -807,6 +825,8 @@ class TestMultiConnectorStats: "num_descriptors": [10, 20], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) } @@ -833,6 +853,8 @@ class TestMultiConnectorStats: "num_descriptors": [10, 20], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) } @@ -862,6 +884,8 @@ class TestMultiConnectorStats: "num_descriptors": [10, 20], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], + "num_kv_expired_reqs": [], } ) } diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 9ab4b0ae95..3b9f607865 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -1682,6 +1682,83 @@ def test_kv_connector_stats_aggregation(): assert all(not isinstance(v, np.generic) for v in cli_stats.values()) +def test_kv_connector_stats_failure_grouping(): + """Transfer, handshake and notification failures are reported as one + transport-failure count, while KV expiry is reported separately: the + former are sporadic lower-transport-layer events, the latter an + autoscaler signal.""" + stats = NixlKVConnectorStats() + assert stats.is_empty() + + stats.record_failed_transfer() + stats.record_failed_handshake() + stats.record_failed_notification() + stats.record_kv_expired_req() + assert not stats.is_empty() + + # No successful transfers: latency stats are zero but the failure + # counts still surface. + reduced = stats.reduce() + assert reduced["Num successful transfers"] == 0 + assert reduced["Num failed transfers"] == 3 + assert reduced["Num KV expired reqs"] == 1 + + +def test_nixl_prom_metrics_group_handshake_with_transfer_failures(): + """vllm:nixl_num_failed_transfers counts handshake and notification + failures too, while vllm:nixl_num_kv_expired_reqs stays a separate + counter.""" + from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( + NixlPromMetrics, + ) + + registry = CollectorRegistry() + + class RegistryGauge(Gauge): + def __init__(self, *args, **kwargs): + super().__init__(*args, registry=registry, **kwargs) + + class RegistryCounter(Counter): + def __init__(self, *args, **kwargs): + super().__init__(*args, registry=registry, **kwargs) + + class RegistryHistogram(Histogram): + def __init__(self, *args, **kwargs): + super().__init__(*args, registry=registry, **kwargs) + + vllm_config = create_vllm_config() + metric_types = { + Gauge: RegistryGauge, + Counter: RegistryCounter, + Histogram: RegistryHistogram, + } + prom = NixlPromMetrics( + vllm_config, + metric_types, + labelnames=["engine"], + per_engine_labelvalues={0: ["engine-0"]}, + ) + + stats = NixlKVConnectorStats() + stats.record_failed_transfer() + stats.record_failed_handshake() + stats.record_failed_notification() + stats.record_kv_expired_req() + prom.observe(stats.data, engine_idx=0) + + def counter_value(name: str) -> float: + for metric in registry.collect(): + for sample in metric.samples: + if sample.name == name: + return sample.value + raise AssertionError(f"metric {name} not found in registry") + + assert counter_value("vllm:nixl_num_failed_transfers_total") == 3.0 + assert counter_value("vllm:nixl_num_kv_expired_reqs_total") == 1.0 + + def test_multi_kv_connector_stats_aggregation(): """Test MultiKVConnectorStats aggregation across TP ranks using KVOutputAggregator (used by MultiprocExecutor). @@ -3029,6 +3106,11 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init): assert request_id in done_recving assert connector.get_block_ids_with_load_errors() == {1, 2, 3} + # Handshake failures are recorded as transport failures, separately + # from KV expiry. + assert connector.connector_worker.xfer_stats.data["num_failed_handshakes"] + assert connector.connector_worker.xfer_stats.data["num_failed_transfers"] == [] + @patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index 7ce84bf685..e35ead0015 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -47,6 +47,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( NixlPushConnectorWorker, ) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( + NixlKVConnectorStats, +) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import ( get_base_request_id, @@ -360,6 +363,7 @@ class _StubWriterWorker(NixlPushConnectorWorker): w._recv_failures = set() w._recving_transfers = defaultdict(list) w._is_hma_required = False + w.xfer_stats = NixlKVConnectorStats() w._reqs_to_process = set() w._reqs_to_send = {} w.consumer_notification_counts_by_req = defaultdict(int) @@ -730,6 +734,10 @@ def test_do_start_push_kv_drops_request_on_handshake_failure(): assert xfer_calls == [] assert len(failures) == 1 assert failures[0]["failure_type"] == "push_handshake_failed" + # The handshake metric is counted once by _ensure_handshake's own + # callback (stubbed out here); the push side must not double-record it. + assert w.xfer_stats.data["num_failed_handshakes"] == [] + assert w.xfer_stats.data["num_failed_transfers"] == [] def test_writer_loop_drains_deferred_push_inbox(): diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 60cf20303c..c2b20ffc59 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -1306,6 +1306,8 @@ class NixlBaseConnectorWorker: error=e, remote_engine_id=eid, ) + # Count once per handshake, regardless of waiting requests. + self.xfer_stats.record_failed_handshake() fut.add_done_callback(done_callback) return fut @@ -2811,13 +2813,17 @@ class NixlBaseConnectorWorker: done_sending.update(self._replicated_pcp_done_sending) self._replicated_pcp_done_sending.clear() - # Process receive failures reported by background threads. + # Process receive failures reported by background threads. Each + # producer already recorded a specific metric (handshake or + # notification), so only the failure bookkeeping runs here. while not self._failed_recv_reqs.empty(): try: req_id = self._failed_recv_reqs.get_nowait() except queue.Empty: break - self._handle_failed_transfer(req_id, None, self._recv_failures) + self._handle_failed_transfer( + req_id, None, self._recv_failures, record_failed_transfer=False + ) failed_recv_reqs: set[ReqId] = set() if self._recv_failures: @@ -3059,10 +3065,27 @@ class NixlBaseConnectorWorker: return True def _handle_failed_transfer( - self, req_id: str, handle: int | None, failed_req_ids: set[str] | None = None + self, + req_id: str, + handle: int | None, + failed_req_ids: set[str] | None = None, + record_failed_transfer: bool = True, ) -> bool: - """Record a failure and release its handle, returning False to retain it.""" - self.xfer_stats.record_failed_transfer() + """Record a failure and release its handle, returning False to retain it. + + Args: + req_id: The request ID. + handle: The transfer handle. + failed_req_ids: Requests observed as failed; each stays listed + until it has no outstanding handles. + record_failed_transfer: Whether to count the failure toward the + transport-failure metric. Callers that already recorded a more + specific metric (eg KV expiry, reported separately from + transport failures) pass False. + + """ + if record_failed_transfer: + self.xfer_stats.record_failed_transfer() if failed_req_ids is not None: failed_req_ids.add(req_id) return handle is None or self._try_release_xfer_handle(req_id, handle) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index 203c6bcfa6..7711a35954 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -149,7 +149,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): engine_id, ) self.xfer_stats.record_kv_expired_req() - self._handle_failed_transfer(req_id, None, self._recv_failures) + # KV expiry is reported separately from transport failures, so only + # the state cleanup side of _handle_failed_transfer runs here. + self._handle_failed_transfer( + req_id, None, self._recv_failures, record_failed_transfer=False + ) return if any(len(group) > 0 for group in meta.local_block_ids): @@ -364,7 +368,20 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): remote_agents = self._remote_agents[meta.remote.engine_id] for rank_to_notify, agent in remote_agents.items(): if rank_to_notify != (0, read_specs[0].remote_rank): - self.nixl_wrapper.send_notif(agent, notif_msg=notif_id) + try: + self.nixl_wrapper.send_notif(agent, notif_msg=notif_id) + except Exception as e: + self._log_failure( + failure_type="notification_failed", + msg="Remote rank will not update request state. " + "This may indicate network issues.", + req_id=req_id, + error=e, + dst_engine_id=meta.remote.engine_id, + remote_rank=rank_to_notify[1], + remote_agent_name=agent, + ) + self.xfer_stats.record_failed_notification() def _read_blocks( self, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py index 01c956a525..fbe85f46a3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py @@ -365,6 +365,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): engine_id, req_id, ) + self.xfer_stats.record_failed_notification() self._failed_recv_reqs.put(req_id) return for rank, agent_name in agents.items(): @@ -377,6 +378,9 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): error=e, remote_rank=rank, ) + self.xfer_stats.record_failed_notification() + # Earlier registrations may still trigger WRITEs into D's blocks. + # Keep the receive pending until those writes are finished. logger.debug( "Sent PUSH_REG for %s to engine %s (%dB)", req_id, engine_id, len(notif_msg) ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py index 2cff3f843b..f6d0684b6b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py @@ -39,6 +39,7 @@ class NixlKVConnectorStats(KVConnectorStats): "num_descriptors": [], "num_failed_transfers": [], "num_failed_notifications": [], + "num_failed_handshakes": [], "num_kv_expired_reqs": [], } @@ -57,6 +58,10 @@ class NixlKVConnectorStats(KVConnectorStats): """Record a failed NIXL notification (send_notif).""" self.data["num_failed_notifications"].append(1) + def record_failed_handshake(self): + """Record a failed NIXL handshake.""" + self.data["num_failed_handshakes"].append(1) + def record_kv_expired_req(self): """Record a request that had its KV blocks expire.""" self.data["num_kv_expired_reqs"].append(1) @@ -72,6 +77,7 @@ class NixlKVConnectorStats(KVConnectorStats): self.num_successful_transfers == 0 and len(self.data["num_failed_transfers"]) == 0 and len(self.data["num_failed_notifications"]) == 0 + and len(self.data["num_failed_handshakes"]) == 0 and len(self.data["num_kv_expired_reqs"]) == 0 ) @@ -84,10 +90,22 @@ class NixlKVConnectorStats(KVConnectorStats): return self def reduce(self) -> dict[str, int | float]: - # Compute compact representative stats suitable for CLI logging + # Compute compact representative stats suitable for CLI logging. + # Failure counts are reported on every interval: transfer, handshake + # and notification failures are grouped as sporadic + # lower-transport-layer events, while KV expiry is reported separately + # as it is an actionable autoscaler signal rather than a transport + # issue. + failure_counts = { + "Num failed transfers": len(self.data["num_failed_transfers"]) + + len(self.data["num_failed_handshakes"]) + + len(self.data["num_failed_notifications"]), + "Num KV expired reqs": len(self.data["num_kv_expired_reqs"]), + } if self.num_successful_transfers == 0: - # CLI logging only reports successful transfers stats. If all requests in - # the interval were unsuccessful, Prom will report failures stats instead. + # Timing / throughput stats only cover successful transfers. If + # all requests in the interval were unsuccessful, the failure + # counts above still surface what happened. return { "Num successful transfers": 0, "Avg xfer time (ms)": 0, @@ -97,6 +115,7 @@ class NixlKVConnectorStats(KVConnectorStats): "Avg MB per transfer": 0, "Throughput (MB/s)": 0, "Avg number of descriptors": 0, + **failure_counts, } xfer_time = np.asarray(self.data["transfer_duration"]) @@ -122,6 +141,7 @@ class NixlKVConnectorStats(KVConnectorStats): "Avg MB per transfer": round(avg_mb.item(), 3), "Throughput (MB/s)": round(throughput_mb_s.item(), 3), "Avg number of descriptors": round(descs.mean().item(), 1), + **failure_counts, } @property @@ -212,15 +232,20 @@ class NixlPromMetrics(KVConnectorPromMetrics): ) counter_nixl_num_failed_transfers = self._counter_cls( name="vllm:nixl_num_failed_transfers", - documentation="Number of failed NIXL KV Cache transfers.", + documentation="Number of failed NIXL KV Cache transfers, including" + " handshake and notification failures. NOTE: KV expiry is tracked" + " separately in vllm:nixl_num_kv_expired_reqs.", labelnames=labelnames, ) self.counter_nixl_num_failed_transfers = create_metric_per_engine( counter_nixl_num_failed_transfers, self.per_engine_labelvalues ) + counter_nixl_num_failed_notifications = self._counter_cls( name="vllm:nixl_num_failed_notifications", - documentation="Number of failed NIXL KV Cache notifications.", + documentation="Number of failed NIXL KV Cache notifications. " + "Retained for compatibility; these failures are also included in " + "vllm:nixl_num_failed_transfers.", labelnames=labelnames, ) self.counter_nixl_num_failed_notifications = create_metric_per_engine( @@ -254,13 +279,26 @@ class NixlPromMetrics(KVConnectorPromMetrics): ): for list_item in transfer_stats_data[list_item_key]: prom_obj[engine_idx].observe(list_item) - for counter_obj, counter_item_key in zip( + for counter_obj, counter_item_keys in zip( [ self.counter_nixl_num_failed_transfers, self.counter_nixl_num_failed_notifications, self.counter_nixl_num_kv_expired_reqs, ], - ["num_failed_transfers", "num_failed_notifications", "num_kv_expired_reqs"], + [ + # Transfer, handshake and notification failures are grouped: + # all are sporadic lower-transport-layer events. KV expiry is + # reported separately as it signals autoscaler behavior, not + # transport health. + ( + "num_failed_transfers", + "num_failed_handshakes", + "num_failed_notifications", + ), + ("num_failed_notifications",), + ("num_kv_expired_reqs",), + ], ): - for list_item in transfer_stats_data[counter_item_key]: - counter_obj[engine_idx].inc(list_item) + for counter_item_key in counter_item_keys: + for list_item in transfer_stats_data[counter_item_key]: + counter_obj[engine_idx].inc(list_item)