[Nixl] Separate transport-failure metrics from KV expiry (#55854)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-09-18 18:08:59 +02:00
committed by GitHub
parent a7156060c1
commit 164dca8d1e
9 changed files with 224 additions and 22 deletions
@@ -1306,6 +1306,8 @@ class NixlBaseConnectorWorker:
error=e,
remote_engine_id=eid,
)
# Count once per handshake, regardless of waiting requests.
self.xfer_stats.record_failed_handshake()
fut.add_done_callback(done_callback)
return fut
@@ -2811,13 +2813,17 @@ class NixlBaseConnectorWorker:
done_sending.update(self._replicated_pcp_done_sending)
self._replicated_pcp_done_sending.clear()
# Process receive failures reported by background threads.
# Process receive failures reported by background threads. Each
# producer already recorded a specific metric (handshake or
# notification), so only the failure bookkeeping runs here.
while not self._failed_recv_reqs.empty():
try:
req_id = self._failed_recv_reqs.get_nowait()
except queue.Empty:
break
self._handle_failed_transfer(req_id, None, self._recv_failures)
self._handle_failed_transfer(
req_id, None, self._recv_failures, record_failed_transfer=False
)
failed_recv_reqs: set[ReqId] = set()
if self._recv_failures:
@@ -3059,10 +3065,27 @@ class NixlBaseConnectorWorker:
return True
def _handle_failed_transfer(
self, req_id: str, handle: int | None, failed_req_ids: set[str] | None = None
self,
req_id: str,
handle: int | None,
failed_req_ids: set[str] | None = None,
record_failed_transfer: bool = True,
) -> bool:
"""Record a failure and release its handle, returning False to retain it."""
self.xfer_stats.record_failed_transfer()
"""Record a failure and release its handle, returning False to retain it.
Args:
req_id: The request ID.
handle: The transfer handle.
failed_req_ids: Requests observed as failed; each stays listed
until it has no outstanding handles.
record_failed_transfer: Whether to count the failure toward the
transport-failure metric. Callers that already recorded a more
specific metric (eg KV expiry, reported separately from
transport failures) pass False.
"""
if record_failed_transfer:
self.xfer_stats.record_failed_transfer()
if failed_req_ids is not None:
failed_req_ids.add(req_id)
return handle is None or self._try_release_xfer_handle(req_id, handle)
@@ -149,7 +149,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
engine_id,
)
self.xfer_stats.record_kv_expired_req()
self._handle_failed_transfer(req_id, None, self._recv_failures)
# KV expiry is reported separately from transport failures, so only
# the state cleanup side of _handle_failed_transfer runs here.
self._handle_failed_transfer(
req_id, None, self._recv_failures, record_failed_transfer=False
)
return
if any(len(group) > 0 for group in meta.local_block_ids):
@@ -364,7 +368,20 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
remote_agents = self._remote_agents[meta.remote.engine_id]
for rank_to_notify, agent in remote_agents.items():
if rank_to_notify != (0, read_specs[0].remote_rank):
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
try:
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
except Exception as e:
self._log_failure(
failure_type="notification_failed",
msg="Remote rank will not update request state. "
"This may indicate network issues.",
req_id=req_id,
error=e,
dst_engine_id=meta.remote.engine_id,
remote_rank=rank_to_notify[1],
remote_agent_name=agent,
)
self.xfer_stats.record_failed_notification()
def _read_blocks(
self,
@@ -365,6 +365,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
engine_id,
req_id,
)
self.xfer_stats.record_failed_notification()
self._failed_recv_reqs.put(req_id)
return
for rank, agent_name in agents.items():
@@ -377,6 +378,9 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
error=e,
remote_rank=rank,
)
self.xfer_stats.record_failed_notification()
# Earlier registrations may still trigger WRITEs into D's blocks.
# Keep the receive pending until those writes are finished.
logger.debug(
"Sent PUSH_REG for %s to engine %s (%dB)", req_id, engine_id, len(notif_msg)
)
@@ -39,6 +39,7 @@ class NixlKVConnectorStats(KVConnectorStats):
"num_descriptors": [],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
}
@@ -57,6 +58,10 @@ class NixlKVConnectorStats(KVConnectorStats):
"""Record a failed NIXL notification (send_notif)."""
self.data["num_failed_notifications"].append(1)
def record_failed_handshake(self):
"""Record a failed NIXL handshake."""
self.data["num_failed_handshakes"].append(1)
def record_kv_expired_req(self):
"""Record a request that had its KV blocks expire."""
self.data["num_kv_expired_reqs"].append(1)
@@ -72,6 +77,7 @@ class NixlKVConnectorStats(KVConnectorStats):
self.num_successful_transfers == 0
and len(self.data["num_failed_transfers"]) == 0
and len(self.data["num_failed_notifications"]) == 0
and len(self.data["num_failed_handshakes"]) == 0
and len(self.data["num_kv_expired_reqs"]) == 0
)
@@ -84,10 +90,22 @@ class NixlKVConnectorStats(KVConnectorStats):
return self
def reduce(self) -> dict[str, int | float]:
# Compute compact representative stats suitable for CLI logging
# Compute compact representative stats suitable for CLI logging.
# Failure counts are reported on every interval: transfer, handshake
# and notification failures are grouped as sporadic
# lower-transport-layer events, while KV expiry is reported separately
# as it is an actionable autoscaler signal rather than a transport
# issue.
failure_counts = {
"Num failed transfers": len(self.data["num_failed_transfers"])
+ len(self.data["num_failed_handshakes"])
+ len(self.data["num_failed_notifications"]),
"Num KV expired reqs": len(self.data["num_kv_expired_reqs"]),
}
if self.num_successful_transfers == 0:
# CLI logging only reports successful transfers stats. If all requests in
# the interval were unsuccessful, Prom will report failures stats instead.
# Timing / throughput stats only cover successful transfers. If
# all requests in the interval were unsuccessful, the failure
# counts above still surface what happened.
return {
"Num successful transfers": 0,
"Avg xfer time (ms)": 0,
@@ -97,6 +115,7 @@ class NixlKVConnectorStats(KVConnectorStats):
"Avg MB per transfer": 0,
"Throughput (MB/s)": 0,
"Avg number of descriptors": 0,
**failure_counts,
}
xfer_time = np.asarray(self.data["transfer_duration"])
@@ -122,6 +141,7 @@ class NixlKVConnectorStats(KVConnectorStats):
"Avg MB per transfer": round(avg_mb.item(), 3),
"Throughput (MB/s)": round(throughput_mb_s.item(), 3),
"Avg number of descriptors": round(descs.mean().item(), 1),
**failure_counts,
}
@property
@@ -212,15 +232,20 @@ class NixlPromMetrics(KVConnectorPromMetrics):
)
counter_nixl_num_failed_transfers = self._counter_cls(
name="vllm:nixl_num_failed_transfers",
documentation="Number of failed NIXL KV Cache transfers.",
documentation="Number of failed NIXL KV Cache transfers, including"
" handshake and notification failures. NOTE: KV expiry is tracked"
" separately in vllm:nixl_num_kv_expired_reqs.",
labelnames=labelnames,
)
self.counter_nixl_num_failed_transfers = create_metric_per_engine(
counter_nixl_num_failed_transfers, self.per_engine_labelvalues
)
counter_nixl_num_failed_notifications = self._counter_cls(
name="vllm:nixl_num_failed_notifications",
documentation="Number of failed NIXL KV Cache notifications.",
documentation="Number of failed NIXL KV Cache notifications. "
"Retained for compatibility; these failures are also included in "
"vllm:nixl_num_failed_transfers.",
labelnames=labelnames,
)
self.counter_nixl_num_failed_notifications = create_metric_per_engine(
@@ -254,13 +279,26 @@ class NixlPromMetrics(KVConnectorPromMetrics):
):
for list_item in transfer_stats_data[list_item_key]:
prom_obj[engine_idx].observe(list_item)
for counter_obj, counter_item_key in zip(
for counter_obj, counter_item_keys in zip(
[
self.counter_nixl_num_failed_transfers,
self.counter_nixl_num_failed_notifications,
self.counter_nixl_num_kv_expired_reqs,
],
["num_failed_transfers", "num_failed_notifications", "num_kv_expired_reqs"],
[
# Transfer, handshake and notification failures are grouped:
# all are sporadic lower-transport-layer events. KV expiry is
# reported separately as it signals autoscaler behavior, not
# transport health.
(
"num_failed_transfers",
"num_failed_handshakes",
"num_failed_notifications",
),
("num_failed_notifications",),
("num_kv_expired_reqs",),
],
):
for list_item in transfer_stats_data[counter_item_key]:
counter_obj[engine_idx].inc(list_item)
for counter_item_key in counter_item_keys:
for list_item in transfer_stats_data[counter_item_key]:
counter_obj[engine_idx].inc(list_item)