[Nixl] Separate transport-failure metrics from KV expiry (#55854)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -1306,6 +1306,8 @@ class NixlBaseConnectorWorker:
|
||||
error=e,
|
||||
remote_engine_id=eid,
|
||||
)
|
||||
# Count once per handshake, regardless of waiting requests.
|
||||
self.xfer_stats.record_failed_handshake()
|
||||
|
||||
fut.add_done_callback(done_callback)
|
||||
return fut
|
||||
@@ -2811,13 +2813,17 @@ class NixlBaseConnectorWorker:
|
||||
done_sending.update(self._replicated_pcp_done_sending)
|
||||
self._replicated_pcp_done_sending.clear()
|
||||
|
||||
# Process receive failures reported by background threads.
|
||||
# Process receive failures reported by background threads. Each
|
||||
# producer already recorded a specific metric (handshake or
|
||||
# notification), so only the failure bookkeeping runs here.
|
||||
while not self._failed_recv_reqs.empty():
|
||||
try:
|
||||
req_id = self._failed_recv_reqs.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self._handle_failed_transfer(req_id, None, self._recv_failures)
|
||||
self._handle_failed_transfer(
|
||||
req_id, None, self._recv_failures, record_failed_transfer=False
|
||||
)
|
||||
|
||||
failed_recv_reqs: set[ReqId] = set()
|
||||
if self._recv_failures:
|
||||
@@ -3059,10 +3065,27 @@ class NixlBaseConnectorWorker:
|
||||
return True
|
||||
|
||||
def _handle_failed_transfer(
|
||||
self, req_id: str, handle: int | None, failed_req_ids: set[str] | None = None
|
||||
self,
|
||||
req_id: str,
|
||||
handle: int | None,
|
||||
failed_req_ids: set[str] | None = None,
|
||||
record_failed_transfer: bool = True,
|
||||
) -> bool:
|
||||
"""Record a failure and release its handle, returning False to retain it."""
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
"""Record a failure and release its handle, returning False to retain it.
|
||||
|
||||
Args:
|
||||
req_id: The request ID.
|
||||
handle: The transfer handle.
|
||||
failed_req_ids: Requests observed as failed; each stays listed
|
||||
until it has no outstanding handles.
|
||||
record_failed_transfer: Whether to count the failure toward the
|
||||
transport-failure metric. Callers that already recorded a more
|
||||
specific metric (eg KV expiry, reported separately from
|
||||
transport failures) pass False.
|
||||
|
||||
"""
|
||||
if record_failed_transfer:
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
if failed_req_ids is not None:
|
||||
failed_req_ids.add(req_id)
|
||||
return handle is None or self._try_release_xfer_handle(req_id, handle)
|
||||
|
||||
@@ -149,7 +149,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
engine_id,
|
||||
)
|
||||
self.xfer_stats.record_kv_expired_req()
|
||||
self._handle_failed_transfer(req_id, None, self._recv_failures)
|
||||
# KV expiry is reported separately from transport failures, so only
|
||||
# the state cleanup side of _handle_failed_transfer runs here.
|
||||
self._handle_failed_transfer(
|
||||
req_id, None, self._recv_failures, record_failed_transfer=False
|
||||
)
|
||||
return
|
||||
|
||||
if any(len(group) > 0 for group in meta.local_block_ids):
|
||||
@@ -364,7 +368,20 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
remote_agents = self._remote_agents[meta.remote.engine_id]
|
||||
for rank_to_notify, agent in remote_agents.items():
|
||||
if rank_to_notify != (0, read_specs[0].remote_rank):
|
||||
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
|
||||
try:
|
||||
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="notification_failed",
|
||||
msg="Remote rank will not update request state. "
|
||||
"This may indicate network issues.",
|
||||
req_id=req_id,
|
||||
error=e,
|
||||
dst_engine_id=meta.remote.engine_id,
|
||||
remote_rank=rank_to_notify[1],
|
||||
remote_agent_name=agent,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
|
||||
def _read_blocks(
|
||||
self,
|
||||
|
||||
@@ -365,6 +365,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
engine_id,
|
||||
req_id,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
self._failed_recv_reqs.put(req_id)
|
||||
return
|
||||
for rank, agent_name in agents.items():
|
||||
@@ -377,6 +378,9 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
error=e,
|
||||
remote_rank=rank,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
# Earlier registrations may still trigger WRITEs into D's blocks.
|
||||
# Keep the receive pending until those writes are finished.
|
||||
logger.debug(
|
||||
"Sent PUSH_REG for %s to engine %s (%dB)", req_id, engine_id, len(notif_msg)
|
||||
)
|
||||
|
||||
@@ -39,6 +39,7 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
"num_descriptors": [],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
|
||||
@@ -57,6 +58,10 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
"""Record a failed NIXL notification (send_notif)."""
|
||||
self.data["num_failed_notifications"].append(1)
|
||||
|
||||
def record_failed_handshake(self):
|
||||
"""Record a failed NIXL handshake."""
|
||||
self.data["num_failed_handshakes"].append(1)
|
||||
|
||||
def record_kv_expired_req(self):
|
||||
"""Record a request that had its KV blocks expire."""
|
||||
self.data["num_kv_expired_reqs"].append(1)
|
||||
@@ -72,6 +77,7 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
self.num_successful_transfers == 0
|
||||
and len(self.data["num_failed_transfers"]) == 0
|
||||
and len(self.data["num_failed_notifications"]) == 0
|
||||
and len(self.data["num_failed_handshakes"]) == 0
|
||||
and len(self.data["num_kv_expired_reqs"]) == 0
|
||||
)
|
||||
|
||||
@@ -84,10 +90,22 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
# Compute compact representative stats suitable for CLI logging
|
||||
# Compute compact representative stats suitable for CLI logging.
|
||||
# Failure counts are reported on every interval: transfer, handshake
|
||||
# and notification failures are grouped as sporadic
|
||||
# lower-transport-layer events, while KV expiry is reported separately
|
||||
# as it is an actionable autoscaler signal rather than a transport
|
||||
# issue.
|
||||
failure_counts = {
|
||||
"Num failed transfers": len(self.data["num_failed_transfers"])
|
||||
+ len(self.data["num_failed_handshakes"])
|
||||
+ len(self.data["num_failed_notifications"]),
|
||||
"Num KV expired reqs": len(self.data["num_kv_expired_reqs"]),
|
||||
}
|
||||
if self.num_successful_transfers == 0:
|
||||
# CLI logging only reports successful transfers stats. If all requests in
|
||||
# the interval were unsuccessful, Prom will report failures stats instead.
|
||||
# Timing / throughput stats only cover successful transfers. If
|
||||
# all requests in the interval were unsuccessful, the failure
|
||||
# counts above still surface what happened.
|
||||
return {
|
||||
"Num successful transfers": 0,
|
||||
"Avg xfer time (ms)": 0,
|
||||
@@ -97,6 +115,7 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
"Avg MB per transfer": 0,
|
||||
"Throughput (MB/s)": 0,
|
||||
"Avg number of descriptors": 0,
|
||||
**failure_counts,
|
||||
}
|
||||
|
||||
xfer_time = np.asarray(self.data["transfer_duration"])
|
||||
@@ -122,6 +141,7 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
"Avg MB per transfer": round(avg_mb.item(), 3),
|
||||
"Throughput (MB/s)": round(throughput_mb_s.item(), 3),
|
||||
"Avg number of descriptors": round(descs.mean().item(), 1),
|
||||
**failure_counts,
|
||||
}
|
||||
|
||||
@property
|
||||
@@ -212,15 +232,20 @@ class NixlPromMetrics(KVConnectorPromMetrics):
|
||||
)
|
||||
counter_nixl_num_failed_transfers = self._counter_cls(
|
||||
name="vllm:nixl_num_failed_transfers",
|
||||
documentation="Number of failed NIXL KV Cache transfers.",
|
||||
documentation="Number of failed NIXL KV Cache transfers, including"
|
||||
" handshake and notification failures. NOTE: KV expiry is tracked"
|
||||
" separately in vllm:nixl_num_kv_expired_reqs.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_nixl_num_failed_transfers = create_metric_per_engine(
|
||||
counter_nixl_num_failed_transfers, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
counter_nixl_num_failed_notifications = self._counter_cls(
|
||||
name="vllm:nixl_num_failed_notifications",
|
||||
documentation="Number of failed NIXL KV Cache notifications.",
|
||||
documentation="Number of failed NIXL KV Cache notifications. "
|
||||
"Retained for compatibility; these failures are also included in "
|
||||
"vllm:nixl_num_failed_transfers.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_nixl_num_failed_notifications = create_metric_per_engine(
|
||||
@@ -254,13 +279,26 @@ class NixlPromMetrics(KVConnectorPromMetrics):
|
||||
):
|
||||
for list_item in transfer_stats_data[list_item_key]:
|
||||
prom_obj[engine_idx].observe(list_item)
|
||||
for counter_obj, counter_item_key in zip(
|
||||
for counter_obj, counter_item_keys in zip(
|
||||
[
|
||||
self.counter_nixl_num_failed_transfers,
|
||||
self.counter_nixl_num_failed_notifications,
|
||||
self.counter_nixl_num_kv_expired_reqs,
|
||||
],
|
||||
["num_failed_transfers", "num_failed_notifications", "num_kv_expired_reqs"],
|
||||
[
|
||||
# Transfer, handshake and notification failures are grouped:
|
||||
# all are sporadic lower-transport-layer events. KV expiry is
|
||||
# reported separately as it signals autoscaler behavior, not
|
||||
# transport health.
|
||||
(
|
||||
"num_failed_transfers",
|
||||
"num_failed_handshakes",
|
||||
"num_failed_notifications",
|
||||
),
|
||||
("num_failed_notifications",),
|
||||
("num_kv_expired_reqs",),
|
||||
],
|
||||
):
|
||||
for list_item in transfer_stats_data[counter_item_key]:
|
||||
counter_obj[engine_idx].inc(list_item)
|
||||
for counter_item_key in counter_item_keys:
|
||||
for list_item in transfer_stats_data[counter_item_key]:
|
||||
counter_obj[engine_idx].inc(list_item)
|
||||
|
||||
Reference in New Issue
Block a user