[Nixl] Separate transport-failure metrics from KV expiry (#55854)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -418,12 +418,13 @@ activity for the last reporting interval. Example output:
|
||||
```text
|
||||
KV Transfer metrics: Num successful transfers=4, Avg xfer time (ms)=1.381,
|
||||
P90 xfer time (ms)=2.601, Avg post time (ms)=0.672, P90 post time (ms)=0.801,
|
||||
Avg MB per transfer=2.25, Throughput (MB/s)=1629.549, Avg number of descriptors=72.0
|
||||
Avg MB per transfer=2.25, Throughput (MB/s)=1629.549, Avg number of descriptors=72.0,
|
||||
Num failed transfers=0, Num KV expired reqs=0
|
||||
```
|
||||
|
||||
The table below describes each field. All timing values cover only the
|
||||
successful transfers recorded in the current interval; failed transfers are
|
||||
counted separately via Prometheus (see
|
||||
successful transfers recorded in the current interval; failures are reported
|
||||
as interval counts in the same line and via Prometheus (see
|
||||
[Prometheus metrics](#prometheus-metrics) below).
|
||||
|
||||
| Metric | Unit | Description |
|
||||
@@ -436,6 +437,8 @@ counted separately via Prometheus (see
|
||||
| `Avg MB per transfer` | MB | Mean payload size per transfer, computed as `total bytes transferred / number of transfers`. Reflects the average KV cache footprint of a single request (sequence length × layers × head dimension × dtype bytes). |
|
||||
| `Throughput (MB/s)` | MB/s | Effective bandwidth over the interval: `total MB transferred / total xfer time (s)` across all successful transfers. This is aggregate throughput, not per-request bandwidth. |
|
||||
| `Avg number of descriptors` | count | Mean number of NIXL memory descriptors (scatter-gather segments) submitted per transfer. More descriptors indicate more fragmented or larger KV cache allocations; very high counts can increase descriptor-registration overhead. |
|
||||
| `Num failed transfers` | count | Failed NIXL transfers, handshakes, and completion notifications (`send_notif`) in the interval, grouped because all are sporadic lower-transport-layer events. |
|
||||
| `Num KV expired reqs` | count | Requests whose KV blocks expired before being read. Reported separately from the failure counts above because it is an autoscaler/lease-tuning signal rather than a transport issue. |
|
||||
|
||||
### Prometheus metrics
|
||||
|
||||
@@ -448,9 +451,9 @@ exported when NixlConnector is active:
|
||||
| `vllm:nixl_post_time_seconds` | Histogram | Time to submit the transfer request to the RDMA backend (seconds). |
|
||||
| `vllm:nixl_bytes_transferred` | Histogram | Bytes moved per transfer. |
|
||||
| `vllm:nixl_num_descriptors` | Histogram | Descriptor count per transfer. |
|
||||
| `vllm:nixl_num_failed_transfers` | Counter | Cumulative count of failed NIXL KV-block transfers. |
|
||||
| `vllm:nixl_num_failed_notifications` | Counter | Cumulative count of failed completion notifications (`send_notif`). |
|
||||
| `vllm:nixl_num_kv_expired_reqs` | Counter | Requests whose KV blocks expired on the prefiller before the decoder read them (tracked on the P instance). |
|
||||
| `vllm:nixl_num_failed_transfers` | Counter | Cumulative count of failed NIXL KV-block transfers, including handshake and notification (`send_notif`) failures. These are grouped because all are sporadic lower-transport-layer events. |
|
||||
| `vllm:nixl_num_failed_notifications` | Counter | Cumulative count of failed notifications (`send_notif`), retained for compatibility. These failures are also included in `vllm:nixl_num_failed_transfers`; do not sum the two counters. |
|
||||
| `vllm:nixl_num_kv_expired_reqs` | Counter | Requests whose KV blocks expired on the prefiller before the decoder read them (tracked on the P instance). Kept separate from the failure counters above: KV expiry is an autoscaler/lease-tuning signal, not a transport issue. |
|
||||
|
||||
!!! tip
|
||||
High `vllm:nixl_num_kv_expired_reqs` indicates that the prefiller's lease
|
||||
|
||||
@@ -969,6 +969,9 @@ def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined):
|
||||
assert worker.xfer_stats.data["num_kv_expired_reqs"] == (
|
||||
[1] if expect_declined else []
|
||||
)
|
||||
# KV expiry must not pollute the transport failure counters.
|
||||
assert worker.xfer_stats.data["num_failed_transfers"] == []
|
||||
assert worker.xfer_stats.data["num_failed_handshakes"] == []
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert "req" in done_recving
|
||||
|
||||
|
||||
@@ -567,6 +567,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10, 20],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -590,6 +592,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
},
|
||||
"MockConnector": {"mock_field": [1, 2, 3]},
|
||||
}
|
||||
@@ -619,6 +623,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -638,6 +644,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
mock_stats = MockConnectorStats(data={"mock_field": [1, 2, 3]})
|
||||
@@ -667,6 +675,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
|
||||
@@ -698,6 +708,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
},
|
||||
"ExampleConnector": {"some_field": [1, 2, 3]},
|
||||
}
|
||||
@@ -735,6 +747,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -750,6 +764,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [20],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -781,6 +797,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -807,6 +825,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10, 20],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -833,6 +853,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10, 20],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -862,6 +884,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10, 20],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1682,6 +1682,83 @@ def test_kv_connector_stats_aggregation():
|
||||
assert all(not isinstance(v, np.generic) for v in cli_stats.values())
|
||||
|
||||
|
||||
def test_kv_connector_stats_failure_grouping():
|
||||
"""Transfer, handshake and notification failures are reported as one
|
||||
transport-failure count, while KV expiry is reported separately: the
|
||||
former are sporadic lower-transport-layer events, the latter an
|
||||
autoscaler signal."""
|
||||
stats = NixlKVConnectorStats()
|
||||
assert stats.is_empty()
|
||||
|
||||
stats.record_failed_transfer()
|
||||
stats.record_failed_handshake()
|
||||
stats.record_failed_notification()
|
||||
stats.record_kv_expired_req()
|
||||
assert not stats.is_empty()
|
||||
|
||||
# No successful transfers: latency stats are zero but the failure
|
||||
# counts still surface.
|
||||
reduced = stats.reduce()
|
||||
assert reduced["Num successful transfers"] == 0
|
||||
assert reduced["Num failed transfers"] == 3
|
||||
assert reduced["Num KV expired reqs"] == 1
|
||||
|
||||
|
||||
def test_nixl_prom_metrics_group_handshake_with_transfer_failures():
|
||||
"""vllm:nixl_num_failed_transfers counts handshake and notification
|
||||
failures too, while vllm:nixl_num_kv_expired_reqs stays a separate
|
||||
counter."""
|
||||
from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
|
||||
NixlPromMetrics,
|
||||
)
|
||||
|
||||
registry = CollectorRegistry()
|
||||
|
||||
class RegistryGauge(Gauge):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, registry=registry, **kwargs)
|
||||
|
||||
class RegistryCounter(Counter):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, registry=registry, **kwargs)
|
||||
|
||||
class RegistryHistogram(Histogram):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, registry=registry, **kwargs)
|
||||
|
||||
vllm_config = create_vllm_config()
|
||||
metric_types = {
|
||||
Gauge: RegistryGauge,
|
||||
Counter: RegistryCounter,
|
||||
Histogram: RegistryHistogram,
|
||||
}
|
||||
prom = NixlPromMetrics(
|
||||
vllm_config,
|
||||
metric_types,
|
||||
labelnames=["engine"],
|
||||
per_engine_labelvalues={0: ["engine-0"]},
|
||||
)
|
||||
|
||||
stats = NixlKVConnectorStats()
|
||||
stats.record_failed_transfer()
|
||||
stats.record_failed_handshake()
|
||||
stats.record_failed_notification()
|
||||
stats.record_kv_expired_req()
|
||||
prom.observe(stats.data, engine_idx=0)
|
||||
|
||||
def counter_value(name: str) -> float:
|
||||
for metric in registry.collect():
|
||||
for sample in metric.samples:
|
||||
if sample.name == name:
|
||||
return sample.value
|
||||
raise AssertionError(f"metric {name} not found in registry")
|
||||
|
||||
assert counter_value("vllm:nixl_num_failed_transfers_total") == 3.0
|
||||
assert counter_value("vllm:nixl_num_kv_expired_reqs_total") == 1.0
|
||||
|
||||
|
||||
def test_multi_kv_connector_stats_aggregation():
|
||||
"""Test MultiKVConnectorStats aggregation across TP ranks using
|
||||
KVOutputAggregator (used by MultiprocExecutor).
|
||||
@@ -3029,6 +3106,11 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
|
||||
assert request_id in done_recving
|
||||
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
|
||||
|
||||
# Handshake failures are recorded as transport failures, separately
|
||||
# from KV expiry.
|
||||
assert connector.connector_worker.xfer_stats.data["num_failed_handshakes"]
|
||||
assert connector.connector_worker.xfer_stats.data["num_failed_transfers"] == []
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
|
||||
@@ -47,6 +47,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
NixlPushConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
|
||||
NixlKVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
|
||||
get_base_request_id,
|
||||
@@ -360,6 +363,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w._recv_failures = set()
|
||||
w._recving_transfers = defaultdict(list)
|
||||
w._is_hma_required = False
|
||||
w.xfer_stats = NixlKVConnectorStats()
|
||||
w._reqs_to_process = set()
|
||||
w._reqs_to_send = {}
|
||||
w.consumer_notification_counts_by_req = defaultdict(int)
|
||||
@@ -730,6 +734,10 @@ def test_do_start_push_kv_drops_request_on_handshake_failure():
|
||||
assert xfer_calls == []
|
||||
assert len(failures) == 1
|
||||
assert failures[0]["failure_type"] == "push_handshake_failed"
|
||||
# The handshake metric is counted once by _ensure_handshake's own
|
||||
# callback (stubbed out here); the push side must not double-record it.
|
||||
assert w.xfer_stats.data["num_failed_handshakes"] == []
|
||||
assert w.xfer_stats.data["num_failed_transfers"] == []
|
||||
|
||||
|
||||
def test_writer_loop_drains_deferred_push_inbox():
|
||||
|
||||
@@ -1306,6 +1306,8 @@ class NixlBaseConnectorWorker:
|
||||
error=e,
|
||||
remote_engine_id=eid,
|
||||
)
|
||||
# Count once per handshake, regardless of waiting requests.
|
||||
self.xfer_stats.record_failed_handshake()
|
||||
|
||||
fut.add_done_callback(done_callback)
|
||||
return fut
|
||||
@@ -2811,13 +2813,17 @@ class NixlBaseConnectorWorker:
|
||||
done_sending.update(self._replicated_pcp_done_sending)
|
||||
self._replicated_pcp_done_sending.clear()
|
||||
|
||||
# Process receive failures reported by background threads.
|
||||
# Process receive failures reported by background threads. Each
|
||||
# producer already recorded a specific metric (handshake or
|
||||
# notification), so only the failure bookkeeping runs here.
|
||||
while not self._failed_recv_reqs.empty():
|
||||
try:
|
||||
req_id = self._failed_recv_reqs.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self._handle_failed_transfer(req_id, None, self._recv_failures)
|
||||
self._handle_failed_transfer(
|
||||
req_id, None, self._recv_failures, record_failed_transfer=False
|
||||
)
|
||||
|
||||
failed_recv_reqs: set[ReqId] = set()
|
||||
if self._recv_failures:
|
||||
@@ -3059,10 +3065,27 @@ class NixlBaseConnectorWorker:
|
||||
return True
|
||||
|
||||
def _handle_failed_transfer(
|
||||
self, req_id: str, handle: int | None, failed_req_ids: set[str] | None = None
|
||||
self,
|
||||
req_id: str,
|
||||
handle: int | None,
|
||||
failed_req_ids: set[str] | None = None,
|
||||
record_failed_transfer: bool = True,
|
||||
) -> bool:
|
||||
"""Record a failure and release its handle, returning False to retain it."""
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
"""Record a failure and release its handle, returning False to retain it.
|
||||
|
||||
Args:
|
||||
req_id: The request ID.
|
||||
handle: The transfer handle.
|
||||
failed_req_ids: Requests observed as failed; each stays listed
|
||||
until it has no outstanding handles.
|
||||
record_failed_transfer: Whether to count the failure toward the
|
||||
transport-failure metric. Callers that already recorded a more
|
||||
specific metric (eg KV expiry, reported separately from
|
||||
transport failures) pass False.
|
||||
|
||||
"""
|
||||
if record_failed_transfer:
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
if failed_req_ids is not None:
|
||||
failed_req_ids.add(req_id)
|
||||
return handle is None or self._try_release_xfer_handle(req_id, handle)
|
||||
|
||||
@@ -149,7 +149,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
engine_id,
|
||||
)
|
||||
self.xfer_stats.record_kv_expired_req()
|
||||
self._handle_failed_transfer(req_id, None, self._recv_failures)
|
||||
# KV expiry is reported separately from transport failures, so only
|
||||
# the state cleanup side of _handle_failed_transfer runs here.
|
||||
self._handle_failed_transfer(
|
||||
req_id, None, self._recv_failures, record_failed_transfer=False
|
||||
)
|
||||
return
|
||||
|
||||
if any(len(group) > 0 for group in meta.local_block_ids):
|
||||
@@ -364,7 +368,20 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
remote_agents = self._remote_agents[meta.remote.engine_id]
|
||||
for rank_to_notify, agent in remote_agents.items():
|
||||
if rank_to_notify != (0, read_specs[0].remote_rank):
|
||||
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
|
||||
try:
|
||||
self.nixl_wrapper.send_notif(agent, notif_msg=notif_id)
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="notification_failed",
|
||||
msg="Remote rank will not update request state. "
|
||||
"This may indicate network issues.",
|
||||
req_id=req_id,
|
||||
error=e,
|
||||
dst_engine_id=meta.remote.engine_id,
|
||||
remote_rank=rank_to_notify[1],
|
||||
remote_agent_name=agent,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
|
||||
def _read_blocks(
|
||||
self,
|
||||
|
||||
@@ -365,6 +365,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
engine_id,
|
||||
req_id,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
self._failed_recv_reqs.put(req_id)
|
||||
return
|
||||
for rank, agent_name in agents.items():
|
||||
@@ -377,6 +378,9 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
error=e,
|
||||
remote_rank=rank,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
# Earlier registrations may still trigger WRITEs into D's blocks.
|
||||
# Keep the receive pending until those writes are finished.
|
||||
logger.debug(
|
||||
"Sent PUSH_REG for %s to engine %s (%dB)", req_id, engine_id, len(notif_msg)
|
||||
)
|
||||
|
||||
@@ -39,6 +39,7 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
"num_descriptors": [],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
|
||||
@@ -57,6 +58,10 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
"""Record a failed NIXL notification (send_notif)."""
|
||||
self.data["num_failed_notifications"].append(1)
|
||||
|
||||
def record_failed_handshake(self):
|
||||
"""Record a failed NIXL handshake."""
|
||||
self.data["num_failed_handshakes"].append(1)
|
||||
|
||||
def record_kv_expired_req(self):
|
||||
"""Record a request that had its KV blocks expire."""
|
||||
self.data["num_kv_expired_reqs"].append(1)
|
||||
@@ -72,6 +77,7 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
self.num_successful_transfers == 0
|
||||
and len(self.data["num_failed_transfers"]) == 0
|
||||
and len(self.data["num_failed_notifications"]) == 0
|
||||
and len(self.data["num_failed_handshakes"]) == 0
|
||||
and len(self.data["num_kv_expired_reqs"]) == 0
|
||||
)
|
||||
|
||||
@@ -84,10 +90,22 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
# Compute compact representative stats suitable for CLI logging
|
||||
# Compute compact representative stats suitable for CLI logging.
|
||||
# Failure counts are reported on every interval: transfer, handshake
|
||||
# and notification failures are grouped as sporadic
|
||||
# lower-transport-layer events, while KV expiry is reported separately
|
||||
# as it is an actionable autoscaler signal rather than a transport
|
||||
# issue.
|
||||
failure_counts = {
|
||||
"Num failed transfers": len(self.data["num_failed_transfers"])
|
||||
+ len(self.data["num_failed_handshakes"])
|
||||
+ len(self.data["num_failed_notifications"]),
|
||||
"Num KV expired reqs": len(self.data["num_kv_expired_reqs"]),
|
||||
}
|
||||
if self.num_successful_transfers == 0:
|
||||
# CLI logging only reports successful transfers stats. If all requests in
|
||||
# the interval were unsuccessful, Prom will report failures stats instead.
|
||||
# Timing / throughput stats only cover successful transfers. If
|
||||
# all requests in the interval were unsuccessful, the failure
|
||||
# counts above still surface what happened.
|
||||
return {
|
||||
"Num successful transfers": 0,
|
||||
"Avg xfer time (ms)": 0,
|
||||
@@ -97,6 +115,7 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
"Avg MB per transfer": 0,
|
||||
"Throughput (MB/s)": 0,
|
||||
"Avg number of descriptors": 0,
|
||||
**failure_counts,
|
||||
}
|
||||
|
||||
xfer_time = np.asarray(self.data["transfer_duration"])
|
||||
@@ -122,6 +141,7 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
"Avg MB per transfer": round(avg_mb.item(), 3),
|
||||
"Throughput (MB/s)": round(throughput_mb_s.item(), 3),
|
||||
"Avg number of descriptors": round(descs.mean().item(), 1),
|
||||
**failure_counts,
|
||||
}
|
||||
|
||||
@property
|
||||
@@ -212,15 +232,20 @@ class NixlPromMetrics(KVConnectorPromMetrics):
|
||||
)
|
||||
counter_nixl_num_failed_transfers = self._counter_cls(
|
||||
name="vllm:nixl_num_failed_transfers",
|
||||
documentation="Number of failed NIXL KV Cache transfers.",
|
||||
documentation="Number of failed NIXL KV Cache transfers, including"
|
||||
" handshake and notification failures. NOTE: KV expiry is tracked"
|
||||
" separately in vllm:nixl_num_kv_expired_reqs.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_nixl_num_failed_transfers = create_metric_per_engine(
|
||||
counter_nixl_num_failed_transfers, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
counter_nixl_num_failed_notifications = self._counter_cls(
|
||||
name="vllm:nixl_num_failed_notifications",
|
||||
documentation="Number of failed NIXL KV Cache notifications.",
|
||||
documentation="Number of failed NIXL KV Cache notifications. "
|
||||
"Retained for compatibility; these failures are also included in "
|
||||
"vllm:nixl_num_failed_transfers.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_nixl_num_failed_notifications = create_metric_per_engine(
|
||||
@@ -254,13 +279,26 @@ class NixlPromMetrics(KVConnectorPromMetrics):
|
||||
):
|
||||
for list_item in transfer_stats_data[list_item_key]:
|
||||
prom_obj[engine_idx].observe(list_item)
|
||||
for counter_obj, counter_item_key in zip(
|
||||
for counter_obj, counter_item_keys in zip(
|
||||
[
|
||||
self.counter_nixl_num_failed_transfers,
|
||||
self.counter_nixl_num_failed_notifications,
|
||||
self.counter_nixl_num_kv_expired_reqs,
|
||||
],
|
||||
["num_failed_transfers", "num_failed_notifications", "num_kv_expired_reqs"],
|
||||
[
|
||||
# Transfer, handshake and notification failures are grouped:
|
||||
# all are sporadic lower-transport-layer events. KV expiry is
|
||||
# reported separately as it signals autoscaler behavior, not
|
||||
# transport health.
|
||||
(
|
||||
"num_failed_transfers",
|
||||
"num_failed_handshakes",
|
||||
"num_failed_notifications",
|
||||
),
|
||||
("num_failed_notifications",),
|
||||
("num_kv_expired_reqs",),
|
||||
],
|
||||
):
|
||||
for list_item in transfer_stats_data[counter_item_key]:
|
||||
counter_obj[engine_idx].inc(list_item)
|
||||
for counter_item_key in counter_item_keys:
|
||||
for list_item in transfer_stats_data[counter_item_key]:
|
||||
counter_obj[engine_idx].inc(list_item)
|
||||
|
||||
Reference in New Issue
Block a user