[Nixl] Separate transport-failure metrics from KV expiry (#55854)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-09-18 18:08:59 +02:00
committed by GitHub
parent a7156060c1
commit 164dca8d1e
9 changed files with 224 additions and 22 deletions
@@ -969,6 +969,9 @@ def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined):
assert worker.xfer_stats.data["num_kv_expired_reqs"] == (
[1] if expect_declined else []
)
# KV expiry must not pollute the transport failure counters.
assert worker.xfer_stats.data["num_failed_transfers"] == []
assert worker.xfer_stats.data["num_failed_handshakes"] == []
_, done_recving = connector.get_finished(finished_req_ids=set())
assert "req" in done_recving
@@ -567,6 +567,8 @@ class TestMultiConnectorStats:
"num_descriptors": [10, 20],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
}
}
@@ -590,6 +592,8 @@ class TestMultiConnectorStats:
"num_descriptors": [10],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
},
"MockConnector": {"mock_field": [1, 2, 3]},
}
@@ -619,6 +623,8 @@ class TestMultiConnectorStats:
"num_descriptors": [10],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
},
}
@@ -638,6 +644,8 @@ class TestMultiConnectorStats:
"num_descriptors": [10],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
}
)
mock_stats = MockConnectorStats(data={"mock_field": [1, 2, 3]})
@@ -667,6 +675,8 @@ class TestMultiConnectorStats:
"num_descriptors": [10],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
}
)
@@ -698,6 +708,8 @@ class TestMultiConnectorStats:
"num_descriptors": [10],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
},
"ExampleConnector": {"some_field": [1, 2, 3]},
}
@@ -735,6 +747,8 @@ class TestMultiConnectorStats:
"num_descriptors": [10],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
}
)
}
@@ -750,6 +764,8 @@ class TestMultiConnectorStats:
"num_descriptors": [20],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
}
)
}
@@ -781,6 +797,8 @@ class TestMultiConnectorStats:
"num_descriptors": [10],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
}
)
}
@@ -807,6 +825,8 @@ class TestMultiConnectorStats:
"num_descriptors": [10, 20],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
}
)
}
@@ -833,6 +853,8 @@ class TestMultiConnectorStats:
"num_descriptors": [10, 20],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
}
)
}
@@ -862,6 +884,8 @@ class TestMultiConnectorStats:
"num_descriptors": [10, 20],
"num_failed_transfers": [],
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
}
)
}
@@ -1682,6 +1682,83 @@ def test_kv_connector_stats_aggregation():
assert all(not isinstance(v, np.generic) for v in cli_stats.values())
def test_kv_connector_stats_failure_grouping():
"""Transfer, handshake and notification failures are reported as one
transport-failure count, while KV expiry is reported separately: the
former are sporadic lower-transport-layer events, the latter an
autoscaler signal."""
stats = NixlKVConnectorStats()
assert stats.is_empty()
stats.record_failed_transfer()
stats.record_failed_handshake()
stats.record_failed_notification()
stats.record_kv_expired_req()
assert not stats.is_empty()
# No successful transfers: latency stats are zero but the failure
# counts still surface.
reduced = stats.reduce()
assert reduced["Num successful transfers"] == 0
assert reduced["Num failed transfers"] == 3
assert reduced["Num KV expired reqs"] == 1
def test_nixl_prom_metrics_group_handshake_with_transfer_failures():
"""vllm:nixl_num_failed_transfers counts handshake and notification
failures too, while vllm:nixl_num_kv_expired_reqs stays a separate
counter."""
from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
NixlPromMetrics,
)
registry = CollectorRegistry()
class RegistryGauge(Gauge):
def __init__(self, *args, **kwargs):
super().__init__(*args, registry=registry, **kwargs)
class RegistryCounter(Counter):
def __init__(self, *args, **kwargs):
super().__init__(*args, registry=registry, **kwargs)
class RegistryHistogram(Histogram):
def __init__(self, *args, **kwargs):
super().__init__(*args, registry=registry, **kwargs)
vllm_config = create_vllm_config()
metric_types = {
Gauge: RegistryGauge,
Counter: RegistryCounter,
Histogram: RegistryHistogram,
}
prom = NixlPromMetrics(
vllm_config,
metric_types,
labelnames=["engine"],
per_engine_labelvalues={0: ["engine-0"]},
)
stats = NixlKVConnectorStats()
stats.record_failed_transfer()
stats.record_failed_handshake()
stats.record_failed_notification()
stats.record_kv_expired_req()
prom.observe(stats.data, engine_idx=0)
def counter_value(name: str) -> float:
for metric in registry.collect():
for sample in metric.samples:
if sample.name == name:
return sample.value
raise AssertionError(f"metric {name} not found in registry")
assert counter_value("vllm:nixl_num_failed_transfers_total") == 3.0
assert counter_value("vllm:nixl_num_kv_expired_reqs_total") == 1.0
def test_multi_kv_connector_stats_aggregation():
"""Test MultiKVConnectorStats aggregation across TP ranks using
KVOutputAggregator (used by MultiprocExecutor).
@@ -3029,6 +3106,11 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
assert request_id in done_recving
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
# Handshake failures are recorded as transport failures, separately
# from KV expiry.
assert connector.connector_worker.xfer_stats.data["num_failed_handshakes"]
assert connector.connector_worker.xfer_stats.data["num_failed_transfers"] == []
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
@@ -47,6 +47,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
NixlPushConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
NixlKVConnectorStats,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
get_base_request_id,
@@ -360,6 +363,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w._recv_failures = set()
w._recving_transfers = defaultdict(list)
w._is_hma_required = False
w.xfer_stats = NixlKVConnectorStats()
w._reqs_to_process = set()
w._reqs_to_send = {}
w.consumer_notification_counts_by_req = defaultdict(int)
@@ -730,6 +734,10 @@ def test_do_start_push_kv_drops_request_on_handshake_failure():
assert xfer_calls == []
assert len(failures) == 1
assert failures[0]["failure_type"] == "push_handshake_failed"
# The handshake metric is counted once by _ensure_handshake's own
# callback (stubbed out here); the push side must not double-record it.
assert w.xfer_stats.data["num_failed_handshakes"] == []
assert w.xfer_stats.data["num_failed_transfers"] == []
def test_writer_loop_drains_deferred_push_inbox():