[Nixl] Separate transport-failure metrics from KV expiry (#55854)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -969,6 +969,9 @@ def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined):
|
||||
assert worker.xfer_stats.data["num_kv_expired_reqs"] == (
|
||||
[1] if expect_declined else []
|
||||
)
|
||||
# KV expiry must not pollute the transport failure counters.
|
||||
assert worker.xfer_stats.data["num_failed_transfers"] == []
|
||||
assert worker.xfer_stats.data["num_failed_handshakes"] == []
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert "req" in done_recving
|
||||
|
||||
|
||||
@@ -567,6 +567,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10, 20],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -590,6 +592,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
},
|
||||
"MockConnector": {"mock_field": [1, 2, 3]},
|
||||
}
|
||||
@@ -619,6 +623,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -638,6 +644,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
mock_stats = MockConnectorStats(data={"mock_field": [1, 2, 3]})
|
||||
@@ -667,6 +675,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
|
||||
@@ -698,6 +708,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
},
|
||||
"ExampleConnector": {"some_field": [1, 2, 3]},
|
||||
}
|
||||
@@ -735,6 +747,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -750,6 +764,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [20],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -781,6 +797,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -807,6 +825,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10, 20],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -833,6 +853,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10, 20],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -862,6 +884,8 @@ class TestMultiConnectorStats:
|
||||
"num_descriptors": [10, 20],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1682,6 +1682,83 @@ def test_kv_connector_stats_aggregation():
|
||||
assert all(not isinstance(v, np.generic) for v in cli_stats.values())
|
||||
|
||||
|
||||
def test_kv_connector_stats_failure_grouping():
|
||||
"""Transfer, handshake and notification failures are reported as one
|
||||
transport-failure count, while KV expiry is reported separately: the
|
||||
former are sporadic lower-transport-layer events, the latter an
|
||||
autoscaler signal."""
|
||||
stats = NixlKVConnectorStats()
|
||||
assert stats.is_empty()
|
||||
|
||||
stats.record_failed_transfer()
|
||||
stats.record_failed_handshake()
|
||||
stats.record_failed_notification()
|
||||
stats.record_kv_expired_req()
|
||||
assert not stats.is_empty()
|
||||
|
||||
# No successful transfers: latency stats are zero but the failure
|
||||
# counts still surface.
|
||||
reduced = stats.reduce()
|
||||
assert reduced["Num successful transfers"] == 0
|
||||
assert reduced["Num failed transfers"] == 3
|
||||
assert reduced["Num KV expired reqs"] == 1
|
||||
|
||||
|
||||
def test_nixl_prom_metrics_group_handshake_with_transfer_failures():
|
||||
"""vllm:nixl_num_failed_transfers counts handshake and notification
|
||||
failures too, while vllm:nixl_num_kv_expired_reqs stays a separate
|
||||
counter."""
|
||||
from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
|
||||
NixlPromMetrics,
|
||||
)
|
||||
|
||||
registry = CollectorRegistry()
|
||||
|
||||
class RegistryGauge(Gauge):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, registry=registry, **kwargs)
|
||||
|
||||
class RegistryCounter(Counter):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, registry=registry, **kwargs)
|
||||
|
||||
class RegistryHistogram(Histogram):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, registry=registry, **kwargs)
|
||||
|
||||
vllm_config = create_vllm_config()
|
||||
metric_types = {
|
||||
Gauge: RegistryGauge,
|
||||
Counter: RegistryCounter,
|
||||
Histogram: RegistryHistogram,
|
||||
}
|
||||
prom = NixlPromMetrics(
|
||||
vllm_config,
|
||||
metric_types,
|
||||
labelnames=["engine"],
|
||||
per_engine_labelvalues={0: ["engine-0"]},
|
||||
)
|
||||
|
||||
stats = NixlKVConnectorStats()
|
||||
stats.record_failed_transfer()
|
||||
stats.record_failed_handshake()
|
||||
stats.record_failed_notification()
|
||||
stats.record_kv_expired_req()
|
||||
prom.observe(stats.data, engine_idx=0)
|
||||
|
||||
def counter_value(name: str) -> float:
|
||||
for metric in registry.collect():
|
||||
for sample in metric.samples:
|
||||
if sample.name == name:
|
||||
return sample.value
|
||||
raise AssertionError(f"metric {name} not found in registry")
|
||||
|
||||
assert counter_value("vllm:nixl_num_failed_transfers_total") == 3.0
|
||||
assert counter_value("vllm:nixl_num_kv_expired_reqs_total") == 1.0
|
||||
|
||||
|
||||
def test_multi_kv_connector_stats_aggregation():
|
||||
"""Test MultiKVConnectorStats aggregation across TP ranks using
|
||||
KVOutputAggregator (used by MultiprocExecutor).
|
||||
@@ -3029,6 +3106,11 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
|
||||
assert request_id in done_recving
|
||||
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
|
||||
|
||||
# Handshake failures are recorded as transport failures, separately
|
||||
# from KV expiry.
|
||||
assert connector.connector_worker.xfer_stats.data["num_failed_handshakes"]
|
||||
assert connector.connector_worker.xfer_stats.data["num_failed_transfers"] == []
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
|
||||
@@ -47,6 +47,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import (
|
||||
NixlPushConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
|
||||
NixlKVConnectorStats,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import (
|
||||
get_base_request_id,
|
||||
@@ -360,6 +363,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w._recv_failures = set()
|
||||
w._recving_transfers = defaultdict(list)
|
||||
w._is_hma_required = False
|
||||
w.xfer_stats = NixlKVConnectorStats()
|
||||
w._reqs_to_process = set()
|
||||
w._reqs_to_send = {}
|
||||
w.consumer_notification_counts_by_req = defaultdict(int)
|
||||
@@ -730,6 +734,10 @@ def test_do_start_push_kv_drops_request_on_handshake_failure():
|
||||
assert xfer_calls == []
|
||||
assert len(failures) == 1
|
||||
assert failures[0]["failure_type"] == "push_handshake_failed"
|
||||
# The handshake metric is counted once by _ensure_handshake's own
|
||||
# callback (stubbed out here); the push side must not double-record it.
|
||||
assert w.xfer_stats.data["num_failed_handshakes"] == []
|
||||
assert w.xfer_stats.data["num_failed_transfers"] == []
|
||||
|
||||
|
||||
def test_writer_loop_drains_deferred_push_inbox():
|
||||
|
||||
Reference in New Issue
Block a user