[KVConnector][NIXL] Count completion notifications that arrive after KV expiry (#58875)
Signed-off-by: Zijing Liu <liuzijing2014@gmail.com>
This commit is contained in:
@@ -454,6 +454,7 @@ exported when NixlConnector is active:
|
||||
| `vllm:nixl_num_failed_transfers` | Counter | Cumulative count of failed NIXL KV-block transfers, including handshake and notification (`send_notif`) failures. These are grouped because all are sporadic lower-transport-layer events. |
|
||||
| `vllm:nixl_num_failed_notifications` | Counter | Cumulative count of failed notifications (`send_notif`), retained for compatibility. These failures are also included in `vllm:nixl_num_failed_transfers`; do not sum the two counters. |
|
||||
| `vllm:nixl_num_kv_expired_reqs` | Counter | Requests whose KV blocks expired on the prefiller before the decoder read them (tracked on the P instance). Kept separate from the failure counters above: KV expiry is an autoscaler/lease-tuning signal, not a transport issue. |
|
||||
| `vllm:nixl_num_notifications_after_expiry` | Counter | Completion notifications that arrive for a request that is no longer tracked, usually because its KV lease expired first. Pull connector: counted on P, where it means the decoder may have read blocks that were already reused; each matches a "Potentially invalid KV blocks" error log. Push connector: counted on whichever instance tracks the request as neither a send nor a receive (P after the lease expired, or D after the receive completed or was abandoned); each matches an "Unrecognized request" error log. Counted per notification, not per request: each remote rank notifies separately, so with TP > 1 or heterogeneous TP one late request counts several times, and the value is not comparable to `vllm:nixl_num_kv_expired_reqs`. |
|
||||
|
||||
!!! tip
|
||||
High `vllm:nixl_num_kv_expired_reqs` indicates that the prefiller's lease
|
||||
|
||||
@@ -601,6 +601,7 @@ class TestMultiConnectorStats:
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -626,6 +627,7 @@ class TestMultiConnectorStats:
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
},
|
||||
"MockConnector": {"mock_field": [1, 2, 3]},
|
||||
}
|
||||
@@ -657,6 +659,7 @@ class TestMultiConnectorStats:
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -678,6 +681,7 @@ class TestMultiConnectorStats:
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
}
|
||||
)
|
||||
mock_stats = MockConnectorStats(data={"mock_field": [1, 2, 3]})
|
||||
@@ -709,6 +713,7 @@ class TestMultiConnectorStats:
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
}
|
||||
)
|
||||
|
||||
@@ -742,6 +747,7 @@ class TestMultiConnectorStats:
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
},
|
||||
"ExampleConnector": {"some_field": [1, 2, 3]},
|
||||
}
|
||||
@@ -781,6 +787,7 @@ class TestMultiConnectorStats:
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -798,6 +805,7 @@ class TestMultiConnectorStats:
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -831,6 +839,7 @@ class TestMultiConnectorStats:
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -859,6 +868,7 @@ class TestMultiConnectorStats:
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -887,6 +897,7 @@ class TestMultiConnectorStats:
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -918,6 +929,7 @@ class TestMultiConnectorStats:
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1698,6 +1698,7 @@ def test_kv_connector_stats_failure_grouping():
|
||||
stats.record_failed_handshake()
|
||||
stats.record_failed_notification()
|
||||
stats.record_kv_expired_req()
|
||||
stats.record_notification_after_expiry()
|
||||
assert not stats.is_empty()
|
||||
|
||||
# No successful transfers: latency stats are zero but the failure
|
||||
@@ -1706,6 +1707,7 @@ def test_kv_connector_stats_failure_grouping():
|
||||
assert reduced["Num successful transfers"] == 0
|
||||
assert reduced["Num failed transfers"] == 3
|
||||
assert reduced["Num KV expired reqs"] == 1
|
||||
assert reduced["Num notifs after expiry"] == 1
|
||||
|
||||
|
||||
def test_nixl_prom_metrics_group_handshake_with_transfer_failures():
|
||||
@@ -1750,6 +1752,7 @@ def test_nixl_prom_metrics_group_handshake_with_transfer_failures():
|
||||
stats.record_failed_handshake()
|
||||
stats.record_failed_notification()
|
||||
stats.record_kv_expired_req()
|
||||
stats.record_notification_after_expiry()
|
||||
prom.observe(stats.data, engine_idx=0)
|
||||
|
||||
def counter_value(name: str) -> float:
|
||||
@@ -1761,6 +1764,33 @@ def test_nixl_prom_metrics_group_handshake_with_transfer_failures():
|
||||
|
||||
assert counter_value("vllm:nixl_num_failed_transfers_total") == 3.0
|
||||
assert counter_value("vllm:nixl_num_kv_expired_reqs_total") == 1.0
|
||||
assert counter_value("vllm:nixl_num_notifications_after_expiry_total") == 1.0
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
)
|
||||
def test_notification_after_expiry_is_counted(default_vllm_config, dist_init):
|
||||
vllm_config = create_vllm_config()
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config, connector.engine_id, hand_shake_latency=0
|
||||
)
|
||||
worker = connector.connector_worker
|
||||
worker._reqs_to_process.add("known")
|
||||
worker._reqs_to_send["known"] = time.perf_counter() + 10
|
||||
worker.nixl_wrapper.get_new_notifs = MagicMock(
|
||||
return_value={"decode-agent": [b"known:1", b"unknown:1"]}
|
||||
)
|
||||
|
||||
assert worker._get_new_notifs() == {"known"}
|
||||
|
||||
stats = connector.get_kv_connector_stats()
|
||||
assert isinstance(stats, NixlKVConnectorStats)
|
||||
assert stats.data["num_notifications_after_expiry"] == [1]
|
||||
|
||||
|
||||
def test_multi_kv_connector_stats_aggregation():
|
||||
|
||||
@@ -1357,6 +1357,7 @@ class TestPushWriterNegative:
|
||||
assert notified == set()
|
||||
# Did not register anywhere.
|
||||
assert "never-heard-of-you" not in w._recving_transfers
|
||||
assert w.xfer_stats.data["num_notifications_after_expiry"] == [1]
|
||||
|
||||
def test_start_load_kv_with_empty_metadata_is_noop(self):
|
||||
"""Empty metadata must not wake the writer or enqueue anything."""
|
||||
|
||||
@@ -660,6 +660,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
req_id not in self._reqs_to_send
|
||||
and req_id not in self._reqs_to_process
|
||||
):
|
||||
self.xfer_stats.record_notification_after_expiry()
|
||||
logger.error(
|
||||
"Potentially invalid KV blocks for "
|
||||
"unrecognized request %s were retrieved by "
|
||||
|
||||
@@ -786,6 +786,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
else:
|
||||
# Not tracked on either side (lease may have expired
|
||||
# before the notif arrived). Log and skip.
|
||||
self.xfer_stats.record_notification_after_expiry()
|
||||
logger.error(
|
||||
"Unrecognized request %s notif (may have expired).",
|
||||
req_id,
|
||||
|
||||
@@ -41,6 +41,7 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
"num_failed_notifications": [],
|
||||
"num_failed_handshakes": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
"num_notifications_after_expiry": [],
|
||||
}
|
||||
|
||||
def record_transfer(self, res: "nixlXferTelemetry"):
|
||||
@@ -66,6 +67,10 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
"""Record a request that had its KV blocks expire."""
|
||||
self.data["num_kv_expired_reqs"].append(1)
|
||||
|
||||
def record_notification_after_expiry(self):
|
||||
"""Record a completion notification that arrived after its request expired."""
|
||||
self.data["num_notifications_after_expiry"].append(1)
|
||||
|
||||
def clone_and_reset(self) -> "NixlKVConnectorStats":
|
||||
old = copy.copy(self)
|
||||
self.reset()
|
||||
@@ -79,6 +84,7 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
and len(self.data["num_failed_notifications"]) == 0
|
||||
and len(self.data["num_failed_handshakes"]) == 0
|
||||
and len(self.data["num_kv_expired_reqs"]) == 0
|
||||
and len(self.data["num_notifications_after_expiry"]) == 0
|
||||
)
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
@@ -95,12 +101,14 @@ class NixlKVConnectorStats(KVConnectorStats):
|
||||
# and notification failures are grouped as sporadic
|
||||
# lower-transport-layer events, while KV expiry is reported separately
|
||||
# as it is an actionable autoscaler signal rather than a transport
|
||||
# issue.
|
||||
# issue. Notifications arriving after expiry are surfaced too, as the
|
||||
# reader may have consumed KV blocks that were already reused.
|
||||
failure_counts = {
|
||||
"Num failed transfers": len(self.data["num_failed_transfers"])
|
||||
+ len(self.data["num_failed_handshakes"])
|
||||
+ len(self.data["num_failed_notifications"]),
|
||||
"Num KV expired reqs": len(self.data["num_kv_expired_reqs"]),
|
||||
"Num notifs after expiry": len(self.data["num_notifications_after_expiry"]),
|
||||
}
|
||||
if self.num_successful_transfers == 0:
|
||||
# Timing / throughput stats only cover successful transfers. If
|
||||
@@ -262,6 +270,18 @@ class NixlPromMetrics(KVConnectorPromMetrics):
|
||||
counter_nixl_num_kv_expired_reqs, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
counter_nixl_num_notifications_after_expiry = self._counter_cls(
|
||||
name="vllm:nixl_num_notifications_after_expiry",
|
||||
documentation="Number of completion notifications for requests that "
|
||||
"were no longer tracked, usually because their KV lease expired. The "
|
||||
"KV blocks may have been reused before the transfer finished. Counted "
|
||||
"per notification (one per remote rank), not per request.",
|
||||
labelnames=labelnames,
|
||||
)
|
||||
self.counter_nixl_num_notifications_after_expiry = create_metric_per_engine(
|
||||
counter_nixl_num_notifications_after_expiry, self.per_engine_labelvalues
|
||||
)
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
for prom_obj, list_item_key in zip(
|
||||
[
|
||||
@@ -284,6 +304,7 @@ class NixlPromMetrics(KVConnectorPromMetrics):
|
||||
self.counter_nixl_num_failed_transfers,
|
||||
self.counter_nixl_num_failed_notifications,
|
||||
self.counter_nixl_num_kv_expired_reqs,
|
||||
self.counter_nixl_num_notifications_after_expiry,
|
||||
],
|
||||
[
|
||||
# Transfer, handshake and notification failures are grouped:
|
||||
@@ -297,6 +318,7 @@ class NixlPromMetrics(KVConnectorPromMetrics):
|
||||
),
|
||||
("num_failed_notifications",),
|
||||
("num_kv_expired_reqs",),
|
||||
("num_notifications_after_expiry",),
|
||||
],
|
||||
):
|
||||
for counter_item_key in counter_item_keys:
|
||||
|
||||
Reference in New Issue
Block a user