[KVConnector][NIXL] Count completion notifications that arrive after KV expiry (#58875)

Signed-off-by: Zijing Liu <liuzijing2014@gmail.com>
This commit is contained in:
Zijing Liu
2026-10-01 21:30:58 +00:00
committed by GitHub
parent ccda9098cc
commit ea84051819
7 changed files with 69 additions and 1 deletions
@@ -660,6 +660,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
req_id not in self._reqs_to_send
and req_id not in self._reqs_to_process
):
self.xfer_stats.record_notification_after_expiry()
logger.error(
"Potentially invalid KV blocks for "
"unrecognized request %s were retrieved by "
@@ -786,6 +786,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
else:
# Not tracked on either side (lease may have expired
# before the notif arrived). Log and skip.
self.xfer_stats.record_notification_after_expiry()
logger.error(
"Unrecognized request %s notif (may have expired).",
req_id,
@@ -41,6 +41,7 @@ class NixlKVConnectorStats(KVConnectorStats):
"num_failed_notifications": [],
"num_failed_handshakes": [],
"num_kv_expired_reqs": [],
"num_notifications_after_expiry": [],
}
def record_transfer(self, res: "nixlXferTelemetry"):
@@ -66,6 +67,10 @@ class NixlKVConnectorStats(KVConnectorStats):
"""Record a request that had its KV blocks expire."""
self.data["num_kv_expired_reqs"].append(1)
def record_notification_after_expiry(self):
"""Record a completion notification that arrived after its request expired."""
self.data["num_notifications_after_expiry"].append(1)
def clone_and_reset(self) -> "NixlKVConnectorStats":
old = copy.copy(self)
self.reset()
@@ -79,6 +84,7 @@ class NixlKVConnectorStats(KVConnectorStats):
and len(self.data["num_failed_notifications"]) == 0
and len(self.data["num_failed_handshakes"]) == 0
and len(self.data["num_kv_expired_reqs"]) == 0
and len(self.data["num_notifications_after_expiry"]) == 0
)
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
@@ -95,12 +101,14 @@ class NixlKVConnectorStats(KVConnectorStats):
# and notification failures are grouped as sporadic
# lower-transport-layer events, while KV expiry is reported separately
# as it is an actionable autoscaler signal rather than a transport
# issue.
# issue. Notifications arriving after expiry are surfaced too, as the
# reader may have consumed KV blocks that were already reused.
failure_counts = {
"Num failed transfers": len(self.data["num_failed_transfers"])
+ len(self.data["num_failed_handshakes"])
+ len(self.data["num_failed_notifications"]),
"Num KV expired reqs": len(self.data["num_kv_expired_reqs"]),
"Num notifs after expiry": len(self.data["num_notifications_after_expiry"]),
}
if self.num_successful_transfers == 0:
# Timing / throughput stats only cover successful transfers. If
@@ -262,6 +270,18 @@ class NixlPromMetrics(KVConnectorPromMetrics):
counter_nixl_num_kv_expired_reqs, self.per_engine_labelvalues
)
counter_nixl_num_notifications_after_expiry = self._counter_cls(
name="vllm:nixl_num_notifications_after_expiry",
documentation="Number of completion notifications for requests that "
"were no longer tracked, usually because their KV lease expired. The "
"KV blocks may have been reused before the transfer finished. Counted "
"per notification (one per remote rank), not per request.",
labelnames=labelnames,
)
self.counter_nixl_num_notifications_after_expiry = create_metric_per_engine(
counter_nixl_num_notifications_after_expiry, self.per_engine_labelvalues
)
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
for prom_obj, list_item_key in zip(
[
@@ -284,6 +304,7 @@ class NixlPromMetrics(KVConnectorPromMetrics):
self.counter_nixl_num_failed_transfers,
self.counter_nixl_num_failed_notifications,
self.counter_nixl_num_kv_expired_reqs,
self.counter_nixl_num_notifications_after_expiry,
],
[
# Transfer, handshake and notification failures are grouped:
@@ -297,6 +318,7 @@ class NixlPromMetrics(KVConnectorPromMetrics):
),
("num_failed_notifications",),
("num_kv_expired_reqs",),
("num_notifications_after_expiry",),
],
):
for counter_item_key in counter_item_keys: