[Bugfix][NIXL] Count heartbeats as remote engine activity (#59873)

Signed-off-by: Maroon Ayoub <mayoub@redhat.com>
This commit is contained in:
Maroon Ayoub
2026-10-05 09:36:13 +00:00
committed by GitHub
parent ae53b06898
commit b1401e0aa7
2 changed files with 53 additions and 1 deletions
@@ -50,6 +50,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
NixlKVConnectorStats, NixlKVConnectorStats,
) )
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
HeartbeatInfo,
RemoteMeta, RemoteMeta,
ReqMeta, ReqMeta,
compute_nixl_compatibility_hash, compute_nixl_compatibility_hash,
@@ -2700,6 +2701,51 @@ def test_engine_with_inflight_transfer_is_not_evicted(default_vllm_config, dist_
assert mock_rem.call_count == 2 assert mock_rem.call_count == 2
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
)
def test_heartbeated_engine_is_not_evicted(default_vllm_config, dist_init, monkeypatch):
"""Heartbeats keep a remote engine's NIXL state alive.
Requests waiting in the D scheduler issue no reads, but their engine is
heartbeated through its remote agents. If the TTL expires anyway, the
next heartbeat evicts the engine and starts a new handshake, which loads
every remote agent again.
"""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker
worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=10.0)
nixl_wrapper = worker.nixl_wrapper
now = {"t": 1000.0}
monkeypatch.setattr(base_worker.time, "perf_counter", lambda: now["t"])
worker._engine_last_active[engine_id] = now["t"]
metadata = NixlConnectorMetadata()
metadata.heartbeat_by_engine = {
engine_id: HeartbeatInfo(
req_ids={"remote-req"}, host="localhost", port=1234, tp_size=2
)
}
with (
patch.object(nixl_wrapper, "send_notif") as mock_send,
patch.object(nixl_wrapper, "remove_remote_agent") as mock_rem,
patch.object(worker, "_handshake_initiation_executor") as mock_executor,
):
# One heartbeat every 5 s for twice the TTL, and no reads.
for _ in range(4):
now["t"] += 5.0
worker._send_heartbeats(metadata)
assert engine_id in worker._remote_agents
mock_rem.assert_not_called()
mock_executor.submit.assert_not_called()
# Every heartbeat reached both remote agents.
assert mock_send.call_count == 8
@patch( @patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper, FakeNixlWrapper,
@@ -3236,6 +3236,9 @@ class NixlBaseConnectorWorker:
): ):
continue # handshake is still pending continue # handshake is still pending
# Refresh the TTL: a heartbeat means the router still paired this P-D,
# so requests are still waiting on this engine
self._engine_last_active[engine_id] = time.perf_counter()
# Build the heartbeat message: "HB:req1,req2,..." # Build the heartbeat message: "HB:req1,req2,..."
hb_msg = ("HB:" + ",".join(hb_info.req_ids)).encode() hb_msg = ("HB:" + ",".join(hb_info.req_ids)).encode()
for agent_name in self._remote_agents[engine_id].values(): for agent_name in self._remote_agents[engine_id].values():
@@ -3586,10 +3589,13 @@ class NixlBaseConnectorWorker:
prevents us from using background threads, though memory usage is not guaranteed prevents us from using background threads, though memory usage is not guaranteed
to be "optimal" until a new handshake is performed. to be "optimal" until a new handshake is performed.
Engines with active transfers or pending handshakes cannot be stale: Engines with active transfers, heartbeats or pending handshakes cannot
be stale:
- Reads stamp _engine_last_active when they are issued, and engines a - Reads stamp _engine_last_active when they are issued, and engines a
transfer is still reading from are held back explicitly, since the transfer is still reading from are held back explicitly, since the
stamp is not refreshed while the read runs. stamp is not refreshed while the read runs.
- Heartbeats stamp it too, so an engine stays registered while this
instance holds requests waiting on it.
- Pending handshakes don't have an _engine_last_active entry yet - Pending handshakes don't have an _engine_last_active entry yet
""" """
# NOTE (NickLucche): This does NOT currently prevent OOMing if a huge number # NOTE (NickLucche): This does NOT currently prevent OOMing if a huge number