[Bugfix][NIXL] Count heartbeats as remote engine activity (#59873)
Signed-off-by: Maroon Ayoub <mayoub@redhat.com>
This commit is contained in:
@@ -50,6 +50,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
|
|||||||
NixlKVConnectorStats,
|
NixlKVConnectorStats,
|
||||||
)
|
)
|
||||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||||
|
HeartbeatInfo,
|
||||||
RemoteMeta,
|
RemoteMeta,
|
||||||
ReqMeta,
|
ReqMeta,
|
||||||
compute_nixl_compatibility_hash,
|
compute_nixl_compatibility_hash,
|
||||||
@@ -2700,6 +2701,51 @@ def test_engine_with_inflight_transfer_is_not_evicted(default_vllm_config, dist_
|
|||||||
assert mock_rem.call_count == 2
|
assert mock_rem.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
@patch(
|
||||||
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||||
|
FakeNixlWrapper,
|
||||||
|
)
|
||||||
|
def test_heartbeated_engine_is_not_evicted(default_vllm_config, dist_init, monkeypatch):
|
||||||
|
"""Heartbeats keep a remote engine's NIXL state alive.
|
||||||
|
|
||||||
|
Requests waiting in the D scheduler issue no reads, but their engine is
|
||||||
|
heartbeated through its remote agents. If the TTL expires anyway, the
|
||||||
|
next heartbeat evicts the engine and starts a new handshake, which loads
|
||||||
|
every remote agent again.
|
||||||
|
"""
|
||||||
|
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker
|
||||||
|
|
||||||
|
worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=10.0)
|
||||||
|
nixl_wrapper = worker.nixl_wrapper
|
||||||
|
|
||||||
|
now = {"t": 1000.0}
|
||||||
|
monkeypatch.setattr(base_worker.time, "perf_counter", lambda: now["t"])
|
||||||
|
worker._engine_last_active[engine_id] = now["t"]
|
||||||
|
|
||||||
|
metadata = NixlConnectorMetadata()
|
||||||
|
metadata.heartbeat_by_engine = {
|
||||||
|
engine_id: HeartbeatInfo(
|
||||||
|
req_ids={"remote-req"}, host="localhost", port=1234, tp_size=2
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(nixl_wrapper, "send_notif") as mock_send,
|
||||||
|
patch.object(nixl_wrapper, "remove_remote_agent") as mock_rem,
|
||||||
|
patch.object(worker, "_handshake_initiation_executor") as mock_executor,
|
||||||
|
):
|
||||||
|
# One heartbeat every 5 s for twice the TTL, and no reads.
|
||||||
|
for _ in range(4):
|
||||||
|
now["t"] += 5.0
|
||||||
|
worker._send_heartbeats(metadata)
|
||||||
|
|
||||||
|
assert engine_id in worker._remote_agents
|
||||||
|
mock_rem.assert_not_called()
|
||||||
|
mock_executor.submit.assert_not_called()
|
||||||
|
# Every heartbeat reached both remote agents.
|
||||||
|
assert mock_send.call_count == 8
|
||||||
|
|
||||||
|
|
||||||
@patch(
|
@patch(
|
||||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||||
FakeNixlWrapper,
|
FakeNixlWrapper,
|
||||||
|
|||||||
@@ -3236,6 +3236,9 @@ class NixlBaseConnectorWorker:
|
|||||||
):
|
):
|
||||||
continue # handshake is still pending
|
continue # handshake is still pending
|
||||||
|
|
||||||
|
# Refresh the TTL: a heartbeat means the router still paired this P-D,
|
||||||
|
# so requests are still waiting on this engine
|
||||||
|
self._engine_last_active[engine_id] = time.perf_counter()
|
||||||
# Build the heartbeat message: "HB:req1,req2,..."
|
# Build the heartbeat message: "HB:req1,req2,..."
|
||||||
hb_msg = ("HB:" + ",".join(hb_info.req_ids)).encode()
|
hb_msg = ("HB:" + ",".join(hb_info.req_ids)).encode()
|
||||||
for agent_name in self._remote_agents[engine_id].values():
|
for agent_name in self._remote_agents[engine_id].values():
|
||||||
@@ -3586,10 +3589,13 @@ class NixlBaseConnectorWorker:
|
|||||||
prevents us from using background threads, though memory usage is not guaranteed
|
prevents us from using background threads, though memory usage is not guaranteed
|
||||||
to be "optimal" until a new handshake is performed.
|
to be "optimal" until a new handshake is performed.
|
||||||
|
|
||||||
Engines with active transfers or pending handshakes cannot be stale:
|
Engines with active transfers, heartbeats or pending handshakes cannot
|
||||||
|
be stale:
|
||||||
- Reads stamp _engine_last_active when they are issued, and engines a
|
- Reads stamp _engine_last_active when they are issued, and engines a
|
||||||
transfer is still reading from are held back explicitly, since the
|
transfer is still reading from are held back explicitly, since the
|
||||||
stamp is not refreshed while the read runs.
|
stamp is not refreshed while the read runs.
|
||||||
|
- Heartbeats stamp it too, so an engine stays registered while this
|
||||||
|
instance holds requests waiting on it.
|
||||||
- Pending handshakes don't have an _engine_last_active entry yet
|
- Pending handshakes don't have an _engine_last_active entry yet
|
||||||
"""
|
"""
|
||||||
# NOTE (NickLucche): This does NOT currently prevent OOMing if a huge number
|
# NOTE (NickLucche): This does NOT currently prevent OOMing if a huge number
|
||||||
|
|||||||
Reference in New Issue
Block a user