[Bugfix][NIXL] Don't evict a remote engine a transfer is still reading from (#54689)
Signed-off-by: Yizheng Jiao <jyizheng@gmail.com>
This commit is contained in:
@@ -2444,6 +2444,48 @@ def test_engine_ttl_eviction(default_vllm_config, dist_init):
|
||||
mock_rem.assert_any_call("agent_1")
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
)
|
||||
def test_engine_with_inflight_transfer_is_not_evicted(default_vllm_config, dist_init):
|
||||
"""A transfer that outlives the TTL must keep its engine registered.
|
||||
|
||||
_engine_last_active is stamped when a read is issued and not refreshed
|
||||
while it runs, so a stalled transfer -- a peer that has lost its NIC holds
|
||||
one indefinitely -- leaves its engine looking idle. Evicting it releases
|
||||
the dlist handle and remote agent the transfer is still reading through.
|
||||
"""
|
||||
worker, engine_id = _setup_worker_with_remote_engine(engine_ttl=10.0)
|
||||
nixl_wrapper = worker.nixl_wrapper
|
||||
|
||||
request_id = "req-still-reading"
|
||||
worker._recving_transfers[request_id] = [MagicMock()]
|
||||
worker._recving_metadata[request_id] = MagicMock(
|
||||
remote=MagicMock(engine_id=engine_id)
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(nixl_wrapper, "release_dlist_handle") as mock_rel,
|
||||
patch.object(nixl_wrapper, "remove_remote_agent") as mock_rem,
|
||||
):
|
||||
worker._engine_last_active[engine_id] = time.perf_counter() - 20.0
|
||||
|
||||
worker._evict_stale_engines()
|
||||
|
||||
assert engine_id in worker._remote_agents
|
||||
assert engine_id in worker.dst_xfer_side_handles
|
||||
mock_rel.assert_not_called()
|
||||
mock_rem.assert_not_called()
|
||||
|
||||
# Once the transfer is done the engine is stale like any other.
|
||||
del worker._recving_transfers[request_id]
|
||||
worker._evict_stale_engines()
|
||||
|
||||
assert engine_id not in worker._remote_agents
|
||||
assert mock_rem.call_count == 2
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FakeNixlWrapper,
|
||||
|
||||
@@ -3143,7 +3143,9 @@ class NixlBaseConnectorWorker:
|
||||
to be "optimal" until a new handshake is performed.
|
||||
|
||||
Engines with active transfers or pending handshakes cannot be stale:
|
||||
- Active transfers touch _engine_last_active in start_load_kv.
|
||||
- Reads stamp _engine_last_active when they are issued, and engines a
|
||||
transfer is still reading from are held back explicitly, since the
|
||||
stamp is not refreshed while the read runs.
|
||||
- Pending handshakes don't have an _engine_last_active entry yet
|
||||
"""
|
||||
# NOTE (NickLucche): This does NOT currently prevent OOMing if a huge number
|
||||
@@ -3154,10 +3156,25 @@ class NixlBaseConnectorWorker:
|
||||
return
|
||||
|
||||
now = time.perf_counter()
|
||||
busy = self._engines_with_inflight_transfers()
|
||||
for eid, last_active in list(self._engine_last_active.items()):
|
||||
if now - last_active > self._engine_ttl:
|
||||
if now - last_active > self._engine_ttl and eid not in busy:
|
||||
self._cleanup_remote_engine(eid)
|
||||
|
||||
def _engines_with_inflight_transfers(self) -> set[EngineId]:
|
||||
"""Remote engines a transfer is still reading from.
|
||||
|
||||
The timestamp is stamped when a read is issued and not refreshed while
|
||||
it runs, so a transfer that outlives the TTL leaves its engine looking
|
||||
idle. A peer that has lost its NIC holds one indefinitely.
|
||||
"""
|
||||
return {
|
||||
meta.remote.engine_id
|
||||
for req_id in self._recving_transfers
|
||||
if (meta := self._recving_metadata.get(req_id)) is not None
|
||||
and meta.remote is not None
|
||||
}
|
||||
|
||||
def _cleanup_remote_engine(
|
||||
self, engine_id: EngineId, *, log_eviction: bool = True
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user