[Security] Fix P2P ZMQ socket exhaustion via unbounded peer sessions (#51504)

Signed-off-by: jperezde <jperezde@redhat.com>
Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
This commit is contained in:
Juan Pérez de Algaba
2026-10-11 11:39:11 -04:00
committed by GitHub
co-authored by Robert Shaw Itay Etelis
parent 239a3fa3a8
commit 93d2cab35e
9 changed files with 711 additions and 41 deletions
+18
View File
@@ -60,6 +60,24 @@ def test_p2p_side_channel_defaults_and_override(monkeypatch: pytest.MonkeyPatch)
assert envs.VLLM_P2P_SIDE_CHANNEL_PORT == 5799
def test_p2p_resource_limit_defaults_and_override(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.delenv("VLLM_P2P_MAX_PEERS", raising=False)
monkeypatch.delenv("VLLM_P2P_HANDSHAKE_TIMEOUT_S", raising=False)
monkeypatch.delenv("VLLM_P2P_IDLE_TIMEOUT_S", raising=False)
assert envs.VLLM_P2P_MAX_PEERS == 128
assert envs.VLLM_P2P_HANDSHAKE_TIMEOUT_S == 30
assert envs.VLLM_P2P_IDLE_TIMEOUT_S == 300
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "64")
monkeypatch.setenv("VLLM_P2P_HANDSHAKE_TIMEOUT_S", "15")
monkeypatch.setenv("VLLM_P2P_IDLE_TIMEOUT_S", "60")
assert envs.VLLM_P2P_MAX_PEERS == 64
assert envs.VLLM_P2P_HANDSHAKE_TIMEOUT_S == 15
assert envs.VLLM_P2P_IDLE_TIMEOUT_S == 60
def test_getattr_with_cache(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_HOST_IP", "1.1.1.1")
monkeypatch.setenv("VLLM_PORT", "1234")
@@ -647,6 +647,7 @@ class _FakeSession:
close_req_ids: list[str] | None = None,
close_stores: list[int] | None = None,
close_failed_serves: list[ReqContext] | None = None,
last_activity_at: float | None = None,
) -> None:
self.peer_id = peer_id
self.alive = alive
@@ -663,6 +664,9 @@ class _FakeSession:
self.stores_added: list[tuple[str, list, object, int]] = []
self.attached: list[object] = []
self.finishes: list[str] = []
self.last_activity_at: float = (
last_activity_at if last_activity_at is not None else time.monotonic()
)
# Mirror P2PSession._server._inflight (transfer_id → handle) and
# P2PSession._client.has_active_loads for the shutdown-drain and
# drain_jobs paths. Tests populate _server._inflight when needed.
@@ -1844,3 +1848,272 @@ class TestBindHostPortDefaults:
mgr_b = self._construct(monkeypatch, host="localhost", port=5710)
assert mgr_a._local_id == mgr_b._local_id == "localhost:5710"
assert mgr_a._nixl_agent_name != mgr_b._nixl_agent_name
# ---------------------------------------------------------------------------
# Tests for session capacity limit (VLLM_P2P_MAX_PEERS)
# ---------------------------------------------------------------------------
class TestSessionCapLimit:
"""_get_or_create_session and _accept_new_peers reject new peers when
the session count reaches VLLM_P2P_MAX_PEERS."""
def _make_with_control(self):
mgr = _make_manager()
created = []
class FakeControl:
def connect(self, peer_id):
conn = _RecordingConn(peer_id)
created.append(conn)
return conn
def poll(self):
return []
class FakeData:
block_len = 4096
base_addr = 0x1000
num_blocks = 16
config_fingerprint = ""
def get_agent_metadata(self):
return b"meta"
def add_remote_peer(self, *a, **k):
pass
def remove_remote_peer(self, pid):
pass
mgr._control = FakeControl() # type: ignore[assignment]
mgr._data = FakeData() # type: ignore[assignment]
return mgr, created
def test_outbound_rejected_at_capacity(self, monkeypatch):
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "2")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
mgr, created = self._make_with_control()
mgr._sessions["peer-a:1"] = _FakeSession(peer_id="peer-a:1") # type: ignore[assignment]
mgr._sessions["peer-b:2"] = _FakeSession(peer_id="peer-b:2") # type: ignore[assignment]
result = mgr._get_or_create_session("peer-c:3")
assert result is None
assert "peer-c:3" not in mgr._sessions
assert len(created) == 0
def test_on_new_request_records_failure_on_cap(self, monkeypatch):
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "1")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
mgr, _ = self._make_with_control()
mgr._sessions["existing:1"] = _FakeSession(peer_id="existing:1") # type: ignore[assignment]
ctx = _req_context(
kv_params=_remote_prefiller_kv_params(
remote_host="10.0.0.2", remote_port=9000, kv_request_id="req-cap"
)
)
mgr.on_new_request(ctx)
assert "req-cap" in mgr._failed_req_ids
def test_inbound_rejected_at_capacity(self, monkeypatch):
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "1")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
mgr, _ = self._make_with_control()
mgr._sessions["existing:1"] = _FakeSession(peer_id="existing:1") # type: ignore[assignment]
conn = _RecordingConn("new-peer:2")
mgr._accept_new_peers([conn])
assert conn.close_calls == 1
assert "new-peer:2" not in mgr._sessions
def test_existing_session_reused_below_cap(self, monkeypatch):
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "1")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
mgr, _ = self._make_with_control()
existing = _FakeSession(peer_id="existing:1")
mgr._sessions["existing:1"] = existing # type: ignore[assignment]
result = mgr._get_or_create_session("existing:1")
assert result is existing
# ---------------------------------------------------------------------------
# Tests for graceful error boundary in _get_or_create_session
# ---------------------------------------------------------------------------
class TestGetOrCreateSessionErrorBoundary:
def test_connect_failure_returns_none(self):
mgr = _make_manager()
class FailingControl:
def connect(self, peer_id):
raise RuntimeError("socket exhausted")
def poll(self):
return []
mgr._control = FailingControl() # type: ignore[assignment]
result = mgr._get_or_create_session("bad:1")
assert result is None
assert "bad:1" not in mgr._sessions
def test_session_construction_failure_closes_conn(self):
"""connect() succeeding but P2PSession construction failing must
close the conn so it is not left registered without a session."""
mgr = _make_manager()
created: list[_RecordingConn] = []
class RaisingConn(_RecordingConn):
def send(self, msg):
raise RuntimeError("send failed during connect")
class Control:
def connect(self, peer_id):
conn = RaisingConn(peer_id)
created.append(conn)
return conn
def poll(self):
return []
class FakeData:
block_len = 4096
base_addr = 0x1000
num_blocks = 16
config_fingerprint = ""
def get_agent_metadata(self):
return b"meta"
mgr._control = Control() # type: ignore[assignment]
mgr._data = FakeData() # type: ignore[assignment]
result = mgr._get_or_create_session("bad:2")
assert result is None
assert "bad:2" not in mgr._sessions
assert len(created) == 1
assert created[0].close_calls == 1
# ---------------------------------------------------------------------------
# Tests for idle session eviction
# ---------------------------------------------------------------------------
class TestIdleSessionEviction:
def _make_with_fake_data(self):
mgr = _make_manager()
class FakeControl:
def poll(self):
return []
class FakeData:
def remove_remote_peer(self, pid):
pass
mgr._control = FakeControl() # type: ignore[assignment]
mgr._data = FakeData() # type: ignore[assignment]
return mgr
def test_idle_session_evicted(self, monkeypatch):
monkeypatch.setenv("VLLM_P2P_IDLE_TIMEOUT_S", "10")
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "1")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_IDLE_TIMEOUT_S", None)
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
mgr = self._make_with_fake_data()
idle_session = _FakeSession(
peer_id="idle:1",
last_activity_at=time.monotonic() - 20,
)
mgr._sessions["idle:1"] = idle_session # type: ignore[assignment]
mgr._reap_idle_sessions()
assert "idle:1" not in mgr._sessions
def test_idle_session_kept_below_headroom(self, monkeypatch):
monkeypatch.setenv("VLLM_P2P_IDLE_TIMEOUT_S", "10")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_IDLE_TIMEOUT_S", None)
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
mgr = self._make_with_fake_data()
idle_session = _FakeSession(
peer_id="idle:1",
last_activity_at=time.monotonic() - 20,
)
mgr._sessions["idle:1"] = idle_session # type: ignore[assignment]
mgr._reap_idle_sessions()
assert "idle:1" in mgr._sessions
def test_active_session_kept(self, monkeypatch):
monkeypatch.setenv("VLLM_P2P_IDLE_TIMEOUT_S", "10")
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "1")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_IDLE_TIMEOUT_S", None)
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
mgr = self._make_with_fake_data()
active = _FakeSession(
peer_id="active:1",
last_activity_at=time.monotonic(),
)
mgr._sessions["active:1"] = active # type: ignore[assignment]
mgr._reap_idle_sessions()
assert "active:1" in mgr._sessions
def test_session_with_pending_work_not_evicted(self, monkeypatch):
monkeypatch.setenv("VLLM_P2P_IDLE_TIMEOUT_S", "1")
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "1")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_IDLE_TIMEOUT_S", None)
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
mgr = self._make_with_fake_data()
busy = _FakeSession(
peer_id="busy:1",
last_activity_at=time.monotonic() - 100,
)
busy._server._inflight[1] = object()
mgr._sessions["busy:1"] = busy # type: ignore[assignment]
mgr._reap_idle_sessions()
assert "busy:1" in mgr._sessions
def test_disabled_when_zero(self, monkeypatch):
monkeypatch.setenv("VLLM_P2P_IDLE_TIMEOUT_S", "0")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_IDLE_TIMEOUT_S", None)
mgr = self._make_with_fake_data()
old = _FakeSession(
peer_id="old:1",
last_activity_at=time.monotonic() - 999999,
)
mgr._sessions["old:1"] = old # type: ignore[assignment]
mgr._reap_idle_sessions()
assert "old:1" in mgr._sessions
@@ -2698,3 +2698,141 @@ class TestTransferDoneMsgValidation:
}
with pytest.raises(ValueError, match="success"):
TransferDoneMsg.validate(msg)
# ---------------------------------------------------------------------------
# Tests for handshake deadline
# ---------------------------------------------------------------------------
class TestHandshakeDeadline:
"""A connected-but-not-ready session is marked dead after the
handshake timeout expires."""
def test_marks_dead_after_timeout(self, monkeypatch):
monkeypatch.setenv("VLLM_P2P_HANDSHAKE_TIMEOUT_S", "1")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_HANDSHAKE_TIMEOUT_S", None)
transport = FakeDataTransport()
conn = FakeConnection(peer_id="stale:9999")
session = P2PSession(
peer_id="stale:9999",
local_id="local:1",
transport=transport,
local_block_len=4096,
local_hash_seed=_DEFAULT_HASH_SEED,
conn=conn,
)
assert session.connected
assert not session.ready
assert session.alive
session._connected_at = time.monotonic() - 5
session.poll()
assert not session.alive
def test_no_timeout_when_ready(self, monkeypatch):
monkeypatch.setenv("VLLM_P2P_HANDSHAKE_TIMEOUT_S", "1")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_HANDSHAKE_TIMEOUT_S", None)
transport = FakeDataTransport()
conn = FakeConnection(peer_id="good:1234")
session = P2PSession(
peer_id="good:1234",
local_id="local:1",
transport=transport,
local_block_len=4096,
local_hash_seed=_DEFAULT_HASH_SEED,
conn=conn,
)
# Complete the handshake by feeding ConnectMsg + ConnectAckMsg.
conn._inbox = [
{
TYPE_KEY: ConnectMsg.TYPE,
ConnectMsg.PEER_ID: "good:1234",
ConnectMsg.AGENT_METADATA: b"meta",
ConnectMsg.BASE_ADDR: 0x1000,
ConnectMsg.NUM_BLOCKS: 16,
ConnectMsg.BLOCK_LEN: 4096,
ConnectMsg.CONFIG_FINGERPRINT: "",
ConnectMsg.HASH_SEED: _DEFAULT_HASH_SEED,
},
{TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: "good:1234"},
]
session.poll()
assert session.ready
session._connected_at = time.monotonic() - 100
session.poll()
assert session.alive
def test_buffered_ack_at_deadline_completes_handshake(self, monkeypatch):
"""A ConnectAck already in the inbox at the deadline must complete
the handshake instead of being discarded by mark_dead."""
monkeypatch.setenv("VLLM_P2P_HANDSHAKE_TIMEOUT_S", "1")
import vllm.envs
vllm.envs.__dict__.pop("VLLM_P2P_HANDSHAKE_TIMEOUT_S", None)
transport = FakeDataTransport()
conn = FakeConnection(peer_id="late-ack:1")
session = P2PSession(
peer_id="late-ack:1",
local_id="local:1",
transport=transport,
local_block_len=4096,
local_hash_seed=_DEFAULT_HASH_SEED,
conn=conn,
)
session._connected_at = time.monotonic() - 5
conn._inbox = [
{
TYPE_KEY: ConnectMsg.TYPE,
ConnectMsg.PEER_ID: "late-ack:1",
ConnectMsg.AGENT_METADATA: b"meta",
ConnectMsg.BASE_ADDR: 0x1000,
ConnectMsg.NUM_BLOCKS: 16,
ConnectMsg.BLOCK_LEN: 4096,
ConnectMsg.CONFIG_FINGERPRINT: "",
ConnectMsg.HASH_SEED: _DEFAULT_HASH_SEED,
},
{TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: "late-ack:1"},
]
session.poll()
assert session.ready
assert session.alive
def test_activity_timestamp_updated_on_recv(self):
transport = FakeDataTransport()
conn = FakeConnection(peer_id="peer:1")
session = P2PSession(
peer_id="peer:1",
local_id="local:1",
transport=transport,
local_block_len=4096,
local_hash_seed=_DEFAULT_HASH_SEED,
conn=conn,
)
before = session._last_activity_at
conn._inbox = [
{
TYPE_KEY: ConnectMsg.TYPE,
ConnectMsg.PEER_ID: "peer:1",
ConnectMsg.AGENT_METADATA: b"meta",
ConnectMsg.BASE_ADDR: 0x1000,
ConnectMsg.NUM_BLOCKS: 16,
ConnectMsg.BLOCK_LEN: 4096,
ConnectMsg.CONFIG_FINGERPRINT: "",
ConnectMsg.HASH_SEED: _DEFAULT_HASH_SEED,
},
]
session.poll()
assert session._last_activity_at >= before
@@ -336,3 +336,47 @@ class TestZmqReconnect:
finally:
transport_a.close()
transport_b.close()
class TestExpandedMonitorEvents:
"""The monitor mask includes EVENT_DISCONNECTED and EVENT_CLOSED so
both are treated as fatal."""
def test_fatal_events_include_disconnected_and_closed(self):
from vllm.v1.kv_offload.tiering.p2p.control.zmq import (
_FATAL_MONITOR_EVENTS,
)
assert zmq.EVENT_DISCONNECTED in _FATAL_MONITOR_EVENTS
assert zmq.EVENT_CLOSED in _FATAL_MONITOR_EVENTS
def test_monitor_mask_covers_fatal_events(self):
from vllm.v1.kv_offload.tiering.p2p.control.zmq import (
_FATAL_MONITOR_EVENTS,
_MONITOR_EVENT_MASK,
)
for ev in _FATAL_MONITOR_EVENTS:
assert _MONITOR_EVENT_MASK & ev == ev
def test_disconnect_detected_after_peer_shutdown(self):
"""When a peer's transport closes, the outbound connection is
eventually marked dead by the monitor."""
transport_a, port_a = _make_transport()
transport_b, _ = _make_transport()
try:
conn_out = transport_b.connect(f"127.0.0.1:{port_a}")
conn_out.send({"type": "hello"})
_wait_for_inbound(transport_a)
transport_a.close()
end = time.monotonic() + 15
while conn_out.alive and time.monotonic() < end:
transport_b.poll()
time.sleep(0.05)
assert not conn_out.alive
finally:
transport_b.close()