[Security] Fix P2P ZMQ socket exhaustion via unbounded peer sessions (#51504)
Signed-off-by: jperezde <jperezde@redhat.com> Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
This commit is contained in:
co-authored by
Robert Shaw
Itay Etelis
parent
239a3fa3a8
commit
93d2cab35e
@@ -60,6 +60,24 @@ def test_p2p_side_channel_defaults_and_override(monkeypatch: pytest.MonkeyPatch)
|
||||
assert envs.VLLM_P2P_SIDE_CHANNEL_PORT == 5799
|
||||
|
||||
|
||||
def test_p2p_resource_limit_defaults_and_override(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.delenv("VLLM_P2P_MAX_PEERS", raising=False)
|
||||
monkeypatch.delenv("VLLM_P2P_HANDSHAKE_TIMEOUT_S", raising=False)
|
||||
monkeypatch.delenv("VLLM_P2P_IDLE_TIMEOUT_S", raising=False)
|
||||
assert envs.VLLM_P2P_MAX_PEERS == 128
|
||||
assert envs.VLLM_P2P_HANDSHAKE_TIMEOUT_S == 30
|
||||
assert envs.VLLM_P2P_IDLE_TIMEOUT_S == 300
|
||||
|
||||
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "64")
|
||||
monkeypatch.setenv("VLLM_P2P_HANDSHAKE_TIMEOUT_S", "15")
|
||||
monkeypatch.setenv("VLLM_P2P_IDLE_TIMEOUT_S", "60")
|
||||
assert envs.VLLM_P2P_MAX_PEERS == 64
|
||||
assert envs.VLLM_P2P_HANDSHAKE_TIMEOUT_S == 15
|
||||
assert envs.VLLM_P2P_IDLE_TIMEOUT_S == 60
|
||||
|
||||
|
||||
def test_getattr_with_cache(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("VLLM_HOST_IP", "1.1.1.1")
|
||||
monkeypatch.setenv("VLLM_PORT", "1234")
|
||||
|
||||
@@ -647,6 +647,7 @@ class _FakeSession:
|
||||
close_req_ids: list[str] | None = None,
|
||||
close_stores: list[int] | None = None,
|
||||
close_failed_serves: list[ReqContext] | None = None,
|
||||
last_activity_at: float | None = None,
|
||||
) -> None:
|
||||
self.peer_id = peer_id
|
||||
self.alive = alive
|
||||
@@ -663,6 +664,9 @@ class _FakeSession:
|
||||
self.stores_added: list[tuple[str, list, object, int]] = []
|
||||
self.attached: list[object] = []
|
||||
self.finishes: list[str] = []
|
||||
self.last_activity_at: float = (
|
||||
last_activity_at if last_activity_at is not None else time.monotonic()
|
||||
)
|
||||
# Mirror P2PSession._server._inflight (transfer_id → handle) and
|
||||
# P2PSession._client.has_active_loads for the shutdown-drain and
|
||||
# drain_jobs paths. Tests populate _server._inflight when needed.
|
||||
@@ -1844,3 +1848,272 @@ class TestBindHostPortDefaults:
|
||||
mgr_b = self._construct(monkeypatch, host="localhost", port=5710)
|
||||
assert mgr_a._local_id == mgr_b._local_id == "localhost:5710"
|
||||
assert mgr_a._nixl_agent_name != mgr_b._nixl_agent_name
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for session capacity limit (VLLM_P2P_MAX_PEERS)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSessionCapLimit:
|
||||
"""_get_or_create_session and _accept_new_peers reject new peers when
|
||||
the session count reaches VLLM_P2P_MAX_PEERS."""
|
||||
|
||||
def _make_with_control(self):
|
||||
mgr = _make_manager()
|
||||
created = []
|
||||
|
||||
class FakeControl:
|
||||
def connect(self, peer_id):
|
||||
conn = _RecordingConn(peer_id)
|
||||
created.append(conn)
|
||||
return conn
|
||||
|
||||
def poll(self):
|
||||
return []
|
||||
|
||||
class FakeData:
|
||||
block_len = 4096
|
||||
base_addr = 0x1000
|
||||
num_blocks = 16
|
||||
config_fingerprint = ""
|
||||
|
||||
def get_agent_metadata(self):
|
||||
return b"meta"
|
||||
|
||||
def add_remote_peer(self, *a, **k):
|
||||
pass
|
||||
|
||||
def remove_remote_peer(self, pid):
|
||||
pass
|
||||
|
||||
mgr._control = FakeControl() # type: ignore[assignment]
|
||||
mgr._data = FakeData() # type: ignore[assignment]
|
||||
return mgr, created
|
||||
|
||||
def test_outbound_rejected_at_capacity(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "2")
|
||||
import vllm.envs
|
||||
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
|
||||
|
||||
mgr, created = self._make_with_control()
|
||||
mgr._sessions["peer-a:1"] = _FakeSession(peer_id="peer-a:1") # type: ignore[assignment]
|
||||
mgr._sessions["peer-b:2"] = _FakeSession(peer_id="peer-b:2") # type: ignore[assignment]
|
||||
|
||||
result = mgr._get_or_create_session("peer-c:3")
|
||||
assert result is None
|
||||
assert "peer-c:3" not in mgr._sessions
|
||||
assert len(created) == 0
|
||||
|
||||
def test_on_new_request_records_failure_on_cap(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "1")
|
||||
import vllm.envs
|
||||
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
|
||||
|
||||
mgr, _ = self._make_with_control()
|
||||
mgr._sessions["existing:1"] = _FakeSession(peer_id="existing:1") # type: ignore[assignment]
|
||||
|
||||
ctx = _req_context(
|
||||
kv_params=_remote_prefiller_kv_params(
|
||||
remote_host="10.0.0.2", remote_port=9000, kv_request_id="req-cap"
|
||||
)
|
||||
)
|
||||
mgr.on_new_request(ctx)
|
||||
assert "req-cap" in mgr._failed_req_ids
|
||||
|
||||
def test_inbound_rejected_at_capacity(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "1")
|
||||
import vllm.envs
|
||||
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
|
||||
|
||||
mgr, _ = self._make_with_control()
|
||||
mgr._sessions["existing:1"] = _FakeSession(peer_id="existing:1") # type: ignore[assignment]
|
||||
|
||||
conn = _RecordingConn("new-peer:2")
|
||||
mgr._accept_new_peers([conn])
|
||||
assert conn.close_calls == 1
|
||||
assert "new-peer:2" not in mgr._sessions
|
||||
|
||||
def test_existing_session_reused_below_cap(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "1")
|
||||
import vllm.envs
|
||||
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
|
||||
|
||||
mgr, _ = self._make_with_control()
|
||||
existing = _FakeSession(peer_id="existing:1")
|
||||
mgr._sessions["existing:1"] = existing # type: ignore[assignment]
|
||||
|
||||
result = mgr._get_or_create_session("existing:1")
|
||||
assert result is existing
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for graceful error boundary in _get_or_create_session
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGetOrCreateSessionErrorBoundary:
|
||||
def test_connect_failure_returns_none(self):
|
||||
mgr = _make_manager()
|
||||
|
||||
class FailingControl:
|
||||
def connect(self, peer_id):
|
||||
raise RuntimeError("socket exhausted")
|
||||
|
||||
def poll(self):
|
||||
return []
|
||||
|
||||
mgr._control = FailingControl() # type: ignore[assignment]
|
||||
result = mgr._get_or_create_session("bad:1")
|
||||
assert result is None
|
||||
assert "bad:1" not in mgr._sessions
|
||||
|
||||
def test_session_construction_failure_closes_conn(self):
|
||||
"""connect() succeeding but P2PSession construction failing must
|
||||
close the conn so it is not left registered without a session."""
|
||||
mgr = _make_manager()
|
||||
created: list[_RecordingConn] = []
|
||||
|
||||
class RaisingConn(_RecordingConn):
|
||||
def send(self, msg):
|
||||
raise RuntimeError("send failed during connect")
|
||||
|
||||
class Control:
|
||||
def connect(self, peer_id):
|
||||
conn = RaisingConn(peer_id)
|
||||
created.append(conn)
|
||||
return conn
|
||||
|
||||
def poll(self):
|
||||
return []
|
||||
|
||||
class FakeData:
|
||||
block_len = 4096
|
||||
base_addr = 0x1000
|
||||
num_blocks = 16
|
||||
config_fingerprint = ""
|
||||
|
||||
def get_agent_metadata(self):
|
||||
return b"meta"
|
||||
|
||||
mgr._control = Control() # type: ignore[assignment]
|
||||
mgr._data = FakeData() # type: ignore[assignment]
|
||||
result = mgr._get_or_create_session("bad:2")
|
||||
assert result is None
|
||||
assert "bad:2" not in mgr._sessions
|
||||
assert len(created) == 1
|
||||
assert created[0].close_calls == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for idle session eviction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestIdleSessionEviction:
|
||||
def _make_with_fake_data(self):
|
||||
mgr = _make_manager()
|
||||
|
||||
class FakeControl:
|
||||
def poll(self):
|
||||
return []
|
||||
|
||||
class FakeData:
|
||||
def remove_remote_peer(self, pid):
|
||||
pass
|
||||
|
||||
mgr._control = FakeControl() # type: ignore[assignment]
|
||||
mgr._data = FakeData() # type: ignore[assignment]
|
||||
return mgr
|
||||
|
||||
def test_idle_session_evicted(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_P2P_IDLE_TIMEOUT_S", "10")
|
||||
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "1")
|
||||
import vllm.envs
|
||||
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_IDLE_TIMEOUT_S", None)
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
|
||||
|
||||
mgr = self._make_with_fake_data()
|
||||
idle_session = _FakeSession(
|
||||
peer_id="idle:1",
|
||||
last_activity_at=time.monotonic() - 20,
|
||||
)
|
||||
mgr._sessions["idle:1"] = idle_session # type: ignore[assignment]
|
||||
|
||||
mgr._reap_idle_sessions()
|
||||
assert "idle:1" not in mgr._sessions
|
||||
|
||||
def test_idle_session_kept_below_headroom(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_P2P_IDLE_TIMEOUT_S", "10")
|
||||
import vllm.envs
|
||||
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_IDLE_TIMEOUT_S", None)
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
|
||||
|
||||
mgr = self._make_with_fake_data()
|
||||
idle_session = _FakeSession(
|
||||
peer_id="idle:1",
|
||||
last_activity_at=time.monotonic() - 20,
|
||||
)
|
||||
mgr._sessions["idle:1"] = idle_session # type: ignore[assignment]
|
||||
|
||||
mgr._reap_idle_sessions()
|
||||
assert "idle:1" in mgr._sessions
|
||||
|
||||
def test_active_session_kept(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_P2P_IDLE_TIMEOUT_S", "10")
|
||||
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "1")
|
||||
import vllm.envs
|
||||
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_IDLE_TIMEOUT_S", None)
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
|
||||
|
||||
mgr = self._make_with_fake_data()
|
||||
active = _FakeSession(
|
||||
peer_id="active:1",
|
||||
last_activity_at=time.monotonic(),
|
||||
)
|
||||
mgr._sessions["active:1"] = active # type: ignore[assignment]
|
||||
|
||||
mgr._reap_idle_sessions()
|
||||
assert "active:1" in mgr._sessions
|
||||
|
||||
def test_session_with_pending_work_not_evicted(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_P2P_IDLE_TIMEOUT_S", "1")
|
||||
monkeypatch.setenv("VLLM_P2P_MAX_PEERS", "1")
|
||||
import vllm.envs
|
||||
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_IDLE_TIMEOUT_S", None)
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_MAX_PEERS", None)
|
||||
|
||||
mgr = self._make_with_fake_data()
|
||||
busy = _FakeSession(
|
||||
peer_id="busy:1",
|
||||
last_activity_at=time.monotonic() - 100,
|
||||
)
|
||||
busy._server._inflight[1] = object()
|
||||
mgr._sessions["busy:1"] = busy # type: ignore[assignment]
|
||||
|
||||
mgr._reap_idle_sessions()
|
||||
assert "busy:1" in mgr._sessions
|
||||
|
||||
def test_disabled_when_zero(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_P2P_IDLE_TIMEOUT_S", "0")
|
||||
import vllm.envs
|
||||
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_IDLE_TIMEOUT_S", None)
|
||||
|
||||
mgr = self._make_with_fake_data()
|
||||
old = _FakeSession(
|
||||
peer_id="old:1",
|
||||
last_activity_at=time.monotonic() - 999999,
|
||||
)
|
||||
mgr._sessions["old:1"] = old # type: ignore[assignment]
|
||||
|
||||
mgr._reap_idle_sessions()
|
||||
assert "old:1" in mgr._sessions
|
||||
|
||||
@@ -2698,3 +2698,141 @@ class TestTransferDoneMsgValidation:
|
||||
}
|
||||
with pytest.raises(ValueError, match="success"):
|
||||
TransferDoneMsg.validate(msg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for handshake deadline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestHandshakeDeadline:
|
||||
"""A connected-but-not-ready session is marked dead after the
|
||||
handshake timeout expires."""
|
||||
|
||||
def test_marks_dead_after_timeout(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_P2P_HANDSHAKE_TIMEOUT_S", "1")
|
||||
import vllm.envs
|
||||
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_HANDSHAKE_TIMEOUT_S", None)
|
||||
|
||||
transport = FakeDataTransport()
|
||||
conn = FakeConnection(peer_id="stale:9999")
|
||||
session = P2PSession(
|
||||
peer_id="stale:9999",
|
||||
local_id="local:1",
|
||||
transport=transport,
|
||||
local_block_len=4096,
|
||||
local_hash_seed=_DEFAULT_HASH_SEED,
|
||||
conn=conn,
|
||||
)
|
||||
|
||||
assert session.connected
|
||||
assert not session.ready
|
||||
assert session.alive
|
||||
|
||||
session._connected_at = time.monotonic() - 5
|
||||
|
||||
session.poll()
|
||||
assert not session.alive
|
||||
|
||||
def test_no_timeout_when_ready(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_P2P_HANDSHAKE_TIMEOUT_S", "1")
|
||||
import vllm.envs
|
||||
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_HANDSHAKE_TIMEOUT_S", None)
|
||||
|
||||
transport = FakeDataTransport()
|
||||
conn = FakeConnection(peer_id="good:1234")
|
||||
session = P2PSession(
|
||||
peer_id="good:1234",
|
||||
local_id="local:1",
|
||||
transport=transport,
|
||||
local_block_len=4096,
|
||||
local_hash_seed=_DEFAULT_HASH_SEED,
|
||||
conn=conn,
|
||||
)
|
||||
|
||||
# Complete the handshake by feeding ConnectMsg + ConnectAckMsg.
|
||||
conn._inbox = [
|
||||
{
|
||||
TYPE_KEY: ConnectMsg.TYPE,
|
||||
ConnectMsg.PEER_ID: "good:1234",
|
||||
ConnectMsg.AGENT_METADATA: b"meta",
|
||||
ConnectMsg.BASE_ADDR: 0x1000,
|
||||
ConnectMsg.NUM_BLOCKS: 16,
|
||||
ConnectMsg.BLOCK_LEN: 4096,
|
||||
ConnectMsg.CONFIG_FINGERPRINT: "",
|
||||
ConnectMsg.HASH_SEED: _DEFAULT_HASH_SEED,
|
||||
},
|
||||
{TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: "good:1234"},
|
||||
]
|
||||
session.poll()
|
||||
assert session.ready
|
||||
|
||||
session._connected_at = time.monotonic() - 100
|
||||
session.poll()
|
||||
assert session.alive
|
||||
|
||||
def test_buffered_ack_at_deadline_completes_handshake(self, monkeypatch):
|
||||
"""A ConnectAck already in the inbox at the deadline must complete
|
||||
the handshake instead of being discarded by mark_dead."""
|
||||
monkeypatch.setenv("VLLM_P2P_HANDSHAKE_TIMEOUT_S", "1")
|
||||
import vllm.envs
|
||||
|
||||
vllm.envs.__dict__.pop("VLLM_P2P_HANDSHAKE_TIMEOUT_S", None)
|
||||
|
||||
transport = FakeDataTransport()
|
||||
conn = FakeConnection(peer_id="late-ack:1")
|
||||
session = P2PSession(
|
||||
peer_id="late-ack:1",
|
||||
local_id="local:1",
|
||||
transport=transport,
|
||||
local_block_len=4096,
|
||||
local_hash_seed=_DEFAULT_HASH_SEED,
|
||||
conn=conn,
|
||||
)
|
||||
session._connected_at = time.monotonic() - 5
|
||||
conn._inbox = [
|
||||
{
|
||||
TYPE_KEY: ConnectMsg.TYPE,
|
||||
ConnectMsg.PEER_ID: "late-ack:1",
|
||||
ConnectMsg.AGENT_METADATA: b"meta",
|
||||
ConnectMsg.BASE_ADDR: 0x1000,
|
||||
ConnectMsg.NUM_BLOCKS: 16,
|
||||
ConnectMsg.BLOCK_LEN: 4096,
|
||||
ConnectMsg.CONFIG_FINGERPRINT: "",
|
||||
ConnectMsg.HASH_SEED: _DEFAULT_HASH_SEED,
|
||||
},
|
||||
{TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: "late-ack:1"},
|
||||
]
|
||||
session.poll()
|
||||
assert session.ready
|
||||
assert session.alive
|
||||
|
||||
def test_activity_timestamp_updated_on_recv(self):
|
||||
transport = FakeDataTransport()
|
||||
conn = FakeConnection(peer_id="peer:1")
|
||||
session = P2PSession(
|
||||
peer_id="peer:1",
|
||||
local_id="local:1",
|
||||
transport=transport,
|
||||
local_block_len=4096,
|
||||
local_hash_seed=_DEFAULT_HASH_SEED,
|
||||
conn=conn,
|
||||
)
|
||||
before = session._last_activity_at
|
||||
|
||||
conn._inbox = [
|
||||
{
|
||||
TYPE_KEY: ConnectMsg.TYPE,
|
||||
ConnectMsg.PEER_ID: "peer:1",
|
||||
ConnectMsg.AGENT_METADATA: b"meta",
|
||||
ConnectMsg.BASE_ADDR: 0x1000,
|
||||
ConnectMsg.NUM_BLOCKS: 16,
|
||||
ConnectMsg.BLOCK_LEN: 4096,
|
||||
ConnectMsg.CONFIG_FINGERPRINT: "",
|
||||
ConnectMsg.HASH_SEED: _DEFAULT_HASH_SEED,
|
||||
},
|
||||
]
|
||||
session.poll()
|
||||
assert session._last_activity_at >= before
|
||||
|
||||
@@ -336,3 +336,47 @@ class TestZmqReconnect:
|
||||
finally:
|
||||
transport_a.close()
|
||||
transport_b.close()
|
||||
|
||||
|
||||
class TestExpandedMonitorEvents:
|
||||
"""The monitor mask includes EVENT_DISCONNECTED and EVENT_CLOSED so
|
||||
both are treated as fatal."""
|
||||
|
||||
def test_fatal_events_include_disconnected_and_closed(self):
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.zmq import (
|
||||
_FATAL_MONITOR_EVENTS,
|
||||
)
|
||||
|
||||
assert zmq.EVENT_DISCONNECTED in _FATAL_MONITOR_EVENTS
|
||||
assert zmq.EVENT_CLOSED in _FATAL_MONITOR_EVENTS
|
||||
|
||||
def test_monitor_mask_covers_fatal_events(self):
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.zmq import (
|
||||
_FATAL_MONITOR_EVENTS,
|
||||
_MONITOR_EVENT_MASK,
|
||||
)
|
||||
|
||||
for ev in _FATAL_MONITOR_EVENTS:
|
||||
assert _MONITOR_EVENT_MASK & ev == ev
|
||||
|
||||
def test_disconnect_detected_after_peer_shutdown(self):
|
||||
"""When a peer's transport closes, the outbound connection is
|
||||
eventually marked dead by the monitor."""
|
||||
transport_a, port_a = _make_transport()
|
||||
transport_b, _ = _make_transport()
|
||||
|
||||
try:
|
||||
conn_out = transport_b.connect(f"127.0.0.1:{port_a}")
|
||||
conn_out.send({"type": "hello"})
|
||||
_wait_for_inbound(transport_a)
|
||||
|
||||
transport_a.close()
|
||||
|
||||
end = time.monotonic() + 15
|
||||
while conn_out.alive and time.monotonic() < end:
|
||||
transport_b.poll()
|
||||
time.sleep(0.05)
|
||||
|
||||
assert not conn_out.alive
|
||||
finally:
|
||||
transport_b.close()
|
||||
|
||||
Reference in New Issue
Block a user