[EC Connector] P2P NIXL + CPU EC Connector (#47941)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
omerpaz95
2026-09-04 11:23:56 +08:00
committed by GitHub
co-authored by Claude Opus 5
parent 69cf055936
commit 8a0a7ee40a
28 changed files with 3521 additions and 72 deletions
@@ -333,3 +333,57 @@ def test_concurrent_alloc_evict():
t.join()
assert not errors, errors
# ── discard ──────────────────────────────────────────────────────────────────
def test_discard_not_ready_returns_blocks():
cache = EmbeddingCache(num_blocks=4)
entry = cache.alloc("h", 3)
assert entry is not None
cache.discard("h")
assert cache.get("h") is None
# All blocks reclaimed: a fresh 4-block alloc now fits.
assert cache.alloc("h2", 4) is not None
def test_discard_rejects_ready_entry():
cache = EmbeddingCache(num_blocks=4)
cache.alloc("h", 1)
cache.mark_ready("h")
with pytest.raises(AssertionError):
cache.discard("h")
def test_discard_rejects_absent_key():
cache = EmbeddingCache(num_blocks=4)
with pytest.raises(KeyError):
cache.discard("nope")
# ── pin_if_ready ─────────────────────────────────────────────────────────────
def test_pin_if_ready_pins_and_returns_entry():
cache = EmbeddingCache(num_blocks=4)
cache.alloc("h", 2)
cache.mark_ready("h")
entry = cache.pin_if_ready("h")
assert entry is not None and entry.ready and len(entry.block_ids) == 2
assert cache.alloc("h2", 4) is None # pinned ⇒ not evictable
def test_pin_if_ready_none_when_absent():
cache = EmbeddingCache(num_blocks=4)
assert cache.pin_if_ready("nope") is None
def test_pin_if_ready_returns_unpinned_entry_when_not_ready():
"""The producer needs "present but not ready" told apart from "absent":
the first is NACKed as retryable, the second as a miss."""
cache = EmbeddingCache(num_blocks=4)
cache.alloc("h", 1) # not-ready
entry = cache.pin_if_ready("h")
assert entry is not None and not entry.ready
assert cache.alloc("h2", 4) is None # not-ready ⇒ still not evictable
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
@@ -34,14 +36,36 @@ def test_worker_role_builds_only_worker(monkeypatch):
assert c.connector_scheduler is None
def test_request_finished_inherited_noop(monkeypatch):
@pytest.mark.parametrize("role", [ECConnectorRole.SCHEDULER, ECConnectorRole.WORKER])
def test_v1_model_runner_is_rejected(monkeypatch, role):
"""The V1 runner never reports save completions, so refuse to start."""
monkeypatch.setattr(
ECCPUConnector, "_make_scheduler", lambda self, cfg: MagicMock()
)
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
assert c.request_finished(MagicMock()) == (False, None)
monkeypatch.setattr(ECCPUConnector, "_make_worker", lambda self, cfg: MagicMock())
cfg = _cfg()
cfg.use_v2_model_runner = False
with pytest.raises(ValueError, match="V2 model runner"):
ECCPUConnector(cfg, role)
def test_factory_registered():
cls = ECConnectorFactory._registry["ECCPUConnector"]()
assert cls is ECCPUConnector
def test_request_finished_forwards_to_scheduler(monkeypatch):
from unittest.mock import MagicMock
import vllm.distributed.ec_transfer.ec_connector.cpu.connector as conn_mod # noqa: F401
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
fake_sched = MagicMock()
fake_sched.request_finished.return_value = (False, {"h1": {"peer_port": 1}})
monkeypatch.setattr(ECCPUConnector, "_make_scheduler", lambda self, cfg: fake_sched)
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
req = MagicMock()
assert c.request_finished(req) == (False, {"h1": {"peer_port": 1}})
fake_sched.request_finished.assert_called_once_with(req)
@@ -64,6 +64,14 @@ _requires_cuda_alike = pytest.mark.skipif(
reason="stalling the compute stream requires a CUDA-like platform",
)
_requires_swap_blocks_batch = pytest.mark.skipif(
not hasattr(torch.ops._C_cache_ops, "swap_blocks_batch"),
reason=(
"installed vllm C++ extension predates the swap_blocks_batch op "
"flush_saves/start_load_caches use; rebuild the extension to run this"
),
)
# Cycles to stall the compute stream for. Must comfortably outlast the host-side
# work the lifetime tests do while a copy is deliberately held back — ~0.5 s on
# a current datacenter GPU.
@@ -194,6 +202,7 @@ def _warm_up_stream_pools(worker: ECCPUWorker) -> None:
# ── save_caches ──────────────────────────────────────────────────────────────
@_requires_swap_blocks_batch
@_requires_accelerator
@pytest.mark.parametrize(
"n_elements,block_ids",
@@ -326,6 +335,7 @@ def test_save_caches_raises_when_allocated_blocks_too_small(make_worker):
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0, 1]}))
@_requires_swap_blocks_batch
@_requires_accelerator
def test_save_caches_batches_multiple_hashes(make_worker):
"""Multiple save_caches calls are batched into a single flush."""
@@ -401,6 +411,7 @@ def test_save_descriptors_hold_high_bit_source_addresses(make_worker):
# ── start_load_caches ────────────────────────────────────────────────────────
@_requires_swap_blocks_batch
@_requires_accelerator
def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker):
"""Single batched load across all hashes with correct byte→dtype→shape."""
@@ -465,6 +476,7 @@ def test_start_load_caches_noop_when_loads_is_empty(make_worker):
assert encoder_cache == {}
@_requires_swap_blocks_batch
@_requires_accelerator
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
"""Every hash in ``meta.loads`` is copied from mmap, including one whose
@@ -497,6 +509,7 @@ def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
assert torch.equal(new.cpu(), src_orig[1:])
@_requires_swap_blocks_batch
@_requires_accelerator
@pytest.mark.parametrize(
"tp_rank,pcp_rank",
@@ -526,6 +539,7 @@ def test_start_load_caches_works_on_all_ranks(make_worker, tp_rank, pcp_rank):
# ── round-trip ───────────────────────────────────────────────────────────────
@_requires_swap_blocks_batch
@_requires_accelerator
def test_save_then_load_round_trips_bytes(make_worker):
"""Full producer→mmap→consumer byte path in one shot."""
@@ -552,6 +566,7 @@ def test_save_then_load_round_trips_bytes(make_worker):
# ── memory lifetime across in-flight copies ─────────────────────────────────
@_requires_swap_blocks_batch
@_requires_cuda_alike
def test_save_survives_encoder_cache_free_before_copy_runs(make_worker):
"""The bytes handed to ``save_caches`` must reach the mmap intact even when
@@ -612,6 +627,7 @@ def test_save_survives_encoder_cache_free_before_copy_runs(make_worker):
)
@_requires_swap_blocks_batch
@_requires_cuda_alike
def test_load_buffer_survives_eviction_while_consumer_read_is_queued(make_worker):
"""A loaded encoder cache entry must keep its bytes for a consumer already
@@ -662,6 +678,7 @@ def test_load_buffer_survives_eviction_while_consumer_read_is_queued(make_worker
# ── buffer recycling ────────────────────────────────────────────────────────
@_requires_swap_blocks_batch
@_requires_accelerator
def test_buffer_pool_is_reused_across_save_steps(make_worker):
"""Once a save copy completes its descriptor buffers return to the pool and
@@ -689,6 +706,7 @@ def test_buffer_pool_is_reused_across_save_steps(make_worker):
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
@_requires_swap_blocks_batch
@_requires_accelerator
def test_buffer_pool_is_reused_across_load_steps(make_worker):
"""Once a load copy completes its descriptor buffers return to the pool and
@@ -793,6 +811,7 @@ def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
# ── e2e: scheduler + worker pipeline ────────────────────────────────────────
@_requires_swap_blocks_batch
@_requires_accelerator
def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
"""Full pipeline: scheduler allocates blocks, worker saves a GPU tensor to
+285
View File
@@ -0,0 +1,285 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for control-layer ABCs, ZmqClientConnection, and ECSession."""
from unittest.mock import MagicMock, patch
import pytest
import zmq
from vllm.distributed.ec_transfer.ec_connector.cpu.control.base import (
ControlConnection,
)
class _ConcreteConn(ControlConnection):
"""Minimal concrete subclass for ABC-instantiation tests."""
def __init__(self):
self._alive = True
self._sent = []
self._inbox = []
@property
def alive(self) -> bool:
return self._alive
def send(self, msg: bytes) -> None:
self._sent.append(msg)
def recv(self) -> list[bytes]:
msgs, self._inbox = self._inbox, []
return msgs
def mark_dead(self) -> None:
self._alive = False
def close(self) -> None:
self._alive = False
def test_control_connection_abc_cannot_be_instantiated_directly():
with pytest.raises(TypeError):
ControlConnection() # type: ignore[abstract]
def test_concrete_connection_send_recv():
conn = _ConcreteConn()
assert conn.alive
conn.send(b"hello")
conn._inbox.append(b"world")
assert conn.recv() == [b"world"]
assert conn._sent == [b"hello"]
def test_mark_dead_sets_alive_false():
conn = _ConcreteConn()
conn.mark_dead()
assert not conn.alive
def test_close_sets_alive_false():
conn = _ConcreteConn()
conn.close()
assert not conn.alive
# ── ZmqClientConnection ───────────────────────────────────────────────────────
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import ( # noqa: E402
ZmqClientConnection,
ZmqClientTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr # noqa: E402
def _conn(messages: list[list[bytes]], monitor=None) -> ZmqClientConnection:
"""Build a ZmqClientConnection whose dealer yields the given frame-lists."""
dealer = MagicMock()
dealer.recv_multipart.side_effect = [*messages, zmq.Again()]
return ZmqClientConnection(dealer=dealer, monitor=monitor)
def test_zmq_client_connection_recv_extracts_payloads_from_envelopes():
conn = _conn([[b"", b"msg1"], [b"", b"msg2"]])
assert conn.recv() == [b"msg1", b"msg2"]
def test_zmq_client_connection_recv_skips_frames_without_empty_delimiter():
# A frame that does not start with b"" is malformed and must be dropped.
conn = _conn([[b"", b"ok"], [b"no-delimiter", b"bad"]])
assert conn.recv() == [b"ok"]
def test_zmq_client_connection_recv_breaks_on_non_again_exception():
# A socket error should stop the drain and return whatever was collected.
dealer = MagicMock()
dealer.recv_multipart.side_effect = [[b"", b"first"], OSError("broken")]
conn = ZmqClientConnection(dealer=dealer, monitor=None)
assert conn.recv() == [b"first"]
def test_zmq_client_connection_send_prefixes_empty_delimiter():
dealer = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=None)
conn.send(b"payload")
dealer.send_multipart.assert_called_once_with([b"", b"payload"])
def test_zmq_client_connection_mark_dead_does_not_close_socket():
# mark_dead signals liveness only; resource teardown is close()'s job.
dealer = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=None)
conn.mark_dead()
assert not conn.alive
dealer.close.assert_not_called()
def test_zmq_client_connection_close_without_monitor():
dealer = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=None)
conn.close()
dealer.close.assert_called_once_with(linger=0)
assert not conn.alive
def test_zmq_client_connection_close_with_monitor_disables_then_closes_both():
# The disable_monitor → monitor.close → dealer.close sequence must be
# respected: closing the dealer before disabling the monitor can cause
# a zmq assertion in some versions.
dealer = MagicMock()
monitor = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=monitor)
conn.close()
dealer.disable_monitor.assert_called_once()
monitor.close.assert_called_once_with(linger=0)
dealer.close.assert_called_once_with(linger=0)
assert not conn.alive
def test_zmq_client_connection_close_is_idempotent():
dealer = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=None)
conn.close()
conn.close() # must not raise
assert not conn.alive
# ── ZmqClientTransport ────────────────────────────────────────────────────────
def _make_transport() -> tuple[ZmqClientTransport, MagicMock]:
"""Return a ZmqClientTransport with a mocked ZMQ context."""
ctx = MagicMock()
ctx.socket.return_value = MagicMock()
with (
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context",
return_value=ctx,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket",
return_value=MagicMock(),
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path",
return_value="tcp://host:0",
),
):
transport = ZmqClientTransport()
transport._ctx = ctx
return transport, ctx
def _inject_conn(transport: ZmqClientTransport, addr: PeerAddr) -> ZmqClientConnection:
"""Inject a ZmqClientConnection directly into the transport pool."""
conn = ZmqClientConnection(dealer=MagicMock(), monitor=MagicMock())
transport._connections[addr] = conn
return conn
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
def test_zmq_client_transport_connect_caches_connection(
mock_ctx_cls, mock_path, mock_sock
):
mock_ctx_cls.return_value = MagicMock()
mock_ctx_cls.return_value.socket.return_value = MagicMock()
mock_sock.return_value = MagicMock()
transport = ZmqClientTransport()
addr: PeerAddr = ("host", 1234)
assert transport.connect(addr) is transport.connect(addr)
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
def test_zmq_client_transport_connect_creates_independent_conns_per_addr(
mock_ctx_cls, mock_path, mock_sock
):
mock_ctx_cls.return_value = MagicMock()
mock_ctx_cls.return_value.socket.return_value = MagicMock()
mock_sock.return_value = MagicMock()
transport = ZmqClientTransport()
conn_a = transport.connect(("host-a", 1111))
conn_b = transport.connect(("host-b", 2222))
assert conn_a is not conn_b
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
def test_zmq_client_transport_poll_dead_detects_disconnected_peer(mock_recv_mon):
transport, _ = _make_transport()
addr: PeerAddr = ("host", 1234)
conn = _inject_conn(transport, addr)
mock_recv_mon.side_effect = [{"event": zmq.EVENT_DISCONNECTED}, zmq.Again()]
dead = transport.poll_dead()
assert addr in dead
assert not conn.alive
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
def test_zmq_client_transport_poll_dead_skips_conn_without_monitor(mock_recv_mon):
transport, _ = _make_transport()
addr: PeerAddr = ("host", 9999)
conn = ZmqClientConnection(dealer=MagicMock(), monitor=None)
transport._connections[addr] = conn
dead = transport.poll_dead()
assert dead == []
mock_recv_mon.assert_not_called()
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
def test_zmq_client_transport_poll_dead_ignores_non_disconnect_events(mock_recv_mon):
transport, _ = _make_transport()
addr: PeerAddr = ("host", 1234)
conn = _inject_conn(transport, addr)
# A CONNECTED event should not trigger a dead notification.
mock_recv_mon.side_effect = [{"event": zmq.EVENT_CONNECTED}, zmq.Again()]
dead = transport.poll_dead()
assert dead == []
assert conn.alive
def test_zmq_client_transport_remove_closes_and_evicts_conn():
transport, _ = _make_transport()
addr: PeerAddr = ("host", 5678)
conn = _inject_conn(transport, addr)
removed = transport.remove(addr)
assert removed is conn
assert addr not in transport._connections
conn.dealer.close.assert_called()
def test_zmq_client_transport_remove_returns_none_for_unknown_addr():
transport, _ = _make_transport()
assert transport.remove(("ghost", 0)) is None
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
def test_zmq_client_transport_close_closes_all_connections(
mock_ctx_cls, mock_path, mock_sock
):
mock_ctx = MagicMock()
mock_ctx_cls.return_value = mock_ctx
mock_ctx.socket.return_value = MagicMock()
dealer_a, dealer_b = MagicMock(), MagicMock()
mock_sock.side_effect = [dealer_a, dealer_b]
transport = ZmqClientTransport()
transport.connect(("a", 1))
transport.connect(("b", 2))
transport.close()
dealer_a.close.assert_called()
dealer_b.close.assert_called()
assert transport._connections == {}
+184
View File
@@ -0,0 +1,184 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for DataTransport ABC and NixlDataTransport."""
from unittest.mock import MagicMock, patch
import pytest
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
DataTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
serialize_mem_descriptor,
)
_NIXL_PATH = "vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl"
# ── helpers ───────────────────────────────────────────────────────────────────
def _make_nixl_mock(prep_side_effect=None) -> MagicMock:
nixl = MagicMock()
nixl.get_agent_metadata.return_value = b"agent-meta"
nixl.get_reg_descs.return_value = MagicMock()
nixl.get_xfer_descs.return_value = MagicMock()
nixl.prep_xfer_dlist.side_effect = prep_side_effect or [77, 88]
nixl.add_remote_agent.return_value = "remote-agent-1"
nixl.get_new_notifs.return_value = {}
nixl.check_xfer_state.return_value = "PROC"
return nixl
def _make_transport(nixl_mock=None):
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
NixlDataTransport,
)
mock = nixl_mock or _make_nixl_mock()
with (
patch(f"{_NIXL_PATH}.NixlWrapper", return_value=mock),
patch(f"{_NIXL_PATH}.nixl_agent_config", return_value=MagicMock()),
):
t = NixlDataTransport("agent", 0, 4, 64, 256)
t._nixl = mock
return t
# ── ABC ───────────────────────────────────────────────────────────────────────
def test_data_transport_abc_cannot_be_instantiated():
with pytest.raises(TypeError):
DataTransport() # type: ignore[abstract]
# ── NixlDataTransport construction ───────────────────────────────────────────
def test_nixl_data_transport_registers_memory_on_init():
nixl = _make_nixl_mock()
_make_transport(nixl)
nixl.register_memory.assert_called_once()
def test_nixl_data_transport_unwinds_registration_if_init_fails():
"""A failure after register_memory must not leave the region pinned.
__init__ propagates, so the caller never gets a transport to deregister.
"""
nixl = _make_nixl_mock(prep_side_effect=RuntimeError("no dlist"))
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
NixlDataTransport,
)
with (
patch(f"{_NIXL_PATH}.NixlWrapper", return_value=nixl),
patch(f"{_NIXL_PATH}.nixl_agent_config", return_value=MagicMock()),
pytest.raises(RuntimeError, match="no dlist"),
):
NixlDataTransport("agent", 0, 4, 64, 256)
nixl.deregister_memory.assert_called_once_with(nixl.get_reg_descs.return_value)
def test_nixl_data_transport_raises_if_nixl_unavailable():
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
NixlDataTransport,
)
with (
patch(f"{_NIXL_PATH}.NixlWrapper", None),
patch(f"{_NIXL_PATH}.nixl_agent_config", None),
pytest.raises(RuntimeError, match="requires NIXL"),
):
NixlDataTransport("a", 0, 1, 64, 64)
# ── public accessors ──────────────────────────────────────────────────────────
def test_get_agent_metadata_returns_nixl_blob():
assert _make_transport().get_agent_metadata() == b"agent-meta"
def test_get_mem_descriptor_roundtrips_block_descs():
t = _make_transport()
raw = t.get_mem_descriptor()
assert isinstance(raw, bytes) and len(raw) > 0
# ── add_remote_peer ───────────────────────────────────────────────────────────
def test_add_remote_peer_returns_agent_name_and_stores_handle_internally():
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
t = _make_transport(nixl)
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
agent_name = t.add_remote_peer(b"meta", mem_desc)
assert agent_name == "remote-agent-1"
# Dlist handle stored internally; accessible via _peer_handles.
assert t._peer_handles["remote-agent-1"] == 88
def test_add_remote_peer_registers_remote_agent():
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
t = _make_transport(nixl)
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
t.add_remote_peer(b"fresh-meta", mem_desc)
nixl.add_remote_agent.assert_called_once_with(b"fresh-meta")
# ── remove_remote_peer ────────────────────────────────────────────────────────
def test_remove_remote_peer_delegates_to_nixl():
t = _make_transport()
t.remove_remote_peer("agent-x")
t._nixl.remove_remote_agent.assert_called_once_with("agent-x")
def test_remove_remote_peer_swallows_exceptions():
t = _make_transport()
t._nixl.remove_remote_agent.side_effect = RuntimeError("gone")
t.remove_remote_peer("agent-x") # must not raise
# ── post_read ─────────────────────────────────────────────────────────────────
def test_post_read_raises_on_index_count_mismatch():
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
t = _make_transport(nixl)
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
agent = t.add_remote_peer(b"meta", mem_desc)
with pytest.raises(ValueError, match="mismatch"):
t.post_read([0, 1], agent, [5], notif_msg=b"sid")
def test_post_read_calls_make_prepped_xfer_and_transfer():
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
t = _make_transport(nixl)
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
agent = t.add_remote_peer(b"meta", mem_desc)
nixl.make_prepped_xfer.return_value = 999
handle = t.post_read([0, 1], agent, [5, 6], notif_msg=b"sid")
nixl.make_prepped_xfer.assert_called_once()
nixl.transfer.assert_called_once_with(999)
assert handle == 999
# ── release_xfer_handle / deregister ─────────────────────────────────────────
def test_release_xfer_handle_swallows_exceptions():
t = _make_transport()
t._nixl.release_xfer_handle.side_effect = RuntimeError("stale")
t.release_xfer_handle(42) # must not raise
def test_deregister_swallows_exceptions():
t = _make_transport()
t._nixl.deregister_memory.side_effect = RuntimeError("already gone")
t.deregister() # must not raise
@@ -7,7 +7,8 @@ No running engine required.
from unittest.mock import MagicMock
from tests.v1.core.utils import create_scheduler
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
@@ -15,6 +16,16 @@ from vllm.outputs import CompletionOutput, RequestOutput
from vllm.sampling_params import SamplingParams
from vllm.v1.request import Request, RequestStatus
# tests.v1.core.utils transitively imports the example KV connector, which
# requires the compiled CUDA flash-attention extensions; skip cleanly rather
# than erroring at collection when those extensions aren't built.
_core_utils = pytest.importorskip(
"tests.v1.core.utils",
reason="requires compiled CUDA flash-attention extensions",
exc_type=ImportError,
)
create_scheduler = _core_utils.create_scheduler
EC_PARAMS: dict = {"mm_hash_abc": {"peer_host": "10.0.0.1", "peer_port": 5501}}
@@ -0,0 +1,41 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import msgspec
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
EC_CONNECTOR_VERSION,
XferAck,
XferReq,
XferStatus,
compute_ec_compatibility_hash,
)
def test_xferreq_roundtrip():
req = XferReq(mm_hash="h1", compatibility_hash="c1", session_id="s1")
data = msgspec.msgpack.encode(req)
out = msgspec.msgpack.decode(data, type=XferReq)
assert out.mm_hash == "h1"
assert out.compatibility_hash == "c1"
assert out.connector_version == EC_CONNECTOR_VERSION
def test_xferack_roundtrip_ok():
ack = XferAck(
mm_hash="h1",
status=XferStatus.OK,
src_block_indices=[1, 2],
agent_metadata=b"meta",
mem_descriptor=b"desc",
)
out = msgspec.msgpack.decode(msgspec.msgpack.encode(ack), type=XferAck)
assert out.status == XferStatus.OK
assert out.src_block_indices == [1, 2]
def test_compat_hash_deterministic_and_sensitive():
a = compute_ec_compatibility_hash("0.1", "m", "float16", 64)
b = compute_ec_compatibility_hash("0.1", "m", "float16", 64)
c = compute_ec_compatibility_hash("0.1", "m", "float16", 128)
assert a == b
assert a != c
@@ -0,0 +1,350 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import uuid
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
_N, _BS, _HID, _ES = 16, 64, 32, 2
class _Pos:
def __init__(self, offset, length):
self.offset, self.length = offset, length
class _Feature:
def __init__(self, mm_hash, length=1):
self.mm_hash = mm_hash
self.identifier = mm_hash
self.mm_position = _Pos(0, length)
class _Request:
def __init__(self, features, params=None, req_id="r1"):
self.mm_features = features
self.request_id = req_id
self.ec_transfer_params = params
class _FakeResults:
def __init__(self):
self.completed = set()
self.tombstoned = set()
self.quarantined = set()
self.retryable = set()
self.settled = []
class _FakeSession:
"""Records started transfers; lets the test flip them to completed."""
def __init__(self):
self.started = []
self._results = _FakeResults()
def start_xfer(self, mm_hash, block_indices, deadline):
self.started.append(mm_hash)
def poll(self, messages, now):
pass
def take_results(self):
r, self._results = self._results, _FakeResults()
return r
def close(self):
pass
class _FakeTransport:
"""Stand-in for ZmqClientTransport so _poll_step runs without ZMQ."""
def poll(self):
return {}
def poll_dead(self):
return []
def close(self):
pass
def _consumer_sched(monkeypatch):
def _region(cfg):
return ECSharedRegion(
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
)
monkeypatch.setattr(sched_mod, "create_ec_shared_region", _region)
# ec_enable_nixl defaults to False, so this builds gate-off; the fields
# below flip on the NIXL consumer state directly.
s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_consumer"))
# Turn on NIXL consumer state without constructing real transports.
s._nixl_enabled = True
s._transport = _FakeTransport()
s._data = None
s._compat_hash = "c"
# _setup_nixl normally computes these from model_config; set them
# directly since this helper builds gate-off then flips fields on.
s._hidden_dim = _HID
s._element_size = _ES
return s
def _params(mm_hash, length):
return {
mm_hash: {
"peer_host": "h",
"peer_port": 1,
"size_bytes": length * _HID * _ES,
}
}
def test_new_remote_read_defers_then_completes(monkeypatch):
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
# Route _start_xfer to our fake session instead of real ZMQ/NIXL, but
# still reserve a real not-ready cache entry so mark_ready works.
def _fake_start(mm_hash, info, size):
entry = s._cache.alloc(mm_hash, 1)
assert entry is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _fake_start)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
# Step 1: unseen remote item -> read started, request deferred.
assert s.ensure_cache_available(req, 0) is False
assert "h1" in s._in_flight
assert "h1" in fake.started
s.build_connector_meta(scheduler_output=None) # re-arms _first_in_batch
# Session reports completion this step.
fake._results.completed.add("h1")
s._sessions[("h", 1)] = fake
# Step 2: poll drains completion -> entry marked ready, request admitted.
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
assert "h1" in s._step_completed
entry = s._cache.get("h1")
assert entry is not None and entry.ready
# The scheduler now allocates encoder input; the ready entry loads through
# the same local path as a natively cached encoding.
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta.loads
assert "h1" not in s._step_completed # cleared by promote
# Step 3: still cached -> admitted directly, no new transfer.
assert s.ensure_cache_available(req, 0) is True
s.shutdown()
def test_no_params_is_ready(monkeypatch):
s = _consumer_sched(monkeypatch)
assert s.ensure_cache_available(_Request([_Feature("x")], params=None), 0) is True
s.shutdown()
def test_tombstoned_read_discards_and_blocks_retry(monkeypatch):
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
def _fake_start(mm_hash, info, size):
entry = s._cache.alloc(mm_hash, 1)
assert entry is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _fake_start)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
# Step 1: start the read.
assert s.ensure_cache_available(req, 0) is False
s.build_connector_meta(scheduler_output=None)
# Producer rejected the read -> tombstoned.
fake._results.tombstoned.add("h1")
s._sessions[("h", 1)] = fake
# Step 2: poll discards the entry and records a tombstone; the tombstone
# is consumed by admit so the request proceeds to local compute.
assert s.ensure_cache_available(req, 0) is True
assert s._cache.get("h1") is None
assert "h1" not in s._in_flight
s.build_connector_meta(scheduler_output=None)
# Step 3: tombstone was consumed, so a fresh transfer starts again.
assert s.ensure_cache_available(req, 0) is False
assert "h1" in s._in_flight
s.shutdown()
def test_retryable_read_re_requests_without_admitting(monkeypatch):
"""A not-ready NACK defers the request and re-requests the read.
Unlike a tombstone it never admits the request, because an admitted request
whose media was rewritten away upstream has no embedding and no way to
recompute one.
"""
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
def _fake_start(mm_hash, info, size):
entry = s._cache.alloc(mm_hash, 1)
assert entry is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _fake_start)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
assert s.ensure_cache_available(req, 0) is False
s.build_connector_meta(scheduler_output=None)
# Producer's save has not landed: retryable, not tombstoned.
fake._results.retryable.add("h1")
s._sessions[("h", 1)] = fake
# The entry is released and the read re-issued in the same step, so the
# request stays deferred rather than being admitted unfetched.
assert s.ensure_cache_available(req, 0) is False
assert "h1" in s._in_flight
assert "h1" not in s._tombstones
assert fake.started.count("h1") == 2
s.shutdown()
def test_size_mismatch_skips_transfer(monkeypatch):
s = _consumer_sched(monkeypatch)
# Advertised size disagrees with pos.length * hidden_dim * element_size.
bad = {"h1": {"peer_host": "h", "peer_port": 1, "size_bytes": 999}}
req = _Request([_Feature("h1", 1)], params=bad)
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
assert s._cache.get("h1") is None
s.shutdown()
def test_metadata_only_entry_admits_without_transfer(monkeypatch):
# A producer that reported placeholder metadata but had no cache entry
# to transfer omits peer_host/peer_port/size_bytes entirely; the consumer
# must fall back to local compute rather than treating that as a size
# mismatch.
s = _consumer_sched(monkeypatch)
metadata_only = {"h1": {"image_grid_thw": [1, 2, 3]}}
req = _Request([_Feature("h1", 1)], params=metadata_only)
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
assert s._cache.get("h1") is None
s.shutdown()
def test_orphan_not_ready_entry_falls_back_no_realloc(monkeypatch):
# Post-quarantine, post-tombstone-consumed orphan: a not-ready cache
# entry exists for the mm_hash, but the hash is tracked in none of
# _in_flight / _step_completed / _tombstones. Its blocks are held by a
# quarantined/settling DMA and must not be re-allocated.
s = _consumer_sched(monkeypatch)
entry = s._cache.alloc("h1", 1)
assert entry is not None and not entry.ready
assert "h1" not in s._in_flight
assert "h1" not in s._step_completed
assert "h1" not in s._tombstones
# Spy on alloc: record calls, delegate to the real allocator (which
# asserts on a duplicate key). A re-alloc for "h1" is the bug.
real_alloc = s._cache.alloc
calls: list[str] = []
def _spy_alloc(key, n):
calls.append(key)
return real_alloc(key, n)
monkeypatch.setattr(s._cache, "alloc", _spy_alloc)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
# Must not raise; request falls back to local compute (admitted True),
# the orphan entry is untouched, and alloc is never called again.
assert s.ensure_cache_available(req, 0) is True
assert calls == []
assert s._cache.get("h1") is entry
assert not entry.ready
s.shutdown()
def test_alloc_failure_falls_back_to_local(monkeypatch):
s = _consumer_sched(monkeypatch)
# Force the cache to reject the allocation.
monkeypatch.setattr(s._cache, "alloc", lambda key, n: None)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
# _start_xfer returns False -> request admitted for local recompute.
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
s.shutdown()
def test_already_computed_feature_admits_without_transfer(monkeypatch):
# A feature entirely within num_computed_tokens is skipped by admit; no
# cache lookup/alloc or transfer should be triggered for it.
s = _consumer_sched(monkeypatch)
calls: list[str] = []
real_alloc = s._cache.alloc
def _spy_alloc(key, n):
calls.append(key)
return real_alloc(key, n)
monkeypatch.setattr(s._cache, "alloc", _spy_alloc)
# Feature spans [0, 4); with num_computed_tokens=4 it is fully computed.
req = _Request([_Feature("h1", 4)], params=_params("h1", 4))
assert s.ensure_cache_available(req, 4) is True
assert "h1" not in s._in_flight
assert calls == []
s.shutdown()
def test_local_ready_entry_admits_without_transfer(monkeypatch):
# A READY entry already present in the cache is a local hit: admit
# returns True and no new transfer is started for that mm_hash.
s = _consumer_sched(monkeypatch)
entry = s._cache.alloc("h1", 1)
assert entry is not None
s._cache.mark_ready("h1")
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
s.shutdown()
def test_in_flight_hash_defers_without_second_transfer(monkeypatch):
# A mm_hash already tracked in _in_flight must defer the request rather
# than starting a duplicate transfer.
s = _consumer_sched(monkeypatch)
started: list[str] = []
def _spy_start_xfer(mm_hash, info, size):
started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _spy_start_xfer)
s._in_flight.add("h1")
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
assert s.ensure_cache_available(req, 0) is False
assert started == []
s.shutdown()
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import uuid
import pytest
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.config.ec_transfer import ECTransferConfig
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
from vllm.distributed.nixl_utils import NixlWrapper
_N = 16
_BS = 64
def _region() -> ECSharedRegion:
return ECSharedRegion(
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
)
def test_ec_enable_nixl_absent_from_extra_config_by_default():
cfg = ECTransferConfig()
assert cfg.get_from_extra_config("ec_enable_nixl", False) is False
def test_ec_enable_nixl_read_from_extra_config():
cfg = ECTransferConfig(
ec_connector="ECCPUConnector",
ec_role="ec_both",
ec_connector_extra_config={"ec_enable_nixl": True},
)
assert cfg.get_from_extra_config("ec_enable_nixl", False) is True
def test_gate_off_builds_no_nixl(monkeypatch):
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
# ec_enable_nixl defaults to False.
s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_both"))
assert s._nixl_enabled is False
assert getattr(s, "_data", None) is None
assert getattr(s, "_producer_session", None) is None
s.shutdown()
def test_string_false_in_extra_config_leaves_nixl_off(monkeypatch):
"""Extra config is not type coerced, and bool("false") is True."""
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
cfg = create_ec_vllm_config(ec_role="ec_both")
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = "false"
s = ECCPUScheduler(cfg)
assert s._nixl_enabled is False
s.shutdown()
@pytest.mark.skipif(NixlWrapper is None, reason="Requires NIXL package")
def test_gate_on_wires_data_transport_and_producer_session(monkeypatch):
# Port 0 lets the OS pick an ephemeral port so the real ZMQ ROUTER bind
# in ProducerSession.start() cannot collide with another test/process.
monkeypatch.setenv("VLLM_EC_SIDE_CHANNEL_HOST", "127.0.0.1")
monkeypatch.setenv("VLLM_EC_SIDE_CHANNEL_PORT", "0")
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
cfg = create_ec_vllm_config(ec_role="ec_both")
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = True
s = ECCPUScheduler(cfg)
try:
assert s._nixl_enabled is True
assert s._data is not None
assert s._compat_hash is not None
assert s._producer_session is not None
assert s._peer_host == "127.0.0.1"
assert s._transport is not None # is_ec_consumer is also True
finally:
s.shutdown()
@@ -0,0 +1,113 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import uuid
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
_N, _BS = 16, 64
class _Pos:
def __init__(self, offset, length):
self.offset, self.length = offset, length
class _Feature:
def __init__(self, mm_hash, length=1):
self.mm_hash = mm_hash
self.identifier = mm_hash
self.mm_position = _Pos(0, length)
self.data = None
self.modality = "image"
class _Request:
def __init__(self, features, req_id="r1"):
self.mm_features = features
self.request_id = req_id
def _sched_gate_off(monkeypatch):
def _region(cfg):
return ECSharedRegion(
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
)
monkeypatch.setattr(sched_mod, "create_ec_shared_region", _region)
return ECCPUScheduler(create_ec_vllm_config(ec_role="ec_both"))
def test_request_finished_gate_off_returns_none(monkeypatch):
s = _sched_gate_off(monkeypatch)
assert s.request_finished(_Request([_Feature("h1")])) == (False, None)
s.shutdown()
def test_request_finished_producer_emits_params(monkeypatch):
s = _sched_gate_off(monkeypatch)
# Simulate NIXL-enabled producer bookkeeping without building real NIXL.
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
# _setup_nixl normally computes these from model_config; set them
# directly since this test builds gate-off then flips fields on.
s._hidden_dim, s._element_size = 32, 2
# feature length=2, hidden_dim=32, element_size=2 -> 128 bytes -> 2 blocks.
entry = s._cache.alloc("h1", 2)
assert entry is not None
s._cache.mark_ready("h1")
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
assert delay is False
assert params == {
"h1": {
"metadata": {},
"peer_host": "1.2.3.4",
"peer_port": 5601,
"size_bytes": 2 * 32 * 2,
}
}
s.shutdown()
def test_request_finished_announces_not_ready_entry(monkeypatch):
"""A save whose GPU->mmap copy hasn't been confirmed complete yet is
still announced: the entry can't be evicted before it's ready, and a read
arriving too early is NACKed NACK_NOT_READY for the consumer to retry —
protecting it is not this method's job."""
s = _sched_gate_off(monkeypatch)
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
s._hidden_dim, s._element_size = 32, 2
s._cache.alloc("h1", 2) # allocated but not marked ready
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
assert delay is False
assert params == {
"h1": {
"metadata": {},
"peer_host": "1.2.3.4",
"peer_port": 5601,
"size_bytes": 2 * 32 * 2,
}
}
s.shutdown()
def test_request_finished_skips_unallocated_entry(monkeypatch):
s = _sched_gate_off(monkeypatch)
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
s._hidden_dim, s._element_size = 32, 2
# No alloc() for "h1" at all — e.g. the cache was full at save time.
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
assert delay is False
# The item's placeholder metadata is still reported even though there's
# no cache entry to transfer (empty "metadata": no fields, no transfer).
assert params == {"h1": {"metadata": {}}}
s.shutdown()
+470
View File
@@ -0,0 +1,470 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ProducerXfer, ProducerSession, ConsumerXfer, ConsumerSession."""
import time
from unittest.mock import MagicMock
import pytest
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
XferAck,
XferReq,
XferStatus,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
_CONSUMER_QUARANTINE_TIMEOUT_S,
ConsumerSession,
ConsumerXfer,
ProducerSession,
ProducerXfer,
XferState,
)
# ── helpers ───────────────────────────────────────────────────────────────────
def _make_data(xfer_state: str = "PROC") -> MagicMock:
data = MagicMock()
data.check_xfer_state.return_value = xfer_state
data.post_read.return_value = 99
return data
def _ok_ack(
mm_hash: str = "h1",
session_id: str = "sess-1",
src_indices: list | None = None,
) -> XferAck:
return XferAck(
mm_hash=mm_hash,
status=XferStatus.OK,
session_id=session_id,
src_block_indices=src_indices or [0, 1],
agent_metadata=b"meta",
mem_descriptor=b"desc",
)
def _xfer(
data: MagicMock | None = None,
deadline: float | None = None,
consumer_session_id: str = "sess-1",
) -> ConsumerXfer:
return ConsumerXfer(
mm_hash="h1",
block_indices=[10, 11],
addr=("host", 1234),
deadline=deadline if deadline is not None else time.monotonic() + 60,
data=data or _make_data(),
consumer_session_id=consumer_session_id,
)
def _started(
data: MagicMock | None = None, deadline: float | None = None
) -> ConsumerXfer:
"""ConsumerXfer that has already received an OK XferAck."""
x = _xfer(data, deadline)
x.handle_ack(_ok_ack(session_id="sess-1", src_indices=[5, 6]), agent_name="agent-1")
return x
# ── ProducerXfer ──────────────────────────────────────────────────────────────
def test_producer_xfer_not_expired_before_deadline():
x = ProducerXfer("h1", [0, 1], deadline=time.monotonic() + 60)
assert not x.is_expired()
def test_producer_xfer_expired_after_deadline():
x = ProducerXfer("h1", [0, 1], deadline=time.monotonic() - 1)
assert x.is_expired()
# ── ProducerSession ───────────────────────────────────────────────────────────
def _xfer_req(mm_hash: str = "h1", session_id: str = "sess-1") -> XferReq:
return XferReq(
mm_hash=mm_hash, compatibility_hash="hash-abc", session_id=session_id
)
def _make_producer_session(cache: EmbeddingCache | None = None) -> ProducerSession:
data = _make_data()
data.get_agent_metadata.return_value = b"meta"
data.get_mem_descriptor.return_value = b"desc"
return ProducerSession(
transport=MagicMock(),
data=data,
cache=cache or EmbeddingCache(num_blocks=8),
compat_hash="hash-abc",
)
def test_producer_session_grant_pins_ready_entry_and_returns_blocks():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
ack = s._grant_or_nack(_xfer_req(mm_hash="h1"))
assert ack.status == XferStatus.OK
entry = cache.get("h1")
assert not entry.evictable # pinned by the grant
assert list(ack.src_block_indices) == list(entry.block_ids)
def test_producer_session_grant_nacks_missing_entry():
s = _make_producer_session()
ack = s._grant_or_nack(_xfer_req(mm_hash="nope"))
assert ack.status == XferStatus.NACK_MISSING
def test_producer_session_grant_nacks_not_ready_entry_as_retryable():
"""An announced entry whose save is still in flight is distinguished from a
miss, so the consumer retries instead of falling back to local compute."""
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2) # not ready
s = _make_producer_session(cache)
ack = s._grant_or_nack(_xfer_req(mm_hash="h1"))
assert ack.status == XferStatus.NACK_NOT_READY
def test_producer_session_grant_nacks_version_mismatch():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
req = XferReq(
mm_hash="h1",
compatibility_hash="hash-abc",
session_id="sess-1",
connector_version=-1,
)
ack = s._grant_or_nack(req)
assert ack.status == XferStatus.NACK_VERSION
entry = cache.get("h1")
assert entry.evictable # nothing was pinned
def test_producer_session_grant_nacks_compat_hash_mismatch():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
req = XferReq(mm_hash="h1", compatibility_hash="wrong-hash", session_id="sess-1")
ack = s._grant_or_nack(req)
assert ack.status == XferStatus.NACK_INCOMPAT
entry = cache.get("h1")
assert entry.evictable # nothing was pinned
def test_producer_session_notif_unpins_via_cache():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
s._grant_or_nack(_xfer_req(mm_hash="h1", session_id="sess-1"))
entry = cache.get("h1")
assert not entry.evictable # pinned
s._data.get_new_notifs.return_value = {"agent": [b"sess-1:h1"]}
s._drain_notifs()
assert entry.evictable # unpinned via cache
assert s._active_xfers == {}
def test_producer_session_sweep_timeouts_unpins_via_cache():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
s._grant_or_nack(_xfer_req(mm_hash="h1", session_id="sess-1"))
key = next(iter(s._active_xfers))
s._active_xfers[key].deadline = time.monotonic() - 1
s._sweep_timeouts()
assert cache.get("h1").evictable
assert s._active_xfers == {}
# ── ConsumerXfer.handle_ack ───────────────────────────────────────────────────
def test_consumer_xfer_handle_ack_ok_starts_read():
data = _make_data()
x = _xfer(data)
assert x.handle_ack(_ok_ack(), agent_name="agent-1") is True
assert x.transfer_handle == 99
def test_consumer_xfer_handle_ack_notif_msg_is_session_id_colon_mm_hash():
data = _make_data()
x = _xfer(data, consumer_session_id="my-sess")
x.handle_ack(_ok_ack(mm_hash="h1"), agent_name="agent-1")
_, kwargs = data.post_read.call_args
assert kwargs["notif_msg"] == b"my-sess:h1"
def test_consumer_xfer_handle_ack_forwards_correct_args():
data = _make_data()
x = _xfer(data)
x.handle_ack(_ok_ack(src_indices=[5, 6]), agent_name="agent-1")
args, _ = data.post_read.call_args
assert args[0] == [10, 11] # local block_indices
assert args[1] == "agent-1" # agent_name (not a raw handle)
assert args[2] == [5, 6] # remote src_block_indices
def test_consumer_xfer_handle_ack_resets_deadline():
original = time.monotonic() + 2
x = _xfer(deadline=original)
x.handle_ack(_ok_ack(), agent_name="agent-1")
assert x.deadline > original
def test_consumer_xfer_handle_ack_nack_returns_false():
data = _make_data()
x = _xfer(data)
assert (
x.handle_ack(XferAck(mm_hash="h1", status=XferStatus.NACK_MISSING), "a")
is False
)
assert x.transfer_handle is None
data.post_read.assert_not_called()
# ── ConsumerXfer.poll — WAITING_ACK ──────────────────────────────────────────
def test_consumer_xfer_poll_waiting_ack_before_deadline():
assert _xfer().poll(time.monotonic()) == XferState.WAITING_ACK
def test_consumer_xfer_poll_waiting_ack_timeout():
assert (
_xfer(deadline=time.monotonic() - 1).poll(time.monotonic())
== XferState.ACK_TIMEOUT
)
# ── ConsumerXfer.poll — READING ───────────────────────────────────────────────
def test_consumer_xfer_poll_reading_proc():
assert _started().poll(time.monotonic()) == XferState.READING
def test_consumer_xfer_poll_reading_done_releases_handle():
data = _make_data(xfer_state="DONE")
x = _started(data)
assert x.poll(time.monotonic()) == XferState.DONE
data.release_xfer_handle.assert_called_once_with(99)
assert x.transfer_handle is None
def test_consumer_xfer_poll_reading_unexpected_state_fails():
data = _make_data(xfer_state="ERR")
x = _started(data)
assert x.poll(time.monotonic()) == XferState.READ_FAILED
data.release_xfer_handle.assert_called_once_with(99)
def test_consumer_xfer_poll_reading_exception_fails_gracefully():
data = _make_data()
data.check_xfer_state.side_effect = RuntimeError("nixl crash")
x = _started(data)
assert x.poll(time.monotonic()) == XferState.READ_FAILED
def test_consumer_xfer_poll_timeout_quarantines_without_releasing():
data = _make_data(xfer_state="PROC")
x = _started(data)
assert x.poll(time.monotonic() + 9999) == XferState.QUARANTINED
data.release_xfer_handle.assert_not_called()
assert x.transfer_handle == 99
# ── ConsumerXfer.poll — QUARANTINED ───────────────────────────────────────────
def test_consumer_xfer_quarantined_proc_stays():
data = _make_data(xfer_state="PROC")
x = _started(data)
x.poll(time.monotonic() + 9999) # → QUARANTINED
assert x.poll(time.monotonic()) == XferState.QUARANTINED
data.release_xfer_handle.assert_not_called()
def test_consumer_xfer_quarantined_terminal_settles():
data = _make_data(xfer_state="PROC")
x = _started(data)
x.poll(time.monotonic() + 9999) # → QUARANTINED
data.check_xfer_state.return_value = "DONE"
assert x.poll(time.monotonic()) == XferState.SETTLED
data.release_xfer_handle.assert_called_once_with(99)
assert x.transfer_handle is None
def test_consumer_xfer_quarantined_exception_settles():
data = _make_data(xfer_state="PROC")
x = _started(data)
x.poll(time.monotonic() + 9999)
data.check_xfer_state.side_effect = RuntimeError("gone")
assert x.poll(time.monotonic()) == XferState.SETTLED
def test_consumer_xfer_quarantined_timeout_evicts_blocks():
"""Quarantine timeout should release handle and return READ_FAILED."""
data = _make_data(xfer_state="PROC")
x = _started(data)
# First poll transitions to QUARANTINED (deadline extended by 60s)
x.poll(time.monotonic() + 9999)
# Second poll with expired quarantine deadline should evict
expired_time = time.monotonic() + 9999 + _CONSUMER_QUARANTINE_TIMEOUT_S + 1
assert x.poll(expired_time) == XferState.READ_FAILED
data.release_xfer_handle.assert_called_once_with(99)
assert x.transfer_handle is None
# ── ConsumerXfer.cancel / release ────────────────────────────────────────────
def test_consumer_xfer_cancel_valid_in_waiting_ack():
_xfer().cancel() # must not raise
def test_consumer_xfer_cancel_raises_when_read_started():
with pytest.raises(AssertionError):
_started().cancel()
def test_consumer_xfer_release_releases_handle():
data = _make_data()
x = _started(data)
x.release()
data.release_xfer_handle.assert_called_once_with(99)
def test_consumer_xfer_release_noop_when_no_handle():
data = _make_data()
_xfer(data).release()
data.release_xfer_handle.assert_not_called()
# ── ConsumerSession ───────────────────────────────────────────────────────────
def _make_consumer_session(data=None):
zmq_conn = MagicMock()
zmq_conn.recv.return_value = []
return ConsumerSession(
addr=("host", 1234),
zmq_conn=zmq_conn,
transport=MagicMock(),
data=data or _make_data(),
compat_hash="hash-abc",
)
def test_consumer_session_has_stable_session_id():
s = _make_consumer_session()
assert isinstance(s._session_id, str) and len(s._session_id) > 0
assert s._session_id == s._session_id # stable
def test_consumer_session_start_xfer_sends_req_with_session_id():
import msgspec
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
raw = s._zmq.send.call_args[0][0]
req = msgspec.msgpack.decode(raw, type=XferReq)
assert req.mm_hash == "h1"
assert req.session_id == s._session_id
def test_consumer_session_poll_done_goes_to_completed():
import msgspec
data = _make_data(xfer_state="PROC")
s = _make_consumer_session(data)
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
ack = XferAck(
mm_hash="h1",
status=XferStatus.OK,
session_id=s._session_id,
src_block_indices=[5, 6],
agent_metadata=b"meta",
mem_descriptor=b"desc",
)
data.add_remote_peer.return_value = "agent-1"
raw_ack = msgspec.msgpack.encode(ack)
data.check_xfer_state.return_value = "DONE"
s.poll([raw_ack], time.monotonic())
results = s.take_results()
assert "h1" in results.completed
assert not results.tombstoned
def test_consumer_session_poll_nack_goes_to_tombstoned():
import msgspec
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
nack = XferAck(mm_hash="h1", status=XferStatus.NACK_MISSING)
s.poll([msgspec.msgpack.encode(nack)], time.monotonic())
results = s.take_results()
assert "h1" in results.tombstoned
assert not results.completed
def test_consumer_session_poll_not_ready_nack_goes_to_retryable():
"""A not-ready NACK must not tombstone: tombstoning admits the request with
no embedding, which is fatal once the media has been rewritten away."""
import msgspec
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
nack = XferAck(mm_hash="h1", status=XferStatus.NACK_NOT_READY)
s.poll([msgspec.msgpack.encode(nack)], time.monotonic())
results = s.take_results()
assert "h1" in results.retryable
assert not results.tombstoned
assert not results.completed
def test_consumer_session_take_results_clears_state():
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() - 1)
s.poll([], time.monotonic())
r1 = s.take_results()
assert "h1" in r1.tombstoned
r2 = s.take_results()
assert not r2.tombstoned # cleared after first take
def test_consumer_session_on_peer_down_cancels_waiting_ack():
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 60)
s.on_peer_down()
results = s.take_results()
assert "h1" in results.retryable
assert not results.tombstoned
+54
View File
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ECCPUConnector scheduler utilities."""
import msgspec
import pytest
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
build_block_descs,
deserialize_mem_descriptor,
serialize_mem_descriptor,
)
# ── build_block_descs ────────────────────────────────────────────────────────
@pytest.mark.parametrize("device_id,expected_dev", [(7, 7), (None, 0)])
def test_build_block_descs(device_id, expected_dev):
kwargs = dict(base_ptr=1000, num_blocks=4, block_size_bytes=256)
if device_id is not None:
kwargs["device_id"] = device_id
descs = build_block_descs(**kwargs)
assert len(descs) == 4
for i, (addr, size, dev) in enumerate(descs):
assert addr == 1000 + i * 256
assert size == 256
assert dev == expected_dev
def test_build_block_descs_zero_blocks_returns_empty():
assert build_block_descs(base_ptr=100, num_blocks=0, block_size_bytes=64) == []
# ── serialize / deserialize_mem_descriptor ───────────────────────────────────
def test_mem_descriptor_roundtrip():
descs = [(100, 64, 0), (164, 64, 0), (228, 64, 1)]
assert deserialize_mem_descriptor(serialize_mem_descriptor(descs)) == descs
@pytest.mark.parametrize(
"bad_value",
[
[(1, 2)], # 2-tuple instead of 3
[("a", "b", "c")], # strings, not ints
],
)
def test_mem_descriptor_rejects_malformed_payload(bad_value):
"""Malformed descriptor lists must fail to decode."""
encoder = msgspec.msgpack.Encoder()
bad_payload = encoder.encode(bad_value)
with pytest.raises((msgspec.DecodeError, msgspec.ValidationError)):
deserialize_mem_descriptor(bad_payload)