diff --git a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py index b68f7d2a11..cc7704d049 100644 --- a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py +++ b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py @@ -297,16 +297,26 @@ async def fanout_encoder_primer( detail=f"Encoder request failed: {detail}", ) - # The encoder reports each item's cache key and grid here. + # The encoder reports each mm_hash's metadata (e.g. the grid) here, + # keyed by the same uuid this proxy assigned above. try: params = (await r.json()).get("ec_transfer_params") or {} - reported = params.get("ec_items") or [] except Exception: logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx) - reported = [] - if reported and idx in item_uuids: - # One item per encoder request, so the first entry is this item's. - item_meta[idx] = {**reported[0], "mm_hash": item_uuids[idx]} + params = {} + if idx in item_uuids: + reported = params.get(item_uuids[idx]) + if reported: + metadata = reported.get("metadata") or {} + if metadata: + item_meta[idx] = {**metadata, "mm_hash": item_uuids[idx]} + # Whatever the encoder reported alongside `metadata` is the + # connector's own handle on the published embedding (for NIXL, + # peer_host/peer_port/size_bytes). The decoder's connector + # looks it up by mm_hash on the request, so carry it through. + orig_request.setdefault("ec_transfer_params", {})[item_uuids[idx]] = ( + reported + ) logger.info( "[%s] All %d encoder requests completed successfully", req_id, len(mm_items) diff --git a/tests/v1/ec_connector/unit/cpu/scheduler/test_embedding_cache.py b/tests/v1/ec_connector/unit/cpu/scheduler/test_embedding_cache.py index 3a4cf86a67..8447532909 100644 --- a/tests/v1/ec_connector/unit/cpu/scheduler/test_embedding_cache.py +++ b/tests/v1/ec_connector/unit/cpu/scheduler/test_embedding_cache.py @@ -333,3 +333,57 @@ def test_concurrent_alloc_evict(): t.join() assert not errors, errors + + +# ── discard ────────────────────────────────────────────────────────────────── + + +def test_discard_not_ready_returns_blocks(): + cache = EmbeddingCache(num_blocks=4) + entry = cache.alloc("h", 3) + assert entry is not None + cache.discard("h") + assert cache.get("h") is None + # All blocks reclaimed: a fresh 4-block alloc now fits. + assert cache.alloc("h2", 4) is not None + + +def test_discard_rejects_ready_entry(): + cache = EmbeddingCache(num_blocks=4) + cache.alloc("h", 1) + cache.mark_ready("h") + with pytest.raises(AssertionError): + cache.discard("h") + + +def test_discard_rejects_absent_key(): + cache = EmbeddingCache(num_blocks=4) + with pytest.raises(KeyError): + cache.discard("nope") + + +# ── pin_if_ready ───────────────────────────────────────────────────────────── + + +def test_pin_if_ready_pins_and_returns_entry(): + cache = EmbeddingCache(num_blocks=4) + cache.alloc("h", 2) + cache.mark_ready("h") + entry = cache.pin_if_ready("h") + assert entry is not None and entry.ready and len(entry.block_ids) == 2 + assert cache.alloc("h2", 4) is None # pinned ⇒ not evictable + + +def test_pin_if_ready_none_when_absent(): + cache = EmbeddingCache(num_blocks=4) + assert cache.pin_if_ready("nope") is None + + +def test_pin_if_ready_returns_unpinned_entry_when_not_ready(): + """The producer needs "present but not ready" told apart from "absent": + the first is NACKed as retryable, the second as a miss.""" + cache = EmbeddingCache(num_blocks=4) + cache.alloc("h", 1) # not-ready + entry = cache.pin_if_ready("h") + assert entry is not None and not entry.ready + assert cache.alloc("h2", 4) is None # not-ready ⇒ still not evictable diff --git a/tests/v1/ec_connector/unit/cpu/test_connector.py b/tests/v1/ec_connector/unit/cpu/test_connector.py index a90c2f72bf..cd5ba91969 100644 --- a/tests/v1/ec_connector/unit/cpu/test_connector.py +++ b/tests/v1/ec_connector/unit/cpu/test_connector.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from unittest.mock import MagicMock +import pytest + from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory @@ -34,14 +36,36 @@ def test_worker_role_builds_only_worker(monkeypatch): assert c.connector_scheduler is None -def test_request_finished_inherited_noop(monkeypatch): +@pytest.mark.parametrize("role", [ECConnectorRole.SCHEDULER, ECConnectorRole.WORKER]) +def test_v1_model_runner_is_rejected(monkeypatch, role): + """The V1 runner never reports save completions, so refuse to start.""" monkeypatch.setattr( ECCPUConnector, "_make_scheduler", lambda self, cfg: MagicMock() ) - c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER) - assert c.request_finished(MagicMock()) == (False, None) + monkeypatch.setattr(ECCPUConnector, "_make_worker", lambda self, cfg: MagicMock()) + cfg = _cfg() + cfg.use_v2_model_runner = False + + with pytest.raises(ValueError, match="V2 model runner"): + ECCPUConnector(cfg, role) def test_factory_registered(): cls = ECConnectorFactory._registry["ECCPUConnector"]() assert cls is ECCPUConnector + + +def test_request_finished_forwards_to_scheduler(monkeypatch): + from unittest.mock import MagicMock + + import vllm.distributed.ec_transfer.ec_connector.cpu.connector as conn_mod # noqa: F401 + from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole + from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector + + fake_sched = MagicMock() + fake_sched.request_finished.return_value = (False, {"h1": {"peer_port": 1}}) + monkeypatch.setattr(ECCPUConnector, "_make_scheduler", lambda self, cfg: fake_sched) + c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER) + req = MagicMock() + assert c.request_finished(req) == (False, {"h1": {"peer_port": 1}}) + fake_sched.request_finished.assert_called_once_with(req) diff --git a/tests/v1/ec_connector/unit/cpu/worker/test_worker.py b/tests/v1/ec_connector/unit/cpu/worker/test_worker.py index 8ae8f81f0a..abc3864c39 100644 --- a/tests/v1/ec_connector/unit/cpu/worker/test_worker.py +++ b/tests/v1/ec_connector/unit/cpu/worker/test_worker.py @@ -64,6 +64,14 @@ _requires_cuda_alike = pytest.mark.skipif( reason="stalling the compute stream requires a CUDA-like platform", ) +_requires_swap_blocks_batch = pytest.mark.skipif( + not hasattr(torch.ops._C_cache_ops, "swap_blocks_batch"), + reason=( + "installed vllm C++ extension predates the swap_blocks_batch op " + "flush_saves/start_load_caches use; rebuild the extension to run this" + ), +) + # Cycles to stall the compute stream for. Must comfortably outlast the host-side # work the lifetime tests do while a copy is deliberately held back — ~0.5 s on # a current datacenter GPU. @@ -194,6 +202,7 @@ def _warm_up_stream_pools(worker: ECCPUWorker) -> None: # ── save_caches ────────────────────────────────────────────────────────────── +@_requires_swap_blocks_batch @_requires_accelerator @pytest.mark.parametrize( "n_elements,block_ids", @@ -326,6 +335,7 @@ def test_save_caches_raises_when_allocated_blocks_too_small(make_worker): worker.save_caches({"h": src}, "h", _meta(saves={"h": [0, 1]})) +@_requires_swap_blocks_batch @_requires_accelerator def test_save_caches_batches_multiple_hashes(make_worker): """Multiple save_caches calls are batched into a single flush.""" @@ -401,6 +411,7 @@ def test_save_descriptors_hold_high_bit_source_addresses(make_worker): # ── start_load_caches ──────────────────────────────────────────────────────── +@_requires_swap_blocks_batch @_requires_accelerator def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker): """Single batched load across all hashes with correct byte→dtype→shape.""" @@ -465,6 +476,7 @@ def test_start_load_caches_noop_when_loads_is_empty(make_worker): assert encoder_cache == {} +@_requires_swap_blocks_batch @_requires_accelerator def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker): """Every hash in ``meta.loads`` is copied from mmap, including one whose @@ -497,6 +509,7 @@ def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker): assert torch.equal(new.cpu(), src_orig[1:]) +@_requires_swap_blocks_batch @_requires_accelerator @pytest.mark.parametrize( "tp_rank,pcp_rank", @@ -526,6 +539,7 @@ def test_start_load_caches_works_on_all_ranks(make_worker, tp_rank, pcp_rank): # ── round-trip ─────────────────────────────────────────────────────────────── +@_requires_swap_blocks_batch @_requires_accelerator def test_save_then_load_round_trips_bytes(make_worker): """Full producer→mmap→consumer byte path in one shot.""" @@ -552,6 +566,7 @@ def test_save_then_load_round_trips_bytes(make_worker): # ── memory lifetime across in-flight copies ───────────────────────────────── +@_requires_swap_blocks_batch @_requires_cuda_alike def test_save_survives_encoder_cache_free_before_copy_runs(make_worker): """The bytes handed to ``save_caches`` must reach the mmap intact even when @@ -612,6 +627,7 @@ def test_save_survives_encoder_cache_free_before_copy_runs(make_worker): ) +@_requires_swap_blocks_batch @_requires_cuda_alike def test_load_buffer_survives_eviction_while_consumer_read_is_queued(make_worker): """A loaded encoder cache entry must keep its bytes for a consumer already @@ -662,6 +678,7 @@ def test_load_buffer_survives_eviction_while_consumer_read_is_queued(make_worker # ── buffer recycling ──────────────────────────────────────────────────────── +@_requires_swap_blocks_batch @_requires_accelerator def test_buffer_pool_is_reused_across_save_steps(make_worker): """Once a save copy completes its descriptor buffers return to the pool and @@ -689,6 +706,7 @@ def test_buffer_pool_is_reused_across_save_steps(make_worker): assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id +@_requires_swap_blocks_batch @_requires_accelerator def test_buffer_pool_is_reused_across_load_steps(make_worker): """Once a load copy completes its descriptor buffers return to the pool and @@ -793,6 +811,7 @@ def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm): # ── e2e: scheduler + worker pipeline ──────────────────────────────────────── +@_requires_swap_blocks_batch @_requires_accelerator def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch): """Full pipeline: scheduler allocates blocks, worker saves a GPU tensor to diff --git a/tests/v1/ec_connector/unit/test_control.py b/tests/v1/ec_connector/unit/test_control.py new file mode 100644 index 0000000000..93701738eb --- /dev/null +++ b/tests/v1/ec_connector/unit/test_control.py @@ -0,0 +1,285 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for control-layer ABCs, ZmqClientConnection, and ECSession.""" + +from unittest.mock import MagicMock, patch + +import pytest +import zmq + +from vllm.distributed.ec_transfer.ec_connector.cpu.control.base import ( + ControlConnection, +) + + +class _ConcreteConn(ControlConnection): + """Minimal concrete subclass for ABC-instantiation tests.""" + + def __init__(self): + self._alive = True + self._sent = [] + self._inbox = [] + + @property + def alive(self) -> bool: + return self._alive + + def send(self, msg: bytes) -> None: + self._sent.append(msg) + + def recv(self) -> list[bytes]: + msgs, self._inbox = self._inbox, [] + return msgs + + def mark_dead(self) -> None: + self._alive = False + + def close(self) -> None: + self._alive = False + + +def test_control_connection_abc_cannot_be_instantiated_directly(): + with pytest.raises(TypeError): + ControlConnection() # type: ignore[abstract] + + +def test_concrete_connection_send_recv(): + conn = _ConcreteConn() + assert conn.alive + conn.send(b"hello") + conn._inbox.append(b"world") + assert conn.recv() == [b"world"] + assert conn._sent == [b"hello"] + + +def test_mark_dead_sets_alive_false(): + conn = _ConcreteConn() + conn.mark_dead() + assert not conn.alive + + +def test_close_sets_alive_false(): + conn = _ConcreteConn() + conn.close() + assert not conn.alive + + +# ── ZmqClientConnection ─────────────────────────────────────────────────────── + + +from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import ( # noqa: E402 + ZmqClientConnection, + ZmqClientTransport, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr # noqa: E402 + + +def _conn(messages: list[list[bytes]], monitor=None) -> ZmqClientConnection: + """Build a ZmqClientConnection whose dealer yields the given frame-lists.""" + dealer = MagicMock() + dealer.recv_multipart.side_effect = [*messages, zmq.Again()] + return ZmqClientConnection(dealer=dealer, monitor=monitor) + + +def test_zmq_client_connection_recv_extracts_payloads_from_envelopes(): + conn = _conn([[b"", b"msg1"], [b"", b"msg2"]]) + assert conn.recv() == [b"msg1", b"msg2"] + + +def test_zmq_client_connection_recv_skips_frames_without_empty_delimiter(): + # A frame that does not start with b"" is malformed and must be dropped. + conn = _conn([[b"", b"ok"], [b"no-delimiter", b"bad"]]) + assert conn.recv() == [b"ok"] + + +def test_zmq_client_connection_recv_breaks_on_non_again_exception(): + # A socket error should stop the drain and return whatever was collected. + dealer = MagicMock() + dealer.recv_multipart.side_effect = [[b"", b"first"], OSError("broken")] + conn = ZmqClientConnection(dealer=dealer, monitor=None) + assert conn.recv() == [b"first"] + + +def test_zmq_client_connection_send_prefixes_empty_delimiter(): + dealer = MagicMock() + conn = ZmqClientConnection(dealer=dealer, monitor=None) + conn.send(b"payload") + dealer.send_multipart.assert_called_once_with([b"", b"payload"]) + + +def test_zmq_client_connection_mark_dead_does_not_close_socket(): + # mark_dead signals liveness only; resource teardown is close()'s job. + dealer = MagicMock() + conn = ZmqClientConnection(dealer=dealer, monitor=None) + conn.mark_dead() + assert not conn.alive + dealer.close.assert_not_called() + + +def test_zmq_client_connection_close_without_monitor(): + dealer = MagicMock() + conn = ZmqClientConnection(dealer=dealer, monitor=None) + conn.close() + dealer.close.assert_called_once_with(linger=0) + assert not conn.alive + + +def test_zmq_client_connection_close_with_monitor_disables_then_closes_both(): + # The disable_monitor → monitor.close → dealer.close sequence must be + # respected: closing the dealer before disabling the monitor can cause + # a zmq assertion in some versions. + dealer = MagicMock() + monitor = MagicMock() + conn = ZmqClientConnection(dealer=dealer, monitor=monitor) + conn.close() + dealer.disable_monitor.assert_called_once() + monitor.close.assert_called_once_with(linger=0) + dealer.close.assert_called_once_with(linger=0) + assert not conn.alive + + +def test_zmq_client_connection_close_is_idempotent(): + dealer = MagicMock() + conn = ZmqClientConnection(dealer=dealer, monitor=None) + conn.close() + conn.close() # must not raise + assert not conn.alive + + +# ── ZmqClientTransport ──────────────────────────────────────────────────────── + + +def _make_transport() -> tuple[ZmqClientTransport, MagicMock]: + """Return a ZmqClientTransport with a mocked ZMQ context.""" + ctx = MagicMock() + ctx.socket.return_value = MagicMock() + with ( + patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context", + return_value=ctx, + ), + patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket", + return_value=MagicMock(), + ), + patch( + "vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path", + return_value="tcp://host:0", + ), + ): + transport = ZmqClientTransport() + transport._ctx = ctx + return transport, ctx + + +def _inject_conn(transport: ZmqClientTransport, addr: PeerAddr) -> ZmqClientConnection: + """Inject a ZmqClientConnection directly into the transport pool.""" + conn = ZmqClientConnection(dealer=MagicMock(), monitor=MagicMock()) + transport._connections[addr] = conn + return conn + + +@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket") +@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path") +@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context") +def test_zmq_client_transport_connect_caches_connection( + mock_ctx_cls, mock_path, mock_sock +): + mock_ctx_cls.return_value = MagicMock() + mock_ctx_cls.return_value.socket.return_value = MagicMock() + mock_sock.return_value = MagicMock() + transport = ZmqClientTransport() + addr: PeerAddr = ("host", 1234) + assert transport.connect(addr) is transport.connect(addr) + + +@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket") +@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path") +@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context") +def test_zmq_client_transport_connect_creates_independent_conns_per_addr( + mock_ctx_cls, mock_path, mock_sock +): + mock_ctx_cls.return_value = MagicMock() + mock_ctx_cls.return_value.socket.return_value = MagicMock() + mock_sock.return_value = MagicMock() + transport = ZmqClientTransport() + conn_a = transport.connect(("host-a", 1111)) + conn_b = transport.connect(("host-b", 2222)) + assert conn_a is not conn_b + + +@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message") +def test_zmq_client_transport_poll_dead_detects_disconnected_peer(mock_recv_mon): + transport, _ = _make_transport() + addr: PeerAddr = ("host", 1234) + conn = _inject_conn(transport, addr) + mock_recv_mon.side_effect = [{"event": zmq.EVENT_DISCONNECTED}, zmq.Again()] + + dead = transport.poll_dead() + + assert addr in dead + assert not conn.alive + + +@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message") +def test_zmq_client_transport_poll_dead_skips_conn_without_monitor(mock_recv_mon): + transport, _ = _make_transport() + addr: PeerAddr = ("host", 9999) + conn = ZmqClientConnection(dealer=MagicMock(), monitor=None) + transport._connections[addr] = conn + + dead = transport.poll_dead() + + assert dead == [] + mock_recv_mon.assert_not_called() + + +@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message") +def test_zmq_client_transport_poll_dead_ignores_non_disconnect_events(mock_recv_mon): + transport, _ = _make_transport() + addr: PeerAddr = ("host", 1234) + conn = _inject_conn(transport, addr) + # A CONNECTED event should not trigger a dead notification. + mock_recv_mon.side_effect = [{"event": zmq.EVENT_CONNECTED}, zmq.Again()] + + dead = transport.poll_dead() + + assert dead == [] + assert conn.alive + + +def test_zmq_client_transport_remove_closes_and_evicts_conn(): + transport, _ = _make_transport() + addr: PeerAddr = ("host", 5678) + conn = _inject_conn(transport, addr) + + removed = transport.remove(addr) + + assert removed is conn + assert addr not in transport._connections + conn.dealer.close.assert_called() + + +def test_zmq_client_transport_remove_returns_none_for_unknown_addr(): + transport, _ = _make_transport() + assert transport.remove(("ghost", 0)) is None + + +@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket") +@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path") +@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context") +def test_zmq_client_transport_close_closes_all_connections( + mock_ctx_cls, mock_path, mock_sock +): + mock_ctx = MagicMock() + mock_ctx_cls.return_value = mock_ctx + mock_ctx.socket.return_value = MagicMock() + dealer_a, dealer_b = MagicMock(), MagicMock() + mock_sock.side_effect = [dealer_a, dealer_b] + transport = ZmqClientTransport() + transport.connect(("a", 1)) + transport.connect(("b", 2)) + transport.close() + dealer_a.close.assert_called() + dealer_b.close.assert_called() + assert transport._connections == {} diff --git a/tests/v1/ec_connector/unit/test_data.py b/tests/v1/ec_connector/unit/test_data.py new file mode 100644 index 0000000000..33f5f11037 --- /dev/null +++ b/tests/v1/ec_connector/unit/test_data.py @@ -0,0 +1,184 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for DataTransport ABC and NixlDataTransport.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import ( + DataTransport, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.utils import ( + serialize_mem_descriptor, +) + +_NIXL_PATH = "vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl" + + +# ── helpers ─────────────────────────────────────────────────────────────────── + + +def _make_nixl_mock(prep_side_effect=None) -> MagicMock: + nixl = MagicMock() + nixl.get_agent_metadata.return_value = b"agent-meta" + nixl.get_reg_descs.return_value = MagicMock() + nixl.get_xfer_descs.return_value = MagicMock() + nixl.prep_xfer_dlist.side_effect = prep_side_effect or [77, 88] + nixl.add_remote_agent.return_value = "remote-agent-1" + nixl.get_new_notifs.return_value = {} + nixl.check_xfer_state.return_value = "PROC" + return nixl + + +def _make_transport(nixl_mock=None): + from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import ( + NixlDataTransport, + ) + + mock = nixl_mock or _make_nixl_mock() + with ( + patch(f"{_NIXL_PATH}.NixlWrapper", return_value=mock), + patch(f"{_NIXL_PATH}.nixl_agent_config", return_value=MagicMock()), + ): + t = NixlDataTransport("agent", 0, 4, 64, 256) + t._nixl = mock + return t + + +# ── ABC ─────────────────────────────────────────────────────────────────────── + + +def test_data_transport_abc_cannot_be_instantiated(): + with pytest.raises(TypeError): + DataTransport() # type: ignore[abstract] + + +# ── NixlDataTransport construction ─────────────────────────────────────────── + + +def test_nixl_data_transport_registers_memory_on_init(): + nixl = _make_nixl_mock() + _make_transport(nixl) + nixl.register_memory.assert_called_once() + + +def test_nixl_data_transport_unwinds_registration_if_init_fails(): + """A failure after register_memory must not leave the region pinned. + + __init__ propagates, so the caller never gets a transport to deregister. + """ + nixl = _make_nixl_mock(prep_side_effect=RuntimeError("no dlist")) + from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import ( + NixlDataTransport, + ) + + with ( + patch(f"{_NIXL_PATH}.NixlWrapper", return_value=nixl), + patch(f"{_NIXL_PATH}.nixl_agent_config", return_value=MagicMock()), + pytest.raises(RuntimeError, match="no dlist"), + ): + NixlDataTransport("agent", 0, 4, 64, 256) + + nixl.deregister_memory.assert_called_once_with(nixl.get_reg_descs.return_value) + + +def test_nixl_data_transport_raises_if_nixl_unavailable(): + from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import ( + NixlDataTransport, + ) + + with ( + patch(f"{_NIXL_PATH}.NixlWrapper", None), + patch(f"{_NIXL_PATH}.nixl_agent_config", None), + pytest.raises(RuntimeError, match="requires NIXL"), + ): + NixlDataTransport("a", 0, 1, 64, 64) + + +# ── public accessors ────────────────────────────────────────────────────────── + + +def test_get_agent_metadata_returns_nixl_blob(): + assert _make_transport().get_agent_metadata() == b"agent-meta" + + +def test_get_mem_descriptor_roundtrips_block_descs(): + t = _make_transport() + raw = t.get_mem_descriptor() + assert isinstance(raw, bytes) and len(raw) > 0 + + +# ── add_remote_peer ─────────────────────────────────────────────────────────── + + +def test_add_remote_peer_returns_agent_name_and_stores_handle_internally(): + nixl = _make_nixl_mock(prep_side_effect=[77, 88]) + t = _make_transport(nixl) + mem_desc = serialize_mem_descriptor([(0, 64, 0)]) + agent_name = t.add_remote_peer(b"meta", mem_desc) + assert agent_name == "remote-agent-1" + # Dlist handle stored internally; accessible via _peer_handles. + assert t._peer_handles["remote-agent-1"] == 88 + + +def test_add_remote_peer_registers_remote_agent(): + nixl = _make_nixl_mock(prep_side_effect=[77, 88]) + t = _make_transport(nixl) + mem_desc = serialize_mem_descriptor([(0, 64, 0)]) + t.add_remote_peer(b"fresh-meta", mem_desc) + nixl.add_remote_agent.assert_called_once_with(b"fresh-meta") + + +# ── remove_remote_peer ──────────────────────────────────────────────────────── + + +def test_remove_remote_peer_delegates_to_nixl(): + t = _make_transport() + t.remove_remote_peer("agent-x") + t._nixl.remove_remote_agent.assert_called_once_with("agent-x") + + +def test_remove_remote_peer_swallows_exceptions(): + t = _make_transport() + t._nixl.remove_remote_agent.side_effect = RuntimeError("gone") + t.remove_remote_peer("agent-x") # must not raise + + +# ── post_read ───────────────────────────────────────────────────────────────── + + +def test_post_read_raises_on_index_count_mismatch(): + nixl = _make_nixl_mock(prep_side_effect=[77, 88]) + t = _make_transport(nixl) + mem_desc = serialize_mem_descriptor([(0, 64, 0)]) + agent = t.add_remote_peer(b"meta", mem_desc) + with pytest.raises(ValueError, match="mismatch"): + t.post_read([0, 1], agent, [5], notif_msg=b"sid") + + +def test_post_read_calls_make_prepped_xfer_and_transfer(): + nixl = _make_nixl_mock(prep_side_effect=[77, 88]) + t = _make_transport(nixl) + mem_desc = serialize_mem_descriptor([(0, 64, 0)]) + agent = t.add_remote_peer(b"meta", mem_desc) + nixl.make_prepped_xfer.return_value = 999 + handle = t.post_read([0, 1], agent, [5, 6], notif_msg=b"sid") + nixl.make_prepped_xfer.assert_called_once() + nixl.transfer.assert_called_once_with(999) + assert handle == 999 + + +# ── release_xfer_handle / deregister ───────────────────────────────────────── + + +def test_release_xfer_handle_swallows_exceptions(): + t = _make_transport() + t._nixl.release_xfer_handle.side_effect = RuntimeError("stale") + t.release_xfer_handle(42) # must not raise + + +def test_deregister_swallows_exceptions(): + t = _make_transport() + t._nixl.deregister_memory.side_effect = RuntimeError("already gone") + t.deregister() # must not raise diff --git a/tests/v1/ec_connector/unit/test_ec_transfer_params.py b/tests/v1/ec_connector/unit/test_ec_transfer_params.py index f161ba203c..127a075b3d 100644 --- a/tests/v1/ec_connector/unit/test_ec_transfer_params.py +++ b/tests/v1/ec_connector/unit/test_ec_transfer_params.py @@ -7,7 +7,8 @@ No running engine required. from unittest.mock import MagicMock -from tests.v1.core.utils import create_scheduler +import pytest + from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ) @@ -15,6 +16,16 @@ from vllm.outputs import CompletionOutput, RequestOutput from vllm.sampling_params import SamplingParams from vllm.v1.request import Request, RequestStatus +# tests.v1.core.utils transitively imports the example KV connector, which +# requires the compiled CUDA flash-attention extensions; skip cleanly rather +# than erroring at collection when those extensions aren't built. +_core_utils = pytest.importorskip( + "tests.v1.core.utils", + reason="requires compiled CUDA flash-attention extensions", + exc_type=ImportError, +) +create_scheduler = _core_utils.create_scheduler + EC_PARAMS: dict = {"mm_hash_abc": {"peer_host": "10.0.0.1", "peer_port": 5501}} diff --git a/tests/v1/ec_connector/unit/test_protocol.py b/tests/v1/ec_connector/unit/test_protocol.py new file mode 100644 index 0000000000..928a1acd6c --- /dev/null +++ b/tests/v1/ec_connector/unit/test_protocol.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import msgspec + +from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import ( + EC_CONNECTOR_VERSION, + XferAck, + XferReq, + XferStatus, + compute_ec_compatibility_hash, +) + + +def test_xferreq_roundtrip(): + req = XferReq(mm_hash="h1", compatibility_hash="c1", session_id="s1") + data = msgspec.msgpack.encode(req) + out = msgspec.msgpack.decode(data, type=XferReq) + assert out.mm_hash == "h1" + assert out.compatibility_hash == "c1" + assert out.connector_version == EC_CONNECTOR_VERSION + + +def test_xferack_roundtrip_ok(): + ack = XferAck( + mm_hash="h1", + status=XferStatus.OK, + src_block_indices=[1, 2], + agent_metadata=b"meta", + mem_descriptor=b"desc", + ) + out = msgspec.msgpack.decode(msgspec.msgpack.encode(ack), type=XferAck) + assert out.status == XferStatus.OK + assert out.src_block_indices == [1, 2] + + +def test_compat_hash_deterministic_and_sensitive(): + a = compute_ec_compatibility_hash("0.1", "m", "float16", 64) + b = compute_ec_compatibility_hash("0.1", "m", "float16", 64) + c = compute_ec_compatibility_hash("0.1", "m", "float16", 128) + assert a == b + assert a != c diff --git a/tests/v1/ec_connector/unit/test_scheduler_nixl_consumer.py b/tests/v1/ec_connector/unit/test_scheduler_nixl_consumer.py new file mode 100644 index 0000000000..0b7d0070b4 --- /dev/null +++ b/tests/v1/ec_connector/unit/test_scheduler_nixl_consumer.py @@ -0,0 +1,350 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import uuid + +import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod +from tests.v1.ec_connector.unit.utils import create_ec_vllm_config +from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import ( + ECSharedRegion, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler + +_N, _BS, _HID, _ES = 16, 64, 32, 2 + + +class _Pos: + def __init__(self, offset, length): + self.offset, self.length = offset, length + + +class _Feature: + def __init__(self, mm_hash, length=1): + self.mm_hash = mm_hash + self.identifier = mm_hash + self.mm_position = _Pos(0, length) + + +class _Request: + def __init__(self, features, params=None, req_id="r1"): + self.mm_features = features + self.request_id = req_id + self.ec_transfer_params = params + + +class _FakeResults: + def __init__(self): + self.completed = set() + self.tombstoned = set() + self.quarantined = set() + self.retryable = set() + self.settled = [] + + +class _FakeSession: + """Records started transfers; lets the test flip them to completed.""" + + def __init__(self): + self.started = [] + self._results = _FakeResults() + + def start_xfer(self, mm_hash, block_indices, deadline): + self.started.append(mm_hash) + + def poll(self, messages, now): + pass + + def take_results(self): + r, self._results = self._results, _FakeResults() + return r + + def close(self): + pass + + +class _FakeTransport: + """Stand-in for ZmqClientTransport so _poll_step runs without ZMQ.""" + + def poll(self): + return {} + + def poll_dead(self): + return [] + + def close(self): + pass + + +def _consumer_sched(monkeypatch): + def _region(cfg): + return ECSharedRegion( + engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS + ) + + monkeypatch.setattr(sched_mod, "create_ec_shared_region", _region) + + # ec_enable_nixl defaults to False, so this builds gate-off; the fields + # below flip on the NIXL consumer state directly. + s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_consumer")) + # Turn on NIXL consumer state without constructing real transports. + s._nixl_enabled = True + s._transport = _FakeTransport() + s._data = None + s._compat_hash = "c" + # _setup_nixl normally computes these from model_config; set them + # directly since this helper builds gate-off then flips fields on. + s._hidden_dim = _HID + s._element_size = _ES + return s + + +def _params(mm_hash, length): + return { + mm_hash: { + "peer_host": "h", + "peer_port": 1, + "size_bytes": length * _HID * _ES, + } + } + + +def test_new_remote_read_defers_then_completes(monkeypatch): + s = _consumer_sched(monkeypatch) + fake = _FakeSession() + + # Route _start_xfer to our fake session instead of real ZMQ/NIXL, but + # still reserve a real not-ready cache entry so mark_ready works. + def _fake_start(mm_hash, info, size): + entry = s._cache.alloc(mm_hash, 1) + assert entry is not None + fake.started.append(mm_hash) + return True + + monkeypatch.setattr(s, "_start_xfer", _fake_start) + req = _Request([_Feature("h1", 1)], params=_params("h1", 1)) + + # Step 1: unseen remote item -> read started, request deferred. + assert s.ensure_cache_available(req, 0) is False + assert "h1" in s._in_flight + assert "h1" in fake.started + s.build_connector_meta(scheduler_output=None) # re-arms _first_in_batch + + # Session reports completion this step. + fake._results.completed.add("h1") + s._sessions[("h", 1)] = fake + + # Step 2: poll drains completion -> entry marked ready, request admitted. + assert s.ensure_cache_available(req, 0) is True + assert "h1" not in s._in_flight + assert "h1" in s._step_completed + entry = s._cache.get("h1") + assert entry is not None and entry.ready + # The scheduler now allocates encoder input; the ready entry loads through + # the same local path as a natively cached encoding. + s.update_state_after_alloc(req, 0) + meta = s.build_connector_meta(scheduler_output=None) + assert "h1" in meta.loads + assert "h1" not in s._step_completed # cleared by promote + + # Step 3: still cached -> admitted directly, no new transfer. + assert s.ensure_cache_available(req, 0) is True + s.shutdown() + + +def test_no_params_is_ready(monkeypatch): + s = _consumer_sched(monkeypatch) + assert s.ensure_cache_available(_Request([_Feature("x")], params=None), 0) is True + s.shutdown() + + +def test_tombstoned_read_discards_and_blocks_retry(monkeypatch): + s = _consumer_sched(monkeypatch) + fake = _FakeSession() + + def _fake_start(mm_hash, info, size): + entry = s._cache.alloc(mm_hash, 1) + assert entry is not None + fake.started.append(mm_hash) + return True + + monkeypatch.setattr(s, "_start_xfer", _fake_start) + req = _Request([_Feature("h1", 1)], params=_params("h1", 1)) + + # Step 1: start the read. + assert s.ensure_cache_available(req, 0) is False + s.build_connector_meta(scheduler_output=None) + + # Producer rejected the read -> tombstoned. + fake._results.tombstoned.add("h1") + s._sessions[("h", 1)] = fake + + # Step 2: poll discards the entry and records a tombstone; the tombstone + # is consumed by admit so the request proceeds to local compute. + assert s.ensure_cache_available(req, 0) is True + assert s._cache.get("h1") is None + assert "h1" not in s._in_flight + s.build_connector_meta(scheduler_output=None) + + # Step 3: tombstone was consumed, so a fresh transfer starts again. + assert s.ensure_cache_available(req, 0) is False + assert "h1" in s._in_flight + s.shutdown() + + +def test_retryable_read_re_requests_without_admitting(monkeypatch): + """A not-ready NACK defers the request and re-requests the read. + + Unlike a tombstone it never admits the request, because an admitted request + whose media was rewritten away upstream has no embedding and no way to + recompute one. + """ + s = _consumer_sched(monkeypatch) + fake = _FakeSession() + + def _fake_start(mm_hash, info, size): + entry = s._cache.alloc(mm_hash, 1) + assert entry is not None + fake.started.append(mm_hash) + return True + + monkeypatch.setattr(s, "_start_xfer", _fake_start) + req = _Request([_Feature("h1", 1)], params=_params("h1", 1)) + + assert s.ensure_cache_available(req, 0) is False + s.build_connector_meta(scheduler_output=None) + + # Producer's save has not landed: retryable, not tombstoned. + fake._results.retryable.add("h1") + s._sessions[("h", 1)] = fake + + # The entry is released and the read re-issued in the same step, so the + # request stays deferred rather than being admitted unfetched. + assert s.ensure_cache_available(req, 0) is False + assert "h1" in s._in_flight + assert "h1" not in s._tombstones + assert fake.started.count("h1") == 2 + s.shutdown() + + +def test_size_mismatch_skips_transfer(monkeypatch): + s = _consumer_sched(monkeypatch) + # Advertised size disagrees with pos.length * hidden_dim * element_size. + bad = {"h1": {"peer_host": "h", "peer_port": 1, "size_bytes": 999}} + req = _Request([_Feature("h1", 1)], params=bad) + assert s.ensure_cache_available(req, 0) is True + assert "h1" not in s._in_flight + assert s._cache.get("h1") is None + s.shutdown() + + +def test_metadata_only_entry_admits_without_transfer(monkeypatch): + # A producer that reported placeholder metadata but had no cache entry + # to transfer omits peer_host/peer_port/size_bytes entirely; the consumer + # must fall back to local compute rather than treating that as a size + # mismatch. + s = _consumer_sched(monkeypatch) + metadata_only = {"h1": {"image_grid_thw": [1, 2, 3]}} + req = _Request([_Feature("h1", 1)], params=metadata_only) + assert s.ensure_cache_available(req, 0) is True + assert "h1" not in s._in_flight + assert s._cache.get("h1") is None + s.shutdown() + + +def test_orphan_not_ready_entry_falls_back_no_realloc(monkeypatch): + # Post-quarantine, post-tombstone-consumed orphan: a not-ready cache + # entry exists for the mm_hash, but the hash is tracked in none of + # _in_flight / _step_completed / _tombstones. Its blocks are held by a + # quarantined/settling DMA and must not be re-allocated. + s = _consumer_sched(monkeypatch) + entry = s._cache.alloc("h1", 1) + assert entry is not None and not entry.ready + assert "h1" not in s._in_flight + assert "h1" not in s._step_completed + assert "h1" not in s._tombstones + + # Spy on alloc: record calls, delegate to the real allocator (which + # asserts on a duplicate key). A re-alloc for "h1" is the bug. + real_alloc = s._cache.alloc + calls: list[str] = [] + + def _spy_alloc(key, n): + calls.append(key) + return real_alloc(key, n) + + monkeypatch.setattr(s._cache, "alloc", _spy_alloc) + req = _Request([_Feature("h1", 1)], params=_params("h1", 1)) + + # Must not raise; request falls back to local compute (admitted True), + # the orphan entry is untouched, and alloc is never called again. + assert s.ensure_cache_available(req, 0) is True + assert calls == [] + assert s._cache.get("h1") is entry + assert not entry.ready + s.shutdown() + + +def test_alloc_failure_falls_back_to_local(monkeypatch): + s = _consumer_sched(monkeypatch) + + # Force the cache to reject the allocation. + monkeypatch.setattr(s._cache, "alloc", lambda key, n: None) + req = _Request([_Feature("h1", 1)], params=_params("h1", 1)) + + # _start_xfer returns False -> request admitted for local recompute. + assert s.ensure_cache_available(req, 0) is True + assert "h1" not in s._in_flight + s.shutdown() + + +def test_already_computed_feature_admits_without_transfer(monkeypatch): + # A feature entirely within num_computed_tokens is skipped by admit; no + # cache lookup/alloc or transfer should be triggered for it. + s = _consumer_sched(monkeypatch) + calls: list[str] = [] + real_alloc = s._cache.alloc + + def _spy_alloc(key, n): + calls.append(key) + return real_alloc(key, n) + + monkeypatch.setattr(s._cache, "alloc", _spy_alloc) + # Feature spans [0, 4); with num_computed_tokens=4 it is fully computed. + req = _Request([_Feature("h1", 4)], params=_params("h1", 4)) + + assert s.ensure_cache_available(req, 4) is True + assert "h1" not in s._in_flight + assert calls == [] + s.shutdown() + + +def test_local_ready_entry_admits_without_transfer(monkeypatch): + # A READY entry already present in the cache is a local hit: admit + # returns True and no new transfer is started for that mm_hash. + s = _consumer_sched(monkeypatch) + entry = s._cache.alloc("h1", 1) + assert entry is not None + s._cache.mark_ready("h1") + req = _Request([_Feature("h1", 1)], params=_params("h1", 1)) + + assert s.ensure_cache_available(req, 0) is True + assert "h1" not in s._in_flight + s.shutdown() + + +def test_in_flight_hash_defers_without_second_transfer(monkeypatch): + # A mm_hash already tracked in _in_flight must defer the request rather + # than starting a duplicate transfer. + s = _consumer_sched(monkeypatch) + started: list[str] = [] + + def _spy_start_xfer(mm_hash, info, size): + started.append(mm_hash) + return True + + monkeypatch.setattr(s, "_start_xfer", _spy_start_xfer) + s._in_flight.add("h1") + req = _Request([_Feature("h1", 1)], params=_params("h1", 1)) + + assert s.ensure_cache_available(req, 0) is False + assert started == [] + s.shutdown() diff --git a/tests/v1/ec_connector/unit/test_scheduler_nixl_ctor.py b/tests/v1/ec_connector/unit/test_scheduler_nixl_ctor.py new file mode 100644 index 0000000000..7d2622c44e --- /dev/null +++ b/tests/v1/ec_connector/unit/test_scheduler_nixl_ctor.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import uuid + +import pytest + +import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod +from tests.v1.ec_connector.unit.utils import create_ec_vllm_config +from vllm.config.ec_transfer import ECTransferConfig +from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import ( + ECSharedRegion, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler +from vllm.distributed.nixl_utils import NixlWrapper + +_N = 16 +_BS = 64 + + +def _region() -> ECSharedRegion: + return ECSharedRegion( + engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS + ) + + +def test_ec_enable_nixl_absent_from_extra_config_by_default(): + cfg = ECTransferConfig() + assert cfg.get_from_extra_config("ec_enable_nixl", False) is False + + +def test_ec_enable_nixl_read_from_extra_config(): + cfg = ECTransferConfig( + ec_connector="ECCPUConnector", + ec_role="ec_both", + ec_connector_extra_config={"ec_enable_nixl": True}, + ) + assert cfg.get_from_extra_config("ec_enable_nixl", False) is True + + +def test_gate_off_builds_no_nixl(monkeypatch): + monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region()) + # ec_enable_nixl defaults to False. + s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_both")) + assert s._nixl_enabled is False + assert getattr(s, "_data", None) is None + assert getattr(s, "_producer_session", None) is None + s.shutdown() + + +def test_string_false_in_extra_config_leaves_nixl_off(monkeypatch): + """Extra config is not type coerced, and bool("false") is True.""" + monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region()) + cfg = create_ec_vllm_config(ec_role="ec_both") + cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = "false" + s = ECCPUScheduler(cfg) + assert s._nixl_enabled is False + s.shutdown() + + +@pytest.mark.skipif(NixlWrapper is None, reason="Requires NIXL package") +def test_gate_on_wires_data_transport_and_producer_session(monkeypatch): + # Port 0 lets the OS pick an ephemeral port so the real ZMQ ROUTER bind + # in ProducerSession.start() cannot collide with another test/process. + monkeypatch.setenv("VLLM_EC_SIDE_CHANNEL_HOST", "127.0.0.1") + monkeypatch.setenv("VLLM_EC_SIDE_CHANNEL_PORT", "0") + monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region()) + cfg = create_ec_vllm_config(ec_role="ec_both") + cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = True + s = ECCPUScheduler(cfg) + try: + assert s._nixl_enabled is True + assert s._data is not None + assert s._compat_hash is not None + assert s._producer_session is not None + assert s._peer_host == "127.0.0.1" + assert s._transport is not None # is_ec_consumer is also True + finally: + s.shutdown() diff --git a/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py b/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py new file mode 100644 index 0000000000..e55ac23769 --- /dev/null +++ b/tests/v1/ec_connector/unit/test_scheduler_nixl_producer.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import uuid + +import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod +from tests.v1.ec_connector.unit.utils import create_ec_vllm_config +from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import ( + ECSharedRegion, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler + +_N, _BS = 16, 64 + + +class _Pos: + def __init__(self, offset, length): + self.offset, self.length = offset, length + + +class _Feature: + def __init__(self, mm_hash, length=1): + self.mm_hash = mm_hash + self.identifier = mm_hash + self.mm_position = _Pos(0, length) + self.data = None + self.modality = "image" + + +class _Request: + def __init__(self, features, req_id="r1"): + self.mm_features = features + self.request_id = req_id + + +def _sched_gate_off(monkeypatch): + def _region(cfg): + return ECSharedRegion( + engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS + ) + + monkeypatch.setattr(sched_mod, "create_ec_shared_region", _region) + return ECCPUScheduler(create_ec_vllm_config(ec_role="ec_both")) + + +def test_request_finished_gate_off_returns_none(monkeypatch): + s = _sched_gate_off(monkeypatch) + assert s.request_finished(_Request([_Feature("h1")])) == (False, None) + s.shutdown() + + +def test_request_finished_producer_emits_params(monkeypatch): + s = _sched_gate_off(monkeypatch) + # Simulate NIXL-enabled producer bookkeeping without building real NIXL. + s._nixl_enabled = True + s._peer_host, s._peer_port = "1.2.3.4", 5601 + # _setup_nixl normally computes these from model_config; set them + # directly since this test builds gate-off then flips fields on. + s._hidden_dim, s._element_size = 32, 2 + # feature length=2, hidden_dim=32, element_size=2 -> 128 bytes -> 2 blocks. + entry = s._cache.alloc("h1", 2) + assert entry is not None + s._cache.mark_ready("h1") + + delay, params = s.request_finished(_Request([_Feature("h1", length=2)])) + assert delay is False + assert params == { + "h1": { + "metadata": {}, + "peer_host": "1.2.3.4", + "peer_port": 5601, + "size_bytes": 2 * 32 * 2, + } + } + s.shutdown() + + +def test_request_finished_announces_not_ready_entry(monkeypatch): + """A save whose GPU->mmap copy hasn't been confirmed complete yet is + still announced: the entry can't be evicted before it's ready, and a read + arriving too early is NACKed NACK_NOT_READY for the consumer to retry — + protecting it is not this method's job.""" + s = _sched_gate_off(monkeypatch) + s._nixl_enabled = True + s._peer_host, s._peer_port = "1.2.3.4", 5601 + s._hidden_dim, s._element_size = 32, 2 + s._cache.alloc("h1", 2) # allocated but not marked ready + + delay, params = s.request_finished(_Request([_Feature("h1", length=2)])) + assert delay is False + assert params == { + "h1": { + "metadata": {}, + "peer_host": "1.2.3.4", + "peer_port": 5601, + "size_bytes": 2 * 32 * 2, + } + } + s.shutdown() + + +def test_request_finished_skips_unallocated_entry(monkeypatch): + s = _sched_gate_off(monkeypatch) + s._nixl_enabled = True + s._peer_host, s._peer_port = "1.2.3.4", 5601 + s._hidden_dim, s._element_size = 32, 2 + # No alloc() for "h1" at all — e.g. the cache was full at save time. + + delay, params = s.request_finished(_Request([_Feature("h1", length=2)])) + assert delay is False + # The item's placeholder metadata is still reported even though there's + # no cache entry to transfer (empty "metadata": no fields, no transfer). + assert params == {"h1": {"metadata": {}}} + s.shutdown() diff --git a/tests/v1/ec_connector/unit/test_session.py b/tests/v1/ec_connector/unit/test_session.py new file mode 100644 index 0000000000..d86e6263a6 --- /dev/null +++ b/tests/v1/ec_connector/unit/test_session.py @@ -0,0 +1,470 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for ProducerXfer, ProducerSession, ConsumerXfer, ConsumerSession.""" + +import time +from unittest.mock import MagicMock + +import pytest + +from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import ( + XferAck, + XferReq, + XferStatus, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import ( + EmbeddingCache, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.session import ( + _CONSUMER_QUARANTINE_TIMEOUT_S, + ConsumerSession, + ConsumerXfer, + ProducerSession, + ProducerXfer, + XferState, +) + +# ── helpers ─────────────────────────────────────────────────────────────────── + + +def _make_data(xfer_state: str = "PROC") -> MagicMock: + data = MagicMock() + data.check_xfer_state.return_value = xfer_state + data.post_read.return_value = 99 + return data + + +def _ok_ack( + mm_hash: str = "h1", + session_id: str = "sess-1", + src_indices: list | None = None, +) -> XferAck: + return XferAck( + mm_hash=mm_hash, + status=XferStatus.OK, + session_id=session_id, + src_block_indices=src_indices or [0, 1], + agent_metadata=b"meta", + mem_descriptor=b"desc", + ) + + +def _xfer( + data: MagicMock | None = None, + deadline: float | None = None, + consumer_session_id: str = "sess-1", +) -> ConsumerXfer: + return ConsumerXfer( + mm_hash="h1", + block_indices=[10, 11], + addr=("host", 1234), + deadline=deadline if deadline is not None else time.monotonic() + 60, + data=data or _make_data(), + consumer_session_id=consumer_session_id, + ) + + +def _started( + data: MagicMock | None = None, deadline: float | None = None +) -> ConsumerXfer: + """ConsumerXfer that has already received an OK XferAck.""" + x = _xfer(data, deadline) + x.handle_ack(_ok_ack(session_id="sess-1", src_indices=[5, 6]), agent_name="agent-1") + return x + + +# ── ProducerXfer ────────────────────────────────────────────────────────────── + + +def test_producer_xfer_not_expired_before_deadline(): + x = ProducerXfer("h1", [0, 1], deadline=time.monotonic() + 60) + assert not x.is_expired() + + +def test_producer_xfer_expired_after_deadline(): + x = ProducerXfer("h1", [0, 1], deadline=time.monotonic() - 1) + assert x.is_expired() + + +# ── ProducerSession ─────────────────────────────────────────────────────────── + + +def _xfer_req(mm_hash: str = "h1", session_id: str = "sess-1") -> XferReq: + return XferReq( + mm_hash=mm_hash, compatibility_hash="hash-abc", session_id=session_id + ) + + +def _make_producer_session(cache: EmbeddingCache | None = None) -> ProducerSession: + data = _make_data() + data.get_agent_metadata.return_value = b"meta" + data.get_mem_descriptor.return_value = b"desc" + return ProducerSession( + transport=MagicMock(), + data=data, + cache=cache or EmbeddingCache(num_blocks=8), + compat_hash="hash-abc", + ) + + +def test_producer_session_grant_pins_ready_entry_and_returns_blocks(): + cache = EmbeddingCache(num_blocks=8) + cache.alloc("h1", 2) + cache.mark_ready("h1") + s = _make_producer_session(cache) + + ack = s._grant_or_nack(_xfer_req(mm_hash="h1")) + + assert ack.status == XferStatus.OK + entry = cache.get("h1") + assert not entry.evictable # pinned by the grant + assert list(ack.src_block_indices) == list(entry.block_ids) + + +def test_producer_session_grant_nacks_missing_entry(): + s = _make_producer_session() + ack = s._grant_or_nack(_xfer_req(mm_hash="nope")) + assert ack.status == XferStatus.NACK_MISSING + + +def test_producer_session_grant_nacks_not_ready_entry_as_retryable(): + """An announced entry whose save is still in flight is distinguished from a + miss, so the consumer retries instead of falling back to local compute.""" + cache = EmbeddingCache(num_blocks=8) + cache.alloc("h1", 2) # not ready + s = _make_producer_session(cache) + ack = s._grant_or_nack(_xfer_req(mm_hash="h1")) + assert ack.status == XferStatus.NACK_NOT_READY + + +def test_producer_session_grant_nacks_version_mismatch(): + cache = EmbeddingCache(num_blocks=8) + cache.alloc("h1", 2) + cache.mark_ready("h1") + s = _make_producer_session(cache) + req = XferReq( + mm_hash="h1", + compatibility_hash="hash-abc", + session_id="sess-1", + connector_version=-1, + ) + + ack = s._grant_or_nack(req) + + assert ack.status == XferStatus.NACK_VERSION + entry = cache.get("h1") + assert entry.evictable # nothing was pinned + + +def test_producer_session_grant_nacks_compat_hash_mismatch(): + cache = EmbeddingCache(num_blocks=8) + cache.alloc("h1", 2) + cache.mark_ready("h1") + s = _make_producer_session(cache) + req = XferReq(mm_hash="h1", compatibility_hash="wrong-hash", session_id="sess-1") + + ack = s._grant_or_nack(req) + + assert ack.status == XferStatus.NACK_INCOMPAT + entry = cache.get("h1") + assert entry.evictable # nothing was pinned + + +def test_producer_session_notif_unpins_via_cache(): + cache = EmbeddingCache(num_blocks=8) + cache.alloc("h1", 2) + cache.mark_ready("h1") + s = _make_producer_session(cache) + s._grant_or_nack(_xfer_req(mm_hash="h1", session_id="sess-1")) + entry = cache.get("h1") + assert not entry.evictable # pinned + + s._data.get_new_notifs.return_value = {"agent": [b"sess-1:h1"]} + s._drain_notifs() + + assert entry.evictable # unpinned via cache + assert s._active_xfers == {} + + +def test_producer_session_sweep_timeouts_unpins_via_cache(): + cache = EmbeddingCache(num_blocks=8) + cache.alloc("h1", 2) + cache.mark_ready("h1") + s = _make_producer_session(cache) + s._grant_or_nack(_xfer_req(mm_hash="h1", session_id="sess-1")) + key = next(iter(s._active_xfers)) + s._active_xfers[key].deadline = time.monotonic() - 1 + + s._sweep_timeouts() + + assert cache.get("h1").evictable + assert s._active_xfers == {} + + +# ── ConsumerXfer.handle_ack ─────────────────────────────────────────────────── + + +def test_consumer_xfer_handle_ack_ok_starts_read(): + data = _make_data() + x = _xfer(data) + assert x.handle_ack(_ok_ack(), agent_name="agent-1") is True + assert x.transfer_handle == 99 + + +def test_consumer_xfer_handle_ack_notif_msg_is_session_id_colon_mm_hash(): + data = _make_data() + x = _xfer(data, consumer_session_id="my-sess") + x.handle_ack(_ok_ack(mm_hash="h1"), agent_name="agent-1") + _, kwargs = data.post_read.call_args + assert kwargs["notif_msg"] == b"my-sess:h1" + + +def test_consumer_xfer_handle_ack_forwards_correct_args(): + data = _make_data() + x = _xfer(data) + x.handle_ack(_ok_ack(src_indices=[5, 6]), agent_name="agent-1") + args, _ = data.post_read.call_args + assert args[0] == [10, 11] # local block_indices + assert args[1] == "agent-1" # agent_name (not a raw handle) + assert args[2] == [5, 6] # remote src_block_indices + + +def test_consumer_xfer_handle_ack_resets_deadline(): + original = time.monotonic() + 2 + x = _xfer(deadline=original) + x.handle_ack(_ok_ack(), agent_name="agent-1") + assert x.deadline > original + + +def test_consumer_xfer_handle_ack_nack_returns_false(): + data = _make_data() + x = _xfer(data) + assert ( + x.handle_ack(XferAck(mm_hash="h1", status=XferStatus.NACK_MISSING), "a") + is False + ) + assert x.transfer_handle is None + data.post_read.assert_not_called() + + +# ── ConsumerXfer.poll — WAITING_ACK ────────────────────────────────────────── + + +def test_consumer_xfer_poll_waiting_ack_before_deadline(): + assert _xfer().poll(time.monotonic()) == XferState.WAITING_ACK + + +def test_consumer_xfer_poll_waiting_ack_timeout(): + assert ( + _xfer(deadline=time.monotonic() - 1).poll(time.monotonic()) + == XferState.ACK_TIMEOUT + ) + + +# ── ConsumerXfer.poll — READING ─────────────────────────────────────────────── + + +def test_consumer_xfer_poll_reading_proc(): + assert _started().poll(time.monotonic()) == XferState.READING + + +def test_consumer_xfer_poll_reading_done_releases_handle(): + data = _make_data(xfer_state="DONE") + x = _started(data) + assert x.poll(time.monotonic()) == XferState.DONE + data.release_xfer_handle.assert_called_once_with(99) + assert x.transfer_handle is None + + +def test_consumer_xfer_poll_reading_unexpected_state_fails(): + data = _make_data(xfer_state="ERR") + x = _started(data) + assert x.poll(time.monotonic()) == XferState.READ_FAILED + data.release_xfer_handle.assert_called_once_with(99) + + +def test_consumer_xfer_poll_reading_exception_fails_gracefully(): + data = _make_data() + data.check_xfer_state.side_effect = RuntimeError("nixl crash") + x = _started(data) + assert x.poll(time.monotonic()) == XferState.READ_FAILED + + +def test_consumer_xfer_poll_timeout_quarantines_without_releasing(): + data = _make_data(xfer_state="PROC") + x = _started(data) + assert x.poll(time.monotonic() + 9999) == XferState.QUARANTINED + data.release_xfer_handle.assert_not_called() + assert x.transfer_handle == 99 + + +# ── ConsumerXfer.poll — QUARANTINED ─────────────────────────────────────────── + + +def test_consumer_xfer_quarantined_proc_stays(): + data = _make_data(xfer_state="PROC") + x = _started(data) + x.poll(time.monotonic() + 9999) # → QUARANTINED + assert x.poll(time.monotonic()) == XferState.QUARANTINED + data.release_xfer_handle.assert_not_called() + + +def test_consumer_xfer_quarantined_terminal_settles(): + data = _make_data(xfer_state="PROC") + x = _started(data) + x.poll(time.monotonic() + 9999) # → QUARANTINED + data.check_xfer_state.return_value = "DONE" + assert x.poll(time.monotonic()) == XferState.SETTLED + data.release_xfer_handle.assert_called_once_with(99) + assert x.transfer_handle is None + + +def test_consumer_xfer_quarantined_exception_settles(): + data = _make_data(xfer_state="PROC") + x = _started(data) + x.poll(time.monotonic() + 9999) + data.check_xfer_state.side_effect = RuntimeError("gone") + assert x.poll(time.monotonic()) == XferState.SETTLED + + +def test_consumer_xfer_quarantined_timeout_evicts_blocks(): + """Quarantine timeout should release handle and return READ_FAILED.""" + + data = _make_data(xfer_state="PROC") + x = _started(data) + # First poll transitions to QUARANTINED (deadline extended by 60s) + x.poll(time.monotonic() + 9999) + # Second poll with expired quarantine deadline should evict + expired_time = time.monotonic() + 9999 + _CONSUMER_QUARANTINE_TIMEOUT_S + 1 + assert x.poll(expired_time) == XferState.READ_FAILED + data.release_xfer_handle.assert_called_once_with(99) + assert x.transfer_handle is None + + +# ── ConsumerXfer.cancel / release ──────────────────────────────────────────── + + +def test_consumer_xfer_cancel_valid_in_waiting_ack(): + _xfer().cancel() # must not raise + + +def test_consumer_xfer_cancel_raises_when_read_started(): + with pytest.raises(AssertionError): + _started().cancel() + + +def test_consumer_xfer_release_releases_handle(): + data = _make_data() + x = _started(data) + x.release() + data.release_xfer_handle.assert_called_once_with(99) + + +def test_consumer_xfer_release_noop_when_no_handle(): + data = _make_data() + _xfer(data).release() + data.release_xfer_handle.assert_not_called() + + +# ── ConsumerSession ─────────────────────────────────────────────────────────── + + +def _make_consumer_session(data=None): + zmq_conn = MagicMock() + zmq_conn.recv.return_value = [] + return ConsumerSession( + addr=("host", 1234), + zmq_conn=zmq_conn, + transport=MagicMock(), + data=data or _make_data(), + compat_hash="hash-abc", + ) + + +def test_consumer_session_has_stable_session_id(): + s = _make_consumer_session() + assert isinstance(s._session_id, str) and len(s._session_id) > 0 + assert s._session_id == s._session_id # stable + + +def test_consumer_session_start_xfer_sends_req_with_session_id(): + import msgspec + + s = _make_consumer_session() + s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10) + raw = s._zmq.send.call_args[0][0] + req = msgspec.msgpack.decode(raw, type=XferReq) + assert req.mm_hash == "h1" + assert req.session_id == s._session_id + + +def test_consumer_session_poll_done_goes_to_completed(): + import msgspec + + data = _make_data(xfer_state="PROC") + s = _make_consumer_session(data) + s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10) + + ack = XferAck( + mm_hash="h1", + status=XferStatus.OK, + session_id=s._session_id, + src_block_indices=[5, 6], + agent_metadata=b"meta", + mem_descriptor=b"desc", + ) + data.add_remote_peer.return_value = "agent-1" + raw_ack = msgspec.msgpack.encode(ack) + + data.check_xfer_state.return_value = "DONE" + s.poll([raw_ack], time.monotonic()) + + results = s.take_results() + assert "h1" in results.completed + assert not results.tombstoned + + +def test_consumer_session_poll_nack_goes_to_tombstoned(): + import msgspec + + s = _make_consumer_session() + s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10) + nack = XferAck(mm_hash="h1", status=XferStatus.NACK_MISSING) + s.poll([msgspec.msgpack.encode(nack)], time.monotonic()) + results = s.take_results() + assert "h1" in results.tombstoned + assert not results.completed + + +def test_consumer_session_poll_not_ready_nack_goes_to_retryable(): + """A not-ready NACK must not tombstone: tombstoning admits the request with + no embedding, which is fatal once the media has been rewritten away.""" + import msgspec + + s = _make_consumer_session() + s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10) + nack = XferAck(mm_hash="h1", status=XferStatus.NACK_NOT_READY) + s.poll([msgspec.msgpack.encode(nack)], time.monotonic()) + results = s.take_results() + assert "h1" in results.retryable + assert not results.tombstoned + assert not results.completed + + +def test_consumer_session_take_results_clears_state(): + s = _make_consumer_session() + s.start_xfer("h1", [0, 1], deadline=time.monotonic() - 1) + s.poll([], time.monotonic()) + r1 = s.take_results() + assert "h1" in r1.tombstoned + r2 = s.take_results() + assert not r2.tombstoned # cleared after first take + + +def test_consumer_session_on_peer_down_cancels_waiting_ack(): + s = _make_consumer_session() + s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 60) + s.on_peer_down() + results = s.take_results() + assert "h1" in results.retryable + assert not results.tombstoned diff --git a/tests/v1/ec_connector/unit/test_utils.py b/tests/v1/ec_connector/unit/test_utils.py new file mode 100644 index 0000000000..92925030bb --- /dev/null +++ b/tests/v1/ec_connector/unit/test_utils.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for ECCPUConnector scheduler utilities.""" + +import msgspec +import pytest + +from vllm.distributed.ec_transfer.ec_connector.cpu.utils import ( + build_block_descs, + deserialize_mem_descriptor, + serialize_mem_descriptor, +) + +# ── build_block_descs ──────────────────────────────────────────────────────── + + +@pytest.mark.parametrize("device_id,expected_dev", [(7, 7), (None, 0)]) +def test_build_block_descs(device_id, expected_dev): + kwargs = dict(base_ptr=1000, num_blocks=4, block_size_bytes=256) + if device_id is not None: + kwargs["device_id"] = device_id + descs = build_block_descs(**kwargs) + assert len(descs) == 4 + for i, (addr, size, dev) in enumerate(descs): + assert addr == 1000 + i * 256 + assert size == 256 + assert dev == expected_dev + + +def test_build_block_descs_zero_blocks_returns_empty(): + assert build_block_descs(base_ptr=100, num_blocks=0, block_size_bytes=64) == [] + + +# ── serialize / deserialize_mem_descriptor ─────────────────────────────────── + + +def test_mem_descriptor_roundtrip(): + descs = [(100, 64, 0), (164, 64, 0), (228, 64, 1)] + assert deserialize_mem_descriptor(serialize_mem_descriptor(descs)) == descs + + +@pytest.mark.parametrize( + "bad_value", + [ + [(1, 2)], # 2-tuple instead of 3 + [("a", "b", "c")], # strings, not ints + ], +) +def test_mem_descriptor_rejects_malformed_payload(bad_value): + """Malformed descriptor lists must fail to decode.""" + encoder = msgspec.msgpack.Encoder() + bad_payload = encoder.encode(bad_value) + with pytest.raises((msgspec.DecodeError, msgspec.ValidationError)): + deserialize_mem_descriptor(bad_payload) diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/connector.py b/vllm/distributed/ec_transfer/ec_connector/cpu/connector.py index c606e98d74..a0996d9035 100644 --- a/vllm/distributed/ec_transfer/ec_connector/cpu/connector.py +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/connector.py @@ -8,7 +8,7 @@ GPU copy plumbing. An ec_both instance reuses encoder outputs it has already offloaded to CPU instead of recomputing them. """ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import torch @@ -37,6 +37,12 @@ class ECCPUConnector(ECConnectorBase): def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole) -> None: super().__init__(vllm_config=vllm_config, role=role) + if not vllm_config.use_v2_model_runner: + raise ValueError( + "ECCPUConnector requires the V2 model runner. " + "Set VLLM_USE_V2_MODEL_RUNNER=1." + ) + self.connector_worker = None self.connector_scheduler = None @@ -108,6 +114,12 @@ class ECCPUConnector(ECConnectorBase): assert self.connector_scheduler is not None return self.connector_scheduler.build_connector_meta(scheduler_output) + def request_finished( + self, request: "Request" + ) -> tuple[bool, "dict[str, Any] | None"]: + assert self.connector_scheduler is not None + return self.connector_scheduler.request_finished(request) + def get_finished( self, finished_req_ids: set[str] ) -> tuple[set[str] | None, set[str] | None]: diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/control/__init__.py b/vllm/distributed/ec_transfer/ec_connector/cpu/control/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/control/base.py b/vllm/distributed/ec_transfer/ec_connector/cpu/control/base.py new file mode 100644 index 0000000000..07114f095d --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/control/base.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Abstract base classes for the EC control plane. + +ControlConnection wraps a single peer channel (one DEALER socket). +""" + +from abc import ABC, abstractmethod + + +class ControlConnection(ABC): + """Bidirectional message channel to a single remote peer. + + All I/O is non-blocking: send enqueues without blocking; recv drains + all buffered inbound messages since the last call. + """ + + @property + @abstractmethod + def alive(self) -> bool: + """True if the connection is still usable.""" + + @abstractmethod + def send(self, msg: bytes) -> None: + """Enqueue msg for delivery. Must not block.""" + + @abstractmethod + def recv(self) -> list[bytes]: + """Drain and return all buffered inbound messages.""" + + @abstractmethod + def mark_dead(self) -> None: + """Signal that the peer disconnected. Sets alive to False.""" + + @abstractmethod + def close(self) -> None: + """Release all resources. Idempotent.""" diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/control/zmq.py b/vllm/distributed/ec_transfer/ec_connector/cpu/control/zmq.py new file mode 100644 index 0000000000..d87bdc7920 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/control/zmq.py @@ -0,0 +1,233 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ZMQ control-plane implementations for ECCPUConnector. + +ZmqClientConnection — one DEALER socket to a single producer peer. +ZmqClientTransport — pool of ZmqClientConnection objects (consumer side). +ZmqServerTransport — ROUTER socket (producer side). +""" + +import contextlib +from dataclasses import dataclass, field + +import zmq +from zmq.utils.monitor import recv_monitor_message + +from vllm.distributed.ec_transfer.ec_connector.cpu.control.base import ( + ControlConnection, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr +from vllm.logger import init_logger +from vllm.utils.network_utils import make_zmq_path, make_zmq_socket + +logger = init_logger(__name__) + +_HEARTBEAT_IVL_MS = 2000 +_HEARTBEAT_TIMEOUT_MS = 4000 +_HEARTBEAT_TTL_MS = 8000 + + +# ── Client side ─────────────────────────────────────────────────────────────── + + +@dataclass +class ZmqClientConnection(ControlConnection): + """One DEALER socket to one producer peer. + + Owns the socket and its optional monitor socket. NIXL state is NOT + stored here — it belongs in ECSession above this layer. + """ + + dealer: zmq.Socket + monitor: zmq.Socket | None + _alive: bool = field(default=True, init=False) + + @property + def alive(self) -> bool: + return self._alive + + def send(self, msg: bytes) -> None: + self.dealer.send_multipart([b"", msg]) + + def recv(self) -> list[bytes]: + msgs: list[bytes] = [] + while True: + try: + frames = self.dealer.recv_multipart(flags=zmq.NOBLOCK) + except zmq.Again: + break + except Exception: + logger.exception("ec: DEALER recv failed") + break + if len(frames) != 2 or frames[0] != b"": + logger.warning( + "ec: dropped malformed frame envelope (%d frames)", len(frames) + ) + continue + msgs.append(frames[1]) + return msgs + + def mark_dead(self) -> None: + self._alive = False + + def close(self) -> None: + if self.monitor is not None: + with contextlib.suppress(Exception): + self.dealer.disable_monitor() + with contextlib.suppress(Exception): + self.monitor.close(linger=0) + with contextlib.suppress(Exception): + self.dealer.close(linger=0) + self._alive = False + + +class ZmqClientTransport: + """Pool of ZmqClientConnection objects — one per producer peer. + + Creates DEALER sockets lazily on first contact. Does not touch NIXL; + NIXL registration is handled by ECCPUConsumer once an XferAck arrives. + """ + + def __init__(self) -> None: + self._ctx = zmq.Context() + self._connections: dict[PeerAddr, ZmqClientConnection] = {} + + def connect(self, addr: PeerAddr) -> ZmqClientConnection: + """Return the existing connection for addr, or create a new one.""" + existing = self._connections.get(addr) + if existing is not None: + return existing + + host, port = addr + dealer = make_zmq_socket( + ctx=self._ctx, + path=make_zmq_path(scheme="tcp", host=host, port=port), + socket_type=zmq.DEALER, + bind=False, + ) + dealer.setsockopt(zmq.HEARTBEAT_IVL, _HEARTBEAT_IVL_MS) + dealer.setsockopt(zmq.HEARTBEAT_TIMEOUT, _HEARTBEAT_TIMEOUT_MS) + dealer.setsockopt(zmq.HEARTBEAT_TTL, _HEARTBEAT_TTL_MS) + + monitor_addr = f"inproc://ec-peer-mon-{host}-{port}" + dealer.monitor(monitor_addr, zmq.EVENT_DISCONNECTED) + monitor = self._ctx.socket(zmq.PAIR) + monitor.connect(monitor_addr) + + conn = ZmqClientConnection(dealer=dealer, monitor=monitor) + self._connections[addr] = conn + return conn + + def poll(self) -> dict[PeerAddr, list[bytes]]: + """Non-blocking drain of messages from every peer's DEALER.""" + msgs: dict[PeerAddr, list[bytes]] = {} + for addr, conn in list(self._connections.items()): + conn_msgs = conn.recv() + msgs[addr] = conn_msgs + return msgs + + def poll_dead(self) -> list[PeerAddr]: + """Non-blocking sweep of peer monitors; return addresses that died.""" + dead: list[PeerAddr] = [] + for addr, conn in list(self._connections.items()): + if conn.monitor is None: + continue + try: + while True: + evt = recv_monitor_message(conn.monitor, flags=zmq.NOBLOCK) + if evt["event"] == zmq.EVENT_DISCONNECTED: + conn.mark_dead() + dead.append(addr) + break + except zmq.Again: + pass + except Exception: + logger.warning("ec: monitor poll failed for %s", addr, exc_info=True) + return dead + + def remove(self, addr: PeerAddr) -> ZmqClientConnection | None: + """Remove and close the connection for addr, returning it (or None).""" + conn = self._connections.pop(addr, None) + if conn is not None: + conn.close() + return conn + + def close(self) -> None: + for conn in list(self._connections.values()): + conn.close() + self._connections.clear() + try: + self._ctx.destroy(linger=0) + except Exception: + logger.debug("ec: zmq context destroy failed", exc_info=True) + + +# ── Server side ─────────────────────────────────────────────────────────────── + + +class ZmqServerTransport: + """ROUTER socket for the producer side. Byte-level only, driven by + ProducerSession. + + poll(timeout_ms) waits up to timeout_ms for messages, then drains all that + are available; timeout_ms=0 drains whatever has already arrived. send() + routes a reply back to a specific peer by ZMQ identity. + """ + + def __init__(self, host: str, port: int) -> None: + self._ctx = zmq.Context() + self._router = make_zmq_socket( + ctx=self._ctx, + path=make_zmq_path(scheme="tcp", host=host, port=port), + socket_type=zmq.ROUTER, + bind=True, + ) + self._poller = zmq.Poller() + self._poller.register(self._router, zmq.POLLIN) + + def poll(self, timeout_ms: int = 0) -> list[tuple[bytes, bytes]]: + """Wait up to timeout_ms for incoming messages, then drain all pending. + + Returns a list of (identity, payload) pairs. + """ + if timeout_ms > 0: + try: + events = dict(self._poller.poll(timeout=timeout_ms)) + except zmq.ContextTerminated: + return [] + except Exception: + logger.exception("ec: router poller failed") + return [] + if self._router not in events: + return [] + + msgs: list[tuple[bytes, bytes]] = [] + while True: + try: + identity, _, payload = self._router.recv_multipart(flags=zmq.NOBLOCK) + msgs.append((identity, payload)) + except zmq.Again: + break + except zmq.ContextTerminated: + break + except Exception: + logger.exception("ec: router recv failed") + break + return msgs + + def send(self, identity: bytes, payload: bytes) -> None: + """Send a reply payload to the peer identified by ZMQ identity.""" + try: + self._router.send_multipart([identity, b"", payload]) + except Exception: + logger.exception("ec: router send failed") + + def close(self) -> None: + try: + self._router.close(linger=0) + except Exception: + logger.debug("ec: router close failed", exc_info=True) + try: + self._ctx.destroy(linger=0) + except Exception: + logger.debug("ec: zmq context destroy failed", exc_info=True) diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/data/__init__.py b/vllm/distributed/ec_transfer/ec_connector/cpu/data/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/data/base.py b/vllm/distributed/ec_transfer/ec_connector/cpu/data/base.py new file mode 100644 index 0000000000..c4250c4ce8 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/data/base.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Abstract base class for the EC data-plane transport (NIXL/UCX).""" + +from abc import ABC, abstractmethod +from typing import Any + + +class DataTransport(ABC): + """Interface wrapping the NIXL data-plane for EC block transfers. + + Peer lifecycle: + add_remote_peer() — register a peer and store its dlist handle internally + remove_remote_peer() — deregister and release the stored handle + post_read() — issue a READ using the stored handle for agent_name + + The dlist handle (prepped xfer descriptor list) is an implementation detail + of the transport and is never exposed to callers. + """ + + @abstractmethod + def get_agent_metadata(self) -> bytes: + """Return this agent's serialized NIXL metadata blob.""" + + @abstractmethod + def get_mem_descriptor(self) -> bytes: + """Return the msgpack-encoded block descriptor list for this agent.""" + + @abstractmethod + def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str: + """Register a remote peer, prep its READ-source dlist, and return agent_name. + + The dlist handle is stored internally and used by post_read(). + """ + + @abstractmethod + def remove_remote_peer(self, agent_name: str) -> None: + """Deregister a previously registered remote peer and release its handle.""" + + @abstractmethod + def post_read( + self, + local_indices: list[int], + agent_name: str, + remote_indices: list[int], + notif_msg: bytes, + ) -> Any: + """Issue a consumer-initiated READ using the stored dlist for agent_name. + + Returns a transfer handle for use with check_xfer_state / release_xfer_handle. + """ + + @abstractmethod + def get_new_notifs(self) -> dict[str, list[bytes]]: + """Drain NIXL completion notifications addressed to this agent.""" + + @abstractmethod + def check_xfer_state(self, handle: Any) -> str: + """Return the NIXL state string: 'DONE', 'PROC', or an error string.""" + + @abstractmethod + def release_xfer_handle(self, handle: Any) -> None: + """Release a transfer handle returned by post_read.""" + + @abstractmethod + def deregister(self) -> None: + """Deregister all memory and tear down the NIXL agent.""" diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/data/nixl.py b/vllm/distributed/ec_transfer/ec_connector/cpu/data/nixl.py new file mode 100644 index 0000000000..48b3514b27 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/data/nixl.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""NIXL-backed data-plane transport for ECCPUConnector.""" + +from typing import Any + +from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import ( + DataTransport, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.utils import ( + build_block_descs, + deserialize_mem_descriptor, + serialize_mem_descriptor, +) +from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config +from vllm.logger import init_logger + +logger = init_logger(__name__) +_NIXL_DRAM = "DRAM" + + +class NixlDataTransport(DataTransport): + """Adapts NixlWrapper to the DataTransport interface. + + Dlist handles (prepped xfer descriptor lists for remote peers) are stored + internally keyed by agent_name. Callers never touch raw NIXL handles — + post_read() resolves the handle from agent_name internally. + + NixlWrapper is not thread-safe, and neither is this class. Every call site + runs on the scheduler thread, which serializes the transport by + construction; transfers are async inside NIXL and are drained from that + same thread via get_new_notifs() / check_xfer_state(). + """ + + def __init__( + self, + agent_name: str, + base_ptr: int, + num_blocks: int, + block_size_bytes: int, + total_size_bytes: int, + num_threads: int = 1, + ) -> None: + if NixlWrapper is None or nixl_agent_config is None: + raise RuntimeError( + "ECCPUConnector requires NIXL; " + "install the `nixl` package or set a different ec_connector." + ) + # agent_name → remote dlist handle (from add_remote_peer, used by post_read) + self._peer_handles: dict[str, int] = {} + # None until the local region is registered, so deregister() is safe on + # a transport whose registration failed part-way. + self._reg_descs: Any = None + + self._nixl = NixlWrapper( + agent_name, + nixl_agent_config(num_threads=num_threads, capture_telemetry=True), + ) + self._register_local_memory( + base_ptr, num_blocks, block_size_bytes, total_size_bytes + ) + + def _register_local_memory( + self, + base_ptr: int, + num_blocks: int, + block_size_bytes: int, + total_size_bytes: int, + ) -> None: + """Register the local region with NIXL and prep its transfer dlist. + + Unwinds the registration if a later step fails: __init__ propagates the + error, so the caller never receives an object it could deregister with. + """ + block_descs = build_block_descs(base_ptr, num_blocks, block_size_bytes) + self._reg_descs = self._nixl.get_reg_descs( + [(base_ptr, total_size_bytes, 0, "")], _NIXL_DRAM + ) + self._nixl.register_memory(self._reg_descs, backends=["UCX"]) + try: + xfer_descs = self._nixl.get_xfer_descs(block_descs, _NIXL_DRAM) + self._local_xfer_handle = self._nixl.prep_xfer_dlist( + "NIXL_INIT_AGENT", xfer_descs + ) + self._agent_metadata = self._nixl.get_agent_metadata() + self._mem_descriptor_bytes = serialize_mem_descriptor(block_descs) + except Exception: + self.deregister() + raise + + def get_agent_metadata(self) -> bytes: + return self._agent_metadata + + def get_mem_descriptor(self) -> bytes: + return self._mem_descriptor_bytes + + def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str: + """Register remote peer, prep its dlist, store handle internally. + + Returns agent_name for use in post_read() and remove_remote_peer(). + """ + agent_name = self._nixl.add_remote_agent(metadata) + remote_blocks = deserialize_mem_descriptor(mem_descriptor) + remote_xfer_descs = self._nixl.get_xfer_descs(remote_blocks, _NIXL_DRAM) + self._peer_handles[agent_name] = self._nixl.prep_xfer_dlist( + agent_name, remote_xfer_descs + ) + return agent_name + + def remove_remote_peer(self, agent_name: str) -> None: + self._peer_handles.pop(agent_name, None) + try: + self._nixl.remove_remote_agent(agent_name) + except Exception: + logger.warning( + "EC: remove_remote_peer failed for %s", agent_name, exc_info=True + ) + + def post_read( + self, + local_indices: list[int], + agent_name: str, + remote_indices: list[int], + notif_msg: bytes, + ) -> Any: + if len(local_indices) != len(remote_indices): + raise ValueError( + f"EC: local/remote block count mismatch " + f"({len(local_indices)} vs {len(remote_indices)})" + ) + remote_handle = self._peer_handles[agent_name] + handle = self._nixl.make_prepped_xfer( + "READ", + self._local_xfer_handle, + local_indices, + remote_handle, + remote_indices, + notif_msg=notif_msg, + ) + self._nixl.transfer(handle) + return handle + + def get_new_notifs(self) -> dict[str, list[bytes]]: + return self._nixl.get_new_notifs() + + def check_xfer_state(self, handle: Any) -> str: + return self._nixl.check_xfer_state(handle) + + def release_xfer_handle(self, handle: Any) -> None: + try: + self._nixl.release_xfer_handle(handle) + except Exception: + logger.warning("EC: release_xfer_handle failed", exc_info=True) + + def deregister(self) -> None: + if self._reg_descs is None: + return + try: + self._nixl.deregister_memory(self._reg_descs) + except Exception: + logger.warning("EC: deregister failed", exc_info=True) + self._reg_descs = None diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/protocol.py b/vllm/distributed/ec_transfer/ec_connector/cpu/protocol.py new file mode 100644 index 0000000000..85bd3cae32 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/protocol.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Scheduler-side wire types for the ECCPUConnector. + +Two peer-to-peer message types exchanged between consumer and producer over +ZMQ (XferReq, XferAck) plus a peer-compatibility fingerprint +(compute_ec_compatibility_hash) sent on every XferReq so a producer refuses +to serve a mismatched peer. +""" + +import enum +import hashlib +import json + +import msgspec + +# Bump when the on-wire shape (XferReq/XferAck) changes in a +# backward-incompatible way. Peers reject mismatched versions. +EC_CONNECTOR_VERSION: int = 1 + + +class XferStatus(enum.IntEnum): + """Outcome the producer reports for an XferReq.""" + + OK = 0 + NACK_MISSING = 1 # producer no longer holds the encoding + NACK_INCOMPAT = 2 # peer-compatibility hash mismatch + NACK_VERSION = 3 # connector wire-version mismatch + NACK_INTERNAL = 4 # producer hit an unexpected error + NACK_NOT_READY = 5 # announced, but its GPU->mmap save has not landed yet + + +# The producer expects a retryable condition to clear on its own, so the +# consumer re-requests the read rather than giving up on the encoding. +RETRYABLE_NACKS = frozenset((XferStatus.NACK_NOT_READY,)) +# Ordinary outcomes rather than signs of a broken peer. +EXPECTED_NACKS = RETRYABLE_NACKS | {XferStatus.NACK_MISSING} + + +class XferReq( # type: ignore[call-arg] + msgspec.Struct, + tag="req", + omit_defaults=True, +): + """Consumer → Producer: I want to READ mm_hash; pin it and tell me where.""" + + mm_hash: str + compatibility_hash: str + # Consumer's session identity. Together with mm_hash it uniquely identifies + # this transfer on both sides: the producer keys its active xfers by + # (session_id, mm_hash), and the NIXL notif_msg carries the same pair. + session_id: str = "" + connector_version: int = EC_CONNECTOR_VERSION + + +class XferAck( # type: ignore[call-arg] + msgspec.Struct, + tag="ack", + omit_defaults=True, +): + """Producer → Consumer: response to an XferReq. + + On status == OK: producer has pinned source blocks and returns their + indices plus fresh NIXL metadata for the consumer to register and READ. + On any NACK_*: optional fields are empty; consumer falls back to local encode. + """ + + mm_hash: str + status: XferStatus + # Consumer's session_id echoed back from XferReq. Together with mm_hash it + # identifies the exact transfer; the consumer uses both to build notif_msg. + session_id: str = "" + src_block_indices: list[int] = [] + agent_metadata: bytes = b"" + mem_descriptor: bytes = b"" + + +def compute_ec_compatibility_hash( + vllm_version: str, + model: str, + dtype: str, + block_size_bytes: int, +) -> str: + """Peer-compatibility fingerprint over the four factors that determine + byte-layout compatibility. Producer NACKs any XferReq whose hash differs. + """ + factors = json.dumps( + [vllm_version, model, dtype, block_size_bytes], + separators=(",", ":"), + ) + return hashlib.sha256(factors.encode("utf-8")).hexdigest() diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/__init__.py b/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/__init__.py index cee946fd26..bc52e1fb62 100644 --- a/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/__init__.py +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/__init__.py @@ -7,16 +7,23 @@ Owns the mmap region and the embedding cache, and handles the producer for the ECCPUConnector. """ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any + +import torch from vllm.distributed.ec_transfer.ec_connector.cpu.common import ( ECCPUConnectorMetadata, ECCPUWorkerMetadata, + _get_encoder_cache_hidden_dim, create_ec_shared_region, ) from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import ( EmbeddingCache, ) +from vllm.distributed.ec_transfer.ec_connector.utils import ( + PlaceholderMetadataResolver, + collect_ec_item_metadata, +) from vllm.logger import init_logger if TYPE_CHECKING: @@ -40,6 +47,7 @@ class ECCPUScheduler: self._region = create_ec_shared_region(vllm_config) # Block allocator + LRU eviction policy for the shared region. self._cache = EmbeddingCache(self._region.num_blocks) + self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config) # mm_hash → block IDs allocated this step for GPU→mmap saves. self._pending_saves: dict[str, list[int]] = {} @@ -63,6 +71,112 @@ class ECCPUScheduler: else 1 ) + self._ec_config = ec_config + # NIXL p2p is an option of this connector, not engine-wide behavior, so + # it lives in extra config. Extra config is not type coerced, so a value + # that arrived as a JSON/CLI string is parsed, not truth-tested. + raw_nixl = ec_config.get_from_extra_config("ec_enable_nixl", False) + self._nixl_enabled = ( + raw_nixl + if isinstance(raw_nixl, bool) + else str(raw_nixl).strip().lower() in ("true", "1", "yes") + ) + # NIXL fields default to None/empty so the gate-off path is untouched. + self._data: Any = None + self._compat_hash: str | None = None + self._first_in_batch = True + self._transport: Any = None + self._producer_session: Any = None + self._sessions: dict = {} + self._in_flight: set[str] = set() + self._tombstones: set[str] = set() + self._step_completed: set[str] = set() + self._peer_host: str | None = None + self._peer_port: int | None = None + # Model shape for size checks + compat hash; only set by + # _setup_nixl (the gate-off path never touches model_config). + self._dtype: torch.dtype | None = None + self._hidden_dim: int = 0 + self._element_size: int = 0 + self._ack_timeout_s: float = 0.0 + if self._nixl_enabled: + self._setup_nixl(vllm_config) + + def _setup_nixl(self, vllm_config: "VllmConfig") -> None: + # Lazy imports keep nixl/zmq off the gate-off path. + from vllm import envs + from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import ( + ZmqClientTransport, + ZmqServerTransport, + ) + from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import ( + NixlDataTransport, + ) + from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import ( + compute_ec_compatibility_hash, + ) + from vllm.distributed.ec_transfer.ec_connector.cpu.session import ( + ProducerSession, + ) + from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config + from vllm.version import __version__ as VLLM_VERSION + + if NixlWrapper is None or nixl_agent_config is None: + raise RuntimeError( + "ec_enable_nixl requires NIXL; install the `nixl` package or " + "remove ec_enable_nixl from ec_connector_extra_config." + ) + engine_id = self._ec_config.engine_id + assert engine_id is not None + self._dtype = vllm_config.model_config.dtype + self._hidden_dim = _get_encoder_cache_hidden_dim(vllm_config) + self._element_size = torch.empty(0, dtype=self._dtype).element_size() + # How long a consumer waits for an XferAck. The producer answers from + # its scheduler step, so its reply latency scales with the encoder's + # batch size: a deployment whose steps run longer must raise this. + self._ack_timeout_s = float( + self._ec_config.get_from_extra_config("consumer_ack_timeout_s", 2.0) + ) + self._compat_hash = compute_ec_compatibility_hash( + vllm_version=VLLM_VERSION, + model=str(vllm_config.model_config.model), + dtype=str(self._dtype), + block_size_bytes=self._region.block_size_bytes, + ) + if self._is_producer: + self._peer_host = envs.VLLM_EC_SIDE_CHANNEL_HOST + self._peer_port = envs.VLLM_EC_SIDE_CHANNEL_PORT + + # Registering the region with NIXL and binding the control sockets are + # the first side effects here. __init__ propagates a failure, so the + # caller never receives a scheduler it could shut down: unwind through + # the same teardown shutdown() uses. + try: + self._data = NixlDataTransport( + agent_name=engine_id, + base_ptr=self._region.blocks.data_ptr(), + num_blocks=self._region.num_blocks, + block_size_bytes=self._region.block_size_bytes, + total_size_bytes=self._region.num_blocks + * self._region.block_size_bytes, + ) + if self._is_producer: + assert self._peer_host is not None + assert self._peer_port is not None + self._producer_session = ProducerSession( + transport=ZmqServerTransport( + host=self._peer_host, port=self._peer_port + ), + data=self._data, + cache=self._cache, + compat_hash=self._compat_hash, + ) + if self._is_consumer: + self._transport = ZmqClientTransport() + except Exception: + self._teardown_nixl() + raise + def has_cache_item(self, identifier: str) -> bool: if not self._is_consumer: return False @@ -72,8 +186,202 @@ class ECCPUScheduler: def ensure_cache_available( self, request: "Request", num_computed_tokens: int ) -> bool: + if not self._nixl_enabled: + return True # CPU offload never blocks. + first = self._first_in_batch + self._first_in_batch = False + if not self._is_consumer: + return True + if first: + self._poll_step() + return self._nixl_consumer_admit(request, num_computed_tokens) + + def _nixl_consumer_admit( + self, request: "Request", num_computed_tokens: int + ) -> bool: + """Admit a request only once all its remote encodings are cached. + + Returns True when every encoding the request needs is either already + ready in the local cache or has no remote source. Any encoding whose + NIXL READ is still in flight (or was just started this step) defers + the request, which the scheduler re-presents on a later step. + """ + params: dict[str, dict[str, Any]] = ( + getattr(request, "ec_transfer_params", None) or {} + ) + if not params: + return True + pending = False + for feature in request.mm_features: + pos = feature.mm_position + if pos.offset + pos.length <= num_computed_tokens: + continue + mm_hash = feature.identifier + entry = self._cache.get(mm_hash) + if entry is not None and entry.ready: + # Local hit: upstream's update_state_after_alloc pins and + # loads it through the same path as a natively cached entry. + continue + if mm_hash in self._in_flight: + pending = True + continue + if mm_hash in self._step_completed: + pending = True + continue + if mm_hash in self._tombstones: + self._tombstones.discard(mm_hash) + continue + if entry is not None: + # Present but not ready and not being fetched: its blocks are + # held by a quarantined/settling DMA and cannot be reused. + # Fall back to local compute this step; never re-alloc a + # mm_hash already in the cache. + continue + info = params.get(mm_hash) + # Absent entirely, or present with only placeholder metadata (no + # cache entry existed on the producer side at request_finished + # time): nothing to fetch, fall back to local compute. + if not info or "peer_host" not in info: + continue + expected = pos.length * self._hidden_dim * self._element_size + if int(info.get("size_bytes", -1)) != expected: + logger.warning( + "EC consumer: size mismatch mm_hash=%s; local encode", mm_hash + ) + continue + try: + started = self._start_xfer(mm_hash, info, expected) + except Exception: + logger.exception( + "EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash + ) + continue + if not started: + continue + self._in_flight.add(mm_hash) + pending = True + return not pending + + def _start_xfer( + self, mm_hash: str, info: "dict[str, Any]", size_bytes: int + ) -> bool: + """Allocate a not-ready cache entry and start a NIXL READ into it. + + Returns True if the transfer was started. Returns False when the + cache cannot accommodate the encoding, in which case the request + falls back to local recomputation. + """ + import time + from math import ceil + + from vllm.distributed.ec_transfer.ec_connector.cpu.session import ( + ConsumerSession, + ) + + n_blocks = max(1, ceil(size_bytes / self._region.block_size_bytes)) + entry = self._cache.alloc(mm_hash, n_blocks) + if entry is None: + logger.debug( + "EC consumer: cache full for mm_hash=%s (%d blocks); local encode", + mm_hash, + n_blocks, + ) + return False + indices = list(entry.block_ids) + addr = (info["peer_host"], int(info["peer_port"])) + if addr not in self._sessions: + zmq_conn = self._transport.connect(addr) + assert self._compat_hash is not None + self._sessions[addr] = ConsumerSession( + addr=addr, + zmq_conn=zmq_conn, + transport=self._transport, + data=self._data, + compat_hash=self._compat_hash, + ) + deadline = time.monotonic() + self._ack_timeout_s + try: + self._sessions[addr].start_xfer(mm_hash, indices, deadline) + except Exception: + self._cache.discard(mm_hash) + raise + logger.debug( + "EC consumer: starting NIXL xfer mm_hash=%s from %s:%d blocks=%d", + mm_hash, + addr[0], + addr[1], + n_blocks, + ) return True + def _poll_step(self) -> None: + import time + + now = time.monotonic() + all_messages = self._transport.poll() + for addr, session in list(self._sessions.items()): + session.poll(all_messages.get(addr, []), now) + for addr in self._transport.poll_dead(): + self._on_peer_down(addr) + for session in self._sessions.values(): + self._process_session_results(session) + + def _process_session_results(self, session: Any) -> None: + r = session.take_results() + for mm_hash in r.completed: + self._in_flight.discard(mm_hash) + self._cache.mark_ready(mm_hash) + self._step_completed.add(mm_hash) + logger.debug("EC consumer: NIXL xfer complete mm_hash=%s", mm_hash) + for mm_hash in r.tombstoned: + self._in_flight.discard(mm_hash) + self._cache.discard(mm_hash) + self._tombstones.add(mm_hash) + logger.debug("EC consumer: NIXL xfer failed mm_hash=%s", mm_hash) + for mm_hash in r.quarantined: + # DMA still running: keep the blocks reserved (entry stays + # not-ready, hence non-evictable) until the xfer settles. + self._in_flight.discard(mm_hash) + self._tombstones.add(mm_hash) + logger.debug( + "EC consumer: NIXL xfer mm_hash=%s quarantined; DMA still running", + mm_hash, + ) + for mm_hash in r.retryable: + # No tombstone: dropping every trace of the attempt is what lets + # the next admit pass re-issue the read as if it were the first. + self._in_flight.discard(mm_hash) + self._cache.discard(mm_hash) + logger.debug( + "EC consumer: NIXL xfer mm_hash=%s retryable; will re-request", + mm_hash, + ) + for mm_hash, _block_indices in r.settled: + self._cache.discard(mm_hash) + logger.debug( + "EC consumer: quarantined NIXL xfer mm_hash=%s settled", mm_hash + ) + + def _on_peer_down(self, addr: Any) -> None: + session = self._sessions.pop(addr, None) + if session is None: + return + session.on_peer_down() + self._process_session_results(session) + session.close() + logger.info("EC consumer: producer peer down addr=%s", addr) + + def _promote_completed_reads(self) -> None: + """Drain the just-completed set built during ``_poll_step``. + + Reads are marked ready in ``_process_session_results``, so a deferred + request is re-admitted by ``ensure_cache_available`` and loaded through + the same local path as a natively cached entry + (``update_state_after_alloc`` -> ``_pending_loads``). No explicit + pin/load bookkeeping is needed here beyond clearing the set. + """ + self._step_completed.clear() + def update_state_after_alloc(self, request: "Request", index: int) -> None: feature = request.mm_features[index] mm_hash = feature.identifier @@ -97,11 +405,17 @@ class ECCPUScheduler: ) -> ECCPUConnectorMetadata: meta = ECCPUConnectorMetadata() if self._is_producer: + if self._nixl_enabled and self._producer_session is not None: + self._producer_session.poll_step() meta.saves = self._pending_saves self._pending_saves = {} if self._is_consumer: + if self._nixl_enabled: + self._promote_completed_reads() meta.loads = self._pending_loads self._pending_loads = {} + if self._nixl_enabled: + self._first_in_batch = True return meta def update_connector_output(self, connector_output: "ECConnectorOutput") -> None: @@ -117,8 +431,15 @@ class ECCPUScheduler: return for mm_hash in meta.completed_saves: entry = self._cache.get(mm_hash) - if entry is not None and not entry.ready: + if entry is None: + logger.debug( + "EC producer: worker reported completed save for unknown " + "mm_hash=%s (already discarded/evicted?)", + mm_hash, + ) + elif not entry.ready: self._cache.mark_ready(mm_hash) + logger.debug("EC producer: mm_hash=%s marked ready", mm_hash) for transfer_id in meta.completed_loads: pending = self._load_acks.get(transfer_id) if pending is None: @@ -131,14 +452,65 @@ class ECCPUScheduler: # as stale rather than releasing the pin a second time. del self._load_acks[transfer_id] self._cache.unpin(mm_hash) + logger.debug("EC consumer: mm_hash=%s unpinned after load", mm_hash) def has_pending_push_work(self) -> bool: - """True while any dispatched save or load has not been confirmed done. + """Keep the engine stepping so this connector's polls keep running. - Keeps the engine stepping so the worker's completion reports are polled - even when no requests are otherwise runnable. + With NIXL enabled the engine tick is the only driver of + ``ProducerSession.poll_step()``, and a producer's work arrives from a + remote consumer, so nothing local would otherwise wake it. Without NIXL + this only has to outlive dispatched saves and loads, so the engine can + quiesce once the worker has confirmed them. """ - return self._cache.has_held_entries() + return self._nixl_enabled or self._cache.has_held_entries() + + def request_finished( + self, request: "Request" + ) -> tuple[bool, "dict[str, Any] | None"]: + if not (self._nixl_enabled and self._is_producer): + return False, None + if not request.mm_features: + return False, None + + items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver) + + for feature in request.mm_features: + mm_hash = feature.identifier + entry = self._cache.get(mm_hash) + if entry is None: + # Never saved, or evicted since. Publishing placeholder + # metadata without an address invites an orchestrator to + # rewrite the media into a reference to an encoding no + # consumer can fetch, leaving the decoder nothing to embed. + # Publish neither, so the media stays on the request. + items[mm_hash]["metadata"] = {} + logger.debug( + "EC producer: mm_hash=%s absent at request_finished; " + "announcing no metadata so the media is not rewritten away", + mm_hash, + ) + continue + # Announce even if the save's GPU->mmap copy hasn't been + # confirmed complete yet: a not-ready entry can't be evicted, so + # it will still be here by the time a consumer's XferReq arrives. + # A read arriving before the save lands is NACKed NACK_NOT_READY, + # which the consumer retries on a later step rather than treating + # as a miss. + size_bytes = ( + feature.mm_position.length * self._hidden_dim * self._element_size + ) + items[mm_hash].update( + peer_host=self._peer_host, + peer_port=self._peer_port, + size_bytes=size_bytes, + ) + logger.debug( + "EC producer: announcing NIXL-readable encodings req_id=%s items=%s", + request.request_id, + items, + ) + return False, items def shutdown(self) -> None: self._pending_saves.clear() @@ -148,7 +520,45 @@ class ECCPUScheduler: self._is_producer = False self._is_consumer = False + if self._nixl_enabled: + self._teardown_nixl() + try: self._region.cleanup() except Exception: logger.debug("ec: region cleanup failed", exc_info=True) + + def _teardown_nixl(self) -> None: + """Close the control sockets and release the NIXL agent. + + Guards each step and clears what it releases, so it is safe on a + scheduler whose NIXL setup failed part-way and safe to call twice. + The shared region is not touched — shutdown() owns that. + """ + if self._producer_session is not None: + try: + self._producer_session.close() + except Exception: + logger.debug("ec: producer session close failed", exc_info=True) + self._producer_session = None + + for session in list(self._sessions.values()): + try: + session.close() + except Exception: + logger.debug("ec: consumer session close failed", exc_info=True) + self._sessions.clear() + + if self._transport is not None: + try: + self._transport.close() + except Exception: + logger.debug("ec: client transport close failed", exc_info=True) + self._transport = None + + if self._data is not None: + try: + self._data.deregister() + except Exception: + logger.debug("ec: deregister failed", exc_info=True) + self._data = None diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/embedding_cache.py b/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/embedding_cache.py index acc42a6ee7..2fd4b01059 100644 --- a/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/embedding_cache.py +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/scheduler/embedding_cache.py @@ -122,6 +122,23 @@ class EmbeddingCache: self._evictable_block_count -= len(entry.block_ids) entry.pin() + def pin_if_ready(self, key: str) -> CacheEntry | None: + """Atomically pin *key* if present and ready; return its entry. + + Returns None when the key is absent. An entry returned with `ready` + False was not pinned: its save is still in flight, which the producer + reports differently from a miss because such a read can be retried. + """ + with self._lock: + entry = self._entries.get(key) + if entry is None or not entry.ready: + return entry + if entry.evictable: + del self._entries_free_list[key] + self._evictable_block_count -= len(entry.block_ids) + entry.pin() + return entry + def unpin(self, key: str) -> None: """Unpin an entry. Asserts currently pinned.""" with self._lock: @@ -134,6 +151,21 @@ class EmbeddingCache: self._entries_free_list[key] = None self._evictable_block_count += len(entry.block_ids) + def discard(self, key: str) -> None: + """Remove a not-ready in-flight entry, returning its blocks to the pool. + + Used when an in-flight fill (e.g. a NIXL READ) fails before the entry + is marked ready. Asserts the entry is present and not ready — ready or + pinned entries are reclaimed through eviction/unpin, not discard. + """ + with self._lock: + entry = self._entries[key] + assert entry._pin_count == -1, ( + f"EmbeddingCache: discard of ready/pinned entry {key!r}" + ) + del self._entries[key] + self._free_blocks.extend(entry.block_ids) + def _evict_until(self, n_blocks: int) -> None: """Evict ready+unpinned entries FIFO until enough space. Lock held.""" while len(self._free_blocks) < n_blocks: diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/session.py b/vllm/distributed/ec_transfer/ec_connector/cpu/session.py new file mode 100644 index 0000000000..54c69a05f3 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/session.py @@ -0,0 +1,633 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Session objects for the ECCPUConnector. + +Layer hierarchy: connector → session → transport (connection) + +Two long-lived sessions: + + ProducerSession — owns ZmqServerTransport. The connector calls poll_step() + once per engine step, which drains the transport without blocking and + feeds the raw bytes to poll(). poll() decodes XferReqs, grants or NACKs + them, pins source blocks, sends XferAck replies, drains NIXL completion + notifications, and sweeps expired xfers. + + ConsumerSession — owns one ZmqClientConnection and the NIXL agent + registration for that peer. The connector calls poll(messages, now) + once per engine step with the raw bytes collected for this peer. + poll() decodes XferAcks, dispatches them to ConsumerXfer objects, + and advances all active xfer state machines. + +Internal data classes: + + ProducerXfer — one pinned read grant, keyed by session_id. + ConsumerXfer — state machine for one NIXL READ, keyed by mm_hash. +""" + +import enum +import logging +import time +import uuid +from dataclasses import dataclass, field +from typing import Any + +import msgspec + +from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import ( + ZmqClientConnection, + ZmqClientTransport, + ZmqServerTransport, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import ( + DataTransport, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import ( + EC_CONNECTOR_VERSION, + EXPECTED_NACKS, + RETRYABLE_NACKS, + XferAck, + XferReq, + XferStatus, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import ( + EmbeddingCache, +) +from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr +from vllm.logger import init_logger + +logger = init_logger(__name__) + +PRODUCER_PIN_LEASE_S = 30.0 +_CONSUMER_READ_TIMEOUT_S = 20.0 +_CONSUMER_QUARANTINE_TIMEOUT_S = 60.0 + + +# ── XferState ───────────────────────────────────────────────────────────────── + + +class XferState(enum.Enum): + """Lifecycle states of a single ConsumerXfer.""" + + WAITING_ACK = "waiting_ack" # XferReq sent, awaiting XferAck + READING = "reading" # NIXL READ in flight + DONE = "done" # READ completed; promote to local cache + ACK_TIMEOUT = "ack_timeout" # XferAck never arrived; free blocks + tombstone + READ_FAILED = "read_failed" # Unexpected NIXL state; free blocks + tombstone + QUARANTINED = "quarantined" # Read timed out, NIXL unabortable; keep blocks + SETTLED = "settled" # Quarantined DMA done; safe to free blocks + + +# ── ProducerXfer ────────────────────────────────────────────────────────────── + + +@dataclass +class ProducerXfer: + """One pinned read grant on the producer side. + + Pure data — lifecycle operations (pin/unpin) are performed by ProducerSession. + Keyed by (consumer_session_id, mm_hash) in ProducerSession._active_xfers. + """ + + mm_hash: str + block_indices: list[int] + deadline: float + + def is_expired(self) -> bool: + return time.monotonic() > self.deadline + + +# ── ConsumerXfer ────────────────────────────────────────────────────────────── + + +@dataclass +class ConsumerXfer: + """State machine for one NIXL READ on the consumer side. + + transfer_handle is the in-flight NIXL handle returned by post_read() — + used for check_xfer_state() / release_xfer_handle(). The dlist handle + for the remote peer lives inside DataTransport, not here. + """ + + mm_hash: str + block_indices: list[int] + addr: PeerAddr + deadline: float + data: DataTransport + consumer_session_id: str # session identity; combined with mm_hash for notif_msg + + transfer_handle: Any | None = field(default=None) + _quarantined: bool = field(default=False, init=False) + + def handle_ack(self, ack: XferAck, agent_name: str) -> bool: + """Issue the NIXL READ on an OK XferAck. Returns False on any NACK.""" + if ack.status != XferStatus.OK: + return False + # notif_msg encodes both session_id and mm_hash so the producer can + # do a direct (session_id, mm_hash) → ProducerXfer lookup on completion. + notif_msg = f"{self.consumer_session_id}:{self.mm_hash}".encode() + self.transfer_handle = self.data.post_read( + self.block_indices, + agent_name, + ack.src_block_indices, + notif_msg=notif_msg, + ) + self.deadline = time.monotonic() + _CONSUMER_READ_TIMEOUT_S + logger.debug( + "EC consumer: NIXL READ posted mm_hash=%s agent=%s local_blocks=%d " + "remote_blocks=%d", + self.mm_hash, + agent_name, + len(self.block_indices), + len(ack.src_block_indices), + ) + return True + + def poll(self, now: float) -> XferState: + if self._quarantined: + return self._poll_quarantined(now) + if self.transfer_handle is None: + return ( + XferState.ACK_TIMEOUT if now > self.deadline else XferState.WAITING_ACK + ) + return self._poll_reading(now) + + def _poll_reading(self, now: float) -> XferState: + assert self.transfer_handle is not None + try: + state = self.data.check_xfer_state(self.transfer_handle) + except Exception: + logger.exception( + "EC Connector: check_xfer_state failed for mm_hash=%s", self.mm_hash + ) + self.data.release_xfer_handle(self.transfer_handle) + self.transfer_handle = None + return XferState.READ_FAILED + if state == "DONE": + self.data.release_xfer_handle(self.transfer_handle) + self.transfer_handle = None + logger.debug("EC consumer: NIXL READ completed mm_hash=%s", self.mm_hash) + return XferState.DONE + if state == "PROC": + if now > self.deadline: + self._quarantined = True + self.deadline = now + _CONSUMER_QUARANTINE_TIMEOUT_S + logger.warning( + "EC Connector: READ for mm_hash=%s from %s:%d timed out; " + "quarantining", + self.mm_hash, + self.addr[0], + self.addr[1], + ) + return XferState.QUARANTINED + return XferState.READING + logger.warning( + "EC Connector: READ for mm_hash=%s unexpected NIXL state %r", + self.mm_hash, + state, + ) + self.data.release_xfer_handle(self.transfer_handle) + self.transfer_handle = None + return XferState.READ_FAILED + + def _poll_quarantined(self, now: float) -> XferState: + if self.transfer_handle is None: + return XferState.SETTLED + # If quarantine has been too long, assume NIXL is stuck and evict blocks. + if now > self.deadline: + self.data.release_xfer_handle(self.transfer_handle) + self.transfer_handle = None + logger.warning( + "EC Connector: quarantined mm_hash=%s from %s:%d timed out; " + "evicting blocks", + self.mm_hash, + self.addr[0], + self.addr[1], + ) + return XferState.READ_FAILED + try: + state = self.data.check_xfer_state(self.transfer_handle) + except Exception: + logger.exception( + "EC Connector: check_xfer_state failed for quarantined mm_hash=%s", + self.mm_hash, + ) + state = None + if state == "PROC": + return XferState.QUARANTINED + self.data.release_xfer_handle(self.transfer_handle) + self.transfer_handle = None + logger.debug( + "EC Connector: quarantined mm_hash=%s settled (state=%s)", + self.mm_hash, + state, + ) + return XferState.SETTLED + + def cancel(self) -> None: + """Cancel a WAITING_ACK xfer (peer down). No tombstone — caller allows retry.""" + assert self.transfer_handle is None, "cancel() requires WAITING_ACK state" + + def release(self) -> None: + """Force-release the transfer handle. Shutdown only.""" + if self.transfer_handle is not None: + self.data.release_xfer_handle(self.transfer_handle) + self.transfer_handle = None + + +# ── ProducerSession ─────────────────────────────────────────────────────────── + + +class ProducerSession: + """Long-lived producer session. Owns ZmqServerTransport. + + poll_step() runs once per engine step: drains the transport without + blocking, decodes XferReqs, grants or NACKs each one, drains NIXL completion + notifications, and sweeps expired xfers. + """ + + def __init__( + self, + transport: ZmqServerTransport, + data: DataTransport, + cache: EmbeddingCache, + compat_hash: str, + ) -> None: + self._transport = transport + self._data = data + self._cache = cache # EmbeddingCache — single block owner + self._compat_hash = compat_hash + + # Key: "{consumer_session_id}:{mm_hash}" — matches the NIXL notif_msg exactly, + # so completion notification lookup is a direct dict pop with no parsing. + self._active_xfers: dict[str, ProducerXfer] = {} + + self._req_decoder = msgspec.msgpack.Decoder(XferReq) + self._req_encoder = msgspec.msgpack.Encoder() + + def close(self) -> None: + for xfer in self._active_xfers.values(): + self._cache.unpin(xfer.mm_hash) # release any still-held pins + self._active_xfers.clear() + self._transport.close() + + def poll_step(self) -> None: + """Drain the transport without blocking and process what arrived.""" + self.poll(self._transport.poll(timeout_ms=0)) + + def poll(self, messages: list[tuple[bytes, bytes]]) -> None: + """Decode and process inbound XferReqs, drain NIXL notifs, sweep timeouts. + + messages: (identity, payload) pairs from ZmqServerTransport.poll(). + """ + for identity, payload in messages: + try: + req = self._req_decoder.decode(payload) + except (msgspec.DecodeError, msgspec.ValidationError): + logger.warning("EC Connector: dropped malformed XferReq") + continue + ack = self._grant_or_nack(req) + self._transport.send(identity, self._req_encoder.encode(ack)) + + self._drain_notifs() + self._sweep_timeouts() + + def _grant_or_nack(self, req: XferReq) -> XferAck: + if req.connector_version != EC_CONNECTOR_VERSION: + logger.warning( + "EC Connector: incompatible version for mm_hash=%s", req.mm_hash + ) + return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_VERSION) + if req.compatibility_hash != self._compat_hash: + logger.warning( + "EC Connector: incompatible compat hash for mm_hash=%s", req.mm_hash + ) + return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INCOMPAT) + if not req.session_id: + # Grants are keyed by (session_id, mm_hash), so an omitted id from + # one peer aliases another's: the first completion would unpin + # blocks the other peer is still reading. + logger.warning( + "EC Connector: XferReq without session_id for mm_hash=%s", req.mm_hash + ) + return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INTERNAL) + entry = self._cache.pin_if_ready(req.mm_hash) + if entry is None or not entry.ready: + status = ( + XferStatus.NACK_MISSING if entry is None else XferStatus.NACK_NOT_READY + ) + logger.log( + logging.INFO if status in EXPECTED_NACKS else logging.WARNING, + "EC Connector: mm_hash=%s not serveable in local cache (%s); NACKing", + req.mm_hash, + status.name, + ) + return XferAck(mm_hash=req.mm_hash, status=status) + block_indices = list(entry.block_ids) + key = f"{req.session_id}:{req.mm_hash}" + if key in self._active_xfers: + # This peer is re-requesting a grant it still holds, having timed + # out waiting. Overwriting it drops the pin taken for it, which + # neither a completion nor the sweep would ever release. + self._cache.unpin(req.mm_hash) + self._active_xfers[key] = ProducerXfer( + mm_hash=req.mm_hash, + block_indices=block_indices, + deadline=time.monotonic() + PRODUCER_PIN_LEASE_S, + ) + logger.debug( + "EC producer: serving NIXL READ request mm_hash=%s key=%s blocks=%d", + req.mm_hash, + key, + len(block_indices), + ) + return XferAck( + mm_hash=req.mm_hash, + status=XferStatus.OK, + session_id=req.session_id, # echo consumer's session_id back + src_block_indices=block_indices, + agent_metadata=self._data.get_agent_metadata(), + mem_descriptor=self._data.get_mem_descriptor(), + ) + + def _drain_notifs(self) -> None: + try: + notifs = self._data.get_new_notifs() + except Exception: + logger.exception("EC Connector: get_new_notifs failed") + return + for msgs in notifs.values(): + for msg in msgs: + try: + key = msg.decode("utf-8") # "{session_id}:{mm_hash}" + except Exception: + logger.exception("EC Connector: Dropped malformed NIXL notif") + continue + xfer = self._active_xfers.pop(key, None) + if xfer is not None: + self._cache.unpin(xfer.mm_hash) + logger.debug( + "EC producer: NIXL READ completed mm_hash=%s key=%s", + xfer.mm_hash, + key, + ) + + def _sweep_timeouts(self) -> None: + for key, xfer in list(self._active_xfers.items()): + if xfer.is_expired(): + logger.warning( + "EC producer: consumer did not complete NIXL READ " + "mm_hash=%s key=%s within the pin lease; releasing " + "pinned blocks", + xfer.mm_hash, + key, + ) + self._cache.unpin(xfer.mm_hash) + del self._active_xfers[key] + + +# ── ConsumerSession ─────────────────────────────────────────────────────────── + + +@dataclass +class ConsumerSessionResults: + """Results from ConsumerSession.take_results().""" + + completed: set[str] # promote to local cache + tombstoned: set[str] # free blocks + prevent retry this step + quarantined: set[str] # timed-out, DMA still running: prevent retry + # but DO NOT free blocks — session holds them + retryable: set[str] # free blocks, allow another attempt (not ready, peer down) + settled: list[tuple[str, list[int]]] # (mm_hash, block_indices) quarantine cleared + + +class ConsumerSession: + """Long-lived per-producer session on the consumer side. + + Owns the ZMQ DEALER connection and the NIXL agent registration for one + producer peer. The connector calls poll(messages, now) once per engine + step with the raw bytes collected from this peer's DEALER socket. + + poll() decodes XferAcks, dispatches them to the right ConsumerXfer, + advances all active xfer state machines, and drains quarantined xfers. + Results accumulate in internal sets; call take_results() to collect them. + """ + + def __init__( + self, + addr: PeerAddr, + zmq_conn: ZmqClientConnection, + transport: ZmqClientTransport, + data: DataTransport, + compat_hash: str, + ) -> None: + self._addr = addr + self._zmq = zmq_conn + self._transport = transport + self._data = data + self._compat_hash = compat_hash + self._session_id = str( + uuid.uuid4() + ) # stable identity for (session_id:mm_hash) keys + + self._nixl_agent_name: str | None = None + self._nixl_metadata_bytes: bytes | None = None + + self._xfers: dict[str, ConsumerXfer] = {} # mm_hash → xfer + self._quarantined: list[ConsumerXfer] = [] # DMA pending, to poll + self._newly_quarantined: set[str] = set() # quarantined since last take_results + + self._completed: set[str] = set() + self._tombstoned: set[str] = set() + self._retryable: set[str] = set() + self._settled: list[tuple[str, list[int]]] = [] + + self._encoder = msgspec.msgpack.Encoder() + self._decoder = msgspec.msgpack.Decoder(XferAck) + + @property + def addr(self) -> PeerAddr: + return self._addr + + def start_xfer( + self, mm_hash: str, block_indices: list[int], deadline: float + ) -> None: + """Create a ConsumerXfer and send the XferReq over the connection.""" + xfer = ConsumerXfer( + mm_hash=mm_hash, + block_indices=block_indices, + addr=self._addr, + deadline=deadline, + data=self._data, + consumer_session_id=self._session_id, + ) + self._zmq.send( + self._encoder.encode( + XferReq( + mm_hash=mm_hash, + compatibility_hash=self._compat_hash, + session_id=self._session_id, + ) + ) + ) + self._xfers[mm_hash] = xfer + logger.debug( + "EC Connector: XferReq sent mm_hash=%s to %s:%d", + mm_hash, + self._addr[0], + self._addr[1], + ) + + def poll(self, messages: list[bytes], now: float) -> None: + """Decode inbound XferAcks, dispatch to xfers, and advance all state machines. + + messages: raw bytes from ZmqClientTransport.poll() for this peer. + Results accumulate until take_results() is called. + """ + for raw in messages: + try: + ack = self._decoder.decode(raw) + except (msgspec.DecodeError, msgspec.ValidationError): + logger.warning( + "EC Connector: dropped malformed XferAck from %s:%d", + self._addr[0], + self._addr[1], + ) + continue + self._dispatch_ack(ack) + + for mm_hash, xfer in list(self._xfers.items()): + state = xfer.poll(now) + if state == XferState.DONE: + self._completed.add(mm_hash) + del self._xfers[mm_hash] + elif state in (XferState.ACK_TIMEOUT, XferState.READ_FAILED): + self._tombstoned.add(mm_hash) + del self._xfers[mm_hash] + elif state == XferState.QUARANTINED: + # DMA may still be writing — caller must NOT free blocks. + self._quarantined.append(xfer) + self._newly_quarantined.add(mm_hash) + del self._xfers[mm_hash] + + self._drain_quarantine(now) + + def _dispatch_ack(self, ack: XferAck) -> None: + xfer = self._xfers.get(ack.mm_hash) + if xfer is None or xfer.transfer_handle is not None: + return # unknown mm_hash or duplicate ack + + if ack.status != XferStatus.OK: + logger.log( + logging.INFO if ack.status in EXPECTED_NACKS else logging.WARNING, + "EC Connector: NACK %s from %s:%d for mm_hash=%s", + ack.status.name, + self._addr[0], + self._addr[1], + ack.mm_hash, + ) + del self._xfers[ack.mm_hash] + # The producer announced this encoding and its save is still in + # flight, so another attempt on a later step can be granted. Every + # other NACK is terminal for this mm_hash. + if ack.status in RETRYABLE_NACKS: + self._retryable.add(ack.mm_hash) + else: + self._tombstoned.add(ack.mm_hash) + return + + try: + agent_name = self._ensure_registered(ack.agent_metadata, ack.mem_descriptor) + except Exception: + logger.exception( + "EC Connector: failed to register peer %s:%d for mm_hash=%s", + self._addr[0], + self._addr[1], + ack.mm_hash, + ) + del self._xfers[ack.mm_hash] + self._tombstoned.add(ack.mm_hash) + return + + if not xfer.handle_ack(ack, agent_name): + del self._xfers[ack.mm_hash] + self._tombstoned.add(ack.mm_hash) + + def _ensure_registered(self, metadata: bytes, mem_descriptor: bytes) -> str: + """Register or re-register the peer in DataTransport. Returns agent_name.""" + if self._nixl_agent_name is not None and self._nixl_metadata_bytes == metadata: + return self._nixl_agent_name + if self._nixl_agent_name is not None: + logger.info( + "EC Connector: producer %s:%d restarted; re-registering", + self._addr[0], + self._addr[1], + ) + self._data.remove_remote_peer(self._nixl_agent_name) + agent_name = self._data.add_remote_peer(metadata, mem_descriptor) + self._nixl_agent_name = agent_name + self._nixl_metadata_bytes = metadata + logger.debug( + "EC Connector: registered peer %s:%d agent=%s", + self._addr[0], + self._addr[1], + agent_name, + ) + return agent_name + + def _drain_quarantine(self, now: float) -> None: + still_pending: list[ConsumerXfer] = [] + for xfer in list(self._quarantined): + state = xfer.poll(now) + if state == XferState.SETTLED: + self._settled.append((xfer.mm_hash, xfer.block_indices)) + else: + still_pending.append(xfer) + self._quarantined = still_pending + + def on_peer_down(self) -> None: + """WAITING_ACK xfers → cancel (retry allowed); READING xfers → quarantine.""" + for mm_hash, xfer in list(self._xfers.items()): + if xfer.transfer_handle is None: + xfer.cancel() + self._retryable.add(mm_hash) + else: + self._quarantined.append(xfer) + self._newly_quarantined.add(mm_hash) + self._xfers.clear() + + def take_results(self) -> ConsumerSessionResults: + """Return and clear all accumulated results since the last call.""" + results = ConsumerSessionResults( + completed=self._completed, + tombstoned=self._tombstoned, + quarantined=self._newly_quarantined, + retryable=self._retryable, + settled=self._settled, + ) + self._completed = set() + self._tombstoned = set() + self._newly_quarantined = set() + self._retryable = set() + self._settled = [] + return results + + def has_in_flight(self) -> bool: + return bool(self._xfers) or bool(self._quarantined) + + def close(self) -> None: + """Release all transfer handles, deregister NIXL peer, close connection. + + Connection teardown is routed through the transport so the pooled + connection is dropped in lock-step with the session — a session and its + DEALER connection have a 1:1 lifecycle, so a later connect() to the same + peer builds a fresh socket instead of handing back this dead one. + """ + for xfer in self._xfers.values(): + xfer.release() + self._xfers.clear() + for xfer in self._quarantined: + xfer.release() + self._quarantined.clear() + if self._nixl_agent_name is not None: + self._data.remove_remote_peer(self._nixl_agent_name) + self._transport.remove(self._addr) diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/utils.py b/vllm/distributed/ec_transfer/ec_connector/cpu/utils.py new file mode 100644 index 0000000000..6a8d8e07e0 --- /dev/null +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/utils.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Supporting utilities for the ECCPUConnector scheduler.""" + +import msgspec + +from vllm.logger import init_logger + +# (host, port) pair identifying a remote peer's ZMQ side-channel endpoint. +PeerAddr = tuple[str, int] + +logger = init_logger(__name__) + + +# Msgpack (de)serialization for the list-of-(addr, size, device_id) +# descriptor form that NIXL's `get_xfer_descs` expects. +_MemDescList = list[tuple[int, int, int]] + +_mem_desc_encoder = msgspec.msgpack.Encoder() +_mem_desc_decoder = msgspec.msgpack.Decoder(_MemDescList) + + +def serialize_mem_descriptor(descs: _MemDescList) -> bytes: + return _mem_desc_encoder.encode(descs) + + +def deserialize_mem_descriptor(payload: bytes) -> _MemDescList: + return _mem_desc_decoder.decode(payload) + + +def build_block_descs( + base_ptr: int, + num_blocks: int, + block_size_bytes: int, + device_id: int = 0, +) -> _MemDescList: + """Per-block `(addr, size, device_id)` tuples for an `ECSharedRegion`. + + Handed to `nixl.get_xfer_descs` to build a dlist where descriptor `i` + addresses block `i` in the mmap. This makes a WRITE to block indices + `[i1, i2, ...]` a simple `make_prepped_xfer(..., [i1, i2])` on both + ends — no per-request address math. + """ + return [ + (base_ptr + i * block_size_bytes, block_size_bytes, device_id) + for i in range(num_blocks) + ] diff --git a/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py b/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py index 56bc09e709..58f29f3dbd 100644 --- a/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py +++ b/vllm/distributed/ec_transfer/ec_connector/cpu/worker/__init__.py @@ -55,7 +55,6 @@ def _coalesce_runs( split. `EmbeddingCache` hands out ascending ids, which is what makes runs findable at all. """ - ids = np.fromiter(block_ids, dtype=np.int64, count=len(block_ids)) breaks = np.flatnonzero(np.diff(ids) != 1) + 1 slots = np.concatenate(([0], breaks)) diff --git a/vllm/distributed/ec_transfer/ec_connector/example_connector.py b/vllm/distributed/ec_transfer/ec_connector/example_connector.py index 5b74dc6776..8ea1684fc9 100644 --- a/vllm/distributed/ec_transfer/ec_connector/example_connector.py +++ b/vllm/distributed/ec_transfer/ec_connector/example_connector.py @@ -5,7 +5,6 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any import safetensors -import torch from vllm.config import VllmConfig from vllm.distributed.ec_transfer.ec_connector.base import ( @@ -13,8 +12,11 @@ from vllm.distributed.ec_transfer.ec_connector.base import ( ECConnectorMetadata, ECConnectorRole, ) +from vllm.distributed.ec_transfer.ec_connector.utils import ( + PlaceholderMetadataResolver, + collect_ec_item_metadata, +) from vllm.logger import init_logger -from vllm.utils.collection_utils import is_list_of from vllm.v1.core.sched.output import SchedulerOutput if TYPE_CHECKING: @@ -52,8 +54,7 @@ class ECExampleConnector(ECConnectorBase): super().__init__(vllm_config=vllm_config, role=role) # req_id -> index self._mm_datas_need_loads: dict[str, int] = {} - self._model_config = vllm_config.model_config - self._metadata_fields_cache: dict[str, set[str]] = {} + self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config) transfer_config = vllm_config.ec_transfer_config if transfer_config is not None: self._storage_path = transfer_config.get_from_extra_config( @@ -169,36 +170,6 @@ class ECExampleConnector(ECConnectorBase): self._mm_datas_need_loads.clear() return meta - def _placeholder_metadata_fields(self, modality: str) -> set[str]: - """Which processed keys this model needs published for `modality`. - - Read from `MultiModalDataParser.embedding_fields`, the same declaration - the consumer's parser requires, so the two cannot drift. An empty set - means the modality cannot be delivered out of band, and the consumer - will process the media itself. - """ - if modality in self._metadata_fields_cache: - return self._metadata_fields_cache[modality] - - fields: set[str] = set() - try: - from vllm.multimodal import MULTIMODAL_REGISTRY - - info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info - fields = info.data_parser.placeholder_metadata_fields(modality) - except Exception: - # Reporting nothing is a safe degradation: the consumer falls back to - # processing the media itself. - logger.warning( - "Could not determine the placeholder metadata fields for " - "modality %s; the consumer will preprocess the media itself.", - modality, - exc_info=True, - ) - - self._metadata_fields_cache[modality] = fields - return fields - def request_finished( self, request: "Request", @@ -214,28 +185,10 @@ class ECExampleConnector(ECConnectorBase): if not self.is_producer: return False, None - items = [] - for feature in request.mm_features: - metadata = {} - # `data` is None for items served from the processor cache, in which - # case the metadata is unavailable here and the consumer has to fall - # back to processing the media itself. - if feature.data is not None: - wanted = self._placeholder_metadata_fields(feature.modality) - for key, value in feature.data.get_data().items(): - if key not in wanted: - continue - if isinstance(value, torch.Tensor): - metadata[key] = value.tolist() - elif is_list_of(value, (int, float)): - # Some metadata (e.g. Qwen3-VL video timestamps) is - # produced as a plain list rather than a tensor. - metadata[key] = value - items.append({"mm_hash": feature.identifier, **metadata}) - + items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver) if not items: return False, None - return False, {"ec_items": items} + return False, items # ============================== # Helper functions diff --git a/vllm/distributed/ec_transfer/ec_connector/utils.py b/vllm/distributed/ec_transfer/ec_connector/utils.py index f5f78e6c3c..46963c2b41 100644 --- a/vllm/distributed/ec_transfer/ec_connector/utils.py +++ b/vllm/distributed/ec_transfer/ec_connector/utils.py @@ -2,8 +2,88 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """EC connector helper utilities.""" +from typing import TYPE_CHECKING, Any + +import torch + +from vllm.logger import init_logger +from vllm.utils.collection_utils import is_list_of from vllm.v1.outputs import ECConnectorOutput, ModelRunnerOutput +if TYPE_CHECKING: + from vllm.config import ModelConfig + from vllm.multimodal.inputs import MultiModalFeatureSpec + +logger = init_logger(__name__) + + +class PlaceholderMetadataResolver: + """Resolves which processed keys a model needs published per modality. + + Reads `MultiModalDataParser.embedding_fields`, the same declaration the + consumer's parser requires, so the two cannot drift. An empty set means + the modality cannot be delivered out of band, and the consumer will + process the media itself. + """ + + def __init__(self, model_config: "ModelConfig") -> None: + self._model_config = model_config + self._cache: dict[str, set[str]] = {} + + def fields_for(self, modality: str) -> set[str]: + if modality in self._cache: + return self._cache[modality] + + fields: set[str] = set() + try: + from vllm.multimodal import MULTIMODAL_REGISTRY + + info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info + fields = info.data_parser.placeholder_metadata_fields(modality) + except Exception: + logger.warning( + "Could not determine the placeholder metadata fields for " + "modality %s; the consumer will preprocess the media itself.", + modality, + exc_info=True, + ) + + self._cache[modality] = fields + return fields + + +def collect_ec_item_metadata( + mm_features: "list[MultiModalFeatureSpec]", + resolver: PlaceholderMetadataResolver, +) -> dict[str, dict[str, Any]]: + """Build one `ec_transfer_params` entry per feature for `request_finished()`. + + Keyed by mm_hash, each entry carries a `metadata` dict with whatever + placeholder fields `resolver` says this model needs published for its + modality, so a consumer can skip the image transform. `data` is None for + items served from the processor cache, in which case the metadata is + unavailable here and the consumer has to fall back to processing the + media itself. A connector that also has transfer coordinates to report + (e.g. NIXL peer_host/peer_port/size_bytes) merges those in alongside + `metadata`, not into it. + """ + items: dict[str, dict[str, Any]] = {} + for feature in mm_features: + metadata: dict[str, Any] = {} + if feature.data is not None: + wanted = resolver.fields_for(feature.modality) + for key, value in feature.data.get_data().items(): + if key not in wanted: + continue + if isinstance(value, torch.Tensor): + metadata[key] = value.tolist() + elif is_list_of(value, (int, float)): + # Some metadata (e.g. Qwen3-VL video timestamps) is + # produced as a plain list rather than a tensor. + metadata[key] = value + items[feature.identifier] = {"metadata": metadata} + return items + class ECOutputAggregator: """Merge every worker's EC connector output onto the single