[EC Connector] P2P NIXL + CPU EC Connector (#47941)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
omerpaz95
2026-09-04 11:23:56 +08:00
committed by GitHub
co-authored by Claude Opus 5
parent 69cf055936
commit 8a0a7ee40a
28 changed files with 3521 additions and 72 deletions
@@ -297,16 +297,26 @@ async def fanout_encoder_primer(
detail=f"Encoder request failed: {detail}",
)
# The encoder reports each item's cache key and grid here.
# The encoder reports each mm_hash's metadata (e.g. the grid) here,
# keyed by the same uuid this proxy assigned above.
try:
params = (await r.json()).get("ec_transfer_params") or {}
reported = params.get("ec_items") or []
except Exception:
logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx)
reported = []
if reported and idx in item_uuids:
# One item per encoder request, so the first entry is this item's.
item_meta[idx] = {**reported[0], "mm_hash": item_uuids[idx]}
params = {}
if idx in item_uuids:
reported = params.get(item_uuids[idx])
if reported:
metadata = reported.get("metadata") or {}
if metadata:
item_meta[idx] = {**metadata, "mm_hash": item_uuids[idx]}
# Whatever the encoder reported alongside `metadata` is the
# connector's own handle on the published embedding (for NIXL,
# peer_host/peer_port/size_bytes). The decoder's connector
# looks it up by mm_hash on the request, so carry it through.
orig_request.setdefault("ec_transfer_params", {})[item_uuids[idx]] = (
reported
)
logger.info(
"[%s] All %d encoder requests completed successfully", req_id, len(mm_items)
@@ -333,3 +333,57 @@ def test_concurrent_alloc_evict():
t.join()
assert not errors, errors
# ── discard ──────────────────────────────────────────────────────────────────
def test_discard_not_ready_returns_blocks():
cache = EmbeddingCache(num_blocks=4)
entry = cache.alloc("h", 3)
assert entry is not None
cache.discard("h")
assert cache.get("h") is None
# All blocks reclaimed: a fresh 4-block alloc now fits.
assert cache.alloc("h2", 4) is not None
def test_discard_rejects_ready_entry():
cache = EmbeddingCache(num_blocks=4)
cache.alloc("h", 1)
cache.mark_ready("h")
with pytest.raises(AssertionError):
cache.discard("h")
def test_discard_rejects_absent_key():
cache = EmbeddingCache(num_blocks=4)
with pytest.raises(KeyError):
cache.discard("nope")
# ── pin_if_ready ─────────────────────────────────────────────────────────────
def test_pin_if_ready_pins_and_returns_entry():
cache = EmbeddingCache(num_blocks=4)
cache.alloc("h", 2)
cache.mark_ready("h")
entry = cache.pin_if_ready("h")
assert entry is not None and entry.ready and len(entry.block_ids) == 2
assert cache.alloc("h2", 4) is None # pinned ⇒ not evictable
def test_pin_if_ready_none_when_absent():
cache = EmbeddingCache(num_blocks=4)
assert cache.pin_if_ready("nope") is None
def test_pin_if_ready_returns_unpinned_entry_when_not_ready():
"""The producer needs "present but not ready" told apart from "absent":
the first is NACKed as retryable, the second as a miss."""
cache = EmbeddingCache(num_blocks=4)
cache.alloc("h", 1) # not-ready
entry = cache.pin_if_ready("h")
assert entry is not None and not entry.ready
assert cache.alloc("h2", 4) is None # not-ready ⇒ still not evictable
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
@@ -34,14 +36,36 @@ def test_worker_role_builds_only_worker(monkeypatch):
assert c.connector_scheduler is None
def test_request_finished_inherited_noop(monkeypatch):
@pytest.mark.parametrize("role", [ECConnectorRole.SCHEDULER, ECConnectorRole.WORKER])
def test_v1_model_runner_is_rejected(monkeypatch, role):
"""The V1 runner never reports save completions, so refuse to start."""
monkeypatch.setattr(
ECCPUConnector, "_make_scheduler", lambda self, cfg: MagicMock()
)
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
assert c.request_finished(MagicMock()) == (False, None)
monkeypatch.setattr(ECCPUConnector, "_make_worker", lambda self, cfg: MagicMock())
cfg = _cfg()
cfg.use_v2_model_runner = False
with pytest.raises(ValueError, match="V2 model runner"):
ECCPUConnector(cfg, role)
def test_factory_registered():
cls = ECConnectorFactory._registry["ECCPUConnector"]()
assert cls is ECCPUConnector
def test_request_finished_forwards_to_scheduler(monkeypatch):
from unittest.mock import MagicMock
import vllm.distributed.ec_transfer.ec_connector.cpu.connector as conn_mod # noqa: F401
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
fake_sched = MagicMock()
fake_sched.request_finished.return_value = (False, {"h1": {"peer_port": 1}})
monkeypatch.setattr(ECCPUConnector, "_make_scheduler", lambda self, cfg: fake_sched)
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
req = MagicMock()
assert c.request_finished(req) == (False, {"h1": {"peer_port": 1}})
fake_sched.request_finished.assert_called_once_with(req)
@@ -64,6 +64,14 @@ _requires_cuda_alike = pytest.mark.skipif(
reason="stalling the compute stream requires a CUDA-like platform",
)
_requires_swap_blocks_batch = pytest.mark.skipif(
not hasattr(torch.ops._C_cache_ops, "swap_blocks_batch"),
reason=(
"installed vllm C++ extension predates the swap_blocks_batch op "
"flush_saves/start_load_caches use; rebuild the extension to run this"
),
)
# Cycles to stall the compute stream for. Must comfortably outlast the host-side
# work the lifetime tests do while a copy is deliberately held back — ~0.5 s on
# a current datacenter GPU.
@@ -194,6 +202,7 @@ def _warm_up_stream_pools(worker: ECCPUWorker) -> None:
# ── save_caches ──────────────────────────────────────────────────────────────
@_requires_swap_blocks_batch
@_requires_accelerator
@pytest.mark.parametrize(
"n_elements,block_ids",
@@ -326,6 +335,7 @@ def test_save_caches_raises_when_allocated_blocks_too_small(make_worker):
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0, 1]}))
@_requires_swap_blocks_batch
@_requires_accelerator
def test_save_caches_batches_multiple_hashes(make_worker):
"""Multiple save_caches calls are batched into a single flush."""
@@ -401,6 +411,7 @@ def test_save_descriptors_hold_high_bit_source_addresses(make_worker):
# ── start_load_caches ────────────────────────────────────────────────────────
@_requires_swap_blocks_batch
@_requires_accelerator
def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker):
"""Single batched load across all hashes with correct byte→dtype→shape."""
@@ -465,6 +476,7 @@ def test_start_load_caches_noop_when_loads_is_empty(make_worker):
assert encoder_cache == {}
@_requires_swap_blocks_batch
@_requires_accelerator
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
"""Every hash in ``meta.loads`` is copied from mmap, including one whose
@@ -497,6 +509,7 @@ def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
assert torch.equal(new.cpu(), src_orig[1:])
@_requires_swap_blocks_batch
@_requires_accelerator
@pytest.mark.parametrize(
"tp_rank,pcp_rank",
@@ -526,6 +539,7 @@ def test_start_load_caches_works_on_all_ranks(make_worker, tp_rank, pcp_rank):
# ── round-trip ───────────────────────────────────────────────────────────────
@_requires_swap_blocks_batch
@_requires_accelerator
def test_save_then_load_round_trips_bytes(make_worker):
"""Full producer→mmap→consumer byte path in one shot."""
@@ -552,6 +566,7 @@ def test_save_then_load_round_trips_bytes(make_worker):
# ── memory lifetime across in-flight copies ─────────────────────────────────
@_requires_swap_blocks_batch
@_requires_cuda_alike
def test_save_survives_encoder_cache_free_before_copy_runs(make_worker):
"""The bytes handed to ``save_caches`` must reach the mmap intact even when
@@ -612,6 +627,7 @@ def test_save_survives_encoder_cache_free_before_copy_runs(make_worker):
)
@_requires_swap_blocks_batch
@_requires_cuda_alike
def test_load_buffer_survives_eviction_while_consumer_read_is_queued(make_worker):
"""A loaded encoder cache entry must keep its bytes for a consumer already
@@ -662,6 +678,7 @@ def test_load_buffer_survives_eviction_while_consumer_read_is_queued(make_worker
# ── buffer recycling ────────────────────────────────────────────────────────
@_requires_swap_blocks_batch
@_requires_accelerator
def test_buffer_pool_is_reused_across_save_steps(make_worker):
"""Once a save copy completes its descriptor buffers return to the pool and
@@ -689,6 +706,7 @@ def test_buffer_pool_is_reused_across_save_steps(make_worker):
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
@_requires_swap_blocks_batch
@_requires_accelerator
def test_buffer_pool_is_reused_across_load_steps(make_worker):
"""Once a load copy completes its descriptor buffers return to the pool and
@@ -793,6 +811,7 @@ def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
# ── e2e: scheduler + worker pipeline ────────────────────────────────────────
@_requires_swap_blocks_batch
@_requires_accelerator
def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
"""Full pipeline: scheduler allocates blocks, worker saves a GPU tensor to
+285
View File
@@ -0,0 +1,285 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for control-layer ABCs, ZmqClientConnection, and ECSession."""
from unittest.mock import MagicMock, patch
import pytest
import zmq
from vllm.distributed.ec_transfer.ec_connector.cpu.control.base import (
ControlConnection,
)
class _ConcreteConn(ControlConnection):
"""Minimal concrete subclass for ABC-instantiation tests."""
def __init__(self):
self._alive = True
self._sent = []
self._inbox = []
@property
def alive(self) -> bool:
return self._alive
def send(self, msg: bytes) -> None:
self._sent.append(msg)
def recv(self) -> list[bytes]:
msgs, self._inbox = self._inbox, []
return msgs
def mark_dead(self) -> None:
self._alive = False
def close(self) -> None:
self._alive = False
def test_control_connection_abc_cannot_be_instantiated_directly():
with pytest.raises(TypeError):
ControlConnection() # type: ignore[abstract]
def test_concrete_connection_send_recv():
conn = _ConcreteConn()
assert conn.alive
conn.send(b"hello")
conn._inbox.append(b"world")
assert conn.recv() == [b"world"]
assert conn._sent == [b"hello"]
def test_mark_dead_sets_alive_false():
conn = _ConcreteConn()
conn.mark_dead()
assert not conn.alive
def test_close_sets_alive_false():
conn = _ConcreteConn()
conn.close()
assert not conn.alive
# ── ZmqClientConnection ───────────────────────────────────────────────────────
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import ( # noqa: E402
ZmqClientConnection,
ZmqClientTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr # noqa: E402
def _conn(messages: list[list[bytes]], monitor=None) -> ZmqClientConnection:
"""Build a ZmqClientConnection whose dealer yields the given frame-lists."""
dealer = MagicMock()
dealer.recv_multipart.side_effect = [*messages, zmq.Again()]
return ZmqClientConnection(dealer=dealer, monitor=monitor)
def test_zmq_client_connection_recv_extracts_payloads_from_envelopes():
conn = _conn([[b"", b"msg1"], [b"", b"msg2"]])
assert conn.recv() == [b"msg1", b"msg2"]
def test_zmq_client_connection_recv_skips_frames_without_empty_delimiter():
# A frame that does not start with b"" is malformed and must be dropped.
conn = _conn([[b"", b"ok"], [b"no-delimiter", b"bad"]])
assert conn.recv() == [b"ok"]
def test_zmq_client_connection_recv_breaks_on_non_again_exception():
# A socket error should stop the drain and return whatever was collected.
dealer = MagicMock()
dealer.recv_multipart.side_effect = [[b"", b"first"], OSError("broken")]
conn = ZmqClientConnection(dealer=dealer, monitor=None)
assert conn.recv() == [b"first"]
def test_zmq_client_connection_send_prefixes_empty_delimiter():
dealer = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=None)
conn.send(b"payload")
dealer.send_multipart.assert_called_once_with([b"", b"payload"])
def test_zmq_client_connection_mark_dead_does_not_close_socket():
# mark_dead signals liveness only; resource teardown is close()'s job.
dealer = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=None)
conn.mark_dead()
assert not conn.alive
dealer.close.assert_not_called()
def test_zmq_client_connection_close_without_monitor():
dealer = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=None)
conn.close()
dealer.close.assert_called_once_with(linger=0)
assert not conn.alive
def test_zmq_client_connection_close_with_monitor_disables_then_closes_both():
# The disable_monitor → monitor.close → dealer.close sequence must be
# respected: closing the dealer before disabling the monitor can cause
# a zmq assertion in some versions.
dealer = MagicMock()
monitor = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=monitor)
conn.close()
dealer.disable_monitor.assert_called_once()
monitor.close.assert_called_once_with(linger=0)
dealer.close.assert_called_once_with(linger=0)
assert not conn.alive
def test_zmq_client_connection_close_is_idempotent():
dealer = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=None)
conn.close()
conn.close() # must not raise
assert not conn.alive
# ── ZmqClientTransport ────────────────────────────────────────────────────────
def _make_transport() -> tuple[ZmqClientTransport, MagicMock]:
"""Return a ZmqClientTransport with a mocked ZMQ context."""
ctx = MagicMock()
ctx.socket.return_value = MagicMock()
with (
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context",
return_value=ctx,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket",
return_value=MagicMock(),
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path",
return_value="tcp://host:0",
),
):
transport = ZmqClientTransport()
transport._ctx = ctx
return transport, ctx
def _inject_conn(transport: ZmqClientTransport, addr: PeerAddr) -> ZmqClientConnection:
"""Inject a ZmqClientConnection directly into the transport pool."""
conn = ZmqClientConnection(dealer=MagicMock(), monitor=MagicMock())
transport._connections[addr] = conn
return conn
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
def test_zmq_client_transport_connect_caches_connection(
mock_ctx_cls, mock_path, mock_sock
):
mock_ctx_cls.return_value = MagicMock()
mock_ctx_cls.return_value.socket.return_value = MagicMock()
mock_sock.return_value = MagicMock()
transport = ZmqClientTransport()
addr: PeerAddr = ("host", 1234)
assert transport.connect(addr) is transport.connect(addr)
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
def test_zmq_client_transport_connect_creates_independent_conns_per_addr(
mock_ctx_cls, mock_path, mock_sock
):
mock_ctx_cls.return_value = MagicMock()
mock_ctx_cls.return_value.socket.return_value = MagicMock()
mock_sock.return_value = MagicMock()
transport = ZmqClientTransport()
conn_a = transport.connect(("host-a", 1111))
conn_b = transport.connect(("host-b", 2222))
assert conn_a is not conn_b
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
def test_zmq_client_transport_poll_dead_detects_disconnected_peer(mock_recv_mon):
transport, _ = _make_transport()
addr: PeerAddr = ("host", 1234)
conn = _inject_conn(transport, addr)
mock_recv_mon.side_effect = [{"event": zmq.EVENT_DISCONNECTED}, zmq.Again()]
dead = transport.poll_dead()
assert addr in dead
assert not conn.alive
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
def test_zmq_client_transport_poll_dead_skips_conn_without_monitor(mock_recv_mon):
transport, _ = _make_transport()
addr: PeerAddr = ("host", 9999)
conn = ZmqClientConnection(dealer=MagicMock(), monitor=None)
transport._connections[addr] = conn
dead = transport.poll_dead()
assert dead == []
mock_recv_mon.assert_not_called()
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
def test_zmq_client_transport_poll_dead_ignores_non_disconnect_events(mock_recv_mon):
transport, _ = _make_transport()
addr: PeerAddr = ("host", 1234)
conn = _inject_conn(transport, addr)
# A CONNECTED event should not trigger a dead notification.
mock_recv_mon.side_effect = [{"event": zmq.EVENT_CONNECTED}, zmq.Again()]
dead = transport.poll_dead()
assert dead == []
assert conn.alive
def test_zmq_client_transport_remove_closes_and_evicts_conn():
transport, _ = _make_transport()
addr: PeerAddr = ("host", 5678)
conn = _inject_conn(transport, addr)
removed = transport.remove(addr)
assert removed is conn
assert addr not in transport._connections
conn.dealer.close.assert_called()
def test_zmq_client_transport_remove_returns_none_for_unknown_addr():
transport, _ = _make_transport()
assert transport.remove(("ghost", 0)) is None
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
def test_zmq_client_transport_close_closes_all_connections(
mock_ctx_cls, mock_path, mock_sock
):
mock_ctx = MagicMock()
mock_ctx_cls.return_value = mock_ctx
mock_ctx.socket.return_value = MagicMock()
dealer_a, dealer_b = MagicMock(), MagicMock()
mock_sock.side_effect = [dealer_a, dealer_b]
transport = ZmqClientTransport()
transport.connect(("a", 1))
transport.connect(("b", 2))
transport.close()
dealer_a.close.assert_called()
dealer_b.close.assert_called()
assert transport._connections == {}
+184
View File
@@ -0,0 +1,184 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for DataTransport ABC and NixlDataTransport."""
from unittest.mock import MagicMock, patch
import pytest
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
DataTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
serialize_mem_descriptor,
)
_NIXL_PATH = "vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl"
# ── helpers ───────────────────────────────────────────────────────────────────
def _make_nixl_mock(prep_side_effect=None) -> MagicMock:
nixl = MagicMock()
nixl.get_agent_metadata.return_value = b"agent-meta"
nixl.get_reg_descs.return_value = MagicMock()
nixl.get_xfer_descs.return_value = MagicMock()
nixl.prep_xfer_dlist.side_effect = prep_side_effect or [77, 88]
nixl.add_remote_agent.return_value = "remote-agent-1"
nixl.get_new_notifs.return_value = {}
nixl.check_xfer_state.return_value = "PROC"
return nixl
def _make_transport(nixl_mock=None):
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
NixlDataTransport,
)
mock = nixl_mock or _make_nixl_mock()
with (
patch(f"{_NIXL_PATH}.NixlWrapper", return_value=mock),
patch(f"{_NIXL_PATH}.nixl_agent_config", return_value=MagicMock()),
):
t = NixlDataTransport("agent", 0, 4, 64, 256)
t._nixl = mock
return t
# ── ABC ───────────────────────────────────────────────────────────────────────
def test_data_transport_abc_cannot_be_instantiated():
with pytest.raises(TypeError):
DataTransport() # type: ignore[abstract]
# ── NixlDataTransport construction ───────────────────────────────────────────
def test_nixl_data_transport_registers_memory_on_init():
nixl = _make_nixl_mock()
_make_transport(nixl)
nixl.register_memory.assert_called_once()
def test_nixl_data_transport_unwinds_registration_if_init_fails():
"""A failure after register_memory must not leave the region pinned.
__init__ propagates, so the caller never gets a transport to deregister.
"""
nixl = _make_nixl_mock(prep_side_effect=RuntimeError("no dlist"))
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
NixlDataTransport,
)
with (
patch(f"{_NIXL_PATH}.NixlWrapper", return_value=nixl),
patch(f"{_NIXL_PATH}.nixl_agent_config", return_value=MagicMock()),
pytest.raises(RuntimeError, match="no dlist"),
):
NixlDataTransport("agent", 0, 4, 64, 256)
nixl.deregister_memory.assert_called_once_with(nixl.get_reg_descs.return_value)
def test_nixl_data_transport_raises_if_nixl_unavailable():
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
NixlDataTransport,
)
with (
patch(f"{_NIXL_PATH}.NixlWrapper", None),
patch(f"{_NIXL_PATH}.nixl_agent_config", None),
pytest.raises(RuntimeError, match="requires NIXL"),
):
NixlDataTransport("a", 0, 1, 64, 64)
# ── public accessors ──────────────────────────────────────────────────────────
def test_get_agent_metadata_returns_nixl_blob():
assert _make_transport().get_agent_metadata() == b"agent-meta"
def test_get_mem_descriptor_roundtrips_block_descs():
t = _make_transport()
raw = t.get_mem_descriptor()
assert isinstance(raw, bytes) and len(raw) > 0
# ── add_remote_peer ───────────────────────────────────────────────────────────
def test_add_remote_peer_returns_agent_name_and_stores_handle_internally():
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
t = _make_transport(nixl)
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
agent_name = t.add_remote_peer(b"meta", mem_desc)
assert agent_name == "remote-agent-1"
# Dlist handle stored internally; accessible via _peer_handles.
assert t._peer_handles["remote-agent-1"] == 88
def test_add_remote_peer_registers_remote_agent():
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
t = _make_transport(nixl)
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
t.add_remote_peer(b"fresh-meta", mem_desc)
nixl.add_remote_agent.assert_called_once_with(b"fresh-meta")
# ── remove_remote_peer ────────────────────────────────────────────────────────
def test_remove_remote_peer_delegates_to_nixl():
t = _make_transport()
t.remove_remote_peer("agent-x")
t._nixl.remove_remote_agent.assert_called_once_with("agent-x")
def test_remove_remote_peer_swallows_exceptions():
t = _make_transport()
t._nixl.remove_remote_agent.side_effect = RuntimeError("gone")
t.remove_remote_peer("agent-x") # must not raise
# ── post_read ─────────────────────────────────────────────────────────────────
def test_post_read_raises_on_index_count_mismatch():
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
t = _make_transport(nixl)
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
agent = t.add_remote_peer(b"meta", mem_desc)
with pytest.raises(ValueError, match="mismatch"):
t.post_read([0, 1], agent, [5], notif_msg=b"sid")
def test_post_read_calls_make_prepped_xfer_and_transfer():
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
t = _make_transport(nixl)
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
agent = t.add_remote_peer(b"meta", mem_desc)
nixl.make_prepped_xfer.return_value = 999
handle = t.post_read([0, 1], agent, [5, 6], notif_msg=b"sid")
nixl.make_prepped_xfer.assert_called_once()
nixl.transfer.assert_called_once_with(999)
assert handle == 999
# ── release_xfer_handle / deregister ─────────────────────────────────────────
def test_release_xfer_handle_swallows_exceptions():
t = _make_transport()
t._nixl.release_xfer_handle.side_effect = RuntimeError("stale")
t.release_xfer_handle(42) # must not raise
def test_deregister_swallows_exceptions():
t = _make_transport()
t._nixl.deregister_memory.side_effect = RuntimeError("already gone")
t.deregister() # must not raise
@@ -7,7 +7,8 @@ No running engine required.
from unittest.mock import MagicMock
from tests.v1.core.utils import create_scheduler
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
@@ -15,6 +16,16 @@ from vllm.outputs import CompletionOutput, RequestOutput
from vllm.sampling_params import SamplingParams
from vllm.v1.request import Request, RequestStatus
# tests.v1.core.utils transitively imports the example KV connector, which
# requires the compiled CUDA flash-attention extensions; skip cleanly rather
# than erroring at collection when those extensions aren't built.
_core_utils = pytest.importorskip(
"tests.v1.core.utils",
reason="requires compiled CUDA flash-attention extensions",
exc_type=ImportError,
)
create_scheduler = _core_utils.create_scheduler
EC_PARAMS: dict = {"mm_hash_abc": {"peer_host": "10.0.0.1", "peer_port": 5501}}
@@ -0,0 +1,41 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import msgspec
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
EC_CONNECTOR_VERSION,
XferAck,
XferReq,
XferStatus,
compute_ec_compatibility_hash,
)
def test_xferreq_roundtrip():
req = XferReq(mm_hash="h1", compatibility_hash="c1", session_id="s1")
data = msgspec.msgpack.encode(req)
out = msgspec.msgpack.decode(data, type=XferReq)
assert out.mm_hash == "h1"
assert out.compatibility_hash == "c1"
assert out.connector_version == EC_CONNECTOR_VERSION
def test_xferack_roundtrip_ok():
ack = XferAck(
mm_hash="h1",
status=XferStatus.OK,
src_block_indices=[1, 2],
agent_metadata=b"meta",
mem_descriptor=b"desc",
)
out = msgspec.msgpack.decode(msgspec.msgpack.encode(ack), type=XferAck)
assert out.status == XferStatus.OK
assert out.src_block_indices == [1, 2]
def test_compat_hash_deterministic_and_sensitive():
a = compute_ec_compatibility_hash("0.1", "m", "float16", 64)
b = compute_ec_compatibility_hash("0.1", "m", "float16", 64)
c = compute_ec_compatibility_hash("0.1", "m", "float16", 128)
assert a == b
assert a != c
@@ -0,0 +1,350 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import uuid
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
_N, _BS, _HID, _ES = 16, 64, 32, 2
class _Pos:
def __init__(self, offset, length):
self.offset, self.length = offset, length
class _Feature:
def __init__(self, mm_hash, length=1):
self.mm_hash = mm_hash
self.identifier = mm_hash
self.mm_position = _Pos(0, length)
class _Request:
def __init__(self, features, params=None, req_id="r1"):
self.mm_features = features
self.request_id = req_id
self.ec_transfer_params = params
class _FakeResults:
def __init__(self):
self.completed = set()
self.tombstoned = set()
self.quarantined = set()
self.retryable = set()
self.settled = []
class _FakeSession:
"""Records started transfers; lets the test flip them to completed."""
def __init__(self):
self.started = []
self._results = _FakeResults()
def start_xfer(self, mm_hash, block_indices, deadline):
self.started.append(mm_hash)
def poll(self, messages, now):
pass
def take_results(self):
r, self._results = self._results, _FakeResults()
return r
def close(self):
pass
class _FakeTransport:
"""Stand-in for ZmqClientTransport so _poll_step runs without ZMQ."""
def poll(self):
return {}
def poll_dead(self):
return []
def close(self):
pass
def _consumer_sched(monkeypatch):
def _region(cfg):
return ECSharedRegion(
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
)
monkeypatch.setattr(sched_mod, "create_ec_shared_region", _region)
# ec_enable_nixl defaults to False, so this builds gate-off; the fields
# below flip on the NIXL consumer state directly.
s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_consumer"))
# Turn on NIXL consumer state without constructing real transports.
s._nixl_enabled = True
s._transport = _FakeTransport()
s._data = None
s._compat_hash = "c"
# _setup_nixl normally computes these from model_config; set them
# directly since this helper builds gate-off then flips fields on.
s._hidden_dim = _HID
s._element_size = _ES
return s
def _params(mm_hash, length):
return {
mm_hash: {
"peer_host": "h",
"peer_port": 1,
"size_bytes": length * _HID * _ES,
}
}
def test_new_remote_read_defers_then_completes(monkeypatch):
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
# Route _start_xfer to our fake session instead of real ZMQ/NIXL, but
# still reserve a real not-ready cache entry so mark_ready works.
def _fake_start(mm_hash, info, size):
entry = s._cache.alloc(mm_hash, 1)
assert entry is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _fake_start)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
# Step 1: unseen remote item -> read started, request deferred.
assert s.ensure_cache_available(req, 0) is False
assert "h1" in s._in_flight
assert "h1" in fake.started
s.build_connector_meta(scheduler_output=None) # re-arms _first_in_batch
# Session reports completion this step.
fake._results.completed.add("h1")
s._sessions[("h", 1)] = fake
# Step 2: poll drains completion -> entry marked ready, request admitted.
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
assert "h1" in s._step_completed
entry = s._cache.get("h1")
assert entry is not None and entry.ready
# The scheduler now allocates encoder input; the ready entry loads through
# the same local path as a natively cached encoding.
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta.loads
assert "h1" not in s._step_completed # cleared by promote
# Step 3: still cached -> admitted directly, no new transfer.
assert s.ensure_cache_available(req, 0) is True
s.shutdown()
def test_no_params_is_ready(monkeypatch):
s = _consumer_sched(monkeypatch)
assert s.ensure_cache_available(_Request([_Feature("x")], params=None), 0) is True
s.shutdown()
def test_tombstoned_read_discards_and_blocks_retry(monkeypatch):
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
def _fake_start(mm_hash, info, size):
entry = s._cache.alloc(mm_hash, 1)
assert entry is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _fake_start)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
# Step 1: start the read.
assert s.ensure_cache_available(req, 0) is False
s.build_connector_meta(scheduler_output=None)
# Producer rejected the read -> tombstoned.
fake._results.tombstoned.add("h1")
s._sessions[("h", 1)] = fake
# Step 2: poll discards the entry and records a tombstone; the tombstone
# is consumed by admit so the request proceeds to local compute.
assert s.ensure_cache_available(req, 0) is True
assert s._cache.get("h1") is None
assert "h1" not in s._in_flight
s.build_connector_meta(scheduler_output=None)
# Step 3: tombstone was consumed, so a fresh transfer starts again.
assert s.ensure_cache_available(req, 0) is False
assert "h1" in s._in_flight
s.shutdown()
def test_retryable_read_re_requests_without_admitting(monkeypatch):
"""A not-ready NACK defers the request and re-requests the read.
Unlike a tombstone it never admits the request, because an admitted request
whose media was rewritten away upstream has no embedding and no way to
recompute one.
"""
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
def _fake_start(mm_hash, info, size):
entry = s._cache.alloc(mm_hash, 1)
assert entry is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _fake_start)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
assert s.ensure_cache_available(req, 0) is False
s.build_connector_meta(scheduler_output=None)
# Producer's save has not landed: retryable, not tombstoned.
fake._results.retryable.add("h1")
s._sessions[("h", 1)] = fake
# The entry is released and the read re-issued in the same step, so the
# request stays deferred rather than being admitted unfetched.
assert s.ensure_cache_available(req, 0) is False
assert "h1" in s._in_flight
assert "h1" not in s._tombstones
assert fake.started.count("h1") == 2
s.shutdown()
def test_size_mismatch_skips_transfer(monkeypatch):
s = _consumer_sched(monkeypatch)
# Advertised size disagrees with pos.length * hidden_dim * element_size.
bad = {"h1": {"peer_host": "h", "peer_port": 1, "size_bytes": 999}}
req = _Request([_Feature("h1", 1)], params=bad)
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
assert s._cache.get("h1") is None
s.shutdown()
def test_metadata_only_entry_admits_without_transfer(monkeypatch):
# A producer that reported placeholder metadata but had no cache entry
# to transfer omits peer_host/peer_port/size_bytes entirely; the consumer
# must fall back to local compute rather than treating that as a size
# mismatch.
s = _consumer_sched(monkeypatch)
metadata_only = {"h1": {"image_grid_thw": [1, 2, 3]}}
req = _Request([_Feature("h1", 1)], params=metadata_only)
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
assert s._cache.get("h1") is None
s.shutdown()
def test_orphan_not_ready_entry_falls_back_no_realloc(monkeypatch):
# Post-quarantine, post-tombstone-consumed orphan: a not-ready cache
# entry exists for the mm_hash, but the hash is tracked in none of
# _in_flight / _step_completed / _tombstones. Its blocks are held by a
# quarantined/settling DMA and must not be re-allocated.
s = _consumer_sched(monkeypatch)
entry = s._cache.alloc("h1", 1)
assert entry is not None and not entry.ready
assert "h1" not in s._in_flight
assert "h1" not in s._step_completed
assert "h1" not in s._tombstones
# Spy on alloc: record calls, delegate to the real allocator (which
# asserts on a duplicate key). A re-alloc for "h1" is the bug.
real_alloc = s._cache.alloc
calls: list[str] = []
def _spy_alloc(key, n):
calls.append(key)
return real_alloc(key, n)
monkeypatch.setattr(s._cache, "alloc", _spy_alloc)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
# Must not raise; request falls back to local compute (admitted True),
# the orphan entry is untouched, and alloc is never called again.
assert s.ensure_cache_available(req, 0) is True
assert calls == []
assert s._cache.get("h1") is entry
assert not entry.ready
s.shutdown()
def test_alloc_failure_falls_back_to_local(monkeypatch):
s = _consumer_sched(monkeypatch)
# Force the cache to reject the allocation.
monkeypatch.setattr(s._cache, "alloc", lambda key, n: None)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
# _start_xfer returns False -> request admitted for local recompute.
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
s.shutdown()
def test_already_computed_feature_admits_without_transfer(monkeypatch):
# A feature entirely within num_computed_tokens is skipped by admit; no
# cache lookup/alloc or transfer should be triggered for it.
s = _consumer_sched(monkeypatch)
calls: list[str] = []
real_alloc = s._cache.alloc
def _spy_alloc(key, n):
calls.append(key)
return real_alloc(key, n)
monkeypatch.setattr(s._cache, "alloc", _spy_alloc)
# Feature spans [0, 4); with num_computed_tokens=4 it is fully computed.
req = _Request([_Feature("h1", 4)], params=_params("h1", 4))
assert s.ensure_cache_available(req, 4) is True
assert "h1" not in s._in_flight
assert calls == []
s.shutdown()
def test_local_ready_entry_admits_without_transfer(monkeypatch):
# A READY entry already present in the cache is a local hit: admit
# returns True and no new transfer is started for that mm_hash.
s = _consumer_sched(monkeypatch)
entry = s._cache.alloc("h1", 1)
assert entry is not None
s._cache.mark_ready("h1")
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
s.shutdown()
def test_in_flight_hash_defers_without_second_transfer(monkeypatch):
# A mm_hash already tracked in _in_flight must defer the request rather
# than starting a duplicate transfer.
s = _consumer_sched(monkeypatch)
started: list[str] = []
def _spy_start_xfer(mm_hash, info, size):
started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _spy_start_xfer)
s._in_flight.add("h1")
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
assert s.ensure_cache_available(req, 0) is False
assert started == []
s.shutdown()
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import uuid
import pytest
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.config.ec_transfer import ECTransferConfig
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
from vllm.distributed.nixl_utils import NixlWrapper
_N = 16
_BS = 64
def _region() -> ECSharedRegion:
return ECSharedRegion(
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
)
def test_ec_enable_nixl_absent_from_extra_config_by_default():
cfg = ECTransferConfig()
assert cfg.get_from_extra_config("ec_enable_nixl", False) is False
def test_ec_enable_nixl_read_from_extra_config():
cfg = ECTransferConfig(
ec_connector="ECCPUConnector",
ec_role="ec_both",
ec_connector_extra_config={"ec_enable_nixl": True},
)
assert cfg.get_from_extra_config("ec_enable_nixl", False) is True
def test_gate_off_builds_no_nixl(monkeypatch):
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
# ec_enable_nixl defaults to False.
s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_both"))
assert s._nixl_enabled is False
assert getattr(s, "_data", None) is None
assert getattr(s, "_producer_session", None) is None
s.shutdown()
def test_string_false_in_extra_config_leaves_nixl_off(monkeypatch):
"""Extra config is not type coerced, and bool("false") is True."""
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
cfg = create_ec_vllm_config(ec_role="ec_both")
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = "false"
s = ECCPUScheduler(cfg)
assert s._nixl_enabled is False
s.shutdown()
@pytest.mark.skipif(NixlWrapper is None, reason="Requires NIXL package")
def test_gate_on_wires_data_transport_and_producer_session(monkeypatch):
# Port 0 lets the OS pick an ephemeral port so the real ZMQ ROUTER bind
# in ProducerSession.start() cannot collide with another test/process.
monkeypatch.setenv("VLLM_EC_SIDE_CHANNEL_HOST", "127.0.0.1")
monkeypatch.setenv("VLLM_EC_SIDE_CHANNEL_PORT", "0")
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
cfg = create_ec_vllm_config(ec_role="ec_both")
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = True
s = ECCPUScheduler(cfg)
try:
assert s._nixl_enabled is True
assert s._data is not None
assert s._compat_hash is not None
assert s._producer_session is not None
assert s._peer_host == "127.0.0.1"
assert s._transport is not None # is_ec_consumer is also True
finally:
s.shutdown()
@@ -0,0 +1,113 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import uuid
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
_N, _BS = 16, 64
class _Pos:
def __init__(self, offset, length):
self.offset, self.length = offset, length
class _Feature:
def __init__(self, mm_hash, length=1):
self.mm_hash = mm_hash
self.identifier = mm_hash
self.mm_position = _Pos(0, length)
self.data = None
self.modality = "image"
class _Request:
def __init__(self, features, req_id="r1"):
self.mm_features = features
self.request_id = req_id
def _sched_gate_off(monkeypatch):
def _region(cfg):
return ECSharedRegion(
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
)
monkeypatch.setattr(sched_mod, "create_ec_shared_region", _region)
return ECCPUScheduler(create_ec_vllm_config(ec_role="ec_both"))
def test_request_finished_gate_off_returns_none(monkeypatch):
s = _sched_gate_off(monkeypatch)
assert s.request_finished(_Request([_Feature("h1")])) == (False, None)
s.shutdown()
def test_request_finished_producer_emits_params(monkeypatch):
s = _sched_gate_off(monkeypatch)
# Simulate NIXL-enabled producer bookkeeping without building real NIXL.
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
# _setup_nixl normally computes these from model_config; set them
# directly since this test builds gate-off then flips fields on.
s._hidden_dim, s._element_size = 32, 2
# feature length=2, hidden_dim=32, element_size=2 -> 128 bytes -> 2 blocks.
entry = s._cache.alloc("h1", 2)
assert entry is not None
s._cache.mark_ready("h1")
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
assert delay is False
assert params == {
"h1": {
"metadata": {},
"peer_host": "1.2.3.4",
"peer_port": 5601,
"size_bytes": 2 * 32 * 2,
}
}
s.shutdown()
def test_request_finished_announces_not_ready_entry(monkeypatch):
"""A save whose GPU->mmap copy hasn't been confirmed complete yet is
still announced: the entry can't be evicted before it's ready, and a read
arriving too early is NACKed NACK_NOT_READY for the consumer to retry —
protecting it is not this method's job."""
s = _sched_gate_off(monkeypatch)
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
s._hidden_dim, s._element_size = 32, 2
s._cache.alloc("h1", 2) # allocated but not marked ready
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
assert delay is False
assert params == {
"h1": {
"metadata": {},
"peer_host": "1.2.3.4",
"peer_port": 5601,
"size_bytes": 2 * 32 * 2,
}
}
s.shutdown()
def test_request_finished_skips_unallocated_entry(monkeypatch):
s = _sched_gate_off(monkeypatch)
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
s._hidden_dim, s._element_size = 32, 2
# No alloc() for "h1" at all — e.g. the cache was full at save time.
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
assert delay is False
# The item's placeholder metadata is still reported even though there's
# no cache entry to transfer (empty "metadata": no fields, no transfer).
assert params == {"h1": {"metadata": {}}}
s.shutdown()
+470
View File
@@ -0,0 +1,470 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ProducerXfer, ProducerSession, ConsumerXfer, ConsumerSession."""
import time
from unittest.mock import MagicMock
import pytest
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
XferAck,
XferReq,
XferStatus,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
_CONSUMER_QUARANTINE_TIMEOUT_S,
ConsumerSession,
ConsumerXfer,
ProducerSession,
ProducerXfer,
XferState,
)
# ── helpers ───────────────────────────────────────────────────────────────────
def _make_data(xfer_state: str = "PROC") -> MagicMock:
data = MagicMock()
data.check_xfer_state.return_value = xfer_state
data.post_read.return_value = 99
return data
def _ok_ack(
mm_hash: str = "h1",
session_id: str = "sess-1",
src_indices: list | None = None,
) -> XferAck:
return XferAck(
mm_hash=mm_hash,
status=XferStatus.OK,
session_id=session_id,
src_block_indices=src_indices or [0, 1],
agent_metadata=b"meta",
mem_descriptor=b"desc",
)
def _xfer(
data: MagicMock | None = None,
deadline: float | None = None,
consumer_session_id: str = "sess-1",
) -> ConsumerXfer:
return ConsumerXfer(
mm_hash="h1",
block_indices=[10, 11],
addr=("host", 1234),
deadline=deadline if deadline is not None else time.monotonic() + 60,
data=data or _make_data(),
consumer_session_id=consumer_session_id,
)
def _started(
data: MagicMock | None = None, deadline: float | None = None
) -> ConsumerXfer:
"""ConsumerXfer that has already received an OK XferAck."""
x = _xfer(data, deadline)
x.handle_ack(_ok_ack(session_id="sess-1", src_indices=[5, 6]), agent_name="agent-1")
return x
# ── ProducerXfer ──────────────────────────────────────────────────────────────
def test_producer_xfer_not_expired_before_deadline():
x = ProducerXfer("h1", [0, 1], deadline=time.monotonic() + 60)
assert not x.is_expired()
def test_producer_xfer_expired_after_deadline():
x = ProducerXfer("h1", [0, 1], deadline=time.monotonic() - 1)
assert x.is_expired()
# ── ProducerSession ───────────────────────────────────────────────────────────
def _xfer_req(mm_hash: str = "h1", session_id: str = "sess-1") -> XferReq:
return XferReq(
mm_hash=mm_hash, compatibility_hash="hash-abc", session_id=session_id
)
def _make_producer_session(cache: EmbeddingCache | None = None) -> ProducerSession:
data = _make_data()
data.get_agent_metadata.return_value = b"meta"
data.get_mem_descriptor.return_value = b"desc"
return ProducerSession(
transport=MagicMock(),
data=data,
cache=cache or EmbeddingCache(num_blocks=8),
compat_hash="hash-abc",
)
def test_producer_session_grant_pins_ready_entry_and_returns_blocks():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
ack = s._grant_or_nack(_xfer_req(mm_hash="h1"))
assert ack.status == XferStatus.OK
entry = cache.get("h1")
assert not entry.evictable # pinned by the grant
assert list(ack.src_block_indices) == list(entry.block_ids)
def test_producer_session_grant_nacks_missing_entry():
s = _make_producer_session()
ack = s._grant_or_nack(_xfer_req(mm_hash="nope"))
assert ack.status == XferStatus.NACK_MISSING
def test_producer_session_grant_nacks_not_ready_entry_as_retryable():
"""An announced entry whose save is still in flight is distinguished from a
miss, so the consumer retries instead of falling back to local compute."""
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2) # not ready
s = _make_producer_session(cache)
ack = s._grant_or_nack(_xfer_req(mm_hash="h1"))
assert ack.status == XferStatus.NACK_NOT_READY
def test_producer_session_grant_nacks_version_mismatch():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
req = XferReq(
mm_hash="h1",
compatibility_hash="hash-abc",
session_id="sess-1",
connector_version=-1,
)
ack = s._grant_or_nack(req)
assert ack.status == XferStatus.NACK_VERSION
entry = cache.get("h1")
assert entry.evictable # nothing was pinned
def test_producer_session_grant_nacks_compat_hash_mismatch():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
req = XferReq(mm_hash="h1", compatibility_hash="wrong-hash", session_id="sess-1")
ack = s._grant_or_nack(req)
assert ack.status == XferStatus.NACK_INCOMPAT
entry = cache.get("h1")
assert entry.evictable # nothing was pinned
def test_producer_session_notif_unpins_via_cache():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
s._grant_or_nack(_xfer_req(mm_hash="h1", session_id="sess-1"))
entry = cache.get("h1")
assert not entry.evictable # pinned
s._data.get_new_notifs.return_value = {"agent": [b"sess-1:h1"]}
s._drain_notifs()
assert entry.evictable # unpinned via cache
assert s._active_xfers == {}
def test_producer_session_sweep_timeouts_unpins_via_cache():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
s._grant_or_nack(_xfer_req(mm_hash="h1", session_id="sess-1"))
key = next(iter(s._active_xfers))
s._active_xfers[key].deadline = time.monotonic() - 1
s._sweep_timeouts()
assert cache.get("h1").evictable
assert s._active_xfers == {}
# ── ConsumerXfer.handle_ack ───────────────────────────────────────────────────
def test_consumer_xfer_handle_ack_ok_starts_read():
data = _make_data()
x = _xfer(data)
assert x.handle_ack(_ok_ack(), agent_name="agent-1") is True
assert x.transfer_handle == 99
def test_consumer_xfer_handle_ack_notif_msg_is_session_id_colon_mm_hash():
data = _make_data()
x = _xfer(data, consumer_session_id="my-sess")
x.handle_ack(_ok_ack(mm_hash="h1"), agent_name="agent-1")
_, kwargs = data.post_read.call_args
assert kwargs["notif_msg"] == b"my-sess:h1"
def test_consumer_xfer_handle_ack_forwards_correct_args():
data = _make_data()
x = _xfer(data)
x.handle_ack(_ok_ack(src_indices=[5, 6]), agent_name="agent-1")
args, _ = data.post_read.call_args
assert args[0] == [10, 11] # local block_indices
assert args[1] == "agent-1" # agent_name (not a raw handle)
assert args[2] == [5, 6] # remote src_block_indices
def test_consumer_xfer_handle_ack_resets_deadline():
original = time.monotonic() + 2
x = _xfer(deadline=original)
x.handle_ack(_ok_ack(), agent_name="agent-1")
assert x.deadline > original
def test_consumer_xfer_handle_ack_nack_returns_false():
data = _make_data()
x = _xfer(data)
assert (
x.handle_ack(XferAck(mm_hash="h1", status=XferStatus.NACK_MISSING), "a")
is False
)
assert x.transfer_handle is None
data.post_read.assert_not_called()
# ── ConsumerXfer.poll — WAITING_ACK ──────────────────────────────────────────
def test_consumer_xfer_poll_waiting_ack_before_deadline():
assert _xfer().poll(time.monotonic()) == XferState.WAITING_ACK
def test_consumer_xfer_poll_waiting_ack_timeout():
assert (
_xfer(deadline=time.monotonic() - 1).poll(time.monotonic())
== XferState.ACK_TIMEOUT
)
# ── ConsumerXfer.poll — READING ───────────────────────────────────────────────
def test_consumer_xfer_poll_reading_proc():
assert _started().poll(time.monotonic()) == XferState.READING
def test_consumer_xfer_poll_reading_done_releases_handle():
data = _make_data(xfer_state="DONE")
x = _started(data)
assert x.poll(time.monotonic()) == XferState.DONE
data.release_xfer_handle.assert_called_once_with(99)
assert x.transfer_handle is None
def test_consumer_xfer_poll_reading_unexpected_state_fails():
data = _make_data(xfer_state="ERR")
x = _started(data)
assert x.poll(time.monotonic()) == XferState.READ_FAILED
data.release_xfer_handle.assert_called_once_with(99)
def test_consumer_xfer_poll_reading_exception_fails_gracefully():
data = _make_data()
data.check_xfer_state.side_effect = RuntimeError("nixl crash")
x = _started(data)
assert x.poll(time.monotonic()) == XferState.READ_FAILED
def test_consumer_xfer_poll_timeout_quarantines_without_releasing():
data = _make_data(xfer_state="PROC")
x = _started(data)
assert x.poll(time.monotonic() + 9999) == XferState.QUARANTINED
data.release_xfer_handle.assert_not_called()
assert x.transfer_handle == 99
# ── ConsumerXfer.poll — QUARANTINED ───────────────────────────────────────────
def test_consumer_xfer_quarantined_proc_stays():
data = _make_data(xfer_state="PROC")
x = _started(data)
x.poll(time.monotonic() + 9999) # → QUARANTINED
assert x.poll(time.monotonic()) == XferState.QUARANTINED
data.release_xfer_handle.assert_not_called()
def test_consumer_xfer_quarantined_terminal_settles():
data = _make_data(xfer_state="PROC")
x = _started(data)
x.poll(time.monotonic() + 9999) # → QUARANTINED
data.check_xfer_state.return_value = "DONE"
assert x.poll(time.monotonic()) == XferState.SETTLED
data.release_xfer_handle.assert_called_once_with(99)
assert x.transfer_handle is None
def test_consumer_xfer_quarantined_exception_settles():
data = _make_data(xfer_state="PROC")
x = _started(data)
x.poll(time.monotonic() + 9999)
data.check_xfer_state.side_effect = RuntimeError("gone")
assert x.poll(time.monotonic()) == XferState.SETTLED
def test_consumer_xfer_quarantined_timeout_evicts_blocks():
"""Quarantine timeout should release handle and return READ_FAILED."""
data = _make_data(xfer_state="PROC")
x = _started(data)
# First poll transitions to QUARANTINED (deadline extended by 60s)
x.poll(time.monotonic() + 9999)
# Second poll with expired quarantine deadline should evict
expired_time = time.monotonic() + 9999 + _CONSUMER_QUARANTINE_TIMEOUT_S + 1
assert x.poll(expired_time) == XferState.READ_FAILED
data.release_xfer_handle.assert_called_once_with(99)
assert x.transfer_handle is None
# ── ConsumerXfer.cancel / release ────────────────────────────────────────────
def test_consumer_xfer_cancel_valid_in_waiting_ack():
_xfer().cancel() # must not raise
def test_consumer_xfer_cancel_raises_when_read_started():
with pytest.raises(AssertionError):
_started().cancel()
def test_consumer_xfer_release_releases_handle():
data = _make_data()
x = _started(data)
x.release()
data.release_xfer_handle.assert_called_once_with(99)
def test_consumer_xfer_release_noop_when_no_handle():
data = _make_data()
_xfer(data).release()
data.release_xfer_handle.assert_not_called()
# ── ConsumerSession ───────────────────────────────────────────────────────────
def _make_consumer_session(data=None):
zmq_conn = MagicMock()
zmq_conn.recv.return_value = []
return ConsumerSession(
addr=("host", 1234),
zmq_conn=zmq_conn,
transport=MagicMock(),
data=data or _make_data(),
compat_hash="hash-abc",
)
def test_consumer_session_has_stable_session_id():
s = _make_consumer_session()
assert isinstance(s._session_id, str) and len(s._session_id) > 0
assert s._session_id == s._session_id # stable
def test_consumer_session_start_xfer_sends_req_with_session_id():
import msgspec
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
raw = s._zmq.send.call_args[0][0]
req = msgspec.msgpack.decode(raw, type=XferReq)
assert req.mm_hash == "h1"
assert req.session_id == s._session_id
def test_consumer_session_poll_done_goes_to_completed():
import msgspec
data = _make_data(xfer_state="PROC")
s = _make_consumer_session(data)
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
ack = XferAck(
mm_hash="h1",
status=XferStatus.OK,
session_id=s._session_id,
src_block_indices=[5, 6],
agent_metadata=b"meta",
mem_descriptor=b"desc",
)
data.add_remote_peer.return_value = "agent-1"
raw_ack = msgspec.msgpack.encode(ack)
data.check_xfer_state.return_value = "DONE"
s.poll([raw_ack], time.monotonic())
results = s.take_results()
assert "h1" in results.completed
assert not results.tombstoned
def test_consumer_session_poll_nack_goes_to_tombstoned():
import msgspec
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
nack = XferAck(mm_hash="h1", status=XferStatus.NACK_MISSING)
s.poll([msgspec.msgpack.encode(nack)], time.monotonic())
results = s.take_results()
assert "h1" in results.tombstoned
assert not results.completed
def test_consumer_session_poll_not_ready_nack_goes_to_retryable():
"""A not-ready NACK must not tombstone: tombstoning admits the request with
no embedding, which is fatal once the media has been rewritten away."""
import msgspec
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
nack = XferAck(mm_hash="h1", status=XferStatus.NACK_NOT_READY)
s.poll([msgspec.msgpack.encode(nack)], time.monotonic())
results = s.take_results()
assert "h1" in results.retryable
assert not results.tombstoned
assert not results.completed
def test_consumer_session_take_results_clears_state():
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() - 1)
s.poll([], time.monotonic())
r1 = s.take_results()
assert "h1" in r1.tombstoned
r2 = s.take_results()
assert not r2.tombstoned # cleared after first take
def test_consumer_session_on_peer_down_cancels_waiting_ack():
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 60)
s.on_peer_down()
results = s.take_results()
assert "h1" in results.retryable
assert not results.tombstoned
+54
View File
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ECCPUConnector scheduler utilities."""
import msgspec
import pytest
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
build_block_descs,
deserialize_mem_descriptor,
serialize_mem_descriptor,
)
# ── build_block_descs ────────────────────────────────────────────────────────
@pytest.mark.parametrize("device_id,expected_dev", [(7, 7), (None, 0)])
def test_build_block_descs(device_id, expected_dev):
kwargs = dict(base_ptr=1000, num_blocks=4, block_size_bytes=256)
if device_id is not None:
kwargs["device_id"] = device_id
descs = build_block_descs(**kwargs)
assert len(descs) == 4
for i, (addr, size, dev) in enumerate(descs):
assert addr == 1000 + i * 256
assert size == 256
assert dev == expected_dev
def test_build_block_descs_zero_blocks_returns_empty():
assert build_block_descs(base_ptr=100, num_blocks=0, block_size_bytes=64) == []
# ── serialize / deserialize_mem_descriptor ───────────────────────────────────
def test_mem_descriptor_roundtrip():
descs = [(100, 64, 0), (164, 64, 0), (228, 64, 1)]
assert deserialize_mem_descriptor(serialize_mem_descriptor(descs)) == descs
@pytest.mark.parametrize(
"bad_value",
[
[(1, 2)], # 2-tuple instead of 3
[("a", "b", "c")], # strings, not ints
],
)
def test_mem_descriptor_rejects_malformed_payload(bad_value):
"""Malformed descriptor lists must fail to decode."""
encoder = msgspec.msgpack.Encoder()
bad_payload = encoder.encode(bad_value)
with pytest.raises((msgspec.DecodeError, msgspec.ValidationError)):
deserialize_mem_descriptor(bad_payload)
@@ -8,7 +8,7 @@ GPU copy plumbing. An ec_both instance reuses encoder outputs it has already
offloaded to CPU instead of recomputing them.
"""
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import torch
@@ -37,6 +37,12 @@ class ECCPUConnector(ECConnectorBase):
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole) -> None:
super().__init__(vllm_config=vllm_config, role=role)
if not vllm_config.use_v2_model_runner:
raise ValueError(
"ECCPUConnector requires the V2 model runner. "
"Set VLLM_USE_V2_MODEL_RUNNER=1."
)
self.connector_worker = None
self.connector_scheduler = None
@@ -108,6 +114,12 @@ class ECCPUConnector(ECConnectorBase):
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def request_finished(
self, request: "Request"
) -> tuple[bool, "dict[str, Any] | None"]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
@@ -0,0 +1,37 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Abstract base classes for the EC control plane.
ControlConnection wraps a single peer channel (one DEALER socket).
"""
from abc import ABC, abstractmethod
class ControlConnection(ABC):
"""Bidirectional message channel to a single remote peer.
All I/O is non-blocking: send enqueues without blocking; recv drains
all buffered inbound messages since the last call.
"""
@property
@abstractmethod
def alive(self) -> bool:
"""True if the connection is still usable."""
@abstractmethod
def send(self, msg: bytes) -> None:
"""Enqueue msg for delivery. Must not block."""
@abstractmethod
def recv(self) -> list[bytes]:
"""Drain and return all buffered inbound messages."""
@abstractmethod
def mark_dead(self) -> None:
"""Signal that the peer disconnected. Sets alive to False."""
@abstractmethod
def close(self) -> None:
"""Release all resources. Idempotent."""
@@ -0,0 +1,233 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ZMQ control-plane implementations for ECCPUConnector.
ZmqClientConnection — one DEALER socket to a single producer peer.
ZmqClientTransport — pool of ZmqClientConnection objects (consumer side).
ZmqServerTransport — ROUTER socket (producer side).
"""
import contextlib
from dataclasses import dataclass, field
import zmq
from zmq.utils.monitor import recv_monitor_message
from vllm.distributed.ec_transfer.ec_connector.cpu.control.base import (
ControlConnection,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr
from vllm.logger import init_logger
from vllm.utils.network_utils import make_zmq_path, make_zmq_socket
logger = init_logger(__name__)
_HEARTBEAT_IVL_MS = 2000
_HEARTBEAT_TIMEOUT_MS = 4000
_HEARTBEAT_TTL_MS = 8000
# ── Client side ───────────────────────────────────────────────────────────────
@dataclass
class ZmqClientConnection(ControlConnection):
"""One DEALER socket to one producer peer.
Owns the socket and its optional monitor socket. NIXL state is NOT
stored here — it belongs in ECSession above this layer.
"""
dealer: zmq.Socket
monitor: zmq.Socket | None
_alive: bool = field(default=True, init=False)
@property
def alive(self) -> bool:
return self._alive
def send(self, msg: bytes) -> None:
self.dealer.send_multipart([b"", msg])
def recv(self) -> list[bytes]:
msgs: list[bytes] = []
while True:
try:
frames = self.dealer.recv_multipart(flags=zmq.NOBLOCK)
except zmq.Again:
break
except Exception:
logger.exception("ec: DEALER recv failed")
break
if len(frames) != 2 or frames[0] != b"":
logger.warning(
"ec: dropped malformed frame envelope (%d frames)", len(frames)
)
continue
msgs.append(frames[1])
return msgs
def mark_dead(self) -> None:
self._alive = False
def close(self) -> None:
if self.monitor is not None:
with contextlib.suppress(Exception):
self.dealer.disable_monitor()
with contextlib.suppress(Exception):
self.monitor.close(linger=0)
with contextlib.suppress(Exception):
self.dealer.close(linger=0)
self._alive = False
class ZmqClientTransport:
"""Pool of ZmqClientConnection objects — one per producer peer.
Creates DEALER sockets lazily on first contact. Does not touch NIXL;
NIXL registration is handled by ECCPUConsumer once an XferAck arrives.
"""
def __init__(self) -> None:
self._ctx = zmq.Context()
self._connections: dict[PeerAddr, ZmqClientConnection] = {}
def connect(self, addr: PeerAddr) -> ZmqClientConnection:
"""Return the existing connection for addr, or create a new one."""
existing = self._connections.get(addr)
if existing is not None:
return existing
host, port = addr
dealer = make_zmq_socket(
ctx=self._ctx,
path=make_zmq_path(scheme="tcp", host=host, port=port),
socket_type=zmq.DEALER,
bind=False,
)
dealer.setsockopt(zmq.HEARTBEAT_IVL, _HEARTBEAT_IVL_MS)
dealer.setsockopt(zmq.HEARTBEAT_TIMEOUT, _HEARTBEAT_TIMEOUT_MS)
dealer.setsockopt(zmq.HEARTBEAT_TTL, _HEARTBEAT_TTL_MS)
monitor_addr = f"inproc://ec-peer-mon-{host}-{port}"
dealer.monitor(monitor_addr, zmq.EVENT_DISCONNECTED)
monitor = self._ctx.socket(zmq.PAIR)
monitor.connect(monitor_addr)
conn = ZmqClientConnection(dealer=dealer, monitor=monitor)
self._connections[addr] = conn
return conn
def poll(self) -> dict[PeerAddr, list[bytes]]:
"""Non-blocking drain of messages from every peer's DEALER."""
msgs: dict[PeerAddr, list[bytes]] = {}
for addr, conn in list(self._connections.items()):
conn_msgs = conn.recv()
msgs[addr] = conn_msgs
return msgs
def poll_dead(self) -> list[PeerAddr]:
"""Non-blocking sweep of peer monitors; return addresses that died."""
dead: list[PeerAddr] = []
for addr, conn in list(self._connections.items()):
if conn.monitor is None:
continue
try:
while True:
evt = recv_monitor_message(conn.monitor, flags=zmq.NOBLOCK)
if evt["event"] == zmq.EVENT_DISCONNECTED:
conn.mark_dead()
dead.append(addr)
break
except zmq.Again:
pass
except Exception:
logger.warning("ec: monitor poll failed for %s", addr, exc_info=True)
return dead
def remove(self, addr: PeerAddr) -> ZmqClientConnection | None:
"""Remove and close the connection for addr, returning it (or None)."""
conn = self._connections.pop(addr, None)
if conn is not None:
conn.close()
return conn
def close(self) -> None:
for conn in list(self._connections.values()):
conn.close()
self._connections.clear()
try:
self._ctx.destroy(linger=0)
except Exception:
logger.debug("ec: zmq context destroy failed", exc_info=True)
# ── Server side ───────────────────────────────────────────────────────────────
class ZmqServerTransport:
"""ROUTER socket for the producer side. Byte-level only, driven by
ProducerSession.
poll(timeout_ms) waits up to timeout_ms for messages, then drains all that
are available; timeout_ms=0 drains whatever has already arrived. send()
routes a reply back to a specific peer by ZMQ identity.
"""
def __init__(self, host: str, port: int) -> None:
self._ctx = zmq.Context()
self._router = make_zmq_socket(
ctx=self._ctx,
path=make_zmq_path(scheme="tcp", host=host, port=port),
socket_type=zmq.ROUTER,
bind=True,
)
self._poller = zmq.Poller()
self._poller.register(self._router, zmq.POLLIN)
def poll(self, timeout_ms: int = 0) -> list[tuple[bytes, bytes]]:
"""Wait up to timeout_ms for incoming messages, then drain all pending.
Returns a list of (identity, payload) pairs.
"""
if timeout_ms > 0:
try:
events = dict(self._poller.poll(timeout=timeout_ms))
except zmq.ContextTerminated:
return []
except Exception:
logger.exception("ec: router poller failed")
return []
if self._router not in events:
return []
msgs: list[tuple[bytes, bytes]] = []
while True:
try:
identity, _, payload = self._router.recv_multipart(flags=zmq.NOBLOCK)
msgs.append((identity, payload))
except zmq.Again:
break
except zmq.ContextTerminated:
break
except Exception:
logger.exception("ec: router recv failed")
break
return msgs
def send(self, identity: bytes, payload: bytes) -> None:
"""Send a reply payload to the peer identified by ZMQ identity."""
try:
self._router.send_multipart([identity, b"", payload])
except Exception:
logger.exception("ec: router send failed")
def close(self) -> None:
try:
self._router.close(linger=0)
except Exception:
logger.debug("ec: router close failed", exc_info=True)
try:
self._ctx.destroy(linger=0)
except Exception:
logger.debug("ec: zmq context destroy failed", exc_info=True)
@@ -0,0 +1,67 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Abstract base class for the EC data-plane transport (NIXL/UCX)."""
from abc import ABC, abstractmethod
from typing import Any
class DataTransport(ABC):
"""Interface wrapping the NIXL data-plane for EC block transfers.
Peer lifecycle:
add_remote_peer() — register a peer and store its dlist handle internally
remove_remote_peer() — deregister and release the stored handle
post_read() — issue a READ using the stored handle for agent_name
The dlist handle (prepped xfer descriptor list) is an implementation detail
of the transport and is never exposed to callers.
"""
@abstractmethod
def get_agent_metadata(self) -> bytes:
"""Return this agent's serialized NIXL metadata blob."""
@abstractmethod
def get_mem_descriptor(self) -> bytes:
"""Return the msgpack-encoded block descriptor list for this agent."""
@abstractmethod
def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str:
"""Register a remote peer, prep its READ-source dlist, and return agent_name.
The dlist handle is stored internally and used by post_read().
"""
@abstractmethod
def remove_remote_peer(self, agent_name: str) -> None:
"""Deregister a previously registered remote peer and release its handle."""
@abstractmethod
def post_read(
self,
local_indices: list[int],
agent_name: str,
remote_indices: list[int],
notif_msg: bytes,
) -> Any:
"""Issue a consumer-initiated READ using the stored dlist for agent_name.
Returns a transfer handle for use with check_xfer_state / release_xfer_handle.
"""
@abstractmethod
def get_new_notifs(self) -> dict[str, list[bytes]]:
"""Drain NIXL completion notifications addressed to this agent."""
@abstractmethod
def check_xfer_state(self, handle: Any) -> str:
"""Return the NIXL state string: 'DONE', 'PROC', or an error string."""
@abstractmethod
def release_xfer_handle(self, handle: Any) -> None:
"""Release a transfer handle returned by post_read."""
@abstractmethod
def deregister(self) -> None:
"""Deregister all memory and tear down the NIXL agent."""
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NIXL-backed data-plane transport for ECCPUConnector."""
from typing import Any
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
DataTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
build_block_descs,
deserialize_mem_descriptor,
serialize_mem_descriptor,
)
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
from vllm.logger import init_logger
logger = init_logger(__name__)
_NIXL_DRAM = "DRAM"
class NixlDataTransport(DataTransport):
"""Adapts NixlWrapper to the DataTransport interface.
Dlist handles (prepped xfer descriptor lists for remote peers) are stored
internally keyed by agent_name. Callers never touch raw NIXL handles —
post_read() resolves the handle from agent_name internally.
NixlWrapper is not thread-safe, and neither is this class. Every call site
runs on the scheduler thread, which serializes the transport by
construction; transfers are async inside NIXL and are drained from that
same thread via get_new_notifs() / check_xfer_state().
"""
def __init__(
self,
agent_name: str,
base_ptr: int,
num_blocks: int,
block_size_bytes: int,
total_size_bytes: int,
num_threads: int = 1,
) -> None:
if NixlWrapper is None or nixl_agent_config is None:
raise RuntimeError(
"ECCPUConnector requires NIXL; "
"install the `nixl` package or set a different ec_connector."
)
# agent_name → remote dlist handle (from add_remote_peer, used by post_read)
self._peer_handles: dict[str, int] = {}
# None until the local region is registered, so deregister() is safe on
# a transport whose registration failed part-way.
self._reg_descs: Any = None
self._nixl = NixlWrapper(
agent_name,
nixl_agent_config(num_threads=num_threads, capture_telemetry=True),
)
self._register_local_memory(
base_ptr, num_blocks, block_size_bytes, total_size_bytes
)
def _register_local_memory(
self,
base_ptr: int,
num_blocks: int,
block_size_bytes: int,
total_size_bytes: int,
) -> None:
"""Register the local region with NIXL and prep its transfer dlist.
Unwinds the registration if a later step fails: __init__ propagates the
error, so the caller never receives an object it could deregister with.
"""
block_descs = build_block_descs(base_ptr, num_blocks, block_size_bytes)
self._reg_descs = self._nixl.get_reg_descs(
[(base_ptr, total_size_bytes, 0, "")], _NIXL_DRAM
)
self._nixl.register_memory(self._reg_descs, backends=["UCX"])
try:
xfer_descs = self._nixl.get_xfer_descs(block_descs, _NIXL_DRAM)
self._local_xfer_handle = self._nixl.prep_xfer_dlist(
"NIXL_INIT_AGENT", xfer_descs
)
self._agent_metadata = self._nixl.get_agent_metadata()
self._mem_descriptor_bytes = serialize_mem_descriptor(block_descs)
except Exception:
self.deregister()
raise
def get_agent_metadata(self) -> bytes:
return self._agent_metadata
def get_mem_descriptor(self) -> bytes:
return self._mem_descriptor_bytes
def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str:
"""Register remote peer, prep its dlist, store handle internally.
Returns agent_name for use in post_read() and remove_remote_peer().
"""
agent_name = self._nixl.add_remote_agent(metadata)
remote_blocks = deserialize_mem_descriptor(mem_descriptor)
remote_xfer_descs = self._nixl.get_xfer_descs(remote_blocks, _NIXL_DRAM)
self._peer_handles[agent_name] = self._nixl.prep_xfer_dlist(
agent_name, remote_xfer_descs
)
return agent_name
def remove_remote_peer(self, agent_name: str) -> None:
self._peer_handles.pop(agent_name, None)
try:
self._nixl.remove_remote_agent(agent_name)
except Exception:
logger.warning(
"EC: remove_remote_peer failed for %s", agent_name, exc_info=True
)
def post_read(
self,
local_indices: list[int],
agent_name: str,
remote_indices: list[int],
notif_msg: bytes,
) -> Any:
if len(local_indices) != len(remote_indices):
raise ValueError(
f"EC: local/remote block count mismatch "
f"({len(local_indices)} vs {len(remote_indices)})"
)
remote_handle = self._peer_handles[agent_name]
handle = self._nixl.make_prepped_xfer(
"READ",
self._local_xfer_handle,
local_indices,
remote_handle,
remote_indices,
notif_msg=notif_msg,
)
self._nixl.transfer(handle)
return handle
def get_new_notifs(self) -> dict[str, list[bytes]]:
return self._nixl.get_new_notifs()
def check_xfer_state(self, handle: Any) -> str:
return self._nixl.check_xfer_state(handle)
def release_xfer_handle(self, handle: Any) -> None:
try:
self._nixl.release_xfer_handle(handle)
except Exception:
logger.warning("EC: release_xfer_handle failed", exc_info=True)
def deregister(self) -> None:
if self._reg_descs is None:
return
try:
self._nixl.deregister_memory(self._reg_descs)
except Exception:
logger.warning("EC: deregister failed", exc_info=True)
self._reg_descs = None
@@ -0,0 +1,91 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scheduler-side wire types for the ECCPUConnector.
Two peer-to-peer message types exchanged between consumer and producer over
ZMQ (XferReq, XferAck) plus a peer-compatibility fingerprint
(compute_ec_compatibility_hash) sent on every XferReq so a producer refuses
to serve a mismatched peer.
"""
import enum
import hashlib
import json
import msgspec
# Bump when the on-wire shape (XferReq/XferAck) changes in a
# backward-incompatible way. Peers reject mismatched versions.
EC_CONNECTOR_VERSION: int = 1
class XferStatus(enum.IntEnum):
"""Outcome the producer reports for an XferReq."""
OK = 0
NACK_MISSING = 1 # producer no longer holds the encoding
NACK_INCOMPAT = 2 # peer-compatibility hash mismatch
NACK_VERSION = 3 # connector wire-version mismatch
NACK_INTERNAL = 4 # producer hit an unexpected error
NACK_NOT_READY = 5 # announced, but its GPU->mmap save has not landed yet
# The producer expects a retryable condition to clear on its own, so the
# consumer re-requests the read rather than giving up on the encoding.
RETRYABLE_NACKS = frozenset((XferStatus.NACK_NOT_READY,))
# Ordinary outcomes rather than signs of a broken peer.
EXPECTED_NACKS = RETRYABLE_NACKS | {XferStatus.NACK_MISSING}
class XferReq( # type: ignore[call-arg]
msgspec.Struct,
tag="req",
omit_defaults=True,
):
"""Consumer → Producer: I want to READ mm_hash; pin it and tell me where."""
mm_hash: str
compatibility_hash: str
# Consumer's session identity. Together with mm_hash it uniquely identifies
# this transfer on both sides: the producer keys its active xfers by
# (session_id, mm_hash), and the NIXL notif_msg carries the same pair.
session_id: str = ""
connector_version: int = EC_CONNECTOR_VERSION
class XferAck( # type: ignore[call-arg]
msgspec.Struct,
tag="ack",
omit_defaults=True,
):
"""Producer → Consumer: response to an XferReq.
On status == OK: producer has pinned source blocks and returns their
indices plus fresh NIXL metadata for the consumer to register and READ.
On any NACK_*: optional fields are empty; consumer falls back to local encode.
"""
mm_hash: str
status: XferStatus
# Consumer's session_id echoed back from XferReq. Together with mm_hash it
# identifies the exact transfer; the consumer uses both to build notif_msg.
session_id: str = ""
src_block_indices: list[int] = []
agent_metadata: bytes = b""
mem_descriptor: bytes = b""
def compute_ec_compatibility_hash(
vllm_version: str,
model: str,
dtype: str,
block_size_bytes: int,
) -> str:
"""Peer-compatibility fingerprint over the four factors that determine
byte-layout compatibility. Producer NACKs any XferReq whose hash differs.
"""
factors = json.dumps(
[vllm_version, model, dtype, block_size_bytes],
separators=(",", ":"),
)
return hashlib.sha256(factors.encode("utf-8")).hexdigest()
@@ -7,16 +7,23 @@ Owns the mmap region and the embedding cache, and handles the producer
for the ECCPUConnector.
"""
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import torch
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
ECCPUWorkerMetadata,
_get_encoder_cache_hidden_dim,
create_ec_shared_region,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
@@ -40,6 +47,7 @@ class ECCPUScheduler:
self._region = create_ec_shared_region(vllm_config)
# Block allocator + LRU eviction policy for the shared region.
self._cache = EmbeddingCache(self._region.num_blocks)
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
# mm_hash → block IDs allocated this step for GPU→mmap saves.
self._pending_saves: dict[str, list[int]] = {}
@@ -63,6 +71,112 @@ class ECCPUScheduler:
else 1
)
self._ec_config = ec_config
# NIXL p2p is an option of this connector, not engine-wide behavior, so
# it lives in extra config. Extra config is not type coerced, so a value
# that arrived as a JSON/CLI string is parsed, not truth-tested.
raw_nixl = ec_config.get_from_extra_config("ec_enable_nixl", False)
self._nixl_enabled = (
raw_nixl
if isinstance(raw_nixl, bool)
else str(raw_nixl).strip().lower() in ("true", "1", "yes")
)
# NIXL fields default to None/empty so the gate-off path is untouched.
self._data: Any = None
self._compat_hash: str | None = None
self._first_in_batch = True
self._transport: Any = None
self._producer_session: Any = None
self._sessions: dict = {}
self._in_flight: set[str] = set()
self._tombstones: set[str] = set()
self._step_completed: set[str] = set()
self._peer_host: str | None = None
self._peer_port: int | None = None
# Model shape for size checks + compat hash; only set by
# _setup_nixl (the gate-off path never touches model_config).
self._dtype: torch.dtype | None = None
self._hidden_dim: int = 0
self._element_size: int = 0
self._ack_timeout_s: float = 0.0
if self._nixl_enabled:
self._setup_nixl(vllm_config)
def _setup_nixl(self, vllm_config: "VllmConfig") -> None:
# Lazy imports keep nixl/zmq off the gate-off path.
from vllm import envs
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import (
ZmqClientTransport,
ZmqServerTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
NixlDataTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
compute_ec_compatibility_hash,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
ProducerSession,
)
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
from vllm.version import __version__ as VLLM_VERSION
if NixlWrapper is None or nixl_agent_config is None:
raise RuntimeError(
"ec_enable_nixl requires NIXL; install the `nixl` package or "
"remove ec_enable_nixl from ec_connector_extra_config."
)
engine_id = self._ec_config.engine_id
assert engine_id is not None
self._dtype = vllm_config.model_config.dtype
self._hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
self._element_size = torch.empty(0, dtype=self._dtype).element_size()
# How long a consumer waits for an XferAck. The producer answers from
# its scheduler step, so its reply latency scales with the encoder's
# batch size: a deployment whose steps run longer must raise this.
self._ack_timeout_s = float(
self._ec_config.get_from_extra_config("consumer_ack_timeout_s", 2.0)
)
self._compat_hash = compute_ec_compatibility_hash(
vllm_version=VLLM_VERSION,
model=str(vllm_config.model_config.model),
dtype=str(self._dtype),
block_size_bytes=self._region.block_size_bytes,
)
if self._is_producer:
self._peer_host = envs.VLLM_EC_SIDE_CHANNEL_HOST
self._peer_port = envs.VLLM_EC_SIDE_CHANNEL_PORT
# Registering the region with NIXL and binding the control sockets are
# the first side effects here. __init__ propagates a failure, so the
# caller never receives a scheduler it could shut down: unwind through
# the same teardown shutdown() uses.
try:
self._data = NixlDataTransport(
agent_name=engine_id,
base_ptr=self._region.blocks.data_ptr(),
num_blocks=self._region.num_blocks,
block_size_bytes=self._region.block_size_bytes,
total_size_bytes=self._region.num_blocks
* self._region.block_size_bytes,
)
if self._is_producer:
assert self._peer_host is not None
assert self._peer_port is not None
self._producer_session = ProducerSession(
transport=ZmqServerTransport(
host=self._peer_host, port=self._peer_port
),
data=self._data,
cache=self._cache,
compat_hash=self._compat_hash,
)
if self._is_consumer:
self._transport = ZmqClientTransport()
except Exception:
self._teardown_nixl()
raise
def has_cache_item(self, identifier: str) -> bool:
if not self._is_consumer:
return False
@@ -72,8 +186,202 @@ class ECCPUScheduler:
def ensure_cache_available(
self, request: "Request", num_computed_tokens: int
) -> bool:
if not self._nixl_enabled:
return True # CPU offload never blocks.
first = self._first_in_batch
self._first_in_batch = False
if not self._is_consumer:
return True
if first:
self._poll_step()
return self._nixl_consumer_admit(request, num_computed_tokens)
def _nixl_consumer_admit(
self, request: "Request", num_computed_tokens: int
) -> bool:
"""Admit a request only once all its remote encodings are cached.
Returns True when every encoding the request needs is either already
ready in the local cache or has no remote source. Any encoding whose
NIXL READ is still in flight (or was just started this step) defers
the request, which the scheduler re-presents on a later step.
"""
params: dict[str, dict[str, Any]] = (
getattr(request, "ec_transfer_params", None) or {}
)
if not params:
return True
pending = False
for feature in request.mm_features:
pos = feature.mm_position
if pos.offset + pos.length <= num_computed_tokens:
continue
mm_hash = feature.identifier
entry = self._cache.get(mm_hash)
if entry is not None and entry.ready:
# Local hit: upstream's update_state_after_alloc pins and
# loads it through the same path as a natively cached entry.
continue
if mm_hash in self._in_flight:
pending = True
continue
if mm_hash in self._step_completed:
pending = True
continue
if mm_hash in self._tombstones:
self._tombstones.discard(mm_hash)
continue
if entry is not None:
# Present but not ready and not being fetched: its blocks are
# held by a quarantined/settling DMA and cannot be reused.
# Fall back to local compute this step; never re-alloc a
# mm_hash already in the cache.
continue
info = params.get(mm_hash)
# Absent entirely, or present with only placeholder metadata (no
# cache entry existed on the producer side at request_finished
# time): nothing to fetch, fall back to local compute.
if not info or "peer_host" not in info:
continue
expected = pos.length * self._hidden_dim * self._element_size
if int(info.get("size_bytes", -1)) != expected:
logger.warning(
"EC consumer: size mismatch mm_hash=%s; local encode", mm_hash
)
continue
try:
started = self._start_xfer(mm_hash, info, expected)
except Exception:
logger.exception(
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
)
continue
if not started:
continue
self._in_flight.add(mm_hash)
pending = True
return not pending
def _start_xfer(
self, mm_hash: str, info: "dict[str, Any]", size_bytes: int
) -> bool:
"""Allocate a not-ready cache entry and start a NIXL READ into it.
Returns True if the transfer was started. Returns False when the
cache cannot accommodate the encoding, in which case the request
falls back to local recomputation.
"""
import time
from math import ceil
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
ConsumerSession,
)
n_blocks = max(1, ceil(size_bytes / self._region.block_size_bytes))
entry = self._cache.alloc(mm_hash, n_blocks)
if entry is None:
logger.debug(
"EC consumer: cache full for mm_hash=%s (%d blocks); local encode",
mm_hash,
n_blocks,
)
return False
indices = list(entry.block_ids)
addr = (info["peer_host"], int(info["peer_port"]))
if addr not in self._sessions:
zmq_conn = self._transport.connect(addr)
assert self._compat_hash is not None
self._sessions[addr] = ConsumerSession(
addr=addr,
zmq_conn=zmq_conn,
transport=self._transport,
data=self._data,
compat_hash=self._compat_hash,
)
deadline = time.monotonic() + self._ack_timeout_s
try:
self._sessions[addr].start_xfer(mm_hash, indices, deadline)
except Exception:
self._cache.discard(mm_hash)
raise
logger.debug(
"EC consumer: starting NIXL xfer mm_hash=%s from %s:%d blocks=%d",
mm_hash,
addr[0],
addr[1],
n_blocks,
)
return True
def _poll_step(self) -> None:
import time
now = time.monotonic()
all_messages = self._transport.poll()
for addr, session in list(self._sessions.items()):
session.poll(all_messages.get(addr, []), now)
for addr in self._transport.poll_dead():
self._on_peer_down(addr)
for session in self._sessions.values():
self._process_session_results(session)
def _process_session_results(self, session: Any) -> None:
r = session.take_results()
for mm_hash in r.completed:
self._in_flight.discard(mm_hash)
self._cache.mark_ready(mm_hash)
self._step_completed.add(mm_hash)
logger.debug("EC consumer: NIXL xfer complete mm_hash=%s", mm_hash)
for mm_hash in r.tombstoned:
self._in_flight.discard(mm_hash)
self._cache.discard(mm_hash)
self._tombstones.add(mm_hash)
logger.debug("EC consumer: NIXL xfer failed mm_hash=%s", mm_hash)
for mm_hash in r.quarantined:
# DMA still running: keep the blocks reserved (entry stays
# not-ready, hence non-evictable) until the xfer settles.
self._in_flight.discard(mm_hash)
self._tombstones.add(mm_hash)
logger.debug(
"EC consumer: NIXL xfer mm_hash=%s quarantined; DMA still running",
mm_hash,
)
for mm_hash in r.retryable:
# No tombstone: dropping every trace of the attempt is what lets
# the next admit pass re-issue the read as if it were the first.
self._in_flight.discard(mm_hash)
self._cache.discard(mm_hash)
logger.debug(
"EC consumer: NIXL xfer mm_hash=%s retryable; will re-request",
mm_hash,
)
for mm_hash, _block_indices in r.settled:
self._cache.discard(mm_hash)
logger.debug(
"EC consumer: quarantined NIXL xfer mm_hash=%s settled", mm_hash
)
def _on_peer_down(self, addr: Any) -> None:
session = self._sessions.pop(addr, None)
if session is None:
return
session.on_peer_down()
self._process_session_results(session)
session.close()
logger.info("EC consumer: producer peer down addr=%s", addr)
def _promote_completed_reads(self) -> None:
"""Drain the just-completed set built during ``_poll_step``.
Reads are marked ready in ``_process_session_results``, so a deferred
request is re-admitted by ``ensure_cache_available`` and loaded through
the same local path as a natively cached entry
(``update_state_after_alloc`` -> ``_pending_loads``). No explicit
pin/load bookkeeping is needed here beyond clearing the set.
"""
self._step_completed.clear()
def update_state_after_alloc(self, request: "Request", index: int) -> None:
feature = request.mm_features[index]
mm_hash = feature.identifier
@@ -97,11 +405,17 @@ class ECCPUScheduler:
) -> ECCPUConnectorMetadata:
meta = ECCPUConnectorMetadata()
if self._is_producer:
if self._nixl_enabled and self._producer_session is not None:
self._producer_session.poll_step()
meta.saves = self._pending_saves
self._pending_saves = {}
if self._is_consumer:
if self._nixl_enabled:
self._promote_completed_reads()
meta.loads = self._pending_loads
self._pending_loads = {}
if self._nixl_enabled:
self._first_in_batch = True
return meta
def update_connector_output(self, connector_output: "ECConnectorOutput") -> None:
@@ -117,8 +431,15 @@ class ECCPUScheduler:
return
for mm_hash in meta.completed_saves:
entry = self._cache.get(mm_hash)
if entry is not None and not entry.ready:
if entry is None:
logger.debug(
"EC producer: worker reported completed save for unknown "
"mm_hash=%s (already discarded/evicted?)",
mm_hash,
)
elif not entry.ready:
self._cache.mark_ready(mm_hash)
logger.debug("EC producer: mm_hash=%s marked ready", mm_hash)
for transfer_id in meta.completed_loads:
pending = self._load_acks.get(transfer_id)
if pending is None:
@@ -131,14 +452,65 @@ class ECCPUScheduler:
# as stale rather than releasing the pin a second time.
del self._load_acks[transfer_id]
self._cache.unpin(mm_hash)
logger.debug("EC consumer: mm_hash=%s unpinned after load", mm_hash)
def has_pending_push_work(self) -> bool:
"""True while any dispatched save or load has not been confirmed done.
"""Keep the engine stepping so this connector's polls keep running.
Keeps the engine stepping so the worker's completion reports are polled
even when no requests are otherwise runnable.
With NIXL enabled the engine tick is the only driver of
``ProducerSession.poll_step()``, and a producer's work arrives from a
remote consumer, so nothing local would otherwise wake it. Without NIXL
this only has to outlive dispatched saves and loads, so the engine can
quiesce once the worker has confirmed them.
"""
return self._cache.has_held_entries()
return self._nixl_enabled or self._cache.has_held_entries()
def request_finished(
self, request: "Request"
) -> tuple[bool, "dict[str, Any] | None"]:
if not (self._nixl_enabled and self._is_producer):
return False, None
if not request.mm_features:
return False, None
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
for feature in request.mm_features:
mm_hash = feature.identifier
entry = self._cache.get(mm_hash)
if entry is None:
# Never saved, or evicted since. Publishing placeholder
# metadata without an address invites an orchestrator to
# rewrite the media into a reference to an encoding no
# consumer can fetch, leaving the decoder nothing to embed.
# Publish neither, so the media stays on the request.
items[mm_hash]["metadata"] = {}
logger.debug(
"EC producer: mm_hash=%s absent at request_finished; "
"announcing no metadata so the media is not rewritten away",
mm_hash,
)
continue
# Announce even if the save's GPU->mmap copy hasn't been
# confirmed complete yet: a not-ready entry can't be evicted, so
# it will still be here by the time a consumer's XferReq arrives.
# A read arriving before the save lands is NACKed NACK_NOT_READY,
# which the consumer retries on a later step rather than treating
# as a miss.
size_bytes = (
feature.mm_position.length * self._hidden_dim * self._element_size
)
items[mm_hash].update(
peer_host=self._peer_host,
peer_port=self._peer_port,
size_bytes=size_bytes,
)
logger.debug(
"EC producer: announcing NIXL-readable encodings req_id=%s items=%s",
request.request_id,
items,
)
return False, items
def shutdown(self) -> None:
self._pending_saves.clear()
@@ -148,7 +520,45 @@ class ECCPUScheduler:
self._is_producer = False
self._is_consumer = False
if self._nixl_enabled:
self._teardown_nixl()
try:
self._region.cleanup()
except Exception:
logger.debug("ec: region cleanup failed", exc_info=True)
def _teardown_nixl(self) -> None:
"""Close the control sockets and release the NIXL agent.
Guards each step and clears what it releases, so it is safe on a
scheduler whose NIXL setup failed part-way and safe to call twice.
The shared region is not touched — shutdown() owns that.
"""
if self._producer_session is not None:
try:
self._producer_session.close()
except Exception:
logger.debug("ec: producer session close failed", exc_info=True)
self._producer_session = None
for session in list(self._sessions.values()):
try:
session.close()
except Exception:
logger.debug("ec: consumer session close failed", exc_info=True)
self._sessions.clear()
if self._transport is not None:
try:
self._transport.close()
except Exception:
logger.debug("ec: client transport close failed", exc_info=True)
self._transport = None
if self._data is not None:
try:
self._data.deregister()
except Exception:
logger.debug("ec: deregister failed", exc_info=True)
self._data = None
@@ -122,6 +122,23 @@ class EmbeddingCache:
self._evictable_block_count -= len(entry.block_ids)
entry.pin()
def pin_if_ready(self, key: str) -> CacheEntry | None:
"""Atomically pin *key* if present and ready; return its entry.
Returns None when the key is absent. An entry returned with `ready`
False was not pinned: its save is still in flight, which the producer
reports differently from a miss because such a read can be retried.
"""
with self._lock:
entry = self._entries.get(key)
if entry is None or not entry.ready:
return entry
if entry.evictable:
del self._entries_free_list[key]
self._evictable_block_count -= len(entry.block_ids)
entry.pin()
return entry
def unpin(self, key: str) -> None:
"""Unpin an entry. Asserts currently pinned."""
with self._lock:
@@ -134,6 +151,21 @@ class EmbeddingCache:
self._entries_free_list[key] = None
self._evictable_block_count += len(entry.block_ids)
def discard(self, key: str) -> None:
"""Remove a not-ready in-flight entry, returning its blocks to the pool.
Used when an in-flight fill (e.g. a NIXL READ) fails before the entry
is marked ready. Asserts the entry is present and not ready — ready or
pinned entries are reclaimed through eviction/unpin, not discard.
"""
with self._lock:
entry = self._entries[key]
assert entry._pin_count == -1, (
f"EmbeddingCache: discard of ready/pinned entry {key!r}"
)
del self._entries[key]
self._free_blocks.extend(entry.block_ids)
def _evict_until(self, n_blocks: int) -> None:
"""Evict ready+unpinned entries FIFO until enough space. Lock held."""
while len(self._free_blocks) < n_blocks:
@@ -0,0 +1,633 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Session objects for the ECCPUConnector.
Layer hierarchy: connector → session → transport (connection)
Two long-lived sessions:
ProducerSession — owns ZmqServerTransport. The connector calls poll_step()
once per engine step, which drains the transport without blocking and
feeds the raw bytes to poll(). poll() decodes XferReqs, grants or NACKs
them, pins source blocks, sends XferAck replies, drains NIXL completion
notifications, and sweeps expired xfers.
ConsumerSession — owns one ZmqClientConnection and the NIXL agent
registration for that peer. The connector calls poll(messages, now)
once per engine step with the raw bytes collected for this peer.
poll() decodes XferAcks, dispatches them to ConsumerXfer objects,
and advances all active xfer state machines.
Internal data classes:
ProducerXfer — one pinned read grant, keyed by session_id.
ConsumerXfer — state machine for one NIXL READ, keyed by mm_hash.
"""
import enum
import logging
import time
import uuid
from dataclasses import dataclass, field
from typing import Any
import msgspec
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import (
ZmqClientConnection,
ZmqClientTransport,
ZmqServerTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
DataTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
EC_CONNECTOR_VERSION,
EXPECTED_NACKS,
RETRYABLE_NACKS,
XferAck,
XferReq,
XferStatus,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr
from vllm.logger import init_logger
logger = init_logger(__name__)
PRODUCER_PIN_LEASE_S = 30.0
_CONSUMER_READ_TIMEOUT_S = 20.0
_CONSUMER_QUARANTINE_TIMEOUT_S = 60.0
# ── XferState ─────────────────────────────────────────────────────────────────
class XferState(enum.Enum):
"""Lifecycle states of a single ConsumerXfer."""
WAITING_ACK = "waiting_ack" # XferReq sent, awaiting XferAck
READING = "reading" # NIXL READ in flight
DONE = "done" # READ completed; promote to local cache
ACK_TIMEOUT = "ack_timeout" # XferAck never arrived; free blocks + tombstone
READ_FAILED = "read_failed" # Unexpected NIXL state; free blocks + tombstone
QUARANTINED = "quarantined" # Read timed out, NIXL unabortable; keep blocks
SETTLED = "settled" # Quarantined DMA done; safe to free blocks
# ── ProducerXfer ──────────────────────────────────────────────────────────────
@dataclass
class ProducerXfer:
"""One pinned read grant on the producer side.
Pure data — lifecycle operations (pin/unpin) are performed by ProducerSession.
Keyed by (consumer_session_id, mm_hash) in ProducerSession._active_xfers.
"""
mm_hash: str
block_indices: list[int]
deadline: float
def is_expired(self) -> bool:
return time.monotonic() > self.deadline
# ── ConsumerXfer ──────────────────────────────────────────────────────────────
@dataclass
class ConsumerXfer:
"""State machine for one NIXL READ on the consumer side.
transfer_handle is the in-flight NIXL handle returned by post_read() —
used for check_xfer_state() / release_xfer_handle(). The dlist handle
for the remote peer lives inside DataTransport, not here.
"""
mm_hash: str
block_indices: list[int]
addr: PeerAddr
deadline: float
data: DataTransport
consumer_session_id: str # session identity; combined with mm_hash for notif_msg
transfer_handle: Any | None = field(default=None)
_quarantined: bool = field(default=False, init=False)
def handle_ack(self, ack: XferAck, agent_name: str) -> bool:
"""Issue the NIXL READ on an OK XferAck. Returns False on any NACK."""
if ack.status != XferStatus.OK:
return False
# notif_msg encodes both session_id and mm_hash so the producer can
# do a direct (session_id, mm_hash) → ProducerXfer lookup on completion.
notif_msg = f"{self.consumer_session_id}:{self.mm_hash}".encode()
self.transfer_handle = self.data.post_read(
self.block_indices,
agent_name,
ack.src_block_indices,
notif_msg=notif_msg,
)
self.deadline = time.monotonic() + _CONSUMER_READ_TIMEOUT_S
logger.debug(
"EC consumer: NIXL READ posted mm_hash=%s agent=%s local_blocks=%d "
"remote_blocks=%d",
self.mm_hash,
agent_name,
len(self.block_indices),
len(ack.src_block_indices),
)
return True
def poll(self, now: float) -> XferState:
if self._quarantined:
return self._poll_quarantined(now)
if self.transfer_handle is None:
return (
XferState.ACK_TIMEOUT if now > self.deadline else XferState.WAITING_ACK
)
return self._poll_reading(now)
def _poll_reading(self, now: float) -> XferState:
assert self.transfer_handle is not None
try:
state = self.data.check_xfer_state(self.transfer_handle)
except Exception:
logger.exception(
"EC Connector: check_xfer_state failed for mm_hash=%s", self.mm_hash
)
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
return XferState.READ_FAILED
if state == "DONE":
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
logger.debug("EC consumer: NIXL READ completed mm_hash=%s", self.mm_hash)
return XferState.DONE
if state == "PROC":
if now > self.deadline:
self._quarantined = True
self.deadline = now + _CONSUMER_QUARANTINE_TIMEOUT_S
logger.warning(
"EC Connector: READ for mm_hash=%s from %s:%d timed out; "
"quarantining",
self.mm_hash,
self.addr[0],
self.addr[1],
)
return XferState.QUARANTINED
return XferState.READING
logger.warning(
"EC Connector: READ for mm_hash=%s unexpected NIXL state %r",
self.mm_hash,
state,
)
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
return XferState.READ_FAILED
def _poll_quarantined(self, now: float) -> XferState:
if self.transfer_handle is None:
return XferState.SETTLED
# If quarantine has been too long, assume NIXL is stuck and evict blocks.
if now > self.deadline:
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
logger.warning(
"EC Connector: quarantined mm_hash=%s from %s:%d timed out; "
"evicting blocks",
self.mm_hash,
self.addr[0],
self.addr[1],
)
return XferState.READ_FAILED
try:
state = self.data.check_xfer_state(self.transfer_handle)
except Exception:
logger.exception(
"EC Connector: check_xfer_state failed for quarantined mm_hash=%s",
self.mm_hash,
)
state = None
if state == "PROC":
return XferState.QUARANTINED
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
logger.debug(
"EC Connector: quarantined mm_hash=%s settled (state=%s)",
self.mm_hash,
state,
)
return XferState.SETTLED
def cancel(self) -> None:
"""Cancel a WAITING_ACK xfer (peer down). No tombstone — caller allows retry."""
assert self.transfer_handle is None, "cancel() requires WAITING_ACK state"
def release(self) -> None:
"""Force-release the transfer handle. Shutdown only."""
if self.transfer_handle is not None:
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
# ── ProducerSession ───────────────────────────────────────────────────────────
class ProducerSession:
"""Long-lived producer session. Owns ZmqServerTransport.
poll_step() runs once per engine step: drains the transport without
blocking, decodes XferReqs, grants or NACKs each one, drains NIXL completion
notifications, and sweeps expired xfers.
"""
def __init__(
self,
transport: ZmqServerTransport,
data: DataTransport,
cache: EmbeddingCache,
compat_hash: str,
) -> None:
self._transport = transport
self._data = data
self._cache = cache # EmbeddingCache — single block owner
self._compat_hash = compat_hash
# Key: "{consumer_session_id}:{mm_hash}" — matches the NIXL notif_msg exactly,
# so completion notification lookup is a direct dict pop with no parsing.
self._active_xfers: dict[str, ProducerXfer] = {}
self._req_decoder = msgspec.msgpack.Decoder(XferReq)
self._req_encoder = msgspec.msgpack.Encoder()
def close(self) -> None:
for xfer in self._active_xfers.values():
self._cache.unpin(xfer.mm_hash) # release any still-held pins
self._active_xfers.clear()
self._transport.close()
def poll_step(self) -> None:
"""Drain the transport without blocking and process what arrived."""
self.poll(self._transport.poll(timeout_ms=0))
def poll(self, messages: list[tuple[bytes, bytes]]) -> None:
"""Decode and process inbound XferReqs, drain NIXL notifs, sweep timeouts.
messages: (identity, payload) pairs from ZmqServerTransport.poll().
"""
for identity, payload in messages:
try:
req = self._req_decoder.decode(payload)
except (msgspec.DecodeError, msgspec.ValidationError):
logger.warning("EC Connector: dropped malformed XferReq")
continue
ack = self._grant_or_nack(req)
self._transport.send(identity, self._req_encoder.encode(ack))
self._drain_notifs()
self._sweep_timeouts()
def _grant_or_nack(self, req: XferReq) -> XferAck:
if req.connector_version != EC_CONNECTOR_VERSION:
logger.warning(
"EC Connector: incompatible version for mm_hash=%s", req.mm_hash
)
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_VERSION)
if req.compatibility_hash != self._compat_hash:
logger.warning(
"EC Connector: incompatible compat hash for mm_hash=%s", req.mm_hash
)
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INCOMPAT)
if not req.session_id:
# Grants are keyed by (session_id, mm_hash), so an omitted id from
# one peer aliases another's: the first completion would unpin
# blocks the other peer is still reading.
logger.warning(
"EC Connector: XferReq without session_id for mm_hash=%s", req.mm_hash
)
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INTERNAL)
entry = self._cache.pin_if_ready(req.mm_hash)
if entry is None or not entry.ready:
status = (
XferStatus.NACK_MISSING if entry is None else XferStatus.NACK_NOT_READY
)
logger.log(
logging.INFO if status in EXPECTED_NACKS else logging.WARNING,
"EC Connector: mm_hash=%s not serveable in local cache (%s); NACKing",
req.mm_hash,
status.name,
)
return XferAck(mm_hash=req.mm_hash, status=status)
block_indices = list(entry.block_ids)
key = f"{req.session_id}:{req.mm_hash}"
if key in self._active_xfers:
# This peer is re-requesting a grant it still holds, having timed
# out waiting. Overwriting it drops the pin taken for it, which
# neither a completion nor the sweep would ever release.
self._cache.unpin(req.mm_hash)
self._active_xfers[key] = ProducerXfer(
mm_hash=req.mm_hash,
block_indices=block_indices,
deadline=time.monotonic() + PRODUCER_PIN_LEASE_S,
)
logger.debug(
"EC producer: serving NIXL READ request mm_hash=%s key=%s blocks=%d",
req.mm_hash,
key,
len(block_indices),
)
return XferAck(
mm_hash=req.mm_hash,
status=XferStatus.OK,
session_id=req.session_id, # echo consumer's session_id back
src_block_indices=block_indices,
agent_metadata=self._data.get_agent_metadata(),
mem_descriptor=self._data.get_mem_descriptor(),
)
def _drain_notifs(self) -> None:
try:
notifs = self._data.get_new_notifs()
except Exception:
logger.exception("EC Connector: get_new_notifs failed")
return
for msgs in notifs.values():
for msg in msgs:
try:
key = msg.decode("utf-8") # "{session_id}:{mm_hash}"
except Exception:
logger.exception("EC Connector: Dropped malformed NIXL notif")
continue
xfer = self._active_xfers.pop(key, None)
if xfer is not None:
self._cache.unpin(xfer.mm_hash)
logger.debug(
"EC producer: NIXL READ completed mm_hash=%s key=%s",
xfer.mm_hash,
key,
)
def _sweep_timeouts(self) -> None:
for key, xfer in list(self._active_xfers.items()):
if xfer.is_expired():
logger.warning(
"EC producer: consumer did not complete NIXL READ "
"mm_hash=%s key=%s within the pin lease; releasing "
"pinned blocks",
xfer.mm_hash,
key,
)
self._cache.unpin(xfer.mm_hash)
del self._active_xfers[key]
# ── ConsumerSession ───────────────────────────────────────────────────────────
@dataclass
class ConsumerSessionResults:
"""Results from ConsumerSession.take_results()."""
completed: set[str] # promote to local cache
tombstoned: set[str] # free blocks + prevent retry this step
quarantined: set[str] # timed-out, DMA still running: prevent retry
# but DO NOT free blocks — session holds them
retryable: set[str] # free blocks, allow another attempt (not ready, peer down)
settled: list[tuple[str, list[int]]] # (mm_hash, block_indices) quarantine cleared
class ConsumerSession:
"""Long-lived per-producer session on the consumer side.
Owns the ZMQ DEALER connection and the NIXL agent registration for one
producer peer. The connector calls poll(messages, now) once per engine
step with the raw bytes collected from this peer's DEALER socket.
poll() decodes XferAcks, dispatches them to the right ConsumerXfer,
advances all active xfer state machines, and drains quarantined xfers.
Results accumulate in internal sets; call take_results() to collect them.
"""
def __init__(
self,
addr: PeerAddr,
zmq_conn: ZmqClientConnection,
transport: ZmqClientTransport,
data: DataTransport,
compat_hash: str,
) -> None:
self._addr = addr
self._zmq = zmq_conn
self._transport = transport
self._data = data
self._compat_hash = compat_hash
self._session_id = str(
uuid.uuid4()
) # stable identity for (session_id:mm_hash) keys
self._nixl_agent_name: str | None = None
self._nixl_metadata_bytes: bytes | None = None
self._xfers: dict[str, ConsumerXfer] = {} # mm_hash → xfer
self._quarantined: list[ConsumerXfer] = [] # DMA pending, to poll
self._newly_quarantined: set[str] = set() # quarantined since last take_results
self._completed: set[str] = set()
self._tombstoned: set[str] = set()
self._retryable: set[str] = set()
self._settled: list[tuple[str, list[int]]] = []
self._encoder = msgspec.msgpack.Encoder()
self._decoder = msgspec.msgpack.Decoder(XferAck)
@property
def addr(self) -> PeerAddr:
return self._addr
def start_xfer(
self, mm_hash: str, block_indices: list[int], deadline: float
) -> None:
"""Create a ConsumerXfer and send the XferReq over the connection."""
xfer = ConsumerXfer(
mm_hash=mm_hash,
block_indices=block_indices,
addr=self._addr,
deadline=deadline,
data=self._data,
consumer_session_id=self._session_id,
)
self._zmq.send(
self._encoder.encode(
XferReq(
mm_hash=mm_hash,
compatibility_hash=self._compat_hash,
session_id=self._session_id,
)
)
)
self._xfers[mm_hash] = xfer
logger.debug(
"EC Connector: XferReq sent mm_hash=%s to %s:%d",
mm_hash,
self._addr[0],
self._addr[1],
)
def poll(self, messages: list[bytes], now: float) -> None:
"""Decode inbound XferAcks, dispatch to xfers, and advance all state machines.
messages: raw bytes from ZmqClientTransport.poll() for this peer.
Results accumulate until take_results() is called.
"""
for raw in messages:
try:
ack = self._decoder.decode(raw)
except (msgspec.DecodeError, msgspec.ValidationError):
logger.warning(
"EC Connector: dropped malformed XferAck from %s:%d",
self._addr[0],
self._addr[1],
)
continue
self._dispatch_ack(ack)
for mm_hash, xfer in list(self._xfers.items()):
state = xfer.poll(now)
if state == XferState.DONE:
self._completed.add(mm_hash)
del self._xfers[mm_hash]
elif state in (XferState.ACK_TIMEOUT, XferState.READ_FAILED):
self._tombstoned.add(mm_hash)
del self._xfers[mm_hash]
elif state == XferState.QUARANTINED:
# DMA may still be writing — caller must NOT free blocks.
self._quarantined.append(xfer)
self._newly_quarantined.add(mm_hash)
del self._xfers[mm_hash]
self._drain_quarantine(now)
def _dispatch_ack(self, ack: XferAck) -> None:
xfer = self._xfers.get(ack.mm_hash)
if xfer is None or xfer.transfer_handle is not None:
return # unknown mm_hash or duplicate ack
if ack.status != XferStatus.OK:
logger.log(
logging.INFO if ack.status in EXPECTED_NACKS else logging.WARNING,
"EC Connector: NACK %s from %s:%d for mm_hash=%s",
ack.status.name,
self._addr[0],
self._addr[1],
ack.mm_hash,
)
del self._xfers[ack.mm_hash]
# The producer announced this encoding and its save is still in
# flight, so another attempt on a later step can be granted. Every
# other NACK is terminal for this mm_hash.
if ack.status in RETRYABLE_NACKS:
self._retryable.add(ack.mm_hash)
else:
self._tombstoned.add(ack.mm_hash)
return
try:
agent_name = self._ensure_registered(ack.agent_metadata, ack.mem_descriptor)
except Exception:
logger.exception(
"EC Connector: failed to register peer %s:%d for mm_hash=%s",
self._addr[0],
self._addr[1],
ack.mm_hash,
)
del self._xfers[ack.mm_hash]
self._tombstoned.add(ack.mm_hash)
return
if not xfer.handle_ack(ack, agent_name):
del self._xfers[ack.mm_hash]
self._tombstoned.add(ack.mm_hash)
def _ensure_registered(self, metadata: bytes, mem_descriptor: bytes) -> str:
"""Register or re-register the peer in DataTransport. Returns agent_name."""
if self._nixl_agent_name is not None and self._nixl_metadata_bytes == metadata:
return self._nixl_agent_name
if self._nixl_agent_name is not None:
logger.info(
"EC Connector: producer %s:%d restarted; re-registering",
self._addr[0],
self._addr[1],
)
self._data.remove_remote_peer(self._nixl_agent_name)
agent_name = self._data.add_remote_peer(metadata, mem_descriptor)
self._nixl_agent_name = agent_name
self._nixl_metadata_bytes = metadata
logger.debug(
"EC Connector: registered peer %s:%d agent=%s",
self._addr[0],
self._addr[1],
agent_name,
)
return agent_name
def _drain_quarantine(self, now: float) -> None:
still_pending: list[ConsumerXfer] = []
for xfer in list(self._quarantined):
state = xfer.poll(now)
if state == XferState.SETTLED:
self._settled.append((xfer.mm_hash, xfer.block_indices))
else:
still_pending.append(xfer)
self._quarantined = still_pending
def on_peer_down(self) -> None:
"""WAITING_ACK xfers → cancel (retry allowed); READING xfers → quarantine."""
for mm_hash, xfer in list(self._xfers.items()):
if xfer.transfer_handle is None:
xfer.cancel()
self._retryable.add(mm_hash)
else:
self._quarantined.append(xfer)
self._newly_quarantined.add(mm_hash)
self._xfers.clear()
def take_results(self) -> ConsumerSessionResults:
"""Return and clear all accumulated results since the last call."""
results = ConsumerSessionResults(
completed=self._completed,
tombstoned=self._tombstoned,
quarantined=self._newly_quarantined,
retryable=self._retryable,
settled=self._settled,
)
self._completed = set()
self._tombstoned = set()
self._newly_quarantined = set()
self._retryable = set()
self._settled = []
return results
def has_in_flight(self) -> bool:
return bool(self._xfers) or bool(self._quarantined)
def close(self) -> None:
"""Release all transfer handles, deregister NIXL peer, close connection.
Connection teardown is routed through the transport so the pooled
connection is dropped in lock-step with the session — a session and its
DEALER connection have a 1:1 lifecycle, so a later connect() to the same
peer builds a fresh socket instead of handing back this dead one.
"""
for xfer in self._xfers.values():
xfer.release()
self._xfers.clear()
for xfer in self._quarantined:
xfer.release()
self._quarantined.clear()
if self._nixl_agent_name is not None:
self._data.remove_remote_peer(self._nixl_agent_name)
self._transport.remove(self._addr)
@@ -0,0 +1,47 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Supporting utilities for the ECCPUConnector scheduler."""
import msgspec
from vllm.logger import init_logger
# (host, port) pair identifying a remote peer's ZMQ side-channel endpoint.
PeerAddr = tuple[str, int]
logger = init_logger(__name__)
# Msgpack (de)serialization for the list-of-(addr, size, device_id)
# descriptor form that NIXL's `get_xfer_descs` expects.
_MemDescList = list[tuple[int, int, int]]
_mem_desc_encoder = msgspec.msgpack.Encoder()
_mem_desc_decoder = msgspec.msgpack.Decoder(_MemDescList)
def serialize_mem_descriptor(descs: _MemDescList) -> bytes:
return _mem_desc_encoder.encode(descs)
def deserialize_mem_descriptor(payload: bytes) -> _MemDescList:
return _mem_desc_decoder.decode(payload)
def build_block_descs(
base_ptr: int,
num_blocks: int,
block_size_bytes: int,
device_id: int = 0,
) -> _MemDescList:
"""Per-block `(addr, size, device_id)` tuples for an `ECSharedRegion`.
Handed to `nixl.get_xfer_descs` to build a dlist where descriptor `i`
addresses block `i` in the mmap. This makes a WRITE to block indices
`[i1, i2, ...]` a simple `make_prepped_xfer(..., [i1, i2])` on both
ends — no per-request address math.
"""
return [
(base_ptr + i * block_size_bytes, block_size_bytes, device_id)
for i in range(num_blocks)
]
@@ -55,7 +55,6 @@ def _coalesce_runs(
split. `EmbeddingCache` hands out ascending ids, which is what makes runs
findable at all.
"""
ids = np.fromiter(block_ids, dtype=np.int64, count=len(block_ids))
breaks = np.flatnonzero(np.diff(ids) != 1) + 1
slots = np.concatenate(([0], breaks))
@@ -5,7 +5,6 @@ from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import safetensors
import torch
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.base import (
@@ -13,8 +12,11 @@ from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
ECConnectorRole,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.logger import init_logger
from vllm.utils.collection_utils import is_list_of
from vllm.v1.core.sched.output import SchedulerOutput
if TYPE_CHECKING:
@@ -52,8 +54,7 @@ class ECExampleConnector(ECConnectorBase):
super().__init__(vllm_config=vllm_config, role=role)
# req_id -> index
self._mm_datas_need_loads: dict[str, int] = {}
self._model_config = vllm_config.model_config
self._metadata_fields_cache: dict[str, set[str]] = {}
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
transfer_config = vllm_config.ec_transfer_config
if transfer_config is not None:
self._storage_path = transfer_config.get_from_extra_config(
@@ -169,36 +170,6 @@ class ECExampleConnector(ECConnectorBase):
self._mm_datas_need_loads.clear()
return meta
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
"""Which processed keys this model needs published for `modality`.
Read from `MultiModalDataParser.embedding_fields`, the same declaration
the consumer's parser requires, so the two cannot drift. An empty set
means the modality cannot be delivered out of band, and the consumer
will process the media itself.
"""
if modality in self._metadata_fields_cache:
return self._metadata_fields_cache[modality]
fields: set[str] = set()
try:
from vllm.multimodal import MULTIMODAL_REGISTRY
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
fields = info.data_parser.placeholder_metadata_fields(modality)
except Exception:
# Reporting nothing is a safe degradation: the consumer falls back to
# processing the media itself.
logger.warning(
"Could not determine the placeholder metadata fields for "
"modality %s; the consumer will preprocess the media itself.",
modality,
exc_info=True,
)
self._metadata_fields_cache[modality] = fields
return fields
def request_finished(
self,
request: "Request",
@@ -214,28 +185,10 @@ class ECExampleConnector(ECConnectorBase):
if not self.is_producer:
return False, None
items = []
for feature in request.mm_features:
metadata = {}
# `data` is None for items served from the processor cache, in which
# case the metadata is unavailable here and the consumer has to fall
# back to processing the media itself.
if feature.data is not None:
wanted = self._placeholder_metadata_fields(feature.modality)
for key, value in feature.data.get_data().items():
if key not in wanted:
continue
if isinstance(value, torch.Tensor):
metadata[key] = value.tolist()
elif is_list_of(value, (int, float)):
# Some metadata (e.g. Qwen3-VL video timestamps) is
# produced as a plain list rather than a tensor.
metadata[key] = value
items.append({"mm_hash": feature.identifier, **metadata})
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
if not items:
return False, None
return False, {"ec_items": items}
return False, items
# ==============================
# Helper functions
@@ -2,8 +2,88 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""EC connector helper utilities."""
from typing import TYPE_CHECKING, Any
import torch
from vllm.logger import init_logger
from vllm.utils.collection_utils import is_list_of
from vllm.v1.outputs import ECConnectorOutput, ModelRunnerOutput
if TYPE_CHECKING:
from vllm.config import ModelConfig
from vllm.multimodal.inputs import MultiModalFeatureSpec
logger = init_logger(__name__)
class PlaceholderMetadataResolver:
"""Resolves which processed keys a model needs published per modality.
Reads `MultiModalDataParser.embedding_fields`, the same declaration the
consumer's parser requires, so the two cannot drift. An empty set means
the modality cannot be delivered out of band, and the consumer will
process the media itself.
"""
def __init__(self, model_config: "ModelConfig") -> None:
self._model_config = model_config
self._cache: dict[str, set[str]] = {}
def fields_for(self, modality: str) -> set[str]:
if modality in self._cache:
return self._cache[modality]
fields: set[str] = set()
try:
from vllm.multimodal import MULTIMODAL_REGISTRY
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
fields = info.data_parser.placeholder_metadata_fields(modality)
except Exception:
logger.warning(
"Could not determine the placeholder metadata fields for "
"modality %s; the consumer will preprocess the media itself.",
modality,
exc_info=True,
)
self._cache[modality] = fields
return fields
def collect_ec_item_metadata(
mm_features: "list[MultiModalFeatureSpec]",
resolver: PlaceholderMetadataResolver,
) -> dict[str, dict[str, Any]]:
"""Build one `ec_transfer_params` entry per feature for `request_finished()`.
Keyed by mm_hash, each entry carries a `metadata` dict with whatever
placeholder fields `resolver` says this model needs published for its
modality, so a consumer can skip the image transform. `data` is None for
items served from the processor cache, in which case the metadata is
unavailable here and the consumer has to fall back to processing the
media itself. A connector that also has transfer coordinates to report
(e.g. NIXL peer_host/peer_port/size_bytes) merges those in alongside
`metadata`, not into it.
"""
items: dict[str, dict[str, Any]] = {}
for feature in mm_features:
metadata: dict[str, Any] = {}
if feature.data is not None:
wanted = resolver.fields_for(feature.modality)
for key, value in feature.data.get_data().items():
if key not in wanted:
continue
if isinstance(value, torch.Tensor):
metadata[key] = value.tolist()
elif is_list_of(value, (int, float)):
# Some metadata (e.g. Qwen3-VL video timestamps) is
# produced as a plain list rather than a tensor.
metadata[key] = value
items[feature.identifier] = {"metadata": metadata}
return items
class ECOutputAggregator:
"""Merge every worker's EC connector output onto the single