[EC Connector] P2P NIXL + CPU EC Connector (#47941)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
69cf055936
commit
8a0a7ee40a
@@ -297,16 +297,26 @@ async def fanout_encoder_primer(
|
||||
detail=f"Encoder request failed: {detail}",
|
||||
)
|
||||
|
||||
# The encoder reports each item's cache key and grid here.
|
||||
# The encoder reports each mm_hash's metadata (e.g. the grid) here,
|
||||
# keyed by the same uuid this proxy assigned above.
|
||||
try:
|
||||
params = (await r.json()).get("ec_transfer_params") or {}
|
||||
reported = params.get("ec_items") or []
|
||||
except Exception:
|
||||
logger.warning("[%s] Could not read encoder metadata #%d", req_id, idx)
|
||||
reported = []
|
||||
if reported and idx in item_uuids:
|
||||
# One item per encoder request, so the first entry is this item's.
|
||||
item_meta[idx] = {**reported[0], "mm_hash": item_uuids[idx]}
|
||||
params = {}
|
||||
if idx in item_uuids:
|
||||
reported = params.get(item_uuids[idx])
|
||||
if reported:
|
||||
metadata = reported.get("metadata") or {}
|
||||
if metadata:
|
||||
item_meta[idx] = {**metadata, "mm_hash": item_uuids[idx]}
|
||||
# Whatever the encoder reported alongside `metadata` is the
|
||||
# connector's own handle on the published embedding (for NIXL,
|
||||
# peer_host/peer_port/size_bytes). The decoder's connector
|
||||
# looks it up by mm_hash on the request, so carry it through.
|
||||
orig_request.setdefault("ec_transfer_params", {})[item_uuids[idx]] = (
|
||||
reported
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"[%s] All %d encoder requests completed successfully", req_id, len(mm_items)
|
||||
|
||||
@@ -333,3 +333,57 @@ def test_concurrent_alloc_evict():
|
||||
t.join()
|
||||
|
||||
assert not errors, errors
|
||||
|
||||
|
||||
# ── discard ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_discard_not_ready_returns_blocks():
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
entry = cache.alloc("h", 3)
|
||||
assert entry is not None
|
||||
cache.discard("h")
|
||||
assert cache.get("h") is None
|
||||
# All blocks reclaimed: a fresh 4-block alloc now fits.
|
||||
assert cache.alloc("h2", 4) is not None
|
||||
|
||||
|
||||
def test_discard_rejects_ready_entry():
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
cache.alloc("h", 1)
|
||||
cache.mark_ready("h")
|
||||
with pytest.raises(AssertionError):
|
||||
cache.discard("h")
|
||||
|
||||
|
||||
def test_discard_rejects_absent_key():
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
with pytest.raises(KeyError):
|
||||
cache.discard("nope")
|
||||
|
||||
|
||||
# ── pin_if_ready ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_pin_if_ready_pins_and_returns_entry():
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
cache.alloc("h", 2)
|
||||
cache.mark_ready("h")
|
||||
entry = cache.pin_if_ready("h")
|
||||
assert entry is not None and entry.ready and len(entry.block_ids) == 2
|
||||
assert cache.alloc("h2", 4) is None # pinned ⇒ not evictable
|
||||
|
||||
|
||||
def test_pin_if_ready_none_when_absent():
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
assert cache.pin_if_ready("nope") is None
|
||||
|
||||
|
||||
def test_pin_if_ready_returns_unpinned_entry_when_not_ready():
|
||||
"""The producer needs "present but not ready" told apart from "absent":
|
||||
the first is NACKed as retryable, the second as a miss."""
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
cache.alloc("h", 1) # not-ready
|
||||
entry = cache.pin_if_ready("h")
|
||||
assert entry is not None and not entry.ready
|
||||
assert cache.alloc("h2", 4) is None # not-ready ⇒ still not evictable
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
|
||||
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
|
||||
@@ -34,14 +36,36 @@ def test_worker_role_builds_only_worker(monkeypatch):
|
||||
assert c.connector_scheduler is None
|
||||
|
||||
|
||||
def test_request_finished_inherited_noop(monkeypatch):
|
||||
@pytest.mark.parametrize("role", [ECConnectorRole.SCHEDULER, ECConnectorRole.WORKER])
|
||||
def test_v1_model_runner_is_rejected(monkeypatch, role):
|
||||
"""The V1 runner never reports save completions, so refuse to start."""
|
||||
monkeypatch.setattr(
|
||||
ECCPUConnector, "_make_scheduler", lambda self, cfg: MagicMock()
|
||||
)
|
||||
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
|
||||
assert c.request_finished(MagicMock()) == (False, None)
|
||||
monkeypatch.setattr(ECCPUConnector, "_make_worker", lambda self, cfg: MagicMock())
|
||||
cfg = _cfg()
|
||||
cfg.use_v2_model_runner = False
|
||||
|
||||
with pytest.raises(ValueError, match="V2 model runner"):
|
||||
ECCPUConnector(cfg, role)
|
||||
|
||||
|
||||
def test_factory_registered():
|
||||
cls = ECConnectorFactory._registry["ECCPUConnector"]()
|
||||
assert cls is ECCPUConnector
|
||||
|
||||
|
||||
def test_request_finished_forwards_to_scheduler(monkeypatch):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.connector as conn_mod # noqa: F401
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
|
||||
|
||||
fake_sched = MagicMock()
|
||||
fake_sched.request_finished.return_value = (False, {"h1": {"peer_port": 1}})
|
||||
monkeypatch.setattr(ECCPUConnector, "_make_scheduler", lambda self, cfg: fake_sched)
|
||||
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
|
||||
req = MagicMock()
|
||||
assert c.request_finished(req) == (False, {"h1": {"peer_port": 1}})
|
||||
fake_sched.request_finished.assert_called_once_with(req)
|
||||
|
||||
@@ -64,6 +64,14 @@ _requires_cuda_alike = pytest.mark.skipif(
|
||||
reason="stalling the compute stream requires a CUDA-like platform",
|
||||
)
|
||||
|
||||
_requires_swap_blocks_batch = pytest.mark.skipif(
|
||||
not hasattr(torch.ops._C_cache_ops, "swap_blocks_batch"),
|
||||
reason=(
|
||||
"installed vllm C++ extension predates the swap_blocks_batch op "
|
||||
"flush_saves/start_load_caches use; rebuild the extension to run this"
|
||||
),
|
||||
)
|
||||
|
||||
# Cycles to stall the compute stream for. Must comfortably outlast the host-side
|
||||
# work the lifetime tests do while a copy is deliberately held back — ~0.5 s on
|
||||
# a current datacenter GPU.
|
||||
@@ -194,6 +202,7 @@ def _warm_up_stream_pools(worker: ECCPUWorker) -> None:
|
||||
# ── save_caches ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_swap_blocks_batch
|
||||
@_requires_accelerator
|
||||
@pytest.mark.parametrize(
|
||||
"n_elements,block_ids",
|
||||
@@ -326,6 +335,7 @@ def test_save_caches_raises_when_allocated_blocks_too_small(make_worker):
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0, 1]}))
|
||||
|
||||
|
||||
@_requires_swap_blocks_batch
|
||||
@_requires_accelerator
|
||||
def test_save_caches_batches_multiple_hashes(make_worker):
|
||||
"""Multiple save_caches calls are batched into a single flush."""
|
||||
@@ -401,6 +411,7 @@ def test_save_descriptors_hold_high_bit_source_addresses(make_worker):
|
||||
# ── start_load_caches ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_swap_blocks_batch
|
||||
@_requires_accelerator
|
||||
def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker):
|
||||
"""Single batched load across all hashes with correct byte→dtype→shape."""
|
||||
@@ -465,6 +476,7 @@ def test_start_load_caches_noop_when_loads_is_empty(make_worker):
|
||||
assert encoder_cache == {}
|
||||
|
||||
|
||||
@_requires_swap_blocks_batch
|
||||
@_requires_accelerator
|
||||
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
|
||||
"""Every hash in ``meta.loads`` is copied from mmap, including one whose
|
||||
@@ -497,6 +509,7 @@ def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
|
||||
assert torch.equal(new.cpu(), src_orig[1:])
|
||||
|
||||
|
||||
@_requires_swap_blocks_batch
|
||||
@_requires_accelerator
|
||||
@pytest.mark.parametrize(
|
||||
"tp_rank,pcp_rank",
|
||||
@@ -526,6 +539,7 @@ def test_start_load_caches_works_on_all_ranks(make_worker, tp_rank, pcp_rank):
|
||||
# ── round-trip ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_swap_blocks_batch
|
||||
@_requires_accelerator
|
||||
def test_save_then_load_round_trips_bytes(make_worker):
|
||||
"""Full producer→mmap→consumer byte path in one shot."""
|
||||
@@ -552,6 +566,7 @@ def test_save_then_load_round_trips_bytes(make_worker):
|
||||
# ── memory lifetime across in-flight copies ─────────────────────────────────
|
||||
|
||||
|
||||
@_requires_swap_blocks_batch
|
||||
@_requires_cuda_alike
|
||||
def test_save_survives_encoder_cache_free_before_copy_runs(make_worker):
|
||||
"""The bytes handed to ``save_caches`` must reach the mmap intact even when
|
||||
@@ -612,6 +627,7 @@ def test_save_survives_encoder_cache_free_before_copy_runs(make_worker):
|
||||
)
|
||||
|
||||
|
||||
@_requires_swap_blocks_batch
|
||||
@_requires_cuda_alike
|
||||
def test_load_buffer_survives_eviction_while_consumer_read_is_queued(make_worker):
|
||||
"""A loaded encoder cache entry must keep its bytes for a consumer already
|
||||
@@ -662,6 +678,7 @@ def test_load_buffer_survives_eviction_while_consumer_read_is_queued(make_worker
|
||||
# ── buffer recycling ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_swap_blocks_batch
|
||||
@_requires_accelerator
|
||||
def test_buffer_pool_is_reused_across_save_steps(make_worker):
|
||||
"""Once a save copy completes its descriptor buffers return to the pool and
|
||||
@@ -689,6 +706,7 @@ def test_buffer_pool_is_reused_across_save_steps(make_worker):
|
||||
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
|
||||
|
||||
|
||||
@_requires_swap_blocks_batch
|
||||
@_requires_accelerator
|
||||
def test_buffer_pool_is_reused_across_load_steps(make_worker):
|
||||
"""Once a load copy completes its descriptor buffers return to the pool and
|
||||
@@ -793,6 +811,7 @@ def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
|
||||
# ── e2e: scheduler + worker pipeline ────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_swap_blocks_batch
|
||||
@_requires_accelerator
|
||||
def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
|
||||
"""Full pipeline: scheduler allocates blocks, worker saves a GPU tensor to
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for control-layer ABCs, ZmqClientConnection, and ECSession."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import zmq
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.control.base import (
|
||||
ControlConnection,
|
||||
)
|
||||
|
||||
|
||||
class _ConcreteConn(ControlConnection):
|
||||
"""Minimal concrete subclass for ABC-instantiation tests."""
|
||||
|
||||
def __init__(self):
|
||||
self._alive = True
|
||||
self._sent = []
|
||||
self._inbox = []
|
||||
|
||||
@property
|
||||
def alive(self) -> bool:
|
||||
return self._alive
|
||||
|
||||
def send(self, msg: bytes) -> None:
|
||||
self._sent.append(msg)
|
||||
|
||||
def recv(self) -> list[bytes]:
|
||||
msgs, self._inbox = self._inbox, []
|
||||
return msgs
|
||||
|
||||
def mark_dead(self) -> None:
|
||||
self._alive = False
|
||||
|
||||
def close(self) -> None:
|
||||
self._alive = False
|
||||
|
||||
|
||||
def test_control_connection_abc_cannot_be_instantiated_directly():
|
||||
with pytest.raises(TypeError):
|
||||
ControlConnection() # type: ignore[abstract]
|
||||
|
||||
|
||||
def test_concrete_connection_send_recv():
|
||||
conn = _ConcreteConn()
|
||||
assert conn.alive
|
||||
conn.send(b"hello")
|
||||
conn._inbox.append(b"world")
|
||||
assert conn.recv() == [b"world"]
|
||||
assert conn._sent == [b"hello"]
|
||||
|
||||
|
||||
def test_mark_dead_sets_alive_false():
|
||||
conn = _ConcreteConn()
|
||||
conn.mark_dead()
|
||||
assert not conn.alive
|
||||
|
||||
|
||||
def test_close_sets_alive_false():
|
||||
conn = _ConcreteConn()
|
||||
conn.close()
|
||||
assert not conn.alive
|
||||
|
||||
|
||||
# ── ZmqClientConnection ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import ( # noqa: E402
|
||||
ZmqClientConnection,
|
||||
ZmqClientTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr # noqa: E402
|
||||
|
||||
|
||||
def _conn(messages: list[list[bytes]], monitor=None) -> ZmqClientConnection:
|
||||
"""Build a ZmqClientConnection whose dealer yields the given frame-lists."""
|
||||
dealer = MagicMock()
|
||||
dealer.recv_multipart.side_effect = [*messages, zmq.Again()]
|
||||
return ZmqClientConnection(dealer=dealer, monitor=monitor)
|
||||
|
||||
|
||||
def test_zmq_client_connection_recv_extracts_payloads_from_envelopes():
|
||||
conn = _conn([[b"", b"msg1"], [b"", b"msg2"]])
|
||||
assert conn.recv() == [b"msg1", b"msg2"]
|
||||
|
||||
|
||||
def test_zmq_client_connection_recv_skips_frames_without_empty_delimiter():
|
||||
# A frame that does not start with b"" is malformed and must be dropped.
|
||||
conn = _conn([[b"", b"ok"], [b"no-delimiter", b"bad"]])
|
||||
assert conn.recv() == [b"ok"]
|
||||
|
||||
|
||||
def test_zmq_client_connection_recv_breaks_on_non_again_exception():
|
||||
# A socket error should stop the drain and return whatever was collected.
|
||||
dealer = MagicMock()
|
||||
dealer.recv_multipart.side_effect = [[b"", b"first"], OSError("broken")]
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=None)
|
||||
assert conn.recv() == [b"first"]
|
||||
|
||||
|
||||
def test_zmq_client_connection_send_prefixes_empty_delimiter():
|
||||
dealer = MagicMock()
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=None)
|
||||
conn.send(b"payload")
|
||||
dealer.send_multipart.assert_called_once_with([b"", b"payload"])
|
||||
|
||||
|
||||
def test_zmq_client_connection_mark_dead_does_not_close_socket():
|
||||
# mark_dead signals liveness only; resource teardown is close()'s job.
|
||||
dealer = MagicMock()
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=None)
|
||||
conn.mark_dead()
|
||||
assert not conn.alive
|
||||
dealer.close.assert_not_called()
|
||||
|
||||
|
||||
def test_zmq_client_connection_close_without_monitor():
|
||||
dealer = MagicMock()
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=None)
|
||||
conn.close()
|
||||
dealer.close.assert_called_once_with(linger=0)
|
||||
assert not conn.alive
|
||||
|
||||
|
||||
def test_zmq_client_connection_close_with_monitor_disables_then_closes_both():
|
||||
# The disable_monitor → monitor.close → dealer.close sequence must be
|
||||
# respected: closing the dealer before disabling the monitor can cause
|
||||
# a zmq assertion in some versions.
|
||||
dealer = MagicMock()
|
||||
monitor = MagicMock()
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=monitor)
|
||||
conn.close()
|
||||
dealer.disable_monitor.assert_called_once()
|
||||
monitor.close.assert_called_once_with(linger=0)
|
||||
dealer.close.assert_called_once_with(linger=0)
|
||||
assert not conn.alive
|
||||
|
||||
|
||||
def test_zmq_client_connection_close_is_idempotent():
|
||||
dealer = MagicMock()
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=None)
|
||||
conn.close()
|
||||
conn.close() # must not raise
|
||||
assert not conn.alive
|
||||
|
||||
|
||||
# ── ZmqClientTransport ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_transport() -> tuple[ZmqClientTransport, MagicMock]:
|
||||
"""Return a ZmqClientTransport with a mocked ZMQ context."""
|
||||
ctx = MagicMock()
|
||||
ctx.socket.return_value = MagicMock()
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context",
|
||||
return_value=ctx,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket",
|
||||
return_value=MagicMock(),
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path",
|
||||
return_value="tcp://host:0",
|
||||
),
|
||||
):
|
||||
transport = ZmqClientTransport()
|
||||
transport._ctx = ctx
|
||||
return transport, ctx
|
||||
|
||||
|
||||
def _inject_conn(transport: ZmqClientTransport, addr: PeerAddr) -> ZmqClientConnection:
|
||||
"""Inject a ZmqClientConnection directly into the transport pool."""
|
||||
conn = ZmqClientConnection(dealer=MagicMock(), monitor=MagicMock())
|
||||
transport._connections[addr] = conn
|
||||
return conn
|
||||
|
||||
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
|
||||
def test_zmq_client_transport_connect_caches_connection(
|
||||
mock_ctx_cls, mock_path, mock_sock
|
||||
):
|
||||
mock_ctx_cls.return_value = MagicMock()
|
||||
mock_ctx_cls.return_value.socket.return_value = MagicMock()
|
||||
mock_sock.return_value = MagicMock()
|
||||
transport = ZmqClientTransport()
|
||||
addr: PeerAddr = ("host", 1234)
|
||||
assert transport.connect(addr) is transport.connect(addr)
|
||||
|
||||
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
|
||||
def test_zmq_client_transport_connect_creates_independent_conns_per_addr(
|
||||
mock_ctx_cls, mock_path, mock_sock
|
||||
):
|
||||
mock_ctx_cls.return_value = MagicMock()
|
||||
mock_ctx_cls.return_value.socket.return_value = MagicMock()
|
||||
mock_sock.return_value = MagicMock()
|
||||
transport = ZmqClientTransport()
|
||||
conn_a = transport.connect(("host-a", 1111))
|
||||
conn_b = transport.connect(("host-b", 2222))
|
||||
assert conn_a is not conn_b
|
||||
|
||||
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
|
||||
def test_zmq_client_transport_poll_dead_detects_disconnected_peer(mock_recv_mon):
|
||||
transport, _ = _make_transport()
|
||||
addr: PeerAddr = ("host", 1234)
|
||||
conn = _inject_conn(transport, addr)
|
||||
mock_recv_mon.side_effect = [{"event": zmq.EVENT_DISCONNECTED}, zmq.Again()]
|
||||
|
||||
dead = transport.poll_dead()
|
||||
|
||||
assert addr in dead
|
||||
assert not conn.alive
|
||||
|
||||
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
|
||||
def test_zmq_client_transport_poll_dead_skips_conn_without_monitor(mock_recv_mon):
|
||||
transport, _ = _make_transport()
|
||||
addr: PeerAddr = ("host", 9999)
|
||||
conn = ZmqClientConnection(dealer=MagicMock(), monitor=None)
|
||||
transport._connections[addr] = conn
|
||||
|
||||
dead = transport.poll_dead()
|
||||
|
||||
assert dead == []
|
||||
mock_recv_mon.assert_not_called()
|
||||
|
||||
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
|
||||
def test_zmq_client_transport_poll_dead_ignores_non_disconnect_events(mock_recv_mon):
|
||||
transport, _ = _make_transport()
|
||||
addr: PeerAddr = ("host", 1234)
|
||||
conn = _inject_conn(transport, addr)
|
||||
# A CONNECTED event should not trigger a dead notification.
|
||||
mock_recv_mon.side_effect = [{"event": zmq.EVENT_CONNECTED}, zmq.Again()]
|
||||
|
||||
dead = transport.poll_dead()
|
||||
|
||||
assert dead == []
|
||||
assert conn.alive
|
||||
|
||||
|
||||
def test_zmq_client_transport_remove_closes_and_evicts_conn():
|
||||
transport, _ = _make_transport()
|
||||
addr: PeerAddr = ("host", 5678)
|
||||
conn = _inject_conn(transport, addr)
|
||||
|
||||
removed = transport.remove(addr)
|
||||
|
||||
assert removed is conn
|
||||
assert addr not in transport._connections
|
||||
conn.dealer.close.assert_called()
|
||||
|
||||
|
||||
def test_zmq_client_transport_remove_returns_none_for_unknown_addr():
|
||||
transport, _ = _make_transport()
|
||||
assert transport.remove(("ghost", 0)) is None
|
||||
|
||||
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
|
||||
def test_zmq_client_transport_close_closes_all_connections(
|
||||
mock_ctx_cls, mock_path, mock_sock
|
||||
):
|
||||
mock_ctx = MagicMock()
|
||||
mock_ctx_cls.return_value = mock_ctx
|
||||
mock_ctx.socket.return_value = MagicMock()
|
||||
dealer_a, dealer_b = MagicMock(), MagicMock()
|
||||
mock_sock.side_effect = [dealer_a, dealer_b]
|
||||
transport = ZmqClientTransport()
|
||||
transport.connect(("a", 1))
|
||||
transport.connect(("b", 2))
|
||||
transport.close()
|
||||
dealer_a.close.assert_called()
|
||||
dealer_b.close.assert_called()
|
||||
assert transport._connections == {}
|
||||
@@ -0,0 +1,184 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for DataTransport ABC and NixlDataTransport."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
|
||||
DataTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
|
||||
serialize_mem_descriptor,
|
||||
)
|
||||
|
||||
_NIXL_PATH = "vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl"
|
||||
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_nixl_mock(prep_side_effect=None) -> MagicMock:
|
||||
nixl = MagicMock()
|
||||
nixl.get_agent_metadata.return_value = b"agent-meta"
|
||||
nixl.get_reg_descs.return_value = MagicMock()
|
||||
nixl.get_xfer_descs.return_value = MagicMock()
|
||||
nixl.prep_xfer_dlist.side_effect = prep_side_effect or [77, 88]
|
||||
nixl.add_remote_agent.return_value = "remote-agent-1"
|
||||
nixl.get_new_notifs.return_value = {}
|
||||
nixl.check_xfer_state.return_value = "PROC"
|
||||
return nixl
|
||||
|
||||
|
||||
def _make_transport(nixl_mock=None):
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
|
||||
NixlDataTransport,
|
||||
)
|
||||
|
||||
mock = nixl_mock or _make_nixl_mock()
|
||||
with (
|
||||
patch(f"{_NIXL_PATH}.NixlWrapper", return_value=mock),
|
||||
patch(f"{_NIXL_PATH}.nixl_agent_config", return_value=MagicMock()),
|
||||
):
|
||||
t = NixlDataTransport("agent", 0, 4, 64, 256)
|
||||
t._nixl = mock
|
||||
return t
|
||||
|
||||
|
||||
# ── ABC ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_data_transport_abc_cannot_be_instantiated():
|
||||
with pytest.raises(TypeError):
|
||||
DataTransport() # type: ignore[abstract]
|
||||
|
||||
|
||||
# ── NixlDataTransport construction ───────────────────────────────────────────
|
||||
|
||||
|
||||
def test_nixl_data_transport_registers_memory_on_init():
|
||||
nixl = _make_nixl_mock()
|
||||
_make_transport(nixl)
|
||||
nixl.register_memory.assert_called_once()
|
||||
|
||||
|
||||
def test_nixl_data_transport_unwinds_registration_if_init_fails():
|
||||
"""A failure after register_memory must not leave the region pinned.
|
||||
|
||||
__init__ propagates, so the caller never gets a transport to deregister.
|
||||
"""
|
||||
nixl = _make_nixl_mock(prep_side_effect=RuntimeError("no dlist"))
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
|
||||
NixlDataTransport,
|
||||
)
|
||||
|
||||
with (
|
||||
patch(f"{_NIXL_PATH}.NixlWrapper", return_value=nixl),
|
||||
patch(f"{_NIXL_PATH}.nixl_agent_config", return_value=MagicMock()),
|
||||
pytest.raises(RuntimeError, match="no dlist"),
|
||||
):
|
||||
NixlDataTransport("agent", 0, 4, 64, 256)
|
||||
|
||||
nixl.deregister_memory.assert_called_once_with(nixl.get_reg_descs.return_value)
|
||||
|
||||
|
||||
def test_nixl_data_transport_raises_if_nixl_unavailable():
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
|
||||
NixlDataTransport,
|
||||
)
|
||||
|
||||
with (
|
||||
patch(f"{_NIXL_PATH}.NixlWrapper", None),
|
||||
patch(f"{_NIXL_PATH}.nixl_agent_config", None),
|
||||
pytest.raises(RuntimeError, match="requires NIXL"),
|
||||
):
|
||||
NixlDataTransport("a", 0, 1, 64, 64)
|
||||
|
||||
|
||||
# ── public accessors ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_get_agent_metadata_returns_nixl_blob():
|
||||
assert _make_transport().get_agent_metadata() == b"agent-meta"
|
||||
|
||||
|
||||
def test_get_mem_descriptor_roundtrips_block_descs():
|
||||
t = _make_transport()
|
||||
raw = t.get_mem_descriptor()
|
||||
assert isinstance(raw, bytes) and len(raw) > 0
|
||||
|
||||
|
||||
# ── add_remote_peer ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_add_remote_peer_returns_agent_name_and_stores_handle_internally():
|
||||
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
|
||||
t = _make_transport(nixl)
|
||||
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
|
||||
agent_name = t.add_remote_peer(b"meta", mem_desc)
|
||||
assert agent_name == "remote-agent-1"
|
||||
# Dlist handle stored internally; accessible via _peer_handles.
|
||||
assert t._peer_handles["remote-agent-1"] == 88
|
||||
|
||||
|
||||
def test_add_remote_peer_registers_remote_agent():
|
||||
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
|
||||
t = _make_transport(nixl)
|
||||
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
|
||||
t.add_remote_peer(b"fresh-meta", mem_desc)
|
||||
nixl.add_remote_agent.assert_called_once_with(b"fresh-meta")
|
||||
|
||||
|
||||
# ── remove_remote_peer ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_remove_remote_peer_delegates_to_nixl():
|
||||
t = _make_transport()
|
||||
t.remove_remote_peer("agent-x")
|
||||
t._nixl.remove_remote_agent.assert_called_once_with("agent-x")
|
||||
|
||||
|
||||
def test_remove_remote_peer_swallows_exceptions():
|
||||
t = _make_transport()
|
||||
t._nixl.remove_remote_agent.side_effect = RuntimeError("gone")
|
||||
t.remove_remote_peer("agent-x") # must not raise
|
||||
|
||||
|
||||
# ── post_read ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_post_read_raises_on_index_count_mismatch():
|
||||
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
|
||||
t = _make_transport(nixl)
|
||||
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
|
||||
agent = t.add_remote_peer(b"meta", mem_desc)
|
||||
with pytest.raises(ValueError, match="mismatch"):
|
||||
t.post_read([0, 1], agent, [5], notif_msg=b"sid")
|
||||
|
||||
|
||||
def test_post_read_calls_make_prepped_xfer_and_transfer():
|
||||
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
|
||||
t = _make_transport(nixl)
|
||||
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
|
||||
agent = t.add_remote_peer(b"meta", mem_desc)
|
||||
nixl.make_prepped_xfer.return_value = 999
|
||||
handle = t.post_read([0, 1], agent, [5, 6], notif_msg=b"sid")
|
||||
nixl.make_prepped_xfer.assert_called_once()
|
||||
nixl.transfer.assert_called_once_with(999)
|
||||
assert handle == 999
|
||||
|
||||
|
||||
# ── release_xfer_handle / deregister ─────────────────────────────────────────
|
||||
|
||||
|
||||
def test_release_xfer_handle_swallows_exceptions():
|
||||
t = _make_transport()
|
||||
t._nixl.release_xfer_handle.side_effect = RuntimeError("stale")
|
||||
t.release_xfer_handle(42) # must not raise
|
||||
|
||||
|
||||
def test_deregister_swallows_exceptions():
|
||||
t = _make_transport()
|
||||
t._nixl.deregister_memory.side_effect = RuntimeError("already gone")
|
||||
t.deregister() # must not raise
|
||||
@@ -7,7 +7,8 @@ No running engine required.
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from tests.v1.core.utils import create_scheduler
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
@@ -15,6 +16,16 @@ from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
# tests.v1.core.utils transitively imports the example KV connector, which
|
||||
# requires the compiled CUDA flash-attention extensions; skip cleanly rather
|
||||
# than erroring at collection when those extensions aren't built.
|
||||
_core_utils = pytest.importorskip(
|
||||
"tests.v1.core.utils",
|
||||
reason="requires compiled CUDA flash-attention extensions",
|
||||
exc_type=ImportError,
|
||||
)
|
||||
create_scheduler = _core_utils.create_scheduler
|
||||
|
||||
EC_PARAMS: dict = {"mm_hash_abc": {"peer_host": "10.0.0.1", "peer_port": 5501}}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import msgspec
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
|
||||
EC_CONNECTOR_VERSION,
|
||||
XferAck,
|
||||
XferReq,
|
||||
XferStatus,
|
||||
compute_ec_compatibility_hash,
|
||||
)
|
||||
|
||||
|
||||
def test_xferreq_roundtrip():
|
||||
req = XferReq(mm_hash="h1", compatibility_hash="c1", session_id="s1")
|
||||
data = msgspec.msgpack.encode(req)
|
||||
out = msgspec.msgpack.decode(data, type=XferReq)
|
||||
assert out.mm_hash == "h1"
|
||||
assert out.compatibility_hash == "c1"
|
||||
assert out.connector_version == EC_CONNECTOR_VERSION
|
||||
|
||||
|
||||
def test_xferack_roundtrip_ok():
|
||||
ack = XferAck(
|
||||
mm_hash="h1",
|
||||
status=XferStatus.OK,
|
||||
src_block_indices=[1, 2],
|
||||
agent_metadata=b"meta",
|
||||
mem_descriptor=b"desc",
|
||||
)
|
||||
out = msgspec.msgpack.decode(msgspec.msgpack.encode(ack), type=XferAck)
|
||||
assert out.status == XferStatus.OK
|
||||
assert out.src_block_indices == [1, 2]
|
||||
|
||||
|
||||
def test_compat_hash_deterministic_and_sensitive():
|
||||
a = compute_ec_compatibility_hash("0.1", "m", "float16", 64)
|
||||
b = compute_ec_compatibility_hash("0.1", "m", "float16", 64)
|
||||
c = compute_ec_compatibility_hash("0.1", "m", "float16", 128)
|
||||
assert a == b
|
||||
assert a != c
|
||||
@@ -0,0 +1,350 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import uuid
|
||||
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
|
||||
|
||||
_N, _BS, _HID, _ES = 16, 64, 32, 2
|
||||
|
||||
|
||||
class _Pos:
|
||||
def __init__(self, offset, length):
|
||||
self.offset, self.length = offset, length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1):
|
||||
self.mm_hash = mm_hash
|
||||
self.identifier = mm_hash
|
||||
self.mm_position = _Pos(0, length)
|
||||
|
||||
|
||||
class _Request:
|
||||
def __init__(self, features, params=None, req_id="r1"):
|
||||
self.mm_features = features
|
||||
self.request_id = req_id
|
||||
self.ec_transfer_params = params
|
||||
|
||||
|
||||
class _FakeResults:
|
||||
def __init__(self):
|
||||
self.completed = set()
|
||||
self.tombstoned = set()
|
||||
self.quarantined = set()
|
||||
self.retryable = set()
|
||||
self.settled = []
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Records started transfers; lets the test flip them to completed."""
|
||||
|
||||
def __init__(self):
|
||||
self.started = []
|
||||
self._results = _FakeResults()
|
||||
|
||||
def start_xfer(self, mm_hash, block_indices, deadline):
|
||||
self.started.append(mm_hash)
|
||||
|
||||
def poll(self, messages, now):
|
||||
pass
|
||||
|
||||
def take_results(self):
|
||||
r, self._results = self._results, _FakeResults()
|
||||
return r
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
class _FakeTransport:
|
||||
"""Stand-in for ZmqClientTransport so _poll_step runs without ZMQ."""
|
||||
|
||||
def poll(self):
|
||||
return {}
|
||||
|
||||
def poll_dead(self):
|
||||
return []
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
def _consumer_sched(monkeypatch):
|
||||
def _region(cfg):
|
||||
return ECSharedRegion(
|
||||
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
|
||||
)
|
||||
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", _region)
|
||||
|
||||
# ec_enable_nixl defaults to False, so this builds gate-off; the fields
|
||||
# below flip on the NIXL consumer state directly.
|
||||
s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_consumer"))
|
||||
# Turn on NIXL consumer state without constructing real transports.
|
||||
s._nixl_enabled = True
|
||||
s._transport = _FakeTransport()
|
||||
s._data = None
|
||||
s._compat_hash = "c"
|
||||
# _setup_nixl normally computes these from model_config; set them
|
||||
# directly since this helper builds gate-off then flips fields on.
|
||||
s._hidden_dim = _HID
|
||||
s._element_size = _ES
|
||||
return s
|
||||
|
||||
|
||||
def _params(mm_hash, length):
|
||||
return {
|
||||
mm_hash: {
|
||||
"peer_host": "h",
|
||||
"peer_port": 1,
|
||||
"size_bytes": length * _HID * _ES,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_new_remote_read_defers_then_completes(monkeypatch):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
|
||||
# Route _start_xfer to our fake session instead of real ZMQ/NIXL, but
|
||||
# still reserve a real not-ready cache entry so mark_ready works.
|
||||
def _fake_start(mm_hash, info, size):
|
||||
entry = s._cache.alloc(mm_hash, 1)
|
||||
assert entry is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(s, "_start_xfer", _fake_start)
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
|
||||
# Step 1: unseen remote item -> read started, request deferred.
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert "h1" in s._in_flight
|
||||
assert "h1" in fake.started
|
||||
s.build_connector_meta(scheduler_output=None) # re-arms _first_in_batch
|
||||
|
||||
# Session reports completion this step.
|
||||
fake._results.completed.add("h1")
|
||||
s._sessions[("h", 1)] = fake
|
||||
|
||||
# Step 2: poll drains completion -> entry marked ready, request admitted.
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
assert "h1" not in s._in_flight
|
||||
assert "h1" in s._step_completed
|
||||
entry = s._cache.get("h1")
|
||||
assert entry is not None and entry.ready
|
||||
# The scheduler now allocates encoder input; the ready entry loads through
|
||||
# the same local path as a natively cached encoding.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta.loads
|
||||
assert "h1" not in s._step_completed # cleared by promote
|
||||
|
||||
# Step 3: still cached -> admitted directly, no new transfer.
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_no_params_is_ready(monkeypatch):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
assert s.ensure_cache_available(_Request([_Feature("x")], params=None), 0) is True
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_tombstoned_read_discards_and_blocks_retry(monkeypatch):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
|
||||
def _fake_start(mm_hash, info, size):
|
||||
entry = s._cache.alloc(mm_hash, 1)
|
||||
assert entry is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(s, "_start_xfer", _fake_start)
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
|
||||
# Step 1: start the read.
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
# Producer rejected the read -> tombstoned.
|
||||
fake._results.tombstoned.add("h1")
|
||||
s._sessions[("h", 1)] = fake
|
||||
|
||||
# Step 2: poll discards the entry and records a tombstone; the tombstone
|
||||
# is consumed by admit so the request proceeds to local compute.
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
assert s._cache.get("h1") is None
|
||||
assert "h1" not in s._in_flight
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
# Step 3: tombstone was consumed, so a fresh transfer starts again.
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert "h1" in s._in_flight
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_retryable_read_re_requests_without_admitting(monkeypatch):
|
||||
"""A not-ready NACK defers the request and re-requests the read.
|
||||
|
||||
Unlike a tombstone it never admits the request, because an admitted request
|
||||
whose media was rewritten away upstream has no embedding and no way to
|
||||
recompute one.
|
||||
"""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
|
||||
def _fake_start(mm_hash, info, size):
|
||||
entry = s._cache.alloc(mm_hash, 1)
|
||||
assert entry is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(s, "_start_xfer", _fake_start)
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
# Producer's save has not landed: retryable, not tombstoned.
|
||||
fake._results.retryable.add("h1")
|
||||
s._sessions[("h", 1)] = fake
|
||||
|
||||
# The entry is released and the read re-issued in the same step, so the
|
||||
# request stays deferred rather than being admitted unfetched.
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert "h1" in s._in_flight
|
||||
assert "h1" not in s._tombstones
|
||||
assert fake.started.count("h1") == 2
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_size_mismatch_skips_transfer(monkeypatch):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
# Advertised size disagrees with pos.length * hidden_dim * element_size.
|
||||
bad = {"h1": {"peer_host": "h", "peer_port": 1, "size_bytes": 999}}
|
||||
req = _Request([_Feature("h1", 1)], params=bad)
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
assert "h1" not in s._in_flight
|
||||
assert s._cache.get("h1") is None
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_metadata_only_entry_admits_without_transfer(monkeypatch):
|
||||
# A producer that reported placeholder metadata but had no cache entry
|
||||
# to transfer omits peer_host/peer_port/size_bytes entirely; the consumer
|
||||
# must fall back to local compute rather than treating that as a size
|
||||
# mismatch.
|
||||
s = _consumer_sched(monkeypatch)
|
||||
metadata_only = {"h1": {"image_grid_thw": [1, 2, 3]}}
|
||||
req = _Request([_Feature("h1", 1)], params=metadata_only)
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
assert "h1" not in s._in_flight
|
||||
assert s._cache.get("h1") is None
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_orphan_not_ready_entry_falls_back_no_realloc(monkeypatch):
|
||||
# Post-quarantine, post-tombstone-consumed orphan: a not-ready cache
|
||||
# entry exists for the mm_hash, but the hash is tracked in none of
|
||||
# _in_flight / _step_completed / _tombstones. Its blocks are held by a
|
||||
# quarantined/settling DMA and must not be re-allocated.
|
||||
s = _consumer_sched(monkeypatch)
|
||||
entry = s._cache.alloc("h1", 1)
|
||||
assert entry is not None and not entry.ready
|
||||
assert "h1" not in s._in_flight
|
||||
assert "h1" not in s._step_completed
|
||||
assert "h1" not in s._tombstones
|
||||
|
||||
# Spy on alloc: record calls, delegate to the real allocator (which
|
||||
# asserts on a duplicate key). A re-alloc for "h1" is the bug.
|
||||
real_alloc = s._cache.alloc
|
||||
calls: list[str] = []
|
||||
|
||||
def _spy_alloc(key, n):
|
||||
calls.append(key)
|
||||
return real_alloc(key, n)
|
||||
|
||||
monkeypatch.setattr(s._cache, "alloc", _spy_alloc)
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
|
||||
# Must not raise; request falls back to local compute (admitted True),
|
||||
# the orphan entry is untouched, and alloc is never called again.
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
assert calls == []
|
||||
assert s._cache.get("h1") is entry
|
||||
assert not entry.ready
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_alloc_failure_falls_back_to_local(monkeypatch):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
|
||||
# Force the cache to reject the allocation.
|
||||
monkeypatch.setattr(s._cache, "alloc", lambda key, n: None)
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
|
||||
# _start_xfer returns False -> request admitted for local recompute.
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
assert "h1" not in s._in_flight
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_already_computed_feature_admits_without_transfer(monkeypatch):
|
||||
# A feature entirely within num_computed_tokens is skipped by admit; no
|
||||
# cache lookup/alloc or transfer should be triggered for it.
|
||||
s = _consumer_sched(monkeypatch)
|
||||
calls: list[str] = []
|
||||
real_alloc = s._cache.alloc
|
||||
|
||||
def _spy_alloc(key, n):
|
||||
calls.append(key)
|
||||
return real_alloc(key, n)
|
||||
|
||||
monkeypatch.setattr(s._cache, "alloc", _spy_alloc)
|
||||
# Feature spans [0, 4); with num_computed_tokens=4 it is fully computed.
|
||||
req = _Request([_Feature("h1", 4)], params=_params("h1", 4))
|
||||
|
||||
assert s.ensure_cache_available(req, 4) is True
|
||||
assert "h1" not in s._in_flight
|
||||
assert calls == []
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_local_ready_entry_admits_without_transfer(monkeypatch):
|
||||
# A READY entry already present in the cache is a local hit: admit
|
||||
# returns True and no new transfer is started for that mm_hash.
|
||||
s = _consumer_sched(monkeypatch)
|
||||
entry = s._cache.alloc("h1", 1)
|
||||
assert entry is not None
|
||||
s._cache.mark_ready("h1")
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
assert "h1" not in s._in_flight
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_in_flight_hash_defers_without_second_transfer(monkeypatch):
|
||||
# A mm_hash already tracked in _in_flight must defer the request rather
|
||||
# than starting a duplicate transfer.
|
||||
s = _consumer_sched(monkeypatch)
|
||||
started: list[str] = []
|
||||
|
||||
def _spy_start_xfer(mm_hash, info, size):
|
||||
started.append(mm_hash)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(s, "_start_xfer", _spy_start_xfer)
|
||||
s._in_flight.add("h1")
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert started == []
|
||||
s.shutdown()
|
||||
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
|
||||
from vllm.config.ec_transfer import ECTransferConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
|
||||
from vllm.distributed.nixl_utils import NixlWrapper
|
||||
|
||||
_N = 16
|
||||
_BS = 64
|
||||
|
||||
|
||||
def _region() -> ECSharedRegion:
|
||||
return ECSharedRegion(
|
||||
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
|
||||
)
|
||||
|
||||
|
||||
def test_ec_enable_nixl_absent_from_extra_config_by_default():
|
||||
cfg = ECTransferConfig()
|
||||
assert cfg.get_from_extra_config("ec_enable_nixl", False) is False
|
||||
|
||||
|
||||
def test_ec_enable_nixl_read_from_extra_config():
|
||||
cfg = ECTransferConfig(
|
||||
ec_connector="ECCPUConnector",
|
||||
ec_role="ec_both",
|
||||
ec_connector_extra_config={"ec_enable_nixl": True},
|
||||
)
|
||||
assert cfg.get_from_extra_config("ec_enable_nixl", False) is True
|
||||
|
||||
|
||||
def test_gate_off_builds_no_nixl(monkeypatch):
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
|
||||
# ec_enable_nixl defaults to False.
|
||||
s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_both"))
|
||||
assert s._nixl_enabled is False
|
||||
assert getattr(s, "_data", None) is None
|
||||
assert getattr(s, "_producer_session", None) is None
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_string_false_in_extra_config_leaves_nixl_off(monkeypatch):
|
||||
"""Extra config is not type coerced, and bool("false") is True."""
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
|
||||
cfg = create_ec_vllm_config(ec_role="ec_both")
|
||||
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = "false"
|
||||
s = ECCPUScheduler(cfg)
|
||||
assert s._nixl_enabled is False
|
||||
s.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.skipif(NixlWrapper is None, reason="Requires NIXL package")
|
||||
def test_gate_on_wires_data_transport_and_producer_session(monkeypatch):
|
||||
# Port 0 lets the OS pick an ephemeral port so the real ZMQ ROUTER bind
|
||||
# in ProducerSession.start() cannot collide with another test/process.
|
||||
monkeypatch.setenv("VLLM_EC_SIDE_CHANNEL_HOST", "127.0.0.1")
|
||||
monkeypatch.setenv("VLLM_EC_SIDE_CHANNEL_PORT", "0")
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
|
||||
cfg = create_ec_vllm_config(ec_role="ec_both")
|
||||
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = True
|
||||
s = ECCPUScheduler(cfg)
|
||||
try:
|
||||
assert s._nixl_enabled is True
|
||||
assert s._data is not None
|
||||
assert s._compat_hash is not None
|
||||
assert s._producer_session is not None
|
||||
assert s._peer_host == "127.0.0.1"
|
||||
assert s._transport is not None # is_ec_consumer is also True
|
||||
finally:
|
||||
s.shutdown()
|
||||
@@ -0,0 +1,113 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import uuid
|
||||
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
|
||||
|
||||
_N, _BS = 16, 64
|
||||
|
||||
|
||||
class _Pos:
|
||||
def __init__(self, offset, length):
|
||||
self.offset, self.length = offset, length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1):
|
||||
self.mm_hash = mm_hash
|
||||
self.identifier = mm_hash
|
||||
self.mm_position = _Pos(0, length)
|
||||
self.data = None
|
||||
self.modality = "image"
|
||||
|
||||
|
||||
class _Request:
|
||||
def __init__(self, features, req_id="r1"):
|
||||
self.mm_features = features
|
||||
self.request_id = req_id
|
||||
|
||||
|
||||
def _sched_gate_off(monkeypatch):
|
||||
def _region(cfg):
|
||||
return ECSharedRegion(
|
||||
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
|
||||
)
|
||||
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", _region)
|
||||
return ECCPUScheduler(create_ec_vllm_config(ec_role="ec_both"))
|
||||
|
||||
|
||||
def test_request_finished_gate_off_returns_none(monkeypatch):
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
assert s.request_finished(_Request([_Feature("h1")])) == (False, None)
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_request_finished_producer_emits_params(monkeypatch):
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
# Simulate NIXL-enabled producer bookkeeping without building real NIXL.
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
# _setup_nixl normally computes these from model_config; set them
|
||||
# directly since this test builds gate-off then flips fields on.
|
||||
s._hidden_dim, s._element_size = 32, 2
|
||||
# feature length=2, hidden_dim=32, element_size=2 -> 128 bytes -> 2 blocks.
|
||||
entry = s._cache.alloc("h1", 2)
|
||||
assert entry is not None
|
||||
s._cache.mark_ready("h1")
|
||||
|
||||
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
|
||||
assert delay is False
|
||||
assert params == {
|
||||
"h1": {
|
||||
"metadata": {},
|
||||
"peer_host": "1.2.3.4",
|
||||
"peer_port": 5601,
|
||||
"size_bytes": 2 * 32 * 2,
|
||||
}
|
||||
}
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_request_finished_announces_not_ready_entry(monkeypatch):
|
||||
"""A save whose GPU->mmap copy hasn't been confirmed complete yet is
|
||||
still announced: the entry can't be evicted before it's ready, and a read
|
||||
arriving too early is NACKed NACK_NOT_READY for the consumer to retry —
|
||||
protecting it is not this method's job."""
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
s._hidden_dim, s._element_size = 32, 2
|
||||
s._cache.alloc("h1", 2) # allocated but not marked ready
|
||||
|
||||
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
|
||||
assert delay is False
|
||||
assert params == {
|
||||
"h1": {
|
||||
"metadata": {},
|
||||
"peer_host": "1.2.3.4",
|
||||
"peer_port": 5601,
|
||||
"size_bytes": 2 * 32 * 2,
|
||||
}
|
||||
}
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_request_finished_skips_unallocated_entry(monkeypatch):
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
s._hidden_dim, s._element_size = 32, 2
|
||||
# No alloc() for "h1" at all — e.g. the cache was full at save time.
|
||||
|
||||
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
|
||||
assert delay is False
|
||||
# The item's placeholder metadata is still reported even though there's
|
||||
# no cache entry to transfer (empty "metadata": no fields, no transfer).
|
||||
assert params == {"h1": {"metadata": {}}}
|
||||
s.shutdown()
|
||||
@@ -0,0 +1,470 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ProducerXfer, ProducerSession, ConsumerXfer, ConsumerSession."""
|
||||
|
||||
import time
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
|
||||
XferAck,
|
||||
XferReq,
|
||||
XferStatus,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
|
||||
_CONSUMER_QUARANTINE_TIMEOUT_S,
|
||||
ConsumerSession,
|
||||
ConsumerXfer,
|
||||
ProducerSession,
|
||||
ProducerXfer,
|
||||
XferState,
|
||||
)
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_data(xfer_state: str = "PROC") -> MagicMock:
|
||||
data = MagicMock()
|
||||
data.check_xfer_state.return_value = xfer_state
|
||||
data.post_read.return_value = 99
|
||||
return data
|
||||
|
||||
|
||||
def _ok_ack(
|
||||
mm_hash: str = "h1",
|
||||
session_id: str = "sess-1",
|
||||
src_indices: list | None = None,
|
||||
) -> XferAck:
|
||||
return XferAck(
|
||||
mm_hash=mm_hash,
|
||||
status=XferStatus.OK,
|
||||
session_id=session_id,
|
||||
src_block_indices=src_indices or [0, 1],
|
||||
agent_metadata=b"meta",
|
||||
mem_descriptor=b"desc",
|
||||
)
|
||||
|
||||
|
||||
def _xfer(
|
||||
data: MagicMock | None = None,
|
||||
deadline: float | None = None,
|
||||
consumer_session_id: str = "sess-1",
|
||||
) -> ConsumerXfer:
|
||||
return ConsumerXfer(
|
||||
mm_hash="h1",
|
||||
block_indices=[10, 11],
|
||||
addr=("host", 1234),
|
||||
deadline=deadline if deadline is not None else time.monotonic() + 60,
|
||||
data=data or _make_data(),
|
||||
consumer_session_id=consumer_session_id,
|
||||
)
|
||||
|
||||
|
||||
def _started(
|
||||
data: MagicMock | None = None, deadline: float | None = None
|
||||
) -> ConsumerXfer:
|
||||
"""ConsumerXfer that has already received an OK XferAck."""
|
||||
x = _xfer(data, deadline)
|
||||
x.handle_ack(_ok_ack(session_id="sess-1", src_indices=[5, 6]), agent_name="agent-1")
|
||||
return x
|
||||
|
||||
|
||||
# ── ProducerXfer ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_producer_xfer_not_expired_before_deadline():
|
||||
x = ProducerXfer("h1", [0, 1], deadline=time.monotonic() + 60)
|
||||
assert not x.is_expired()
|
||||
|
||||
|
||||
def test_producer_xfer_expired_after_deadline():
|
||||
x = ProducerXfer("h1", [0, 1], deadline=time.monotonic() - 1)
|
||||
assert x.is_expired()
|
||||
|
||||
|
||||
# ── ProducerSession ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _xfer_req(mm_hash: str = "h1", session_id: str = "sess-1") -> XferReq:
|
||||
return XferReq(
|
||||
mm_hash=mm_hash, compatibility_hash="hash-abc", session_id=session_id
|
||||
)
|
||||
|
||||
|
||||
def _make_producer_session(cache: EmbeddingCache | None = None) -> ProducerSession:
|
||||
data = _make_data()
|
||||
data.get_agent_metadata.return_value = b"meta"
|
||||
data.get_mem_descriptor.return_value = b"desc"
|
||||
return ProducerSession(
|
||||
transport=MagicMock(),
|
||||
data=data,
|
||||
cache=cache or EmbeddingCache(num_blocks=8),
|
||||
compat_hash="hash-abc",
|
||||
)
|
||||
|
||||
|
||||
def test_producer_session_grant_pins_ready_entry_and_returns_blocks():
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2)
|
||||
cache.mark_ready("h1")
|
||||
s = _make_producer_session(cache)
|
||||
|
||||
ack = s._grant_or_nack(_xfer_req(mm_hash="h1"))
|
||||
|
||||
assert ack.status == XferStatus.OK
|
||||
entry = cache.get("h1")
|
||||
assert not entry.evictable # pinned by the grant
|
||||
assert list(ack.src_block_indices) == list(entry.block_ids)
|
||||
|
||||
|
||||
def test_producer_session_grant_nacks_missing_entry():
|
||||
s = _make_producer_session()
|
||||
ack = s._grant_or_nack(_xfer_req(mm_hash="nope"))
|
||||
assert ack.status == XferStatus.NACK_MISSING
|
||||
|
||||
|
||||
def test_producer_session_grant_nacks_not_ready_entry_as_retryable():
|
||||
"""An announced entry whose save is still in flight is distinguished from a
|
||||
miss, so the consumer retries instead of falling back to local compute."""
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2) # not ready
|
||||
s = _make_producer_session(cache)
|
||||
ack = s._grant_or_nack(_xfer_req(mm_hash="h1"))
|
||||
assert ack.status == XferStatus.NACK_NOT_READY
|
||||
|
||||
|
||||
def test_producer_session_grant_nacks_version_mismatch():
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2)
|
||||
cache.mark_ready("h1")
|
||||
s = _make_producer_session(cache)
|
||||
req = XferReq(
|
||||
mm_hash="h1",
|
||||
compatibility_hash="hash-abc",
|
||||
session_id="sess-1",
|
||||
connector_version=-1,
|
||||
)
|
||||
|
||||
ack = s._grant_or_nack(req)
|
||||
|
||||
assert ack.status == XferStatus.NACK_VERSION
|
||||
entry = cache.get("h1")
|
||||
assert entry.evictable # nothing was pinned
|
||||
|
||||
|
||||
def test_producer_session_grant_nacks_compat_hash_mismatch():
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2)
|
||||
cache.mark_ready("h1")
|
||||
s = _make_producer_session(cache)
|
||||
req = XferReq(mm_hash="h1", compatibility_hash="wrong-hash", session_id="sess-1")
|
||||
|
||||
ack = s._grant_or_nack(req)
|
||||
|
||||
assert ack.status == XferStatus.NACK_INCOMPAT
|
||||
entry = cache.get("h1")
|
||||
assert entry.evictable # nothing was pinned
|
||||
|
||||
|
||||
def test_producer_session_notif_unpins_via_cache():
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2)
|
||||
cache.mark_ready("h1")
|
||||
s = _make_producer_session(cache)
|
||||
s._grant_or_nack(_xfer_req(mm_hash="h1", session_id="sess-1"))
|
||||
entry = cache.get("h1")
|
||||
assert not entry.evictable # pinned
|
||||
|
||||
s._data.get_new_notifs.return_value = {"agent": [b"sess-1:h1"]}
|
||||
s._drain_notifs()
|
||||
|
||||
assert entry.evictable # unpinned via cache
|
||||
assert s._active_xfers == {}
|
||||
|
||||
|
||||
def test_producer_session_sweep_timeouts_unpins_via_cache():
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2)
|
||||
cache.mark_ready("h1")
|
||||
s = _make_producer_session(cache)
|
||||
s._grant_or_nack(_xfer_req(mm_hash="h1", session_id="sess-1"))
|
||||
key = next(iter(s._active_xfers))
|
||||
s._active_xfers[key].deadline = time.monotonic() - 1
|
||||
|
||||
s._sweep_timeouts()
|
||||
|
||||
assert cache.get("h1").evictable
|
||||
assert s._active_xfers == {}
|
||||
|
||||
|
||||
# ── ConsumerXfer.handle_ack ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_consumer_xfer_handle_ack_ok_starts_read():
|
||||
data = _make_data()
|
||||
x = _xfer(data)
|
||||
assert x.handle_ack(_ok_ack(), agent_name="agent-1") is True
|
||||
assert x.transfer_handle == 99
|
||||
|
||||
|
||||
def test_consumer_xfer_handle_ack_notif_msg_is_session_id_colon_mm_hash():
|
||||
data = _make_data()
|
||||
x = _xfer(data, consumer_session_id="my-sess")
|
||||
x.handle_ack(_ok_ack(mm_hash="h1"), agent_name="agent-1")
|
||||
_, kwargs = data.post_read.call_args
|
||||
assert kwargs["notif_msg"] == b"my-sess:h1"
|
||||
|
||||
|
||||
def test_consumer_xfer_handle_ack_forwards_correct_args():
|
||||
data = _make_data()
|
||||
x = _xfer(data)
|
||||
x.handle_ack(_ok_ack(src_indices=[5, 6]), agent_name="agent-1")
|
||||
args, _ = data.post_read.call_args
|
||||
assert args[0] == [10, 11] # local block_indices
|
||||
assert args[1] == "agent-1" # agent_name (not a raw handle)
|
||||
assert args[2] == [5, 6] # remote src_block_indices
|
||||
|
||||
|
||||
def test_consumer_xfer_handle_ack_resets_deadline():
|
||||
original = time.monotonic() + 2
|
||||
x = _xfer(deadline=original)
|
||||
x.handle_ack(_ok_ack(), agent_name="agent-1")
|
||||
assert x.deadline > original
|
||||
|
||||
|
||||
def test_consumer_xfer_handle_ack_nack_returns_false():
|
||||
data = _make_data()
|
||||
x = _xfer(data)
|
||||
assert (
|
||||
x.handle_ack(XferAck(mm_hash="h1", status=XferStatus.NACK_MISSING), "a")
|
||||
is False
|
||||
)
|
||||
assert x.transfer_handle is None
|
||||
data.post_read.assert_not_called()
|
||||
|
||||
|
||||
# ── ConsumerXfer.poll — WAITING_ACK ──────────────────────────────────────────
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_waiting_ack_before_deadline():
|
||||
assert _xfer().poll(time.monotonic()) == XferState.WAITING_ACK
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_waiting_ack_timeout():
|
||||
assert (
|
||||
_xfer(deadline=time.monotonic() - 1).poll(time.monotonic())
|
||||
== XferState.ACK_TIMEOUT
|
||||
)
|
||||
|
||||
|
||||
# ── ConsumerXfer.poll — READING ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_reading_proc():
|
||||
assert _started().poll(time.monotonic()) == XferState.READING
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_reading_done_releases_handle():
|
||||
data = _make_data(xfer_state="DONE")
|
||||
x = _started(data)
|
||||
assert x.poll(time.monotonic()) == XferState.DONE
|
||||
data.release_xfer_handle.assert_called_once_with(99)
|
||||
assert x.transfer_handle is None
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_reading_unexpected_state_fails():
|
||||
data = _make_data(xfer_state="ERR")
|
||||
x = _started(data)
|
||||
assert x.poll(time.monotonic()) == XferState.READ_FAILED
|
||||
data.release_xfer_handle.assert_called_once_with(99)
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_reading_exception_fails_gracefully():
|
||||
data = _make_data()
|
||||
data.check_xfer_state.side_effect = RuntimeError("nixl crash")
|
||||
x = _started(data)
|
||||
assert x.poll(time.monotonic()) == XferState.READ_FAILED
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_timeout_quarantines_without_releasing():
|
||||
data = _make_data(xfer_state="PROC")
|
||||
x = _started(data)
|
||||
assert x.poll(time.monotonic() + 9999) == XferState.QUARANTINED
|
||||
data.release_xfer_handle.assert_not_called()
|
||||
assert x.transfer_handle == 99
|
||||
|
||||
|
||||
# ── ConsumerXfer.poll — QUARANTINED ───────────────────────────────────────────
|
||||
|
||||
|
||||
def test_consumer_xfer_quarantined_proc_stays():
|
||||
data = _make_data(xfer_state="PROC")
|
||||
x = _started(data)
|
||||
x.poll(time.monotonic() + 9999) # → QUARANTINED
|
||||
assert x.poll(time.monotonic()) == XferState.QUARANTINED
|
||||
data.release_xfer_handle.assert_not_called()
|
||||
|
||||
|
||||
def test_consumer_xfer_quarantined_terminal_settles():
|
||||
data = _make_data(xfer_state="PROC")
|
||||
x = _started(data)
|
||||
x.poll(time.monotonic() + 9999) # → QUARANTINED
|
||||
data.check_xfer_state.return_value = "DONE"
|
||||
assert x.poll(time.monotonic()) == XferState.SETTLED
|
||||
data.release_xfer_handle.assert_called_once_with(99)
|
||||
assert x.transfer_handle is None
|
||||
|
||||
|
||||
def test_consumer_xfer_quarantined_exception_settles():
|
||||
data = _make_data(xfer_state="PROC")
|
||||
x = _started(data)
|
||||
x.poll(time.monotonic() + 9999)
|
||||
data.check_xfer_state.side_effect = RuntimeError("gone")
|
||||
assert x.poll(time.monotonic()) == XferState.SETTLED
|
||||
|
||||
|
||||
def test_consumer_xfer_quarantined_timeout_evicts_blocks():
|
||||
"""Quarantine timeout should release handle and return READ_FAILED."""
|
||||
|
||||
data = _make_data(xfer_state="PROC")
|
||||
x = _started(data)
|
||||
# First poll transitions to QUARANTINED (deadline extended by 60s)
|
||||
x.poll(time.monotonic() + 9999)
|
||||
# Second poll with expired quarantine deadline should evict
|
||||
expired_time = time.monotonic() + 9999 + _CONSUMER_QUARANTINE_TIMEOUT_S + 1
|
||||
assert x.poll(expired_time) == XferState.READ_FAILED
|
||||
data.release_xfer_handle.assert_called_once_with(99)
|
||||
assert x.transfer_handle is None
|
||||
|
||||
|
||||
# ── ConsumerXfer.cancel / release ────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_consumer_xfer_cancel_valid_in_waiting_ack():
|
||||
_xfer().cancel() # must not raise
|
||||
|
||||
|
||||
def test_consumer_xfer_cancel_raises_when_read_started():
|
||||
with pytest.raises(AssertionError):
|
||||
_started().cancel()
|
||||
|
||||
|
||||
def test_consumer_xfer_release_releases_handle():
|
||||
data = _make_data()
|
||||
x = _started(data)
|
||||
x.release()
|
||||
data.release_xfer_handle.assert_called_once_with(99)
|
||||
|
||||
|
||||
def test_consumer_xfer_release_noop_when_no_handle():
|
||||
data = _make_data()
|
||||
_xfer(data).release()
|
||||
data.release_xfer_handle.assert_not_called()
|
||||
|
||||
|
||||
# ── ConsumerSession ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_consumer_session(data=None):
|
||||
zmq_conn = MagicMock()
|
||||
zmq_conn.recv.return_value = []
|
||||
return ConsumerSession(
|
||||
addr=("host", 1234),
|
||||
zmq_conn=zmq_conn,
|
||||
transport=MagicMock(),
|
||||
data=data or _make_data(),
|
||||
compat_hash="hash-abc",
|
||||
)
|
||||
|
||||
|
||||
def test_consumer_session_has_stable_session_id():
|
||||
s = _make_consumer_session()
|
||||
assert isinstance(s._session_id, str) and len(s._session_id) > 0
|
||||
assert s._session_id == s._session_id # stable
|
||||
|
||||
|
||||
def test_consumer_session_start_xfer_sends_req_with_session_id():
|
||||
import msgspec
|
||||
|
||||
s = _make_consumer_session()
|
||||
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
|
||||
raw = s._zmq.send.call_args[0][0]
|
||||
req = msgspec.msgpack.decode(raw, type=XferReq)
|
||||
assert req.mm_hash == "h1"
|
||||
assert req.session_id == s._session_id
|
||||
|
||||
|
||||
def test_consumer_session_poll_done_goes_to_completed():
|
||||
import msgspec
|
||||
|
||||
data = _make_data(xfer_state="PROC")
|
||||
s = _make_consumer_session(data)
|
||||
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
|
||||
|
||||
ack = XferAck(
|
||||
mm_hash="h1",
|
||||
status=XferStatus.OK,
|
||||
session_id=s._session_id,
|
||||
src_block_indices=[5, 6],
|
||||
agent_metadata=b"meta",
|
||||
mem_descriptor=b"desc",
|
||||
)
|
||||
data.add_remote_peer.return_value = "agent-1"
|
||||
raw_ack = msgspec.msgpack.encode(ack)
|
||||
|
||||
data.check_xfer_state.return_value = "DONE"
|
||||
s.poll([raw_ack], time.monotonic())
|
||||
|
||||
results = s.take_results()
|
||||
assert "h1" in results.completed
|
||||
assert not results.tombstoned
|
||||
|
||||
|
||||
def test_consumer_session_poll_nack_goes_to_tombstoned():
|
||||
import msgspec
|
||||
|
||||
s = _make_consumer_session()
|
||||
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
|
||||
nack = XferAck(mm_hash="h1", status=XferStatus.NACK_MISSING)
|
||||
s.poll([msgspec.msgpack.encode(nack)], time.monotonic())
|
||||
results = s.take_results()
|
||||
assert "h1" in results.tombstoned
|
||||
assert not results.completed
|
||||
|
||||
|
||||
def test_consumer_session_poll_not_ready_nack_goes_to_retryable():
|
||||
"""A not-ready NACK must not tombstone: tombstoning admits the request with
|
||||
no embedding, which is fatal once the media has been rewritten away."""
|
||||
import msgspec
|
||||
|
||||
s = _make_consumer_session()
|
||||
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
|
||||
nack = XferAck(mm_hash="h1", status=XferStatus.NACK_NOT_READY)
|
||||
s.poll([msgspec.msgpack.encode(nack)], time.monotonic())
|
||||
results = s.take_results()
|
||||
assert "h1" in results.retryable
|
||||
assert not results.tombstoned
|
||||
assert not results.completed
|
||||
|
||||
|
||||
def test_consumer_session_take_results_clears_state():
|
||||
s = _make_consumer_session()
|
||||
s.start_xfer("h1", [0, 1], deadline=time.monotonic() - 1)
|
||||
s.poll([], time.monotonic())
|
||||
r1 = s.take_results()
|
||||
assert "h1" in r1.tombstoned
|
||||
r2 = s.take_results()
|
||||
assert not r2.tombstoned # cleared after first take
|
||||
|
||||
|
||||
def test_consumer_session_on_peer_down_cancels_waiting_ack():
|
||||
s = _make_consumer_session()
|
||||
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 60)
|
||||
s.on_peer_down()
|
||||
results = s.take_results()
|
||||
assert "h1" in results.retryable
|
||||
assert not results.tombstoned
|
||||
@@ -0,0 +1,54 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ECCPUConnector scheduler utilities."""
|
||||
|
||||
import msgspec
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
|
||||
build_block_descs,
|
||||
deserialize_mem_descriptor,
|
||||
serialize_mem_descriptor,
|
||||
)
|
||||
|
||||
# ── build_block_descs ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device_id,expected_dev", [(7, 7), (None, 0)])
|
||||
def test_build_block_descs(device_id, expected_dev):
|
||||
kwargs = dict(base_ptr=1000, num_blocks=4, block_size_bytes=256)
|
||||
if device_id is not None:
|
||||
kwargs["device_id"] = device_id
|
||||
descs = build_block_descs(**kwargs)
|
||||
assert len(descs) == 4
|
||||
for i, (addr, size, dev) in enumerate(descs):
|
||||
assert addr == 1000 + i * 256
|
||||
assert size == 256
|
||||
assert dev == expected_dev
|
||||
|
||||
|
||||
def test_build_block_descs_zero_blocks_returns_empty():
|
||||
assert build_block_descs(base_ptr=100, num_blocks=0, block_size_bytes=64) == []
|
||||
|
||||
|
||||
# ── serialize / deserialize_mem_descriptor ───────────────────────────────────
|
||||
|
||||
|
||||
def test_mem_descriptor_roundtrip():
|
||||
descs = [(100, 64, 0), (164, 64, 0), (228, 64, 1)]
|
||||
assert deserialize_mem_descriptor(serialize_mem_descriptor(descs)) == descs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad_value",
|
||||
[
|
||||
[(1, 2)], # 2-tuple instead of 3
|
||||
[("a", "b", "c")], # strings, not ints
|
||||
],
|
||||
)
|
||||
def test_mem_descriptor_rejects_malformed_payload(bad_value):
|
||||
"""Malformed descriptor lists must fail to decode."""
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
bad_payload = encoder.encode(bad_value)
|
||||
with pytest.raises((msgspec.DecodeError, msgspec.ValidationError)):
|
||||
deserialize_mem_descriptor(bad_payload)
|
||||
@@ -8,7 +8,7 @@ GPU copy plumbing. An ec_both instance reuses encoder outputs it has already
|
||||
offloaded to CPU instead of recomputing them.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -37,6 +37,12 @@ class ECCPUConnector(ECConnectorBase):
|
||||
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole) -> None:
|
||||
super().__init__(vllm_config=vllm_config, role=role)
|
||||
|
||||
if not vllm_config.use_v2_model_runner:
|
||||
raise ValueError(
|
||||
"ECCPUConnector requires the V2 model runner. "
|
||||
"Set VLLM_USE_V2_MODEL_RUNNER=1."
|
||||
)
|
||||
|
||||
self.connector_worker = None
|
||||
self.connector_scheduler = None
|
||||
|
||||
@@ -108,6 +114,12 @@ class ECCPUConnector(ECConnectorBase):
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def request_finished(
|
||||
self, request: "Request"
|
||||
) -> tuple[bool, "dict[str, Any] | None"]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Abstract base classes for the EC control plane.
|
||||
|
||||
ControlConnection wraps a single peer channel (one DEALER socket).
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class ControlConnection(ABC):
|
||||
"""Bidirectional message channel to a single remote peer.
|
||||
|
||||
All I/O is non-blocking: send enqueues without blocking; recv drains
|
||||
all buffered inbound messages since the last call.
|
||||
"""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def alive(self) -> bool:
|
||||
"""True if the connection is still usable."""
|
||||
|
||||
@abstractmethod
|
||||
def send(self, msg: bytes) -> None:
|
||||
"""Enqueue msg for delivery. Must not block."""
|
||||
|
||||
@abstractmethod
|
||||
def recv(self) -> list[bytes]:
|
||||
"""Drain and return all buffered inbound messages."""
|
||||
|
||||
@abstractmethod
|
||||
def mark_dead(self) -> None:
|
||||
"""Signal that the peer disconnected. Sets alive to False."""
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Release all resources. Idempotent."""
|
||||
@@ -0,0 +1,233 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ZMQ control-plane implementations for ECCPUConnector.
|
||||
|
||||
ZmqClientConnection — one DEALER socket to a single producer peer.
|
||||
ZmqClientTransport — pool of ZmqClientConnection objects (consumer side).
|
||||
ZmqServerTransport — ROUTER socket (producer side).
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import zmq
|
||||
from zmq.utils.monitor import recv_monitor_message
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.control.base import (
|
||||
ControlConnection,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import make_zmq_path, make_zmq_socket
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_HEARTBEAT_IVL_MS = 2000
|
||||
_HEARTBEAT_TIMEOUT_MS = 4000
|
||||
_HEARTBEAT_TTL_MS = 8000
|
||||
|
||||
|
||||
# ── Client side ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class ZmqClientConnection(ControlConnection):
|
||||
"""One DEALER socket to one producer peer.
|
||||
|
||||
Owns the socket and its optional monitor socket. NIXL state is NOT
|
||||
stored here — it belongs in ECSession above this layer.
|
||||
"""
|
||||
|
||||
dealer: zmq.Socket
|
||||
monitor: zmq.Socket | None
|
||||
_alive: bool = field(default=True, init=False)
|
||||
|
||||
@property
|
||||
def alive(self) -> bool:
|
||||
return self._alive
|
||||
|
||||
def send(self, msg: bytes) -> None:
|
||||
self.dealer.send_multipart([b"", msg])
|
||||
|
||||
def recv(self) -> list[bytes]:
|
||||
msgs: list[bytes] = []
|
||||
while True:
|
||||
try:
|
||||
frames = self.dealer.recv_multipart(flags=zmq.NOBLOCK)
|
||||
except zmq.Again:
|
||||
break
|
||||
except Exception:
|
||||
logger.exception("ec: DEALER recv failed")
|
||||
break
|
||||
if len(frames) != 2 or frames[0] != b"":
|
||||
logger.warning(
|
||||
"ec: dropped malformed frame envelope (%d frames)", len(frames)
|
||||
)
|
||||
continue
|
||||
msgs.append(frames[1])
|
||||
return msgs
|
||||
|
||||
def mark_dead(self) -> None:
|
||||
self._alive = False
|
||||
|
||||
def close(self) -> None:
|
||||
if self.monitor is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
self.dealer.disable_monitor()
|
||||
with contextlib.suppress(Exception):
|
||||
self.monitor.close(linger=0)
|
||||
with contextlib.suppress(Exception):
|
||||
self.dealer.close(linger=0)
|
||||
self._alive = False
|
||||
|
||||
|
||||
class ZmqClientTransport:
|
||||
"""Pool of ZmqClientConnection objects — one per producer peer.
|
||||
|
||||
Creates DEALER sockets lazily on first contact. Does not touch NIXL;
|
||||
NIXL registration is handled by ECCPUConsumer once an XferAck arrives.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._ctx = zmq.Context()
|
||||
self._connections: dict[PeerAddr, ZmqClientConnection] = {}
|
||||
|
||||
def connect(self, addr: PeerAddr) -> ZmqClientConnection:
|
||||
"""Return the existing connection for addr, or create a new one."""
|
||||
existing = self._connections.get(addr)
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
host, port = addr
|
||||
dealer = make_zmq_socket(
|
||||
ctx=self._ctx,
|
||||
path=make_zmq_path(scheme="tcp", host=host, port=port),
|
||||
socket_type=zmq.DEALER,
|
||||
bind=False,
|
||||
)
|
||||
dealer.setsockopt(zmq.HEARTBEAT_IVL, _HEARTBEAT_IVL_MS)
|
||||
dealer.setsockopt(zmq.HEARTBEAT_TIMEOUT, _HEARTBEAT_TIMEOUT_MS)
|
||||
dealer.setsockopt(zmq.HEARTBEAT_TTL, _HEARTBEAT_TTL_MS)
|
||||
|
||||
monitor_addr = f"inproc://ec-peer-mon-{host}-{port}"
|
||||
dealer.monitor(monitor_addr, zmq.EVENT_DISCONNECTED)
|
||||
monitor = self._ctx.socket(zmq.PAIR)
|
||||
monitor.connect(monitor_addr)
|
||||
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=monitor)
|
||||
self._connections[addr] = conn
|
||||
return conn
|
||||
|
||||
def poll(self) -> dict[PeerAddr, list[bytes]]:
|
||||
"""Non-blocking drain of messages from every peer's DEALER."""
|
||||
msgs: dict[PeerAddr, list[bytes]] = {}
|
||||
for addr, conn in list(self._connections.items()):
|
||||
conn_msgs = conn.recv()
|
||||
msgs[addr] = conn_msgs
|
||||
return msgs
|
||||
|
||||
def poll_dead(self) -> list[PeerAddr]:
|
||||
"""Non-blocking sweep of peer monitors; return addresses that died."""
|
||||
dead: list[PeerAddr] = []
|
||||
for addr, conn in list(self._connections.items()):
|
||||
if conn.monitor is None:
|
||||
continue
|
||||
try:
|
||||
while True:
|
||||
evt = recv_monitor_message(conn.monitor, flags=zmq.NOBLOCK)
|
||||
if evt["event"] == zmq.EVENT_DISCONNECTED:
|
||||
conn.mark_dead()
|
||||
dead.append(addr)
|
||||
break
|
||||
except zmq.Again:
|
||||
pass
|
||||
except Exception:
|
||||
logger.warning("ec: monitor poll failed for %s", addr, exc_info=True)
|
||||
return dead
|
||||
|
||||
def remove(self, addr: PeerAddr) -> ZmqClientConnection | None:
|
||||
"""Remove and close the connection for addr, returning it (or None)."""
|
||||
conn = self._connections.pop(addr, None)
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
return conn
|
||||
|
||||
def close(self) -> None:
|
||||
for conn in list(self._connections.values()):
|
||||
conn.close()
|
||||
self._connections.clear()
|
||||
try:
|
||||
self._ctx.destroy(linger=0)
|
||||
except Exception:
|
||||
logger.debug("ec: zmq context destroy failed", exc_info=True)
|
||||
|
||||
|
||||
# ── Server side ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class ZmqServerTransport:
|
||||
"""ROUTER socket for the producer side. Byte-level only, driven by
|
||||
ProducerSession.
|
||||
|
||||
poll(timeout_ms) waits up to timeout_ms for messages, then drains all that
|
||||
are available; timeout_ms=0 drains whatever has already arrived. send()
|
||||
routes a reply back to a specific peer by ZMQ identity.
|
||||
"""
|
||||
|
||||
def __init__(self, host: str, port: int) -> None:
|
||||
self._ctx = zmq.Context()
|
||||
self._router = make_zmq_socket(
|
||||
ctx=self._ctx,
|
||||
path=make_zmq_path(scheme="tcp", host=host, port=port),
|
||||
socket_type=zmq.ROUTER,
|
||||
bind=True,
|
||||
)
|
||||
self._poller = zmq.Poller()
|
||||
self._poller.register(self._router, zmq.POLLIN)
|
||||
|
||||
def poll(self, timeout_ms: int = 0) -> list[tuple[bytes, bytes]]:
|
||||
"""Wait up to timeout_ms for incoming messages, then drain all pending.
|
||||
|
||||
Returns a list of (identity, payload) pairs.
|
||||
"""
|
||||
if timeout_ms > 0:
|
||||
try:
|
||||
events = dict(self._poller.poll(timeout=timeout_ms))
|
||||
except zmq.ContextTerminated:
|
||||
return []
|
||||
except Exception:
|
||||
logger.exception("ec: router poller failed")
|
||||
return []
|
||||
if self._router not in events:
|
||||
return []
|
||||
|
||||
msgs: list[tuple[bytes, bytes]] = []
|
||||
while True:
|
||||
try:
|
||||
identity, _, payload = self._router.recv_multipart(flags=zmq.NOBLOCK)
|
||||
msgs.append((identity, payload))
|
||||
except zmq.Again:
|
||||
break
|
||||
except zmq.ContextTerminated:
|
||||
break
|
||||
except Exception:
|
||||
logger.exception("ec: router recv failed")
|
||||
break
|
||||
return msgs
|
||||
|
||||
def send(self, identity: bytes, payload: bytes) -> None:
|
||||
"""Send a reply payload to the peer identified by ZMQ identity."""
|
||||
try:
|
||||
self._router.send_multipart([identity, b"", payload])
|
||||
except Exception:
|
||||
logger.exception("ec: router send failed")
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
self._router.close(linger=0)
|
||||
except Exception:
|
||||
logger.debug("ec: router close failed", exc_info=True)
|
||||
try:
|
||||
self._ctx.destroy(linger=0)
|
||||
except Exception:
|
||||
logger.debug("ec: zmq context destroy failed", exc_info=True)
|
||||
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Abstract base class for the EC data-plane transport (NIXL/UCX)."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class DataTransport(ABC):
|
||||
"""Interface wrapping the NIXL data-plane for EC block transfers.
|
||||
|
||||
Peer lifecycle:
|
||||
add_remote_peer() — register a peer and store its dlist handle internally
|
||||
remove_remote_peer() — deregister and release the stored handle
|
||||
post_read() — issue a READ using the stored handle for agent_name
|
||||
|
||||
The dlist handle (prepped xfer descriptor list) is an implementation detail
|
||||
of the transport and is never exposed to callers.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_agent_metadata(self) -> bytes:
|
||||
"""Return this agent's serialized NIXL metadata blob."""
|
||||
|
||||
@abstractmethod
|
||||
def get_mem_descriptor(self) -> bytes:
|
||||
"""Return the msgpack-encoded block descriptor list for this agent."""
|
||||
|
||||
@abstractmethod
|
||||
def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str:
|
||||
"""Register a remote peer, prep its READ-source dlist, and return agent_name.
|
||||
|
||||
The dlist handle is stored internally and used by post_read().
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def remove_remote_peer(self, agent_name: str) -> None:
|
||||
"""Deregister a previously registered remote peer and release its handle."""
|
||||
|
||||
@abstractmethod
|
||||
def post_read(
|
||||
self,
|
||||
local_indices: list[int],
|
||||
agent_name: str,
|
||||
remote_indices: list[int],
|
||||
notif_msg: bytes,
|
||||
) -> Any:
|
||||
"""Issue a consumer-initiated READ using the stored dlist for agent_name.
|
||||
|
||||
Returns a transfer handle for use with check_xfer_state / release_xfer_handle.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_new_notifs(self) -> dict[str, list[bytes]]:
|
||||
"""Drain NIXL completion notifications addressed to this agent."""
|
||||
|
||||
@abstractmethod
|
||||
def check_xfer_state(self, handle: Any) -> str:
|
||||
"""Return the NIXL state string: 'DONE', 'PROC', or an error string."""
|
||||
|
||||
@abstractmethod
|
||||
def release_xfer_handle(self, handle: Any) -> None:
|
||||
"""Release a transfer handle returned by post_read."""
|
||||
|
||||
@abstractmethod
|
||||
def deregister(self) -> None:
|
||||
"""Deregister all memory and tear down the NIXL agent."""
|
||||
@@ -0,0 +1,162 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NIXL-backed data-plane transport for ECCPUConnector."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
|
||||
DataTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
|
||||
build_block_descs,
|
||||
deserialize_mem_descriptor,
|
||||
serialize_mem_descriptor,
|
||||
)
|
||||
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_NIXL_DRAM = "DRAM"
|
||||
|
||||
|
||||
class NixlDataTransport(DataTransport):
|
||||
"""Adapts NixlWrapper to the DataTransport interface.
|
||||
|
||||
Dlist handles (prepped xfer descriptor lists for remote peers) are stored
|
||||
internally keyed by agent_name. Callers never touch raw NIXL handles —
|
||||
post_read() resolves the handle from agent_name internally.
|
||||
|
||||
NixlWrapper is not thread-safe, and neither is this class. Every call site
|
||||
runs on the scheduler thread, which serializes the transport by
|
||||
construction; transfers are async inside NIXL and are drained from that
|
||||
same thread via get_new_notifs() / check_xfer_state().
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent_name: str,
|
||||
base_ptr: int,
|
||||
num_blocks: int,
|
||||
block_size_bytes: int,
|
||||
total_size_bytes: int,
|
||||
num_threads: int = 1,
|
||||
) -> None:
|
||||
if NixlWrapper is None or nixl_agent_config is None:
|
||||
raise RuntimeError(
|
||||
"ECCPUConnector requires NIXL; "
|
||||
"install the `nixl` package or set a different ec_connector."
|
||||
)
|
||||
# agent_name → remote dlist handle (from add_remote_peer, used by post_read)
|
||||
self._peer_handles: dict[str, int] = {}
|
||||
# None until the local region is registered, so deregister() is safe on
|
||||
# a transport whose registration failed part-way.
|
||||
self._reg_descs: Any = None
|
||||
|
||||
self._nixl = NixlWrapper(
|
||||
agent_name,
|
||||
nixl_agent_config(num_threads=num_threads, capture_telemetry=True),
|
||||
)
|
||||
self._register_local_memory(
|
||||
base_ptr, num_blocks, block_size_bytes, total_size_bytes
|
||||
)
|
||||
|
||||
def _register_local_memory(
|
||||
self,
|
||||
base_ptr: int,
|
||||
num_blocks: int,
|
||||
block_size_bytes: int,
|
||||
total_size_bytes: int,
|
||||
) -> None:
|
||||
"""Register the local region with NIXL and prep its transfer dlist.
|
||||
|
||||
Unwinds the registration if a later step fails: __init__ propagates the
|
||||
error, so the caller never receives an object it could deregister with.
|
||||
"""
|
||||
block_descs = build_block_descs(base_ptr, num_blocks, block_size_bytes)
|
||||
self._reg_descs = self._nixl.get_reg_descs(
|
||||
[(base_ptr, total_size_bytes, 0, "")], _NIXL_DRAM
|
||||
)
|
||||
self._nixl.register_memory(self._reg_descs, backends=["UCX"])
|
||||
try:
|
||||
xfer_descs = self._nixl.get_xfer_descs(block_descs, _NIXL_DRAM)
|
||||
self._local_xfer_handle = self._nixl.prep_xfer_dlist(
|
||||
"NIXL_INIT_AGENT", xfer_descs
|
||||
)
|
||||
self._agent_metadata = self._nixl.get_agent_metadata()
|
||||
self._mem_descriptor_bytes = serialize_mem_descriptor(block_descs)
|
||||
except Exception:
|
||||
self.deregister()
|
||||
raise
|
||||
|
||||
def get_agent_metadata(self) -> bytes:
|
||||
return self._agent_metadata
|
||||
|
||||
def get_mem_descriptor(self) -> bytes:
|
||||
return self._mem_descriptor_bytes
|
||||
|
||||
def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str:
|
||||
"""Register remote peer, prep its dlist, store handle internally.
|
||||
|
||||
Returns agent_name for use in post_read() and remove_remote_peer().
|
||||
"""
|
||||
agent_name = self._nixl.add_remote_agent(metadata)
|
||||
remote_blocks = deserialize_mem_descriptor(mem_descriptor)
|
||||
remote_xfer_descs = self._nixl.get_xfer_descs(remote_blocks, _NIXL_DRAM)
|
||||
self._peer_handles[agent_name] = self._nixl.prep_xfer_dlist(
|
||||
agent_name, remote_xfer_descs
|
||||
)
|
||||
return agent_name
|
||||
|
||||
def remove_remote_peer(self, agent_name: str) -> None:
|
||||
self._peer_handles.pop(agent_name, None)
|
||||
try:
|
||||
self._nixl.remove_remote_agent(agent_name)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"EC: remove_remote_peer failed for %s", agent_name, exc_info=True
|
||||
)
|
||||
|
||||
def post_read(
|
||||
self,
|
||||
local_indices: list[int],
|
||||
agent_name: str,
|
||||
remote_indices: list[int],
|
||||
notif_msg: bytes,
|
||||
) -> Any:
|
||||
if len(local_indices) != len(remote_indices):
|
||||
raise ValueError(
|
||||
f"EC: local/remote block count mismatch "
|
||||
f"({len(local_indices)} vs {len(remote_indices)})"
|
||||
)
|
||||
remote_handle = self._peer_handles[agent_name]
|
||||
handle = self._nixl.make_prepped_xfer(
|
||||
"READ",
|
||||
self._local_xfer_handle,
|
||||
local_indices,
|
||||
remote_handle,
|
||||
remote_indices,
|
||||
notif_msg=notif_msg,
|
||||
)
|
||||
self._nixl.transfer(handle)
|
||||
return handle
|
||||
|
||||
def get_new_notifs(self) -> dict[str, list[bytes]]:
|
||||
return self._nixl.get_new_notifs()
|
||||
|
||||
def check_xfer_state(self, handle: Any) -> str:
|
||||
return self._nixl.check_xfer_state(handle)
|
||||
|
||||
def release_xfer_handle(self, handle: Any) -> None:
|
||||
try:
|
||||
self._nixl.release_xfer_handle(handle)
|
||||
except Exception:
|
||||
logger.warning("EC: release_xfer_handle failed", exc_info=True)
|
||||
|
||||
def deregister(self) -> None:
|
||||
if self._reg_descs is None:
|
||||
return
|
||||
try:
|
||||
self._nixl.deregister_memory(self._reg_descs)
|
||||
except Exception:
|
||||
logger.warning("EC: deregister failed", exc_info=True)
|
||||
self._reg_descs = None
|
||||
@@ -0,0 +1,91 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Scheduler-side wire types for the ECCPUConnector.
|
||||
|
||||
Two peer-to-peer message types exchanged between consumer and producer over
|
||||
ZMQ (XferReq, XferAck) plus a peer-compatibility fingerprint
|
||||
(compute_ec_compatibility_hash) sent on every XferReq so a producer refuses
|
||||
to serve a mismatched peer.
|
||||
"""
|
||||
|
||||
import enum
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
import msgspec
|
||||
|
||||
# Bump when the on-wire shape (XferReq/XferAck) changes in a
|
||||
# backward-incompatible way. Peers reject mismatched versions.
|
||||
EC_CONNECTOR_VERSION: int = 1
|
||||
|
||||
|
||||
class XferStatus(enum.IntEnum):
|
||||
"""Outcome the producer reports for an XferReq."""
|
||||
|
||||
OK = 0
|
||||
NACK_MISSING = 1 # producer no longer holds the encoding
|
||||
NACK_INCOMPAT = 2 # peer-compatibility hash mismatch
|
||||
NACK_VERSION = 3 # connector wire-version mismatch
|
||||
NACK_INTERNAL = 4 # producer hit an unexpected error
|
||||
NACK_NOT_READY = 5 # announced, but its GPU->mmap save has not landed yet
|
||||
|
||||
|
||||
# The producer expects a retryable condition to clear on its own, so the
|
||||
# consumer re-requests the read rather than giving up on the encoding.
|
||||
RETRYABLE_NACKS = frozenset((XferStatus.NACK_NOT_READY,))
|
||||
# Ordinary outcomes rather than signs of a broken peer.
|
||||
EXPECTED_NACKS = RETRYABLE_NACKS | {XferStatus.NACK_MISSING}
|
||||
|
||||
|
||||
class XferReq( # type: ignore[call-arg]
|
||||
msgspec.Struct,
|
||||
tag="req",
|
||||
omit_defaults=True,
|
||||
):
|
||||
"""Consumer → Producer: I want to READ mm_hash; pin it and tell me where."""
|
||||
|
||||
mm_hash: str
|
||||
compatibility_hash: str
|
||||
# Consumer's session identity. Together with mm_hash it uniquely identifies
|
||||
# this transfer on both sides: the producer keys its active xfers by
|
||||
# (session_id, mm_hash), and the NIXL notif_msg carries the same pair.
|
||||
session_id: str = ""
|
||||
connector_version: int = EC_CONNECTOR_VERSION
|
||||
|
||||
|
||||
class XferAck( # type: ignore[call-arg]
|
||||
msgspec.Struct,
|
||||
tag="ack",
|
||||
omit_defaults=True,
|
||||
):
|
||||
"""Producer → Consumer: response to an XferReq.
|
||||
|
||||
On status == OK: producer has pinned source blocks and returns their
|
||||
indices plus fresh NIXL metadata for the consumer to register and READ.
|
||||
On any NACK_*: optional fields are empty; consumer falls back to local encode.
|
||||
"""
|
||||
|
||||
mm_hash: str
|
||||
status: XferStatus
|
||||
# Consumer's session_id echoed back from XferReq. Together with mm_hash it
|
||||
# identifies the exact transfer; the consumer uses both to build notif_msg.
|
||||
session_id: str = ""
|
||||
src_block_indices: list[int] = []
|
||||
agent_metadata: bytes = b""
|
||||
mem_descriptor: bytes = b""
|
||||
|
||||
|
||||
def compute_ec_compatibility_hash(
|
||||
vllm_version: str,
|
||||
model: str,
|
||||
dtype: str,
|
||||
block_size_bytes: int,
|
||||
) -> str:
|
||||
"""Peer-compatibility fingerprint over the four factors that determine
|
||||
byte-layout compatibility. Producer NACKs any XferReq whose hash differs.
|
||||
"""
|
||||
factors = json.dumps(
|
||||
[vllm_version, model, dtype, block_size_bytes],
|
||||
separators=(",", ":"),
|
||||
)
|
||||
return hashlib.sha256(factors.encode("utf-8")).hexdigest()
|
||||
@@ -7,16 +7,23 @@ Owns the mmap region and the embedding cache, and handles the producer
|
||||
for the ECCPUConnector.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
ECCPUWorkerMetadata,
|
||||
_get_encoder_cache_hidden_dim,
|
||||
create_ec_shared_region,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
PlaceholderMetadataResolver,
|
||||
collect_ec_item_metadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -40,6 +47,7 @@ class ECCPUScheduler:
|
||||
self._region = create_ec_shared_region(vllm_config)
|
||||
# Block allocator + LRU eviction policy for the shared region.
|
||||
self._cache = EmbeddingCache(self._region.num_blocks)
|
||||
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
|
||||
|
||||
# mm_hash → block IDs allocated this step for GPU→mmap saves.
|
||||
self._pending_saves: dict[str, list[int]] = {}
|
||||
@@ -63,6 +71,112 @@ class ECCPUScheduler:
|
||||
else 1
|
||||
)
|
||||
|
||||
self._ec_config = ec_config
|
||||
# NIXL p2p is an option of this connector, not engine-wide behavior, so
|
||||
# it lives in extra config. Extra config is not type coerced, so a value
|
||||
# that arrived as a JSON/CLI string is parsed, not truth-tested.
|
||||
raw_nixl = ec_config.get_from_extra_config("ec_enable_nixl", False)
|
||||
self._nixl_enabled = (
|
||||
raw_nixl
|
||||
if isinstance(raw_nixl, bool)
|
||||
else str(raw_nixl).strip().lower() in ("true", "1", "yes")
|
||||
)
|
||||
# NIXL fields default to None/empty so the gate-off path is untouched.
|
||||
self._data: Any = None
|
||||
self._compat_hash: str | None = None
|
||||
self._first_in_batch = True
|
||||
self._transport: Any = None
|
||||
self._producer_session: Any = None
|
||||
self._sessions: dict = {}
|
||||
self._in_flight: set[str] = set()
|
||||
self._tombstones: set[str] = set()
|
||||
self._step_completed: set[str] = set()
|
||||
self._peer_host: str | None = None
|
||||
self._peer_port: int | None = None
|
||||
# Model shape for size checks + compat hash; only set by
|
||||
# _setup_nixl (the gate-off path never touches model_config).
|
||||
self._dtype: torch.dtype | None = None
|
||||
self._hidden_dim: int = 0
|
||||
self._element_size: int = 0
|
||||
self._ack_timeout_s: float = 0.0
|
||||
if self._nixl_enabled:
|
||||
self._setup_nixl(vllm_config)
|
||||
|
||||
def _setup_nixl(self, vllm_config: "VllmConfig") -> None:
|
||||
# Lazy imports keep nixl/zmq off the gate-off path.
|
||||
from vllm import envs
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import (
|
||||
ZmqClientTransport,
|
||||
ZmqServerTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
|
||||
NixlDataTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
|
||||
compute_ec_compatibility_hash,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
|
||||
ProducerSession,
|
||||
)
|
||||
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
if NixlWrapper is None or nixl_agent_config is None:
|
||||
raise RuntimeError(
|
||||
"ec_enable_nixl requires NIXL; install the `nixl` package or "
|
||||
"remove ec_enable_nixl from ec_connector_extra_config."
|
||||
)
|
||||
engine_id = self._ec_config.engine_id
|
||||
assert engine_id is not None
|
||||
self._dtype = vllm_config.model_config.dtype
|
||||
self._hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
|
||||
self._element_size = torch.empty(0, dtype=self._dtype).element_size()
|
||||
# How long a consumer waits for an XferAck. The producer answers from
|
||||
# its scheduler step, so its reply latency scales with the encoder's
|
||||
# batch size: a deployment whose steps run longer must raise this.
|
||||
self._ack_timeout_s = float(
|
||||
self._ec_config.get_from_extra_config("consumer_ack_timeout_s", 2.0)
|
||||
)
|
||||
self._compat_hash = compute_ec_compatibility_hash(
|
||||
vllm_version=VLLM_VERSION,
|
||||
model=str(vllm_config.model_config.model),
|
||||
dtype=str(self._dtype),
|
||||
block_size_bytes=self._region.block_size_bytes,
|
||||
)
|
||||
if self._is_producer:
|
||||
self._peer_host = envs.VLLM_EC_SIDE_CHANNEL_HOST
|
||||
self._peer_port = envs.VLLM_EC_SIDE_CHANNEL_PORT
|
||||
|
||||
# Registering the region with NIXL and binding the control sockets are
|
||||
# the first side effects here. __init__ propagates a failure, so the
|
||||
# caller never receives a scheduler it could shut down: unwind through
|
||||
# the same teardown shutdown() uses.
|
||||
try:
|
||||
self._data = NixlDataTransport(
|
||||
agent_name=engine_id,
|
||||
base_ptr=self._region.blocks.data_ptr(),
|
||||
num_blocks=self._region.num_blocks,
|
||||
block_size_bytes=self._region.block_size_bytes,
|
||||
total_size_bytes=self._region.num_blocks
|
||||
* self._region.block_size_bytes,
|
||||
)
|
||||
if self._is_producer:
|
||||
assert self._peer_host is not None
|
||||
assert self._peer_port is not None
|
||||
self._producer_session = ProducerSession(
|
||||
transport=ZmqServerTransport(
|
||||
host=self._peer_host, port=self._peer_port
|
||||
),
|
||||
data=self._data,
|
||||
cache=self._cache,
|
||||
compat_hash=self._compat_hash,
|
||||
)
|
||||
if self._is_consumer:
|
||||
self._transport = ZmqClientTransport()
|
||||
except Exception:
|
||||
self._teardown_nixl()
|
||||
raise
|
||||
|
||||
def has_cache_item(self, identifier: str) -> bool:
|
||||
if not self._is_consumer:
|
||||
return False
|
||||
@@ -72,8 +186,202 @@ class ECCPUScheduler:
|
||||
def ensure_cache_available(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
if not self._nixl_enabled:
|
||||
return True # CPU offload never blocks.
|
||||
first = self._first_in_batch
|
||||
self._first_in_batch = False
|
||||
if not self._is_consumer:
|
||||
return True
|
||||
if first:
|
||||
self._poll_step()
|
||||
return self._nixl_consumer_admit(request, num_computed_tokens)
|
||||
|
||||
def _nixl_consumer_admit(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
"""Admit a request only once all its remote encodings are cached.
|
||||
|
||||
Returns True when every encoding the request needs is either already
|
||||
ready in the local cache or has no remote source. Any encoding whose
|
||||
NIXL READ is still in flight (or was just started this step) defers
|
||||
the request, which the scheduler re-presents on a later step.
|
||||
"""
|
||||
params: dict[str, dict[str, Any]] = (
|
||||
getattr(request, "ec_transfer_params", None) or {}
|
||||
)
|
||||
if not params:
|
||||
return True
|
||||
pending = False
|
||||
for feature in request.mm_features:
|
||||
pos = feature.mm_position
|
||||
if pos.offset + pos.length <= num_computed_tokens:
|
||||
continue
|
||||
mm_hash = feature.identifier
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is not None and entry.ready:
|
||||
# Local hit: upstream's update_state_after_alloc pins and
|
||||
# loads it through the same path as a natively cached entry.
|
||||
continue
|
||||
if mm_hash in self._in_flight:
|
||||
pending = True
|
||||
continue
|
||||
if mm_hash in self._step_completed:
|
||||
pending = True
|
||||
continue
|
||||
if mm_hash in self._tombstones:
|
||||
self._tombstones.discard(mm_hash)
|
||||
continue
|
||||
if entry is not None:
|
||||
# Present but not ready and not being fetched: its blocks are
|
||||
# held by a quarantined/settling DMA and cannot be reused.
|
||||
# Fall back to local compute this step; never re-alloc a
|
||||
# mm_hash already in the cache.
|
||||
continue
|
||||
info = params.get(mm_hash)
|
||||
# Absent entirely, or present with only placeholder metadata (no
|
||||
# cache entry existed on the producer side at request_finished
|
||||
# time): nothing to fetch, fall back to local compute.
|
||||
if not info or "peer_host" not in info:
|
||||
continue
|
||||
expected = pos.length * self._hidden_dim * self._element_size
|
||||
if int(info.get("size_bytes", -1)) != expected:
|
||||
logger.warning(
|
||||
"EC consumer: size mismatch mm_hash=%s; local encode", mm_hash
|
||||
)
|
||||
continue
|
||||
try:
|
||||
started = self._start_xfer(mm_hash, info, expected)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
|
||||
)
|
||||
continue
|
||||
if not started:
|
||||
continue
|
||||
self._in_flight.add(mm_hash)
|
||||
pending = True
|
||||
return not pending
|
||||
|
||||
def _start_xfer(
|
||||
self, mm_hash: str, info: "dict[str, Any]", size_bytes: int
|
||||
) -> bool:
|
||||
"""Allocate a not-ready cache entry and start a NIXL READ into it.
|
||||
|
||||
Returns True if the transfer was started. Returns False when the
|
||||
cache cannot accommodate the encoding, in which case the request
|
||||
falls back to local recomputation.
|
||||
"""
|
||||
import time
|
||||
from math import ceil
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
|
||||
ConsumerSession,
|
||||
)
|
||||
|
||||
n_blocks = max(1, ceil(size_bytes / self._region.block_size_bytes))
|
||||
entry = self._cache.alloc(mm_hash, n_blocks)
|
||||
if entry is None:
|
||||
logger.debug(
|
||||
"EC consumer: cache full for mm_hash=%s (%d blocks); local encode",
|
||||
mm_hash,
|
||||
n_blocks,
|
||||
)
|
||||
return False
|
||||
indices = list(entry.block_ids)
|
||||
addr = (info["peer_host"], int(info["peer_port"]))
|
||||
if addr not in self._sessions:
|
||||
zmq_conn = self._transport.connect(addr)
|
||||
assert self._compat_hash is not None
|
||||
self._sessions[addr] = ConsumerSession(
|
||||
addr=addr,
|
||||
zmq_conn=zmq_conn,
|
||||
transport=self._transport,
|
||||
data=self._data,
|
||||
compat_hash=self._compat_hash,
|
||||
)
|
||||
deadline = time.monotonic() + self._ack_timeout_s
|
||||
try:
|
||||
self._sessions[addr].start_xfer(mm_hash, indices, deadline)
|
||||
except Exception:
|
||||
self._cache.discard(mm_hash)
|
||||
raise
|
||||
logger.debug(
|
||||
"EC consumer: starting NIXL xfer mm_hash=%s from %s:%d blocks=%d",
|
||||
mm_hash,
|
||||
addr[0],
|
||||
addr[1],
|
||||
n_blocks,
|
||||
)
|
||||
return True
|
||||
|
||||
def _poll_step(self) -> None:
|
||||
import time
|
||||
|
||||
now = time.monotonic()
|
||||
all_messages = self._transport.poll()
|
||||
for addr, session in list(self._sessions.items()):
|
||||
session.poll(all_messages.get(addr, []), now)
|
||||
for addr in self._transport.poll_dead():
|
||||
self._on_peer_down(addr)
|
||||
for session in self._sessions.values():
|
||||
self._process_session_results(session)
|
||||
|
||||
def _process_session_results(self, session: Any) -> None:
|
||||
r = session.take_results()
|
||||
for mm_hash in r.completed:
|
||||
self._in_flight.discard(mm_hash)
|
||||
self._cache.mark_ready(mm_hash)
|
||||
self._step_completed.add(mm_hash)
|
||||
logger.debug("EC consumer: NIXL xfer complete mm_hash=%s", mm_hash)
|
||||
for mm_hash in r.tombstoned:
|
||||
self._in_flight.discard(mm_hash)
|
||||
self._cache.discard(mm_hash)
|
||||
self._tombstones.add(mm_hash)
|
||||
logger.debug("EC consumer: NIXL xfer failed mm_hash=%s", mm_hash)
|
||||
for mm_hash in r.quarantined:
|
||||
# DMA still running: keep the blocks reserved (entry stays
|
||||
# not-ready, hence non-evictable) until the xfer settles.
|
||||
self._in_flight.discard(mm_hash)
|
||||
self._tombstones.add(mm_hash)
|
||||
logger.debug(
|
||||
"EC consumer: NIXL xfer mm_hash=%s quarantined; DMA still running",
|
||||
mm_hash,
|
||||
)
|
||||
for mm_hash in r.retryable:
|
||||
# No tombstone: dropping every trace of the attempt is what lets
|
||||
# the next admit pass re-issue the read as if it were the first.
|
||||
self._in_flight.discard(mm_hash)
|
||||
self._cache.discard(mm_hash)
|
||||
logger.debug(
|
||||
"EC consumer: NIXL xfer mm_hash=%s retryable; will re-request",
|
||||
mm_hash,
|
||||
)
|
||||
for mm_hash, _block_indices in r.settled:
|
||||
self._cache.discard(mm_hash)
|
||||
logger.debug(
|
||||
"EC consumer: quarantined NIXL xfer mm_hash=%s settled", mm_hash
|
||||
)
|
||||
|
||||
def _on_peer_down(self, addr: Any) -> None:
|
||||
session = self._sessions.pop(addr, None)
|
||||
if session is None:
|
||||
return
|
||||
session.on_peer_down()
|
||||
self._process_session_results(session)
|
||||
session.close()
|
||||
logger.info("EC consumer: producer peer down addr=%s", addr)
|
||||
|
||||
def _promote_completed_reads(self) -> None:
|
||||
"""Drain the just-completed set built during ``_poll_step``.
|
||||
|
||||
Reads are marked ready in ``_process_session_results``, so a deferred
|
||||
request is re-admitted by ``ensure_cache_available`` and loaded through
|
||||
the same local path as a natively cached entry
|
||||
(``update_state_after_alloc`` -> ``_pending_loads``). No explicit
|
||||
pin/load bookkeeping is needed here beyond clearing the set.
|
||||
"""
|
||||
self._step_completed.clear()
|
||||
|
||||
def update_state_after_alloc(self, request: "Request", index: int) -> None:
|
||||
feature = request.mm_features[index]
|
||||
mm_hash = feature.identifier
|
||||
@@ -97,11 +405,17 @@ class ECCPUScheduler:
|
||||
) -> ECCPUConnectorMetadata:
|
||||
meta = ECCPUConnectorMetadata()
|
||||
if self._is_producer:
|
||||
if self._nixl_enabled and self._producer_session is not None:
|
||||
self._producer_session.poll_step()
|
||||
meta.saves = self._pending_saves
|
||||
self._pending_saves = {}
|
||||
if self._is_consumer:
|
||||
if self._nixl_enabled:
|
||||
self._promote_completed_reads()
|
||||
meta.loads = self._pending_loads
|
||||
self._pending_loads = {}
|
||||
if self._nixl_enabled:
|
||||
self._first_in_batch = True
|
||||
return meta
|
||||
|
||||
def update_connector_output(self, connector_output: "ECConnectorOutput") -> None:
|
||||
@@ -117,8 +431,15 @@ class ECCPUScheduler:
|
||||
return
|
||||
for mm_hash in meta.completed_saves:
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is not None and not entry.ready:
|
||||
if entry is None:
|
||||
logger.debug(
|
||||
"EC producer: worker reported completed save for unknown "
|
||||
"mm_hash=%s (already discarded/evicted?)",
|
||||
mm_hash,
|
||||
)
|
||||
elif not entry.ready:
|
||||
self._cache.mark_ready(mm_hash)
|
||||
logger.debug("EC producer: mm_hash=%s marked ready", mm_hash)
|
||||
for transfer_id in meta.completed_loads:
|
||||
pending = self._load_acks.get(transfer_id)
|
||||
if pending is None:
|
||||
@@ -131,14 +452,65 @@ class ECCPUScheduler:
|
||||
# as stale rather than releasing the pin a second time.
|
||||
del self._load_acks[transfer_id]
|
||||
self._cache.unpin(mm_hash)
|
||||
logger.debug("EC consumer: mm_hash=%s unpinned after load", mm_hash)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
"""True while any dispatched save or load has not been confirmed done.
|
||||
"""Keep the engine stepping so this connector's polls keep running.
|
||||
|
||||
Keeps the engine stepping so the worker's completion reports are polled
|
||||
even when no requests are otherwise runnable.
|
||||
With NIXL enabled the engine tick is the only driver of
|
||||
``ProducerSession.poll_step()``, and a producer's work arrives from a
|
||||
remote consumer, so nothing local would otherwise wake it. Without NIXL
|
||||
this only has to outlive dispatched saves and loads, so the engine can
|
||||
quiesce once the worker has confirmed them.
|
||||
"""
|
||||
return self._cache.has_held_entries()
|
||||
return self._nixl_enabled or self._cache.has_held_entries()
|
||||
|
||||
def request_finished(
|
||||
self, request: "Request"
|
||||
) -> tuple[bool, "dict[str, Any] | None"]:
|
||||
if not (self._nixl_enabled and self._is_producer):
|
||||
return False, None
|
||||
if not request.mm_features:
|
||||
return False, None
|
||||
|
||||
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
|
||||
|
||||
for feature in request.mm_features:
|
||||
mm_hash = feature.identifier
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is None:
|
||||
# Never saved, or evicted since. Publishing placeholder
|
||||
# metadata without an address invites an orchestrator to
|
||||
# rewrite the media into a reference to an encoding no
|
||||
# consumer can fetch, leaving the decoder nothing to embed.
|
||||
# Publish neither, so the media stays on the request.
|
||||
items[mm_hash]["metadata"] = {}
|
||||
logger.debug(
|
||||
"EC producer: mm_hash=%s absent at request_finished; "
|
||||
"announcing no metadata so the media is not rewritten away",
|
||||
mm_hash,
|
||||
)
|
||||
continue
|
||||
# Announce even if the save's GPU->mmap copy hasn't been
|
||||
# confirmed complete yet: a not-ready entry can't be evicted, so
|
||||
# it will still be here by the time a consumer's XferReq arrives.
|
||||
# A read arriving before the save lands is NACKed NACK_NOT_READY,
|
||||
# which the consumer retries on a later step rather than treating
|
||||
# as a miss.
|
||||
size_bytes = (
|
||||
feature.mm_position.length * self._hidden_dim * self._element_size
|
||||
)
|
||||
items[mm_hash].update(
|
||||
peer_host=self._peer_host,
|
||||
peer_port=self._peer_port,
|
||||
size_bytes=size_bytes,
|
||||
)
|
||||
logger.debug(
|
||||
"EC producer: announcing NIXL-readable encodings req_id=%s items=%s",
|
||||
request.request_id,
|
||||
items,
|
||||
)
|
||||
return False, items
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._pending_saves.clear()
|
||||
@@ -148,7 +520,45 @@ class ECCPUScheduler:
|
||||
self._is_producer = False
|
||||
self._is_consumer = False
|
||||
|
||||
if self._nixl_enabled:
|
||||
self._teardown_nixl()
|
||||
|
||||
try:
|
||||
self._region.cleanup()
|
||||
except Exception:
|
||||
logger.debug("ec: region cleanup failed", exc_info=True)
|
||||
|
||||
def _teardown_nixl(self) -> None:
|
||||
"""Close the control sockets and release the NIXL agent.
|
||||
|
||||
Guards each step and clears what it releases, so it is safe on a
|
||||
scheduler whose NIXL setup failed part-way and safe to call twice.
|
||||
The shared region is not touched — shutdown() owns that.
|
||||
"""
|
||||
if self._producer_session is not None:
|
||||
try:
|
||||
self._producer_session.close()
|
||||
except Exception:
|
||||
logger.debug("ec: producer session close failed", exc_info=True)
|
||||
self._producer_session = None
|
||||
|
||||
for session in list(self._sessions.values()):
|
||||
try:
|
||||
session.close()
|
||||
except Exception:
|
||||
logger.debug("ec: consumer session close failed", exc_info=True)
|
||||
self._sessions.clear()
|
||||
|
||||
if self._transport is not None:
|
||||
try:
|
||||
self._transport.close()
|
||||
except Exception:
|
||||
logger.debug("ec: client transport close failed", exc_info=True)
|
||||
self._transport = None
|
||||
|
||||
if self._data is not None:
|
||||
try:
|
||||
self._data.deregister()
|
||||
except Exception:
|
||||
logger.debug("ec: deregister failed", exc_info=True)
|
||||
self._data = None
|
||||
|
||||
@@ -122,6 +122,23 @@ class EmbeddingCache:
|
||||
self._evictable_block_count -= len(entry.block_ids)
|
||||
entry.pin()
|
||||
|
||||
def pin_if_ready(self, key: str) -> CacheEntry | None:
|
||||
"""Atomically pin *key* if present and ready; return its entry.
|
||||
|
||||
Returns None when the key is absent. An entry returned with `ready`
|
||||
False was not pinned: its save is still in flight, which the producer
|
||||
reports differently from a miss because such a read can be retried.
|
||||
"""
|
||||
with self._lock:
|
||||
entry = self._entries.get(key)
|
||||
if entry is None or not entry.ready:
|
||||
return entry
|
||||
if entry.evictable:
|
||||
del self._entries_free_list[key]
|
||||
self._evictable_block_count -= len(entry.block_ids)
|
||||
entry.pin()
|
||||
return entry
|
||||
|
||||
def unpin(self, key: str) -> None:
|
||||
"""Unpin an entry. Asserts currently pinned."""
|
||||
with self._lock:
|
||||
@@ -134,6 +151,21 @@ class EmbeddingCache:
|
||||
self._entries_free_list[key] = None
|
||||
self._evictable_block_count += len(entry.block_ids)
|
||||
|
||||
def discard(self, key: str) -> None:
|
||||
"""Remove a not-ready in-flight entry, returning its blocks to the pool.
|
||||
|
||||
Used when an in-flight fill (e.g. a NIXL READ) fails before the entry
|
||||
is marked ready. Asserts the entry is present and not ready — ready or
|
||||
pinned entries are reclaimed through eviction/unpin, not discard.
|
||||
"""
|
||||
with self._lock:
|
||||
entry = self._entries[key]
|
||||
assert entry._pin_count == -1, (
|
||||
f"EmbeddingCache: discard of ready/pinned entry {key!r}"
|
||||
)
|
||||
del self._entries[key]
|
||||
self._free_blocks.extend(entry.block_ids)
|
||||
|
||||
def _evict_until(self, n_blocks: int) -> None:
|
||||
"""Evict ready+unpinned entries FIFO until enough space. Lock held."""
|
||||
while len(self._free_blocks) < n_blocks:
|
||||
|
||||
@@ -0,0 +1,633 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Session objects for the ECCPUConnector.
|
||||
|
||||
Layer hierarchy: connector → session → transport (connection)
|
||||
|
||||
Two long-lived sessions:
|
||||
|
||||
ProducerSession — owns ZmqServerTransport. The connector calls poll_step()
|
||||
once per engine step, which drains the transport without blocking and
|
||||
feeds the raw bytes to poll(). poll() decodes XferReqs, grants or NACKs
|
||||
them, pins source blocks, sends XferAck replies, drains NIXL completion
|
||||
notifications, and sweeps expired xfers.
|
||||
|
||||
ConsumerSession — owns one ZmqClientConnection and the NIXL agent
|
||||
registration for that peer. The connector calls poll(messages, now)
|
||||
once per engine step with the raw bytes collected for this peer.
|
||||
poll() decodes XferAcks, dispatches them to ConsumerXfer objects,
|
||||
and advances all active xfer state machines.
|
||||
|
||||
Internal data classes:
|
||||
|
||||
ProducerXfer — one pinned read grant, keyed by session_id.
|
||||
ConsumerXfer — state machine for one NIXL READ, keyed by mm_hash.
|
||||
"""
|
||||
|
||||
import enum
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import (
|
||||
ZmqClientConnection,
|
||||
ZmqClientTransport,
|
||||
ZmqServerTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
|
||||
DataTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
|
||||
EC_CONNECTOR_VERSION,
|
||||
EXPECTED_NACKS,
|
||||
RETRYABLE_NACKS,
|
||||
XferAck,
|
||||
XferReq,
|
||||
XferStatus,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
PRODUCER_PIN_LEASE_S = 30.0
|
||||
_CONSUMER_READ_TIMEOUT_S = 20.0
|
||||
_CONSUMER_QUARANTINE_TIMEOUT_S = 60.0
|
||||
|
||||
|
||||
# ── XferState ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class XferState(enum.Enum):
|
||||
"""Lifecycle states of a single ConsumerXfer."""
|
||||
|
||||
WAITING_ACK = "waiting_ack" # XferReq sent, awaiting XferAck
|
||||
READING = "reading" # NIXL READ in flight
|
||||
DONE = "done" # READ completed; promote to local cache
|
||||
ACK_TIMEOUT = "ack_timeout" # XferAck never arrived; free blocks + tombstone
|
||||
READ_FAILED = "read_failed" # Unexpected NIXL state; free blocks + tombstone
|
||||
QUARANTINED = "quarantined" # Read timed out, NIXL unabortable; keep blocks
|
||||
SETTLED = "settled" # Quarantined DMA done; safe to free blocks
|
||||
|
||||
|
||||
# ── ProducerXfer ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProducerXfer:
|
||||
"""One pinned read grant on the producer side.
|
||||
|
||||
Pure data — lifecycle operations (pin/unpin) are performed by ProducerSession.
|
||||
Keyed by (consumer_session_id, mm_hash) in ProducerSession._active_xfers.
|
||||
"""
|
||||
|
||||
mm_hash: str
|
||||
block_indices: list[int]
|
||||
deadline: float
|
||||
|
||||
def is_expired(self) -> bool:
|
||||
return time.monotonic() > self.deadline
|
||||
|
||||
|
||||
# ── ConsumerXfer ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConsumerXfer:
|
||||
"""State machine for one NIXL READ on the consumer side.
|
||||
|
||||
transfer_handle is the in-flight NIXL handle returned by post_read() —
|
||||
used for check_xfer_state() / release_xfer_handle(). The dlist handle
|
||||
for the remote peer lives inside DataTransport, not here.
|
||||
"""
|
||||
|
||||
mm_hash: str
|
||||
block_indices: list[int]
|
||||
addr: PeerAddr
|
||||
deadline: float
|
||||
data: DataTransport
|
||||
consumer_session_id: str # session identity; combined with mm_hash for notif_msg
|
||||
|
||||
transfer_handle: Any | None = field(default=None)
|
||||
_quarantined: bool = field(default=False, init=False)
|
||||
|
||||
def handle_ack(self, ack: XferAck, agent_name: str) -> bool:
|
||||
"""Issue the NIXL READ on an OK XferAck. Returns False on any NACK."""
|
||||
if ack.status != XferStatus.OK:
|
||||
return False
|
||||
# notif_msg encodes both session_id and mm_hash so the producer can
|
||||
# do a direct (session_id, mm_hash) → ProducerXfer lookup on completion.
|
||||
notif_msg = f"{self.consumer_session_id}:{self.mm_hash}".encode()
|
||||
self.transfer_handle = self.data.post_read(
|
||||
self.block_indices,
|
||||
agent_name,
|
||||
ack.src_block_indices,
|
||||
notif_msg=notif_msg,
|
||||
)
|
||||
self.deadline = time.monotonic() + _CONSUMER_READ_TIMEOUT_S
|
||||
logger.debug(
|
||||
"EC consumer: NIXL READ posted mm_hash=%s agent=%s local_blocks=%d "
|
||||
"remote_blocks=%d",
|
||||
self.mm_hash,
|
||||
agent_name,
|
||||
len(self.block_indices),
|
||||
len(ack.src_block_indices),
|
||||
)
|
||||
return True
|
||||
|
||||
def poll(self, now: float) -> XferState:
|
||||
if self._quarantined:
|
||||
return self._poll_quarantined(now)
|
||||
if self.transfer_handle is None:
|
||||
return (
|
||||
XferState.ACK_TIMEOUT if now > self.deadline else XferState.WAITING_ACK
|
||||
)
|
||||
return self._poll_reading(now)
|
||||
|
||||
def _poll_reading(self, now: float) -> XferState:
|
||||
assert self.transfer_handle is not None
|
||||
try:
|
||||
state = self.data.check_xfer_state(self.transfer_handle)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"EC Connector: check_xfer_state failed for mm_hash=%s", self.mm_hash
|
||||
)
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
return XferState.READ_FAILED
|
||||
if state == "DONE":
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
logger.debug("EC consumer: NIXL READ completed mm_hash=%s", self.mm_hash)
|
||||
return XferState.DONE
|
||||
if state == "PROC":
|
||||
if now > self.deadline:
|
||||
self._quarantined = True
|
||||
self.deadline = now + _CONSUMER_QUARANTINE_TIMEOUT_S
|
||||
logger.warning(
|
||||
"EC Connector: READ for mm_hash=%s from %s:%d timed out; "
|
||||
"quarantining",
|
||||
self.mm_hash,
|
||||
self.addr[0],
|
||||
self.addr[1],
|
||||
)
|
||||
return XferState.QUARANTINED
|
||||
return XferState.READING
|
||||
logger.warning(
|
||||
"EC Connector: READ for mm_hash=%s unexpected NIXL state %r",
|
||||
self.mm_hash,
|
||||
state,
|
||||
)
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
return XferState.READ_FAILED
|
||||
|
||||
def _poll_quarantined(self, now: float) -> XferState:
|
||||
if self.transfer_handle is None:
|
||||
return XferState.SETTLED
|
||||
# If quarantine has been too long, assume NIXL is stuck and evict blocks.
|
||||
if now > self.deadline:
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
logger.warning(
|
||||
"EC Connector: quarantined mm_hash=%s from %s:%d timed out; "
|
||||
"evicting blocks",
|
||||
self.mm_hash,
|
||||
self.addr[0],
|
||||
self.addr[1],
|
||||
)
|
||||
return XferState.READ_FAILED
|
||||
try:
|
||||
state = self.data.check_xfer_state(self.transfer_handle)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"EC Connector: check_xfer_state failed for quarantined mm_hash=%s",
|
||||
self.mm_hash,
|
||||
)
|
||||
state = None
|
||||
if state == "PROC":
|
||||
return XferState.QUARANTINED
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
logger.debug(
|
||||
"EC Connector: quarantined mm_hash=%s settled (state=%s)",
|
||||
self.mm_hash,
|
||||
state,
|
||||
)
|
||||
return XferState.SETTLED
|
||||
|
||||
def cancel(self) -> None:
|
||||
"""Cancel a WAITING_ACK xfer (peer down). No tombstone — caller allows retry."""
|
||||
assert self.transfer_handle is None, "cancel() requires WAITING_ACK state"
|
||||
|
||||
def release(self) -> None:
|
||||
"""Force-release the transfer handle. Shutdown only."""
|
||||
if self.transfer_handle is not None:
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
|
||||
|
||||
# ── ProducerSession ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class ProducerSession:
|
||||
"""Long-lived producer session. Owns ZmqServerTransport.
|
||||
|
||||
poll_step() runs once per engine step: drains the transport without
|
||||
blocking, decodes XferReqs, grants or NACKs each one, drains NIXL completion
|
||||
notifications, and sweeps expired xfers.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
transport: ZmqServerTransport,
|
||||
data: DataTransport,
|
||||
cache: EmbeddingCache,
|
||||
compat_hash: str,
|
||||
) -> None:
|
||||
self._transport = transport
|
||||
self._data = data
|
||||
self._cache = cache # EmbeddingCache — single block owner
|
||||
self._compat_hash = compat_hash
|
||||
|
||||
# Key: "{consumer_session_id}:{mm_hash}" — matches the NIXL notif_msg exactly,
|
||||
# so completion notification lookup is a direct dict pop with no parsing.
|
||||
self._active_xfers: dict[str, ProducerXfer] = {}
|
||||
|
||||
self._req_decoder = msgspec.msgpack.Decoder(XferReq)
|
||||
self._req_encoder = msgspec.msgpack.Encoder()
|
||||
|
||||
def close(self) -> None:
|
||||
for xfer in self._active_xfers.values():
|
||||
self._cache.unpin(xfer.mm_hash) # release any still-held pins
|
||||
self._active_xfers.clear()
|
||||
self._transport.close()
|
||||
|
||||
def poll_step(self) -> None:
|
||||
"""Drain the transport without blocking and process what arrived."""
|
||||
self.poll(self._transport.poll(timeout_ms=0))
|
||||
|
||||
def poll(self, messages: list[tuple[bytes, bytes]]) -> None:
|
||||
"""Decode and process inbound XferReqs, drain NIXL notifs, sweep timeouts.
|
||||
|
||||
messages: (identity, payload) pairs from ZmqServerTransport.poll().
|
||||
"""
|
||||
for identity, payload in messages:
|
||||
try:
|
||||
req = self._req_decoder.decode(payload)
|
||||
except (msgspec.DecodeError, msgspec.ValidationError):
|
||||
logger.warning("EC Connector: dropped malformed XferReq")
|
||||
continue
|
||||
ack = self._grant_or_nack(req)
|
||||
self._transport.send(identity, self._req_encoder.encode(ack))
|
||||
|
||||
self._drain_notifs()
|
||||
self._sweep_timeouts()
|
||||
|
||||
def _grant_or_nack(self, req: XferReq) -> XferAck:
|
||||
if req.connector_version != EC_CONNECTOR_VERSION:
|
||||
logger.warning(
|
||||
"EC Connector: incompatible version for mm_hash=%s", req.mm_hash
|
||||
)
|
||||
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_VERSION)
|
||||
if req.compatibility_hash != self._compat_hash:
|
||||
logger.warning(
|
||||
"EC Connector: incompatible compat hash for mm_hash=%s", req.mm_hash
|
||||
)
|
||||
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INCOMPAT)
|
||||
if not req.session_id:
|
||||
# Grants are keyed by (session_id, mm_hash), so an omitted id from
|
||||
# one peer aliases another's: the first completion would unpin
|
||||
# blocks the other peer is still reading.
|
||||
logger.warning(
|
||||
"EC Connector: XferReq without session_id for mm_hash=%s", req.mm_hash
|
||||
)
|
||||
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INTERNAL)
|
||||
entry = self._cache.pin_if_ready(req.mm_hash)
|
||||
if entry is None or not entry.ready:
|
||||
status = (
|
||||
XferStatus.NACK_MISSING if entry is None else XferStatus.NACK_NOT_READY
|
||||
)
|
||||
logger.log(
|
||||
logging.INFO if status in EXPECTED_NACKS else logging.WARNING,
|
||||
"EC Connector: mm_hash=%s not serveable in local cache (%s); NACKing",
|
||||
req.mm_hash,
|
||||
status.name,
|
||||
)
|
||||
return XferAck(mm_hash=req.mm_hash, status=status)
|
||||
block_indices = list(entry.block_ids)
|
||||
key = f"{req.session_id}:{req.mm_hash}"
|
||||
if key in self._active_xfers:
|
||||
# This peer is re-requesting a grant it still holds, having timed
|
||||
# out waiting. Overwriting it drops the pin taken for it, which
|
||||
# neither a completion nor the sweep would ever release.
|
||||
self._cache.unpin(req.mm_hash)
|
||||
self._active_xfers[key] = ProducerXfer(
|
||||
mm_hash=req.mm_hash,
|
||||
block_indices=block_indices,
|
||||
deadline=time.monotonic() + PRODUCER_PIN_LEASE_S,
|
||||
)
|
||||
logger.debug(
|
||||
"EC producer: serving NIXL READ request mm_hash=%s key=%s blocks=%d",
|
||||
req.mm_hash,
|
||||
key,
|
||||
len(block_indices),
|
||||
)
|
||||
return XferAck(
|
||||
mm_hash=req.mm_hash,
|
||||
status=XferStatus.OK,
|
||||
session_id=req.session_id, # echo consumer's session_id back
|
||||
src_block_indices=block_indices,
|
||||
agent_metadata=self._data.get_agent_metadata(),
|
||||
mem_descriptor=self._data.get_mem_descriptor(),
|
||||
)
|
||||
|
||||
def _drain_notifs(self) -> None:
|
||||
try:
|
||||
notifs = self._data.get_new_notifs()
|
||||
except Exception:
|
||||
logger.exception("EC Connector: get_new_notifs failed")
|
||||
return
|
||||
for msgs in notifs.values():
|
||||
for msg in msgs:
|
||||
try:
|
||||
key = msg.decode("utf-8") # "{session_id}:{mm_hash}"
|
||||
except Exception:
|
||||
logger.exception("EC Connector: Dropped malformed NIXL notif")
|
||||
continue
|
||||
xfer = self._active_xfers.pop(key, None)
|
||||
if xfer is not None:
|
||||
self._cache.unpin(xfer.mm_hash)
|
||||
logger.debug(
|
||||
"EC producer: NIXL READ completed mm_hash=%s key=%s",
|
||||
xfer.mm_hash,
|
||||
key,
|
||||
)
|
||||
|
||||
def _sweep_timeouts(self) -> None:
|
||||
for key, xfer in list(self._active_xfers.items()):
|
||||
if xfer.is_expired():
|
||||
logger.warning(
|
||||
"EC producer: consumer did not complete NIXL READ "
|
||||
"mm_hash=%s key=%s within the pin lease; releasing "
|
||||
"pinned blocks",
|
||||
xfer.mm_hash,
|
||||
key,
|
||||
)
|
||||
self._cache.unpin(xfer.mm_hash)
|
||||
del self._active_xfers[key]
|
||||
|
||||
|
||||
# ── ConsumerSession ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConsumerSessionResults:
|
||||
"""Results from ConsumerSession.take_results()."""
|
||||
|
||||
completed: set[str] # promote to local cache
|
||||
tombstoned: set[str] # free blocks + prevent retry this step
|
||||
quarantined: set[str] # timed-out, DMA still running: prevent retry
|
||||
# but DO NOT free blocks — session holds them
|
||||
retryable: set[str] # free blocks, allow another attempt (not ready, peer down)
|
||||
settled: list[tuple[str, list[int]]] # (mm_hash, block_indices) quarantine cleared
|
||||
|
||||
|
||||
class ConsumerSession:
|
||||
"""Long-lived per-producer session on the consumer side.
|
||||
|
||||
Owns the ZMQ DEALER connection and the NIXL agent registration for one
|
||||
producer peer. The connector calls poll(messages, now) once per engine
|
||||
step with the raw bytes collected from this peer's DEALER socket.
|
||||
|
||||
poll() decodes XferAcks, dispatches them to the right ConsumerXfer,
|
||||
advances all active xfer state machines, and drains quarantined xfers.
|
||||
Results accumulate in internal sets; call take_results() to collect them.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
addr: PeerAddr,
|
||||
zmq_conn: ZmqClientConnection,
|
||||
transport: ZmqClientTransport,
|
||||
data: DataTransport,
|
||||
compat_hash: str,
|
||||
) -> None:
|
||||
self._addr = addr
|
||||
self._zmq = zmq_conn
|
||||
self._transport = transport
|
||||
self._data = data
|
||||
self._compat_hash = compat_hash
|
||||
self._session_id = str(
|
||||
uuid.uuid4()
|
||||
) # stable identity for (session_id:mm_hash) keys
|
||||
|
||||
self._nixl_agent_name: str | None = None
|
||||
self._nixl_metadata_bytes: bytes | None = None
|
||||
|
||||
self._xfers: dict[str, ConsumerXfer] = {} # mm_hash → xfer
|
||||
self._quarantined: list[ConsumerXfer] = [] # DMA pending, to poll
|
||||
self._newly_quarantined: set[str] = set() # quarantined since last take_results
|
||||
|
||||
self._completed: set[str] = set()
|
||||
self._tombstoned: set[str] = set()
|
||||
self._retryable: set[str] = set()
|
||||
self._settled: list[tuple[str, list[int]]] = []
|
||||
|
||||
self._encoder = msgspec.msgpack.Encoder()
|
||||
self._decoder = msgspec.msgpack.Decoder(XferAck)
|
||||
|
||||
@property
|
||||
def addr(self) -> PeerAddr:
|
||||
return self._addr
|
||||
|
||||
def start_xfer(
|
||||
self, mm_hash: str, block_indices: list[int], deadline: float
|
||||
) -> None:
|
||||
"""Create a ConsumerXfer and send the XferReq over the connection."""
|
||||
xfer = ConsumerXfer(
|
||||
mm_hash=mm_hash,
|
||||
block_indices=block_indices,
|
||||
addr=self._addr,
|
||||
deadline=deadline,
|
||||
data=self._data,
|
||||
consumer_session_id=self._session_id,
|
||||
)
|
||||
self._zmq.send(
|
||||
self._encoder.encode(
|
||||
XferReq(
|
||||
mm_hash=mm_hash,
|
||||
compatibility_hash=self._compat_hash,
|
||||
session_id=self._session_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
self._xfers[mm_hash] = xfer
|
||||
logger.debug(
|
||||
"EC Connector: XferReq sent mm_hash=%s to %s:%d",
|
||||
mm_hash,
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
)
|
||||
|
||||
def poll(self, messages: list[bytes], now: float) -> None:
|
||||
"""Decode inbound XferAcks, dispatch to xfers, and advance all state machines.
|
||||
|
||||
messages: raw bytes from ZmqClientTransport.poll() for this peer.
|
||||
Results accumulate until take_results() is called.
|
||||
"""
|
||||
for raw in messages:
|
||||
try:
|
||||
ack = self._decoder.decode(raw)
|
||||
except (msgspec.DecodeError, msgspec.ValidationError):
|
||||
logger.warning(
|
||||
"EC Connector: dropped malformed XferAck from %s:%d",
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
)
|
||||
continue
|
||||
self._dispatch_ack(ack)
|
||||
|
||||
for mm_hash, xfer in list(self._xfers.items()):
|
||||
state = xfer.poll(now)
|
||||
if state == XferState.DONE:
|
||||
self._completed.add(mm_hash)
|
||||
del self._xfers[mm_hash]
|
||||
elif state in (XferState.ACK_TIMEOUT, XferState.READ_FAILED):
|
||||
self._tombstoned.add(mm_hash)
|
||||
del self._xfers[mm_hash]
|
||||
elif state == XferState.QUARANTINED:
|
||||
# DMA may still be writing — caller must NOT free blocks.
|
||||
self._quarantined.append(xfer)
|
||||
self._newly_quarantined.add(mm_hash)
|
||||
del self._xfers[mm_hash]
|
||||
|
||||
self._drain_quarantine(now)
|
||||
|
||||
def _dispatch_ack(self, ack: XferAck) -> None:
|
||||
xfer = self._xfers.get(ack.mm_hash)
|
||||
if xfer is None or xfer.transfer_handle is not None:
|
||||
return # unknown mm_hash or duplicate ack
|
||||
|
||||
if ack.status != XferStatus.OK:
|
||||
logger.log(
|
||||
logging.INFO if ack.status in EXPECTED_NACKS else logging.WARNING,
|
||||
"EC Connector: NACK %s from %s:%d for mm_hash=%s",
|
||||
ack.status.name,
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
ack.mm_hash,
|
||||
)
|
||||
del self._xfers[ack.mm_hash]
|
||||
# The producer announced this encoding and its save is still in
|
||||
# flight, so another attempt on a later step can be granted. Every
|
||||
# other NACK is terminal for this mm_hash.
|
||||
if ack.status in RETRYABLE_NACKS:
|
||||
self._retryable.add(ack.mm_hash)
|
||||
else:
|
||||
self._tombstoned.add(ack.mm_hash)
|
||||
return
|
||||
|
||||
try:
|
||||
agent_name = self._ensure_registered(ack.agent_metadata, ack.mem_descriptor)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"EC Connector: failed to register peer %s:%d for mm_hash=%s",
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
ack.mm_hash,
|
||||
)
|
||||
del self._xfers[ack.mm_hash]
|
||||
self._tombstoned.add(ack.mm_hash)
|
||||
return
|
||||
|
||||
if not xfer.handle_ack(ack, agent_name):
|
||||
del self._xfers[ack.mm_hash]
|
||||
self._tombstoned.add(ack.mm_hash)
|
||||
|
||||
def _ensure_registered(self, metadata: bytes, mem_descriptor: bytes) -> str:
|
||||
"""Register or re-register the peer in DataTransport. Returns agent_name."""
|
||||
if self._nixl_agent_name is not None and self._nixl_metadata_bytes == metadata:
|
||||
return self._nixl_agent_name
|
||||
if self._nixl_agent_name is not None:
|
||||
logger.info(
|
||||
"EC Connector: producer %s:%d restarted; re-registering",
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
)
|
||||
self._data.remove_remote_peer(self._nixl_agent_name)
|
||||
agent_name = self._data.add_remote_peer(metadata, mem_descriptor)
|
||||
self._nixl_agent_name = agent_name
|
||||
self._nixl_metadata_bytes = metadata
|
||||
logger.debug(
|
||||
"EC Connector: registered peer %s:%d agent=%s",
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
agent_name,
|
||||
)
|
||||
return agent_name
|
||||
|
||||
def _drain_quarantine(self, now: float) -> None:
|
||||
still_pending: list[ConsumerXfer] = []
|
||||
for xfer in list(self._quarantined):
|
||||
state = xfer.poll(now)
|
||||
if state == XferState.SETTLED:
|
||||
self._settled.append((xfer.mm_hash, xfer.block_indices))
|
||||
else:
|
||||
still_pending.append(xfer)
|
||||
self._quarantined = still_pending
|
||||
|
||||
def on_peer_down(self) -> None:
|
||||
"""WAITING_ACK xfers → cancel (retry allowed); READING xfers → quarantine."""
|
||||
for mm_hash, xfer in list(self._xfers.items()):
|
||||
if xfer.transfer_handle is None:
|
||||
xfer.cancel()
|
||||
self._retryable.add(mm_hash)
|
||||
else:
|
||||
self._quarantined.append(xfer)
|
||||
self._newly_quarantined.add(mm_hash)
|
||||
self._xfers.clear()
|
||||
|
||||
def take_results(self) -> ConsumerSessionResults:
|
||||
"""Return and clear all accumulated results since the last call."""
|
||||
results = ConsumerSessionResults(
|
||||
completed=self._completed,
|
||||
tombstoned=self._tombstoned,
|
||||
quarantined=self._newly_quarantined,
|
||||
retryable=self._retryable,
|
||||
settled=self._settled,
|
||||
)
|
||||
self._completed = set()
|
||||
self._tombstoned = set()
|
||||
self._newly_quarantined = set()
|
||||
self._retryable = set()
|
||||
self._settled = []
|
||||
return results
|
||||
|
||||
def has_in_flight(self) -> bool:
|
||||
return bool(self._xfers) or bool(self._quarantined)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release all transfer handles, deregister NIXL peer, close connection.
|
||||
|
||||
Connection teardown is routed through the transport so the pooled
|
||||
connection is dropped in lock-step with the session — a session and its
|
||||
DEALER connection have a 1:1 lifecycle, so a later connect() to the same
|
||||
peer builds a fresh socket instead of handing back this dead one.
|
||||
"""
|
||||
for xfer in self._xfers.values():
|
||||
xfer.release()
|
||||
self._xfers.clear()
|
||||
for xfer in self._quarantined:
|
||||
xfer.release()
|
||||
self._quarantined.clear()
|
||||
if self._nixl_agent_name is not None:
|
||||
self._data.remove_remote_peer(self._nixl_agent_name)
|
||||
self._transport.remove(self._addr)
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Supporting utilities for the ECCPUConnector scheduler."""
|
||||
|
||||
import msgspec
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
# (host, port) pair identifying a remote peer's ZMQ side-channel endpoint.
|
||||
PeerAddr = tuple[str, int]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Msgpack (de)serialization for the list-of-(addr, size, device_id)
|
||||
# descriptor form that NIXL's `get_xfer_descs` expects.
|
||||
_MemDescList = list[tuple[int, int, int]]
|
||||
|
||||
_mem_desc_encoder = msgspec.msgpack.Encoder()
|
||||
_mem_desc_decoder = msgspec.msgpack.Decoder(_MemDescList)
|
||||
|
||||
|
||||
def serialize_mem_descriptor(descs: _MemDescList) -> bytes:
|
||||
return _mem_desc_encoder.encode(descs)
|
||||
|
||||
|
||||
def deserialize_mem_descriptor(payload: bytes) -> _MemDescList:
|
||||
return _mem_desc_decoder.decode(payload)
|
||||
|
||||
|
||||
def build_block_descs(
|
||||
base_ptr: int,
|
||||
num_blocks: int,
|
||||
block_size_bytes: int,
|
||||
device_id: int = 0,
|
||||
) -> _MemDescList:
|
||||
"""Per-block `(addr, size, device_id)` tuples for an `ECSharedRegion`.
|
||||
|
||||
Handed to `nixl.get_xfer_descs` to build a dlist where descriptor `i`
|
||||
addresses block `i` in the mmap. This makes a WRITE to block indices
|
||||
`[i1, i2, ...]` a simple `make_prepped_xfer(..., [i1, i2])` on both
|
||||
ends — no per-request address math.
|
||||
"""
|
||||
return [
|
||||
(base_ptr + i * block_size_bytes, block_size_bytes, device_id)
|
||||
for i in range(num_blocks)
|
||||
]
|
||||
@@ -55,7 +55,6 @@ def _coalesce_runs(
|
||||
split. `EmbeddingCache` hands out ascending ids, which is what makes runs
|
||||
findable at all.
|
||||
"""
|
||||
|
||||
ids = np.fromiter(block_ids, dtype=np.int64, count=len(block_ids))
|
||||
breaks = np.flatnonzero(np.diff(ids) != 1) + 1
|
||||
slots = np.concatenate(([0], breaks))
|
||||
|
||||
@@ -5,7 +5,6 @@ from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import safetensors
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
@@ -13,8 +12,11 @@ from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorMetadata,
|
||||
ECConnectorRole,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
PlaceholderMetadataResolver,
|
||||
collect_ec_item_metadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -52,8 +54,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
super().__init__(vllm_config=vllm_config, role=role)
|
||||
# req_id -> index
|
||||
self._mm_datas_need_loads: dict[str, int] = {}
|
||||
self._model_config = vllm_config.model_config
|
||||
self._metadata_fields_cache: dict[str, set[str]] = {}
|
||||
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
|
||||
transfer_config = vllm_config.ec_transfer_config
|
||||
if transfer_config is not None:
|
||||
self._storage_path = transfer_config.get_from_extra_config(
|
||||
@@ -169,36 +170,6 @@ class ECExampleConnector(ECConnectorBase):
|
||||
self._mm_datas_need_loads.clear()
|
||||
return meta
|
||||
|
||||
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
|
||||
"""Which processed keys this model needs published for `modality`.
|
||||
|
||||
Read from `MultiModalDataParser.embedding_fields`, the same declaration
|
||||
the consumer's parser requires, so the two cannot drift. An empty set
|
||||
means the modality cannot be delivered out of band, and the consumer
|
||||
will process the media itself.
|
||||
"""
|
||||
if modality in self._metadata_fields_cache:
|
||||
return self._metadata_fields_cache[modality]
|
||||
|
||||
fields: set[str] = set()
|
||||
try:
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
|
||||
fields = info.data_parser.placeholder_metadata_fields(modality)
|
||||
except Exception:
|
||||
# Reporting nothing is a safe degradation: the consumer falls back to
|
||||
# processing the media itself.
|
||||
logger.warning(
|
||||
"Could not determine the placeholder metadata fields for "
|
||||
"modality %s; the consumer will preprocess the media itself.",
|
||||
modality,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
self._metadata_fields_cache[modality] = fields
|
||||
return fields
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
@@ -214,28 +185,10 @@ class ECExampleConnector(ECConnectorBase):
|
||||
if not self.is_producer:
|
||||
return False, None
|
||||
|
||||
items = []
|
||||
for feature in request.mm_features:
|
||||
metadata = {}
|
||||
# `data` is None for items served from the processor cache, in which
|
||||
# case the metadata is unavailable here and the consumer has to fall
|
||||
# back to processing the media itself.
|
||||
if feature.data is not None:
|
||||
wanted = self._placeholder_metadata_fields(feature.modality)
|
||||
for key, value in feature.data.get_data().items():
|
||||
if key not in wanted:
|
||||
continue
|
||||
if isinstance(value, torch.Tensor):
|
||||
metadata[key] = value.tolist()
|
||||
elif is_list_of(value, (int, float)):
|
||||
# Some metadata (e.g. Qwen3-VL video timestamps) is
|
||||
# produced as a plain list rather than a tensor.
|
||||
metadata[key] = value
|
||||
items.append({"mm_hash": feature.identifier, **metadata})
|
||||
|
||||
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
|
||||
if not items:
|
||||
return False, None
|
||||
return False, {"ec_items": items}
|
||||
return False, items
|
||||
|
||||
# ==============================
|
||||
# Helper functions
|
||||
|
||||
@@ -2,8 +2,88 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""EC connector helper utilities."""
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.v1.outputs import ECConnectorOutput, ModelRunnerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class PlaceholderMetadataResolver:
|
||||
"""Resolves which processed keys a model needs published per modality.
|
||||
|
||||
Reads `MultiModalDataParser.embedding_fields`, the same declaration the
|
||||
consumer's parser requires, so the two cannot drift. An empty set means
|
||||
the modality cannot be delivered out of band, and the consumer will
|
||||
process the media itself.
|
||||
"""
|
||||
|
||||
def __init__(self, model_config: "ModelConfig") -> None:
|
||||
self._model_config = model_config
|
||||
self._cache: dict[str, set[str]] = {}
|
||||
|
||||
def fields_for(self, modality: str) -> set[str]:
|
||||
if modality in self._cache:
|
||||
return self._cache[modality]
|
||||
|
||||
fields: set[str] = set()
|
||||
try:
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
|
||||
fields = info.data_parser.placeholder_metadata_fields(modality)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Could not determine the placeholder metadata fields for "
|
||||
"modality %s; the consumer will preprocess the media itself.",
|
||||
modality,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
self._cache[modality] = fields
|
||||
return fields
|
||||
|
||||
|
||||
def collect_ec_item_metadata(
|
||||
mm_features: "list[MultiModalFeatureSpec]",
|
||||
resolver: PlaceholderMetadataResolver,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Build one `ec_transfer_params` entry per feature for `request_finished()`.
|
||||
|
||||
Keyed by mm_hash, each entry carries a `metadata` dict with whatever
|
||||
placeholder fields `resolver` says this model needs published for its
|
||||
modality, so a consumer can skip the image transform. `data` is None for
|
||||
items served from the processor cache, in which case the metadata is
|
||||
unavailable here and the consumer has to fall back to processing the
|
||||
media itself. A connector that also has transfer coordinates to report
|
||||
(e.g. NIXL peer_host/peer_port/size_bytes) merges those in alongside
|
||||
`metadata`, not into it.
|
||||
"""
|
||||
items: dict[str, dict[str, Any]] = {}
|
||||
for feature in mm_features:
|
||||
metadata: dict[str, Any] = {}
|
||||
if feature.data is not None:
|
||||
wanted = resolver.fields_for(feature.modality)
|
||||
for key, value in feature.data.get_data().items():
|
||||
if key not in wanted:
|
||||
continue
|
||||
if isinstance(value, torch.Tensor):
|
||||
metadata[key] = value.tolist()
|
||||
elif is_list_of(value, (int, float)):
|
||||
# Some metadata (e.g. Qwen3-VL video timestamps) is
|
||||
# produced as a plain list rather than a tensor.
|
||||
metadata[key] = value
|
||||
items[feature.identifier] = {"metadata": metadata}
|
||||
return items
|
||||
|
||||
|
||||
class ECOutputAggregator:
|
||||
"""Merge every worker's EC connector output onto the single
|
||||
|
||||
Reference in New Issue
Block a user