[EC Connector] P2P NIXL + CPU EC Connector (#47941)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
omerpaz95
2026-09-04 11:23:56 +08:00
committed by GitHub
co-authored by Claude Opus 5
parent 69cf055936
commit 8a0a7ee40a
28 changed files with 3521 additions and 72 deletions
@@ -8,7 +8,7 @@ GPU copy plumbing. An ec_both instance reuses encoder outputs it has already
offloaded to CPU instead of recomputing them.
"""
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import torch
@@ -37,6 +37,12 @@ class ECCPUConnector(ECConnectorBase):
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole) -> None:
super().__init__(vllm_config=vllm_config, role=role)
if not vllm_config.use_v2_model_runner:
raise ValueError(
"ECCPUConnector requires the V2 model runner. "
"Set VLLM_USE_V2_MODEL_RUNNER=1."
)
self.connector_worker = None
self.connector_scheduler = None
@@ -108,6 +114,12 @@ class ECCPUConnector(ECConnectorBase):
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def request_finished(
self, request: "Request"
) -> tuple[bool, "dict[str, Any] | None"]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
@@ -0,0 +1,37 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Abstract base classes for the EC control plane.
ControlConnection wraps a single peer channel (one DEALER socket).
"""
from abc import ABC, abstractmethod
class ControlConnection(ABC):
"""Bidirectional message channel to a single remote peer.
All I/O is non-blocking: send enqueues without blocking; recv drains
all buffered inbound messages since the last call.
"""
@property
@abstractmethod
def alive(self) -> bool:
"""True if the connection is still usable."""
@abstractmethod
def send(self, msg: bytes) -> None:
"""Enqueue msg for delivery. Must not block."""
@abstractmethod
def recv(self) -> list[bytes]:
"""Drain and return all buffered inbound messages."""
@abstractmethod
def mark_dead(self) -> None:
"""Signal that the peer disconnected. Sets alive to False."""
@abstractmethod
def close(self) -> None:
"""Release all resources. Idempotent."""
@@ -0,0 +1,233 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ZMQ control-plane implementations for ECCPUConnector.
ZmqClientConnection — one DEALER socket to a single producer peer.
ZmqClientTransport — pool of ZmqClientConnection objects (consumer side).
ZmqServerTransport — ROUTER socket (producer side).
"""
import contextlib
from dataclasses import dataclass, field
import zmq
from zmq.utils.monitor import recv_monitor_message
from vllm.distributed.ec_transfer.ec_connector.cpu.control.base import (
ControlConnection,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr
from vllm.logger import init_logger
from vllm.utils.network_utils import make_zmq_path, make_zmq_socket
logger = init_logger(__name__)
_HEARTBEAT_IVL_MS = 2000
_HEARTBEAT_TIMEOUT_MS = 4000
_HEARTBEAT_TTL_MS = 8000
# ── Client side ───────────────────────────────────────────────────────────────
@dataclass
class ZmqClientConnection(ControlConnection):
"""One DEALER socket to one producer peer.
Owns the socket and its optional monitor socket. NIXL state is NOT
stored here — it belongs in ECSession above this layer.
"""
dealer: zmq.Socket
monitor: zmq.Socket | None
_alive: bool = field(default=True, init=False)
@property
def alive(self) -> bool:
return self._alive
def send(self, msg: bytes) -> None:
self.dealer.send_multipart([b"", msg])
def recv(self) -> list[bytes]:
msgs: list[bytes] = []
while True:
try:
frames = self.dealer.recv_multipart(flags=zmq.NOBLOCK)
except zmq.Again:
break
except Exception:
logger.exception("ec: DEALER recv failed")
break
if len(frames) != 2 or frames[0] != b"":
logger.warning(
"ec: dropped malformed frame envelope (%d frames)", len(frames)
)
continue
msgs.append(frames[1])
return msgs
def mark_dead(self) -> None:
self._alive = False
def close(self) -> None:
if self.monitor is not None:
with contextlib.suppress(Exception):
self.dealer.disable_monitor()
with contextlib.suppress(Exception):
self.monitor.close(linger=0)
with contextlib.suppress(Exception):
self.dealer.close(linger=0)
self._alive = False
class ZmqClientTransport:
"""Pool of ZmqClientConnection objects — one per producer peer.
Creates DEALER sockets lazily on first contact. Does not touch NIXL;
NIXL registration is handled by ECCPUConsumer once an XferAck arrives.
"""
def __init__(self) -> None:
self._ctx = zmq.Context()
self._connections: dict[PeerAddr, ZmqClientConnection] = {}
def connect(self, addr: PeerAddr) -> ZmqClientConnection:
"""Return the existing connection for addr, or create a new one."""
existing = self._connections.get(addr)
if existing is not None:
return existing
host, port = addr
dealer = make_zmq_socket(
ctx=self._ctx,
path=make_zmq_path(scheme="tcp", host=host, port=port),
socket_type=zmq.DEALER,
bind=False,
)
dealer.setsockopt(zmq.HEARTBEAT_IVL, _HEARTBEAT_IVL_MS)
dealer.setsockopt(zmq.HEARTBEAT_TIMEOUT, _HEARTBEAT_TIMEOUT_MS)
dealer.setsockopt(zmq.HEARTBEAT_TTL, _HEARTBEAT_TTL_MS)
monitor_addr = f"inproc://ec-peer-mon-{host}-{port}"
dealer.monitor(monitor_addr, zmq.EVENT_DISCONNECTED)
monitor = self._ctx.socket(zmq.PAIR)
monitor.connect(monitor_addr)
conn = ZmqClientConnection(dealer=dealer, monitor=monitor)
self._connections[addr] = conn
return conn
def poll(self) -> dict[PeerAddr, list[bytes]]:
"""Non-blocking drain of messages from every peer's DEALER."""
msgs: dict[PeerAddr, list[bytes]] = {}
for addr, conn in list(self._connections.items()):
conn_msgs = conn.recv()
msgs[addr] = conn_msgs
return msgs
def poll_dead(self) -> list[PeerAddr]:
"""Non-blocking sweep of peer monitors; return addresses that died."""
dead: list[PeerAddr] = []
for addr, conn in list(self._connections.items()):
if conn.monitor is None:
continue
try:
while True:
evt = recv_monitor_message(conn.monitor, flags=zmq.NOBLOCK)
if evt["event"] == zmq.EVENT_DISCONNECTED:
conn.mark_dead()
dead.append(addr)
break
except zmq.Again:
pass
except Exception:
logger.warning("ec: monitor poll failed for %s", addr, exc_info=True)
return dead
def remove(self, addr: PeerAddr) -> ZmqClientConnection | None:
"""Remove and close the connection for addr, returning it (or None)."""
conn = self._connections.pop(addr, None)
if conn is not None:
conn.close()
return conn
def close(self) -> None:
for conn in list(self._connections.values()):
conn.close()
self._connections.clear()
try:
self._ctx.destroy(linger=0)
except Exception:
logger.debug("ec: zmq context destroy failed", exc_info=True)
# ── Server side ───────────────────────────────────────────────────────────────
class ZmqServerTransport:
"""ROUTER socket for the producer side. Byte-level only, driven by
ProducerSession.
poll(timeout_ms) waits up to timeout_ms for messages, then drains all that
are available; timeout_ms=0 drains whatever has already arrived. send()
routes a reply back to a specific peer by ZMQ identity.
"""
def __init__(self, host: str, port: int) -> None:
self._ctx = zmq.Context()
self._router = make_zmq_socket(
ctx=self._ctx,
path=make_zmq_path(scheme="tcp", host=host, port=port),
socket_type=zmq.ROUTER,
bind=True,
)
self._poller = zmq.Poller()
self._poller.register(self._router, zmq.POLLIN)
def poll(self, timeout_ms: int = 0) -> list[tuple[bytes, bytes]]:
"""Wait up to timeout_ms for incoming messages, then drain all pending.
Returns a list of (identity, payload) pairs.
"""
if timeout_ms > 0:
try:
events = dict(self._poller.poll(timeout=timeout_ms))
except zmq.ContextTerminated:
return []
except Exception:
logger.exception("ec: router poller failed")
return []
if self._router not in events:
return []
msgs: list[tuple[bytes, bytes]] = []
while True:
try:
identity, _, payload = self._router.recv_multipart(flags=zmq.NOBLOCK)
msgs.append((identity, payload))
except zmq.Again:
break
except zmq.ContextTerminated:
break
except Exception:
logger.exception("ec: router recv failed")
break
return msgs
def send(self, identity: bytes, payload: bytes) -> None:
"""Send a reply payload to the peer identified by ZMQ identity."""
try:
self._router.send_multipart([identity, b"", payload])
except Exception:
logger.exception("ec: router send failed")
def close(self) -> None:
try:
self._router.close(linger=0)
except Exception:
logger.debug("ec: router close failed", exc_info=True)
try:
self._ctx.destroy(linger=0)
except Exception:
logger.debug("ec: zmq context destroy failed", exc_info=True)
@@ -0,0 +1,67 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Abstract base class for the EC data-plane transport (NIXL/UCX)."""
from abc import ABC, abstractmethod
from typing import Any
class DataTransport(ABC):
"""Interface wrapping the NIXL data-plane for EC block transfers.
Peer lifecycle:
add_remote_peer() — register a peer and store its dlist handle internally
remove_remote_peer() — deregister and release the stored handle
post_read() — issue a READ using the stored handle for agent_name
The dlist handle (prepped xfer descriptor list) is an implementation detail
of the transport and is never exposed to callers.
"""
@abstractmethod
def get_agent_metadata(self) -> bytes:
"""Return this agent's serialized NIXL metadata blob."""
@abstractmethod
def get_mem_descriptor(self) -> bytes:
"""Return the msgpack-encoded block descriptor list for this agent."""
@abstractmethod
def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str:
"""Register a remote peer, prep its READ-source dlist, and return agent_name.
The dlist handle is stored internally and used by post_read().
"""
@abstractmethod
def remove_remote_peer(self, agent_name: str) -> None:
"""Deregister a previously registered remote peer and release its handle."""
@abstractmethod
def post_read(
self,
local_indices: list[int],
agent_name: str,
remote_indices: list[int],
notif_msg: bytes,
) -> Any:
"""Issue a consumer-initiated READ using the stored dlist for agent_name.
Returns a transfer handle for use with check_xfer_state / release_xfer_handle.
"""
@abstractmethod
def get_new_notifs(self) -> dict[str, list[bytes]]:
"""Drain NIXL completion notifications addressed to this agent."""
@abstractmethod
def check_xfer_state(self, handle: Any) -> str:
"""Return the NIXL state string: 'DONE', 'PROC', or an error string."""
@abstractmethod
def release_xfer_handle(self, handle: Any) -> None:
"""Release a transfer handle returned by post_read."""
@abstractmethod
def deregister(self) -> None:
"""Deregister all memory and tear down the NIXL agent."""
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NIXL-backed data-plane transport for ECCPUConnector."""
from typing import Any
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
DataTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
build_block_descs,
deserialize_mem_descriptor,
serialize_mem_descriptor,
)
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
from vllm.logger import init_logger
logger = init_logger(__name__)
_NIXL_DRAM = "DRAM"
class NixlDataTransport(DataTransport):
"""Adapts NixlWrapper to the DataTransport interface.
Dlist handles (prepped xfer descriptor lists for remote peers) are stored
internally keyed by agent_name. Callers never touch raw NIXL handles —
post_read() resolves the handle from agent_name internally.
NixlWrapper is not thread-safe, and neither is this class. Every call site
runs on the scheduler thread, which serializes the transport by
construction; transfers are async inside NIXL and are drained from that
same thread via get_new_notifs() / check_xfer_state().
"""
def __init__(
self,
agent_name: str,
base_ptr: int,
num_blocks: int,
block_size_bytes: int,
total_size_bytes: int,
num_threads: int = 1,
) -> None:
if NixlWrapper is None or nixl_agent_config is None:
raise RuntimeError(
"ECCPUConnector requires NIXL; "
"install the `nixl` package or set a different ec_connector."
)
# agent_name → remote dlist handle (from add_remote_peer, used by post_read)
self._peer_handles: dict[str, int] = {}
# None until the local region is registered, so deregister() is safe on
# a transport whose registration failed part-way.
self._reg_descs: Any = None
self._nixl = NixlWrapper(
agent_name,
nixl_agent_config(num_threads=num_threads, capture_telemetry=True),
)
self._register_local_memory(
base_ptr, num_blocks, block_size_bytes, total_size_bytes
)
def _register_local_memory(
self,
base_ptr: int,
num_blocks: int,
block_size_bytes: int,
total_size_bytes: int,
) -> None:
"""Register the local region with NIXL and prep its transfer dlist.
Unwinds the registration if a later step fails: __init__ propagates the
error, so the caller never receives an object it could deregister with.
"""
block_descs = build_block_descs(base_ptr, num_blocks, block_size_bytes)
self._reg_descs = self._nixl.get_reg_descs(
[(base_ptr, total_size_bytes, 0, "")], _NIXL_DRAM
)
self._nixl.register_memory(self._reg_descs, backends=["UCX"])
try:
xfer_descs = self._nixl.get_xfer_descs(block_descs, _NIXL_DRAM)
self._local_xfer_handle = self._nixl.prep_xfer_dlist(
"NIXL_INIT_AGENT", xfer_descs
)
self._agent_metadata = self._nixl.get_agent_metadata()
self._mem_descriptor_bytes = serialize_mem_descriptor(block_descs)
except Exception:
self.deregister()
raise
def get_agent_metadata(self) -> bytes:
return self._agent_metadata
def get_mem_descriptor(self) -> bytes:
return self._mem_descriptor_bytes
def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str:
"""Register remote peer, prep its dlist, store handle internally.
Returns agent_name for use in post_read() and remove_remote_peer().
"""
agent_name = self._nixl.add_remote_agent(metadata)
remote_blocks = deserialize_mem_descriptor(mem_descriptor)
remote_xfer_descs = self._nixl.get_xfer_descs(remote_blocks, _NIXL_DRAM)
self._peer_handles[agent_name] = self._nixl.prep_xfer_dlist(
agent_name, remote_xfer_descs
)
return agent_name
def remove_remote_peer(self, agent_name: str) -> None:
self._peer_handles.pop(agent_name, None)
try:
self._nixl.remove_remote_agent(agent_name)
except Exception:
logger.warning(
"EC: remove_remote_peer failed for %s", agent_name, exc_info=True
)
def post_read(
self,
local_indices: list[int],
agent_name: str,
remote_indices: list[int],
notif_msg: bytes,
) -> Any:
if len(local_indices) != len(remote_indices):
raise ValueError(
f"EC: local/remote block count mismatch "
f"({len(local_indices)} vs {len(remote_indices)})"
)
remote_handle = self._peer_handles[agent_name]
handle = self._nixl.make_prepped_xfer(
"READ",
self._local_xfer_handle,
local_indices,
remote_handle,
remote_indices,
notif_msg=notif_msg,
)
self._nixl.transfer(handle)
return handle
def get_new_notifs(self) -> dict[str, list[bytes]]:
return self._nixl.get_new_notifs()
def check_xfer_state(self, handle: Any) -> str:
return self._nixl.check_xfer_state(handle)
def release_xfer_handle(self, handle: Any) -> None:
try:
self._nixl.release_xfer_handle(handle)
except Exception:
logger.warning("EC: release_xfer_handle failed", exc_info=True)
def deregister(self) -> None:
if self._reg_descs is None:
return
try:
self._nixl.deregister_memory(self._reg_descs)
except Exception:
logger.warning("EC: deregister failed", exc_info=True)
self._reg_descs = None
@@ -0,0 +1,91 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scheduler-side wire types for the ECCPUConnector.
Two peer-to-peer message types exchanged between consumer and producer over
ZMQ (XferReq, XferAck) plus a peer-compatibility fingerprint
(compute_ec_compatibility_hash) sent on every XferReq so a producer refuses
to serve a mismatched peer.
"""
import enum
import hashlib
import json
import msgspec
# Bump when the on-wire shape (XferReq/XferAck) changes in a
# backward-incompatible way. Peers reject mismatched versions.
EC_CONNECTOR_VERSION: int = 1
class XferStatus(enum.IntEnum):
"""Outcome the producer reports for an XferReq."""
OK = 0
NACK_MISSING = 1 # producer no longer holds the encoding
NACK_INCOMPAT = 2 # peer-compatibility hash mismatch
NACK_VERSION = 3 # connector wire-version mismatch
NACK_INTERNAL = 4 # producer hit an unexpected error
NACK_NOT_READY = 5 # announced, but its GPU->mmap save has not landed yet
# The producer expects a retryable condition to clear on its own, so the
# consumer re-requests the read rather than giving up on the encoding.
RETRYABLE_NACKS = frozenset((XferStatus.NACK_NOT_READY,))
# Ordinary outcomes rather than signs of a broken peer.
EXPECTED_NACKS = RETRYABLE_NACKS | {XferStatus.NACK_MISSING}
class XferReq( # type: ignore[call-arg]
msgspec.Struct,
tag="req",
omit_defaults=True,
):
"""Consumer → Producer: I want to READ mm_hash; pin it and tell me where."""
mm_hash: str
compatibility_hash: str
# Consumer's session identity. Together with mm_hash it uniquely identifies
# this transfer on both sides: the producer keys its active xfers by
# (session_id, mm_hash), and the NIXL notif_msg carries the same pair.
session_id: str = ""
connector_version: int = EC_CONNECTOR_VERSION
class XferAck( # type: ignore[call-arg]
msgspec.Struct,
tag="ack",
omit_defaults=True,
):
"""Producer → Consumer: response to an XferReq.
On status == OK: producer has pinned source blocks and returns their
indices plus fresh NIXL metadata for the consumer to register and READ.
On any NACK_*: optional fields are empty; consumer falls back to local encode.
"""
mm_hash: str
status: XferStatus
# Consumer's session_id echoed back from XferReq. Together with mm_hash it
# identifies the exact transfer; the consumer uses both to build notif_msg.
session_id: str = ""
src_block_indices: list[int] = []
agent_metadata: bytes = b""
mem_descriptor: bytes = b""
def compute_ec_compatibility_hash(
vllm_version: str,
model: str,
dtype: str,
block_size_bytes: int,
) -> str:
"""Peer-compatibility fingerprint over the four factors that determine
byte-layout compatibility. Producer NACKs any XferReq whose hash differs.
"""
factors = json.dumps(
[vllm_version, model, dtype, block_size_bytes],
separators=(",", ":"),
)
return hashlib.sha256(factors.encode("utf-8")).hexdigest()
@@ -7,16 +7,23 @@ Owns the mmap region and the embedding cache, and handles the producer
for the ECCPUConnector.
"""
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import torch
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
ECCPUWorkerMetadata,
_get_encoder_cache_hidden_dim,
create_ec_shared_region,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
@@ -40,6 +47,7 @@ class ECCPUScheduler:
self._region = create_ec_shared_region(vllm_config)
# Block allocator + LRU eviction policy for the shared region.
self._cache = EmbeddingCache(self._region.num_blocks)
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
# mm_hash → block IDs allocated this step for GPU→mmap saves.
self._pending_saves: dict[str, list[int]] = {}
@@ -63,6 +71,112 @@ class ECCPUScheduler:
else 1
)
self._ec_config = ec_config
# NIXL p2p is an option of this connector, not engine-wide behavior, so
# it lives in extra config. Extra config is not type coerced, so a value
# that arrived as a JSON/CLI string is parsed, not truth-tested.
raw_nixl = ec_config.get_from_extra_config("ec_enable_nixl", False)
self._nixl_enabled = (
raw_nixl
if isinstance(raw_nixl, bool)
else str(raw_nixl).strip().lower() in ("true", "1", "yes")
)
# NIXL fields default to None/empty so the gate-off path is untouched.
self._data: Any = None
self._compat_hash: str | None = None
self._first_in_batch = True
self._transport: Any = None
self._producer_session: Any = None
self._sessions: dict = {}
self._in_flight: set[str] = set()
self._tombstones: set[str] = set()
self._step_completed: set[str] = set()
self._peer_host: str | None = None
self._peer_port: int | None = None
# Model shape for size checks + compat hash; only set by
# _setup_nixl (the gate-off path never touches model_config).
self._dtype: torch.dtype | None = None
self._hidden_dim: int = 0
self._element_size: int = 0
self._ack_timeout_s: float = 0.0
if self._nixl_enabled:
self._setup_nixl(vllm_config)
def _setup_nixl(self, vllm_config: "VllmConfig") -> None:
# Lazy imports keep nixl/zmq off the gate-off path.
from vllm import envs
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import (
ZmqClientTransport,
ZmqServerTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
NixlDataTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
compute_ec_compatibility_hash,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
ProducerSession,
)
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
from vllm.version import __version__ as VLLM_VERSION
if NixlWrapper is None or nixl_agent_config is None:
raise RuntimeError(
"ec_enable_nixl requires NIXL; install the `nixl` package or "
"remove ec_enable_nixl from ec_connector_extra_config."
)
engine_id = self._ec_config.engine_id
assert engine_id is not None
self._dtype = vllm_config.model_config.dtype
self._hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
self._element_size = torch.empty(0, dtype=self._dtype).element_size()
# How long a consumer waits for an XferAck. The producer answers from
# its scheduler step, so its reply latency scales with the encoder's
# batch size: a deployment whose steps run longer must raise this.
self._ack_timeout_s = float(
self._ec_config.get_from_extra_config("consumer_ack_timeout_s", 2.0)
)
self._compat_hash = compute_ec_compatibility_hash(
vllm_version=VLLM_VERSION,
model=str(vllm_config.model_config.model),
dtype=str(self._dtype),
block_size_bytes=self._region.block_size_bytes,
)
if self._is_producer:
self._peer_host = envs.VLLM_EC_SIDE_CHANNEL_HOST
self._peer_port = envs.VLLM_EC_SIDE_CHANNEL_PORT
# Registering the region with NIXL and binding the control sockets are
# the first side effects here. __init__ propagates a failure, so the
# caller never receives a scheduler it could shut down: unwind through
# the same teardown shutdown() uses.
try:
self._data = NixlDataTransport(
agent_name=engine_id,
base_ptr=self._region.blocks.data_ptr(),
num_blocks=self._region.num_blocks,
block_size_bytes=self._region.block_size_bytes,
total_size_bytes=self._region.num_blocks
* self._region.block_size_bytes,
)
if self._is_producer:
assert self._peer_host is not None
assert self._peer_port is not None
self._producer_session = ProducerSession(
transport=ZmqServerTransport(
host=self._peer_host, port=self._peer_port
),
data=self._data,
cache=self._cache,
compat_hash=self._compat_hash,
)
if self._is_consumer:
self._transport = ZmqClientTransport()
except Exception:
self._teardown_nixl()
raise
def has_cache_item(self, identifier: str) -> bool:
if not self._is_consumer:
return False
@@ -72,8 +186,202 @@ class ECCPUScheduler:
def ensure_cache_available(
self, request: "Request", num_computed_tokens: int
) -> bool:
if not self._nixl_enabled:
return True # CPU offload never blocks.
first = self._first_in_batch
self._first_in_batch = False
if not self._is_consumer:
return True
if first:
self._poll_step()
return self._nixl_consumer_admit(request, num_computed_tokens)
def _nixl_consumer_admit(
self, request: "Request", num_computed_tokens: int
) -> bool:
"""Admit a request only once all its remote encodings are cached.
Returns True when every encoding the request needs is either already
ready in the local cache or has no remote source. Any encoding whose
NIXL READ is still in flight (or was just started this step) defers
the request, which the scheduler re-presents on a later step.
"""
params: dict[str, dict[str, Any]] = (
getattr(request, "ec_transfer_params", None) or {}
)
if not params:
return True
pending = False
for feature in request.mm_features:
pos = feature.mm_position
if pos.offset + pos.length <= num_computed_tokens:
continue
mm_hash = feature.identifier
entry = self._cache.get(mm_hash)
if entry is not None and entry.ready:
# Local hit: upstream's update_state_after_alloc pins and
# loads it through the same path as a natively cached entry.
continue
if mm_hash in self._in_flight:
pending = True
continue
if mm_hash in self._step_completed:
pending = True
continue
if mm_hash in self._tombstones:
self._tombstones.discard(mm_hash)
continue
if entry is not None:
# Present but not ready and not being fetched: its blocks are
# held by a quarantined/settling DMA and cannot be reused.
# Fall back to local compute this step; never re-alloc a
# mm_hash already in the cache.
continue
info = params.get(mm_hash)
# Absent entirely, or present with only placeholder metadata (no
# cache entry existed on the producer side at request_finished
# time): nothing to fetch, fall back to local compute.
if not info or "peer_host" not in info:
continue
expected = pos.length * self._hidden_dim * self._element_size
if int(info.get("size_bytes", -1)) != expected:
logger.warning(
"EC consumer: size mismatch mm_hash=%s; local encode", mm_hash
)
continue
try:
started = self._start_xfer(mm_hash, info, expected)
except Exception:
logger.exception(
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
)
continue
if not started:
continue
self._in_flight.add(mm_hash)
pending = True
return not pending
def _start_xfer(
self, mm_hash: str, info: "dict[str, Any]", size_bytes: int
) -> bool:
"""Allocate a not-ready cache entry and start a NIXL READ into it.
Returns True if the transfer was started. Returns False when the
cache cannot accommodate the encoding, in which case the request
falls back to local recomputation.
"""
import time
from math import ceil
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
ConsumerSession,
)
n_blocks = max(1, ceil(size_bytes / self._region.block_size_bytes))
entry = self._cache.alloc(mm_hash, n_blocks)
if entry is None:
logger.debug(
"EC consumer: cache full for mm_hash=%s (%d blocks); local encode",
mm_hash,
n_blocks,
)
return False
indices = list(entry.block_ids)
addr = (info["peer_host"], int(info["peer_port"]))
if addr not in self._sessions:
zmq_conn = self._transport.connect(addr)
assert self._compat_hash is not None
self._sessions[addr] = ConsumerSession(
addr=addr,
zmq_conn=zmq_conn,
transport=self._transport,
data=self._data,
compat_hash=self._compat_hash,
)
deadline = time.monotonic() + self._ack_timeout_s
try:
self._sessions[addr].start_xfer(mm_hash, indices, deadline)
except Exception:
self._cache.discard(mm_hash)
raise
logger.debug(
"EC consumer: starting NIXL xfer mm_hash=%s from %s:%d blocks=%d",
mm_hash,
addr[0],
addr[1],
n_blocks,
)
return True
def _poll_step(self) -> None:
import time
now = time.monotonic()
all_messages = self._transport.poll()
for addr, session in list(self._sessions.items()):
session.poll(all_messages.get(addr, []), now)
for addr in self._transport.poll_dead():
self._on_peer_down(addr)
for session in self._sessions.values():
self._process_session_results(session)
def _process_session_results(self, session: Any) -> None:
r = session.take_results()
for mm_hash in r.completed:
self._in_flight.discard(mm_hash)
self._cache.mark_ready(mm_hash)
self._step_completed.add(mm_hash)
logger.debug("EC consumer: NIXL xfer complete mm_hash=%s", mm_hash)
for mm_hash in r.tombstoned:
self._in_flight.discard(mm_hash)
self._cache.discard(mm_hash)
self._tombstones.add(mm_hash)
logger.debug("EC consumer: NIXL xfer failed mm_hash=%s", mm_hash)
for mm_hash in r.quarantined:
# DMA still running: keep the blocks reserved (entry stays
# not-ready, hence non-evictable) until the xfer settles.
self._in_flight.discard(mm_hash)
self._tombstones.add(mm_hash)
logger.debug(
"EC consumer: NIXL xfer mm_hash=%s quarantined; DMA still running",
mm_hash,
)
for mm_hash in r.retryable:
# No tombstone: dropping every trace of the attempt is what lets
# the next admit pass re-issue the read as if it were the first.
self._in_flight.discard(mm_hash)
self._cache.discard(mm_hash)
logger.debug(
"EC consumer: NIXL xfer mm_hash=%s retryable; will re-request",
mm_hash,
)
for mm_hash, _block_indices in r.settled:
self._cache.discard(mm_hash)
logger.debug(
"EC consumer: quarantined NIXL xfer mm_hash=%s settled", mm_hash
)
def _on_peer_down(self, addr: Any) -> None:
session = self._sessions.pop(addr, None)
if session is None:
return
session.on_peer_down()
self._process_session_results(session)
session.close()
logger.info("EC consumer: producer peer down addr=%s", addr)
def _promote_completed_reads(self) -> None:
"""Drain the just-completed set built during ``_poll_step``.
Reads are marked ready in ``_process_session_results``, so a deferred
request is re-admitted by ``ensure_cache_available`` and loaded through
the same local path as a natively cached entry
(``update_state_after_alloc`` -> ``_pending_loads``). No explicit
pin/load bookkeeping is needed here beyond clearing the set.
"""
self._step_completed.clear()
def update_state_after_alloc(self, request: "Request", index: int) -> None:
feature = request.mm_features[index]
mm_hash = feature.identifier
@@ -97,11 +405,17 @@ class ECCPUScheduler:
) -> ECCPUConnectorMetadata:
meta = ECCPUConnectorMetadata()
if self._is_producer:
if self._nixl_enabled and self._producer_session is not None:
self._producer_session.poll_step()
meta.saves = self._pending_saves
self._pending_saves = {}
if self._is_consumer:
if self._nixl_enabled:
self._promote_completed_reads()
meta.loads = self._pending_loads
self._pending_loads = {}
if self._nixl_enabled:
self._first_in_batch = True
return meta
def update_connector_output(self, connector_output: "ECConnectorOutput") -> None:
@@ -117,8 +431,15 @@ class ECCPUScheduler:
return
for mm_hash in meta.completed_saves:
entry = self._cache.get(mm_hash)
if entry is not None and not entry.ready:
if entry is None:
logger.debug(
"EC producer: worker reported completed save for unknown "
"mm_hash=%s (already discarded/evicted?)",
mm_hash,
)
elif not entry.ready:
self._cache.mark_ready(mm_hash)
logger.debug("EC producer: mm_hash=%s marked ready", mm_hash)
for transfer_id in meta.completed_loads:
pending = self._load_acks.get(transfer_id)
if pending is None:
@@ -131,14 +452,65 @@ class ECCPUScheduler:
# as stale rather than releasing the pin a second time.
del self._load_acks[transfer_id]
self._cache.unpin(mm_hash)
logger.debug("EC consumer: mm_hash=%s unpinned after load", mm_hash)
def has_pending_push_work(self) -> bool:
"""True while any dispatched save or load has not been confirmed done.
"""Keep the engine stepping so this connector's polls keep running.
Keeps the engine stepping so the worker's completion reports are polled
even when no requests are otherwise runnable.
With NIXL enabled the engine tick is the only driver of
``ProducerSession.poll_step()``, and a producer's work arrives from a
remote consumer, so nothing local would otherwise wake it. Without NIXL
this only has to outlive dispatched saves and loads, so the engine can
quiesce once the worker has confirmed them.
"""
return self._cache.has_held_entries()
return self._nixl_enabled or self._cache.has_held_entries()
def request_finished(
self, request: "Request"
) -> tuple[bool, "dict[str, Any] | None"]:
if not (self._nixl_enabled and self._is_producer):
return False, None
if not request.mm_features:
return False, None
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
for feature in request.mm_features:
mm_hash = feature.identifier
entry = self._cache.get(mm_hash)
if entry is None:
# Never saved, or evicted since. Publishing placeholder
# metadata without an address invites an orchestrator to
# rewrite the media into a reference to an encoding no
# consumer can fetch, leaving the decoder nothing to embed.
# Publish neither, so the media stays on the request.
items[mm_hash]["metadata"] = {}
logger.debug(
"EC producer: mm_hash=%s absent at request_finished; "
"announcing no metadata so the media is not rewritten away",
mm_hash,
)
continue
# Announce even if the save's GPU->mmap copy hasn't been
# confirmed complete yet: a not-ready entry can't be evicted, so
# it will still be here by the time a consumer's XferReq arrives.
# A read arriving before the save lands is NACKed NACK_NOT_READY,
# which the consumer retries on a later step rather than treating
# as a miss.
size_bytes = (
feature.mm_position.length * self._hidden_dim * self._element_size
)
items[mm_hash].update(
peer_host=self._peer_host,
peer_port=self._peer_port,
size_bytes=size_bytes,
)
logger.debug(
"EC producer: announcing NIXL-readable encodings req_id=%s items=%s",
request.request_id,
items,
)
return False, items
def shutdown(self) -> None:
self._pending_saves.clear()
@@ -148,7 +520,45 @@ class ECCPUScheduler:
self._is_producer = False
self._is_consumer = False
if self._nixl_enabled:
self._teardown_nixl()
try:
self._region.cleanup()
except Exception:
logger.debug("ec: region cleanup failed", exc_info=True)
def _teardown_nixl(self) -> None:
"""Close the control sockets and release the NIXL agent.
Guards each step and clears what it releases, so it is safe on a
scheduler whose NIXL setup failed part-way and safe to call twice.
The shared region is not touched — shutdown() owns that.
"""
if self._producer_session is not None:
try:
self._producer_session.close()
except Exception:
logger.debug("ec: producer session close failed", exc_info=True)
self._producer_session = None
for session in list(self._sessions.values()):
try:
session.close()
except Exception:
logger.debug("ec: consumer session close failed", exc_info=True)
self._sessions.clear()
if self._transport is not None:
try:
self._transport.close()
except Exception:
logger.debug("ec: client transport close failed", exc_info=True)
self._transport = None
if self._data is not None:
try:
self._data.deregister()
except Exception:
logger.debug("ec: deregister failed", exc_info=True)
self._data = None
@@ -122,6 +122,23 @@ class EmbeddingCache:
self._evictable_block_count -= len(entry.block_ids)
entry.pin()
def pin_if_ready(self, key: str) -> CacheEntry | None:
"""Atomically pin *key* if present and ready; return its entry.
Returns None when the key is absent. An entry returned with `ready`
False was not pinned: its save is still in flight, which the producer
reports differently from a miss because such a read can be retried.
"""
with self._lock:
entry = self._entries.get(key)
if entry is None or not entry.ready:
return entry
if entry.evictable:
del self._entries_free_list[key]
self._evictable_block_count -= len(entry.block_ids)
entry.pin()
return entry
def unpin(self, key: str) -> None:
"""Unpin an entry. Asserts currently pinned."""
with self._lock:
@@ -134,6 +151,21 @@ class EmbeddingCache:
self._entries_free_list[key] = None
self._evictable_block_count += len(entry.block_ids)
def discard(self, key: str) -> None:
"""Remove a not-ready in-flight entry, returning its blocks to the pool.
Used when an in-flight fill (e.g. a NIXL READ) fails before the entry
is marked ready. Asserts the entry is present and not ready — ready or
pinned entries are reclaimed through eviction/unpin, not discard.
"""
with self._lock:
entry = self._entries[key]
assert entry._pin_count == -1, (
f"EmbeddingCache: discard of ready/pinned entry {key!r}"
)
del self._entries[key]
self._free_blocks.extend(entry.block_ids)
def _evict_until(self, n_blocks: int) -> None:
"""Evict ready+unpinned entries FIFO until enough space. Lock held."""
while len(self._free_blocks) < n_blocks:
@@ -0,0 +1,633 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Session objects for the ECCPUConnector.
Layer hierarchy: connector → session → transport (connection)
Two long-lived sessions:
ProducerSession — owns ZmqServerTransport. The connector calls poll_step()
once per engine step, which drains the transport without blocking and
feeds the raw bytes to poll(). poll() decodes XferReqs, grants or NACKs
them, pins source blocks, sends XferAck replies, drains NIXL completion
notifications, and sweeps expired xfers.
ConsumerSession — owns one ZmqClientConnection and the NIXL agent
registration for that peer. The connector calls poll(messages, now)
once per engine step with the raw bytes collected for this peer.
poll() decodes XferAcks, dispatches them to ConsumerXfer objects,
and advances all active xfer state machines.
Internal data classes:
ProducerXfer — one pinned read grant, keyed by session_id.
ConsumerXfer — state machine for one NIXL READ, keyed by mm_hash.
"""
import enum
import logging
import time
import uuid
from dataclasses import dataclass, field
from typing import Any
import msgspec
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import (
ZmqClientConnection,
ZmqClientTransport,
ZmqServerTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
DataTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
EC_CONNECTOR_VERSION,
EXPECTED_NACKS,
RETRYABLE_NACKS,
XferAck,
XferReq,
XferStatus,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr
from vllm.logger import init_logger
logger = init_logger(__name__)
PRODUCER_PIN_LEASE_S = 30.0
_CONSUMER_READ_TIMEOUT_S = 20.0
_CONSUMER_QUARANTINE_TIMEOUT_S = 60.0
# ── XferState ─────────────────────────────────────────────────────────────────
class XferState(enum.Enum):
"""Lifecycle states of a single ConsumerXfer."""
WAITING_ACK = "waiting_ack" # XferReq sent, awaiting XferAck
READING = "reading" # NIXL READ in flight
DONE = "done" # READ completed; promote to local cache
ACK_TIMEOUT = "ack_timeout" # XferAck never arrived; free blocks + tombstone
READ_FAILED = "read_failed" # Unexpected NIXL state; free blocks + tombstone
QUARANTINED = "quarantined" # Read timed out, NIXL unabortable; keep blocks
SETTLED = "settled" # Quarantined DMA done; safe to free blocks
# ── ProducerXfer ──────────────────────────────────────────────────────────────
@dataclass
class ProducerXfer:
"""One pinned read grant on the producer side.
Pure data — lifecycle operations (pin/unpin) are performed by ProducerSession.
Keyed by (consumer_session_id, mm_hash) in ProducerSession._active_xfers.
"""
mm_hash: str
block_indices: list[int]
deadline: float
def is_expired(self) -> bool:
return time.monotonic() > self.deadline
# ── ConsumerXfer ──────────────────────────────────────────────────────────────
@dataclass
class ConsumerXfer:
"""State machine for one NIXL READ on the consumer side.
transfer_handle is the in-flight NIXL handle returned by post_read() —
used for check_xfer_state() / release_xfer_handle(). The dlist handle
for the remote peer lives inside DataTransport, not here.
"""
mm_hash: str
block_indices: list[int]
addr: PeerAddr
deadline: float
data: DataTransport
consumer_session_id: str # session identity; combined with mm_hash for notif_msg
transfer_handle: Any | None = field(default=None)
_quarantined: bool = field(default=False, init=False)
def handle_ack(self, ack: XferAck, agent_name: str) -> bool:
"""Issue the NIXL READ on an OK XferAck. Returns False on any NACK."""
if ack.status != XferStatus.OK:
return False
# notif_msg encodes both session_id and mm_hash so the producer can
# do a direct (session_id, mm_hash) → ProducerXfer lookup on completion.
notif_msg = f"{self.consumer_session_id}:{self.mm_hash}".encode()
self.transfer_handle = self.data.post_read(
self.block_indices,
agent_name,
ack.src_block_indices,
notif_msg=notif_msg,
)
self.deadline = time.monotonic() + _CONSUMER_READ_TIMEOUT_S
logger.debug(
"EC consumer: NIXL READ posted mm_hash=%s agent=%s local_blocks=%d "
"remote_blocks=%d",
self.mm_hash,
agent_name,
len(self.block_indices),
len(ack.src_block_indices),
)
return True
def poll(self, now: float) -> XferState:
if self._quarantined:
return self._poll_quarantined(now)
if self.transfer_handle is None:
return (
XferState.ACK_TIMEOUT if now > self.deadline else XferState.WAITING_ACK
)
return self._poll_reading(now)
def _poll_reading(self, now: float) -> XferState:
assert self.transfer_handle is not None
try:
state = self.data.check_xfer_state(self.transfer_handle)
except Exception:
logger.exception(
"EC Connector: check_xfer_state failed for mm_hash=%s", self.mm_hash
)
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
return XferState.READ_FAILED
if state == "DONE":
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
logger.debug("EC consumer: NIXL READ completed mm_hash=%s", self.mm_hash)
return XferState.DONE
if state == "PROC":
if now > self.deadline:
self._quarantined = True
self.deadline = now + _CONSUMER_QUARANTINE_TIMEOUT_S
logger.warning(
"EC Connector: READ for mm_hash=%s from %s:%d timed out; "
"quarantining",
self.mm_hash,
self.addr[0],
self.addr[1],
)
return XferState.QUARANTINED
return XferState.READING
logger.warning(
"EC Connector: READ for mm_hash=%s unexpected NIXL state %r",
self.mm_hash,
state,
)
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
return XferState.READ_FAILED
def _poll_quarantined(self, now: float) -> XferState:
if self.transfer_handle is None:
return XferState.SETTLED
# If quarantine has been too long, assume NIXL is stuck and evict blocks.
if now > self.deadline:
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
logger.warning(
"EC Connector: quarantined mm_hash=%s from %s:%d timed out; "
"evicting blocks",
self.mm_hash,
self.addr[0],
self.addr[1],
)
return XferState.READ_FAILED
try:
state = self.data.check_xfer_state(self.transfer_handle)
except Exception:
logger.exception(
"EC Connector: check_xfer_state failed for quarantined mm_hash=%s",
self.mm_hash,
)
state = None
if state == "PROC":
return XferState.QUARANTINED
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
logger.debug(
"EC Connector: quarantined mm_hash=%s settled (state=%s)",
self.mm_hash,
state,
)
return XferState.SETTLED
def cancel(self) -> None:
"""Cancel a WAITING_ACK xfer (peer down). No tombstone — caller allows retry."""
assert self.transfer_handle is None, "cancel() requires WAITING_ACK state"
def release(self) -> None:
"""Force-release the transfer handle. Shutdown only."""
if self.transfer_handle is not None:
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
# ── ProducerSession ───────────────────────────────────────────────────────────
class ProducerSession:
"""Long-lived producer session. Owns ZmqServerTransport.
poll_step() runs once per engine step: drains the transport without
blocking, decodes XferReqs, grants or NACKs each one, drains NIXL completion
notifications, and sweeps expired xfers.
"""
def __init__(
self,
transport: ZmqServerTransport,
data: DataTransport,
cache: EmbeddingCache,
compat_hash: str,
) -> None:
self._transport = transport
self._data = data
self._cache = cache # EmbeddingCache — single block owner
self._compat_hash = compat_hash
# Key: "{consumer_session_id}:{mm_hash}" — matches the NIXL notif_msg exactly,
# so completion notification lookup is a direct dict pop with no parsing.
self._active_xfers: dict[str, ProducerXfer] = {}
self._req_decoder = msgspec.msgpack.Decoder(XferReq)
self._req_encoder = msgspec.msgpack.Encoder()
def close(self) -> None:
for xfer in self._active_xfers.values():
self._cache.unpin(xfer.mm_hash) # release any still-held pins
self._active_xfers.clear()
self._transport.close()
def poll_step(self) -> None:
"""Drain the transport without blocking and process what arrived."""
self.poll(self._transport.poll(timeout_ms=0))
def poll(self, messages: list[tuple[bytes, bytes]]) -> None:
"""Decode and process inbound XferReqs, drain NIXL notifs, sweep timeouts.
messages: (identity, payload) pairs from ZmqServerTransport.poll().
"""
for identity, payload in messages:
try:
req = self._req_decoder.decode(payload)
except (msgspec.DecodeError, msgspec.ValidationError):
logger.warning("EC Connector: dropped malformed XferReq")
continue
ack = self._grant_or_nack(req)
self._transport.send(identity, self._req_encoder.encode(ack))
self._drain_notifs()
self._sweep_timeouts()
def _grant_or_nack(self, req: XferReq) -> XferAck:
if req.connector_version != EC_CONNECTOR_VERSION:
logger.warning(
"EC Connector: incompatible version for mm_hash=%s", req.mm_hash
)
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_VERSION)
if req.compatibility_hash != self._compat_hash:
logger.warning(
"EC Connector: incompatible compat hash for mm_hash=%s", req.mm_hash
)
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INCOMPAT)
if not req.session_id:
# Grants are keyed by (session_id, mm_hash), so an omitted id from
# one peer aliases another's: the first completion would unpin
# blocks the other peer is still reading.
logger.warning(
"EC Connector: XferReq without session_id for mm_hash=%s", req.mm_hash
)
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INTERNAL)
entry = self._cache.pin_if_ready(req.mm_hash)
if entry is None or not entry.ready:
status = (
XferStatus.NACK_MISSING if entry is None else XferStatus.NACK_NOT_READY
)
logger.log(
logging.INFO if status in EXPECTED_NACKS else logging.WARNING,
"EC Connector: mm_hash=%s not serveable in local cache (%s); NACKing",
req.mm_hash,
status.name,
)
return XferAck(mm_hash=req.mm_hash, status=status)
block_indices = list(entry.block_ids)
key = f"{req.session_id}:{req.mm_hash}"
if key in self._active_xfers:
# This peer is re-requesting a grant it still holds, having timed
# out waiting. Overwriting it drops the pin taken for it, which
# neither a completion nor the sweep would ever release.
self._cache.unpin(req.mm_hash)
self._active_xfers[key] = ProducerXfer(
mm_hash=req.mm_hash,
block_indices=block_indices,
deadline=time.monotonic() + PRODUCER_PIN_LEASE_S,
)
logger.debug(
"EC producer: serving NIXL READ request mm_hash=%s key=%s blocks=%d",
req.mm_hash,
key,
len(block_indices),
)
return XferAck(
mm_hash=req.mm_hash,
status=XferStatus.OK,
session_id=req.session_id, # echo consumer's session_id back
src_block_indices=block_indices,
agent_metadata=self._data.get_agent_metadata(),
mem_descriptor=self._data.get_mem_descriptor(),
)
def _drain_notifs(self) -> None:
try:
notifs = self._data.get_new_notifs()
except Exception:
logger.exception("EC Connector: get_new_notifs failed")
return
for msgs in notifs.values():
for msg in msgs:
try:
key = msg.decode("utf-8") # "{session_id}:{mm_hash}"
except Exception:
logger.exception("EC Connector: Dropped malformed NIXL notif")
continue
xfer = self._active_xfers.pop(key, None)
if xfer is not None:
self._cache.unpin(xfer.mm_hash)
logger.debug(
"EC producer: NIXL READ completed mm_hash=%s key=%s",
xfer.mm_hash,
key,
)
def _sweep_timeouts(self) -> None:
for key, xfer in list(self._active_xfers.items()):
if xfer.is_expired():
logger.warning(
"EC producer: consumer did not complete NIXL READ "
"mm_hash=%s key=%s within the pin lease; releasing "
"pinned blocks",
xfer.mm_hash,
key,
)
self._cache.unpin(xfer.mm_hash)
del self._active_xfers[key]
# ── ConsumerSession ───────────────────────────────────────────────────────────
@dataclass
class ConsumerSessionResults:
"""Results from ConsumerSession.take_results()."""
completed: set[str] # promote to local cache
tombstoned: set[str] # free blocks + prevent retry this step
quarantined: set[str] # timed-out, DMA still running: prevent retry
# but DO NOT free blocks — session holds them
retryable: set[str] # free blocks, allow another attempt (not ready, peer down)
settled: list[tuple[str, list[int]]] # (mm_hash, block_indices) quarantine cleared
class ConsumerSession:
"""Long-lived per-producer session on the consumer side.
Owns the ZMQ DEALER connection and the NIXL agent registration for one
producer peer. The connector calls poll(messages, now) once per engine
step with the raw bytes collected from this peer's DEALER socket.
poll() decodes XferAcks, dispatches them to the right ConsumerXfer,
advances all active xfer state machines, and drains quarantined xfers.
Results accumulate in internal sets; call take_results() to collect them.
"""
def __init__(
self,
addr: PeerAddr,
zmq_conn: ZmqClientConnection,
transport: ZmqClientTransport,
data: DataTransport,
compat_hash: str,
) -> None:
self._addr = addr
self._zmq = zmq_conn
self._transport = transport
self._data = data
self._compat_hash = compat_hash
self._session_id = str(
uuid.uuid4()
) # stable identity for (session_id:mm_hash) keys
self._nixl_agent_name: str | None = None
self._nixl_metadata_bytes: bytes | None = None
self._xfers: dict[str, ConsumerXfer] = {} # mm_hash → xfer
self._quarantined: list[ConsumerXfer] = [] # DMA pending, to poll
self._newly_quarantined: set[str] = set() # quarantined since last take_results
self._completed: set[str] = set()
self._tombstoned: set[str] = set()
self._retryable: set[str] = set()
self._settled: list[tuple[str, list[int]]] = []
self._encoder = msgspec.msgpack.Encoder()
self._decoder = msgspec.msgpack.Decoder(XferAck)
@property
def addr(self) -> PeerAddr:
return self._addr
def start_xfer(
self, mm_hash: str, block_indices: list[int], deadline: float
) -> None:
"""Create a ConsumerXfer and send the XferReq over the connection."""
xfer = ConsumerXfer(
mm_hash=mm_hash,
block_indices=block_indices,
addr=self._addr,
deadline=deadline,
data=self._data,
consumer_session_id=self._session_id,
)
self._zmq.send(
self._encoder.encode(
XferReq(
mm_hash=mm_hash,
compatibility_hash=self._compat_hash,
session_id=self._session_id,
)
)
)
self._xfers[mm_hash] = xfer
logger.debug(
"EC Connector: XferReq sent mm_hash=%s to %s:%d",
mm_hash,
self._addr[0],
self._addr[1],
)
def poll(self, messages: list[bytes], now: float) -> None:
"""Decode inbound XferAcks, dispatch to xfers, and advance all state machines.
messages: raw bytes from ZmqClientTransport.poll() for this peer.
Results accumulate until take_results() is called.
"""
for raw in messages:
try:
ack = self._decoder.decode(raw)
except (msgspec.DecodeError, msgspec.ValidationError):
logger.warning(
"EC Connector: dropped malformed XferAck from %s:%d",
self._addr[0],
self._addr[1],
)
continue
self._dispatch_ack(ack)
for mm_hash, xfer in list(self._xfers.items()):
state = xfer.poll(now)
if state == XferState.DONE:
self._completed.add(mm_hash)
del self._xfers[mm_hash]
elif state in (XferState.ACK_TIMEOUT, XferState.READ_FAILED):
self._tombstoned.add(mm_hash)
del self._xfers[mm_hash]
elif state == XferState.QUARANTINED:
# DMA may still be writing — caller must NOT free blocks.
self._quarantined.append(xfer)
self._newly_quarantined.add(mm_hash)
del self._xfers[mm_hash]
self._drain_quarantine(now)
def _dispatch_ack(self, ack: XferAck) -> None:
xfer = self._xfers.get(ack.mm_hash)
if xfer is None or xfer.transfer_handle is not None:
return # unknown mm_hash or duplicate ack
if ack.status != XferStatus.OK:
logger.log(
logging.INFO if ack.status in EXPECTED_NACKS else logging.WARNING,
"EC Connector: NACK %s from %s:%d for mm_hash=%s",
ack.status.name,
self._addr[0],
self._addr[1],
ack.mm_hash,
)
del self._xfers[ack.mm_hash]
# The producer announced this encoding and its save is still in
# flight, so another attempt on a later step can be granted. Every
# other NACK is terminal for this mm_hash.
if ack.status in RETRYABLE_NACKS:
self._retryable.add(ack.mm_hash)
else:
self._tombstoned.add(ack.mm_hash)
return
try:
agent_name = self._ensure_registered(ack.agent_metadata, ack.mem_descriptor)
except Exception:
logger.exception(
"EC Connector: failed to register peer %s:%d for mm_hash=%s",
self._addr[0],
self._addr[1],
ack.mm_hash,
)
del self._xfers[ack.mm_hash]
self._tombstoned.add(ack.mm_hash)
return
if not xfer.handle_ack(ack, agent_name):
del self._xfers[ack.mm_hash]
self._tombstoned.add(ack.mm_hash)
def _ensure_registered(self, metadata: bytes, mem_descriptor: bytes) -> str:
"""Register or re-register the peer in DataTransport. Returns agent_name."""
if self._nixl_agent_name is not None and self._nixl_metadata_bytes == metadata:
return self._nixl_agent_name
if self._nixl_agent_name is not None:
logger.info(
"EC Connector: producer %s:%d restarted; re-registering",
self._addr[0],
self._addr[1],
)
self._data.remove_remote_peer(self._nixl_agent_name)
agent_name = self._data.add_remote_peer(metadata, mem_descriptor)
self._nixl_agent_name = agent_name
self._nixl_metadata_bytes = metadata
logger.debug(
"EC Connector: registered peer %s:%d agent=%s",
self._addr[0],
self._addr[1],
agent_name,
)
return agent_name
def _drain_quarantine(self, now: float) -> None:
still_pending: list[ConsumerXfer] = []
for xfer in list(self._quarantined):
state = xfer.poll(now)
if state == XferState.SETTLED:
self._settled.append((xfer.mm_hash, xfer.block_indices))
else:
still_pending.append(xfer)
self._quarantined = still_pending
def on_peer_down(self) -> None:
"""WAITING_ACK xfers → cancel (retry allowed); READING xfers → quarantine."""
for mm_hash, xfer in list(self._xfers.items()):
if xfer.transfer_handle is None:
xfer.cancel()
self._retryable.add(mm_hash)
else:
self._quarantined.append(xfer)
self._newly_quarantined.add(mm_hash)
self._xfers.clear()
def take_results(self) -> ConsumerSessionResults:
"""Return and clear all accumulated results since the last call."""
results = ConsumerSessionResults(
completed=self._completed,
tombstoned=self._tombstoned,
quarantined=self._newly_quarantined,
retryable=self._retryable,
settled=self._settled,
)
self._completed = set()
self._tombstoned = set()
self._newly_quarantined = set()
self._retryable = set()
self._settled = []
return results
def has_in_flight(self) -> bool:
return bool(self._xfers) or bool(self._quarantined)
def close(self) -> None:
"""Release all transfer handles, deregister NIXL peer, close connection.
Connection teardown is routed through the transport so the pooled
connection is dropped in lock-step with the session — a session and its
DEALER connection have a 1:1 lifecycle, so a later connect() to the same
peer builds a fresh socket instead of handing back this dead one.
"""
for xfer in self._xfers.values():
xfer.release()
self._xfers.clear()
for xfer in self._quarantined:
xfer.release()
self._quarantined.clear()
if self._nixl_agent_name is not None:
self._data.remove_remote_peer(self._nixl_agent_name)
self._transport.remove(self._addr)
@@ -0,0 +1,47 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Supporting utilities for the ECCPUConnector scheduler."""
import msgspec
from vllm.logger import init_logger
# (host, port) pair identifying a remote peer's ZMQ side-channel endpoint.
PeerAddr = tuple[str, int]
logger = init_logger(__name__)
# Msgpack (de)serialization for the list-of-(addr, size, device_id)
# descriptor form that NIXL's `get_xfer_descs` expects.
_MemDescList = list[tuple[int, int, int]]
_mem_desc_encoder = msgspec.msgpack.Encoder()
_mem_desc_decoder = msgspec.msgpack.Decoder(_MemDescList)
def serialize_mem_descriptor(descs: _MemDescList) -> bytes:
return _mem_desc_encoder.encode(descs)
def deserialize_mem_descriptor(payload: bytes) -> _MemDescList:
return _mem_desc_decoder.decode(payload)
def build_block_descs(
base_ptr: int,
num_blocks: int,
block_size_bytes: int,
device_id: int = 0,
) -> _MemDescList:
"""Per-block `(addr, size, device_id)` tuples for an `ECSharedRegion`.
Handed to `nixl.get_xfer_descs` to build a dlist where descriptor `i`
addresses block `i` in the mmap. This makes a WRITE to block indices
`[i1, i2, ...]` a simple `make_prepped_xfer(..., [i1, i2])` on both
ends — no per-request address math.
"""
return [
(base_ptr + i * block_size_bytes, block_size_bytes, device_id)
for i in range(num_blocks)
]
@@ -55,7 +55,6 @@ def _coalesce_runs(
split. `EmbeddingCache` hands out ascending ids, which is what makes runs
findable at all.
"""
ids = np.fromiter(block_ids, dtype=np.int64, count=len(block_ids))
breaks = np.flatnonzero(np.diff(ids) != 1) + 1
slots = np.concatenate(([0], breaks))
@@ -5,7 +5,6 @@ from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import safetensors
import torch
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.base import (
@@ -13,8 +12,11 @@ from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
ECConnectorRole,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.logger import init_logger
from vllm.utils.collection_utils import is_list_of
from vllm.v1.core.sched.output import SchedulerOutput
if TYPE_CHECKING:
@@ -52,8 +54,7 @@ class ECExampleConnector(ECConnectorBase):
super().__init__(vllm_config=vllm_config, role=role)
# req_id -> index
self._mm_datas_need_loads: dict[str, int] = {}
self._model_config = vllm_config.model_config
self._metadata_fields_cache: dict[str, set[str]] = {}
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
transfer_config = vllm_config.ec_transfer_config
if transfer_config is not None:
self._storage_path = transfer_config.get_from_extra_config(
@@ -169,36 +170,6 @@ class ECExampleConnector(ECConnectorBase):
self._mm_datas_need_loads.clear()
return meta
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
"""Which processed keys this model needs published for `modality`.
Read from `MultiModalDataParser.embedding_fields`, the same declaration
the consumer's parser requires, so the two cannot drift. An empty set
means the modality cannot be delivered out of band, and the consumer
will process the media itself.
"""
if modality in self._metadata_fields_cache:
return self._metadata_fields_cache[modality]
fields: set[str] = set()
try:
from vllm.multimodal import MULTIMODAL_REGISTRY
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
fields = info.data_parser.placeholder_metadata_fields(modality)
except Exception:
# Reporting nothing is a safe degradation: the consumer falls back to
# processing the media itself.
logger.warning(
"Could not determine the placeholder metadata fields for "
"modality %s; the consumer will preprocess the media itself.",
modality,
exc_info=True,
)
self._metadata_fields_cache[modality] = fields
return fields
def request_finished(
self,
request: "Request",
@@ -214,28 +185,10 @@ class ECExampleConnector(ECConnectorBase):
if not self.is_producer:
return False, None
items = []
for feature in request.mm_features:
metadata = {}
# `data` is None for items served from the processor cache, in which
# case the metadata is unavailable here and the consumer has to fall
# back to processing the media itself.
if feature.data is not None:
wanted = self._placeholder_metadata_fields(feature.modality)
for key, value in feature.data.get_data().items():
if key not in wanted:
continue
if isinstance(value, torch.Tensor):
metadata[key] = value.tolist()
elif is_list_of(value, (int, float)):
# Some metadata (e.g. Qwen3-VL video timestamps) is
# produced as a plain list rather than a tensor.
metadata[key] = value
items.append({"mm_hash": feature.identifier, **metadata})
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
if not items:
return False, None
return False, {"ec_items": items}
return False, items
# ==============================
# Helper functions
@@ -2,8 +2,88 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""EC connector helper utilities."""
from typing import TYPE_CHECKING, Any
import torch
from vllm.logger import init_logger
from vllm.utils.collection_utils import is_list_of
from vllm.v1.outputs import ECConnectorOutput, ModelRunnerOutput
if TYPE_CHECKING:
from vllm.config import ModelConfig
from vllm.multimodal.inputs import MultiModalFeatureSpec
logger = init_logger(__name__)
class PlaceholderMetadataResolver:
"""Resolves which processed keys a model needs published per modality.
Reads `MultiModalDataParser.embedding_fields`, the same declaration the
consumer's parser requires, so the two cannot drift. An empty set means
the modality cannot be delivered out of band, and the consumer will
process the media itself.
"""
def __init__(self, model_config: "ModelConfig") -> None:
self._model_config = model_config
self._cache: dict[str, set[str]] = {}
def fields_for(self, modality: str) -> set[str]:
if modality in self._cache:
return self._cache[modality]
fields: set[str] = set()
try:
from vllm.multimodal import MULTIMODAL_REGISTRY
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
fields = info.data_parser.placeholder_metadata_fields(modality)
except Exception:
logger.warning(
"Could not determine the placeholder metadata fields for "
"modality %s; the consumer will preprocess the media itself.",
modality,
exc_info=True,
)
self._cache[modality] = fields
return fields
def collect_ec_item_metadata(
mm_features: "list[MultiModalFeatureSpec]",
resolver: PlaceholderMetadataResolver,
) -> dict[str, dict[str, Any]]:
"""Build one `ec_transfer_params` entry per feature for `request_finished()`.
Keyed by mm_hash, each entry carries a `metadata` dict with whatever
placeholder fields `resolver` says this model needs published for its
modality, so a consumer can skip the image transform. `data` is None for
items served from the processor cache, in which case the metadata is
unavailable here and the consumer has to fall back to processing the
media itself. A connector that also has transfer coordinates to report
(e.g. NIXL peer_host/peer_port/size_bytes) merges those in alongside
`metadata`, not into it.
"""
items: dict[str, dict[str, Any]] = {}
for feature in mm_features:
metadata: dict[str, Any] = {}
if feature.data is not None:
wanted = resolver.fields_for(feature.modality)
for key, value in feature.data.get_data().items():
if key not in wanted:
continue
if isinstance(value, torch.Tensor):
metadata[key] = value.tolist()
elif is_list_of(value, (int, float)):
# Some metadata (e.g. Qwen3-VL video timestamps) is
# produced as a plain list rather than a tensor.
metadata[key] = value
items[feature.identifier] = {"metadata": metadata}
return items
class ECOutputAggregator:
"""Merge every worker's EC connector output onto the single