[EC Connector] P2P NIXL + CPU EC Connector (#47941)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
69cf055936
commit
8a0a7ee40a
@@ -8,7 +8,7 @@ GPU copy plumbing. An ec_both instance reuses encoder outputs it has already
|
||||
offloaded to CPU instead of recomputing them.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -37,6 +37,12 @@ class ECCPUConnector(ECConnectorBase):
|
||||
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole) -> None:
|
||||
super().__init__(vllm_config=vllm_config, role=role)
|
||||
|
||||
if not vllm_config.use_v2_model_runner:
|
||||
raise ValueError(
|
||||
"ECCPUConnector requires the V2 model runner. "
|
||||
"Set VLLM_USE_V2_MODEL_RUNNER=1."
|
||||
)
|
||||
|
||||
self.connector_worker = None
|
||||
self.connector_scheduler = None
|
||||
|
||||
@@ -108,6 +114,12 @@ class ECCPUConnector(ECConnectorBase):
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def request_finished(
|
||||
self, request: "Request"
|
||||
) -> tuple[bool, "dict[str, Any] | None"]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Abstract base classes for the EC control plane.
|
||||
|
||||
ControlConnection wraps a single peer channel (one DEALER socket).
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class ControlConnection(ABC):
|
||||
"""Bidirectional message channel to a single remote peer.
|
||||
|
||||
All I/O is non-blocking: send enqueues without blocking; recv drains
|
||||
all buffered inbound messages since the last call.
|
||||
"""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def alive(self) -> bool:
|
||||
"""True if the connection is still usable."""
|
||||
|
||||
@abstractmethod
|
||||
def send(self, msg: bytes) -> None:
|
||||
"""Enqueue msg for delivery. Must not block."""
|
||||
|
||||
@abstractmethod
|
||||
def recv(self) -> list[bytes]:
|
||||
"""Drain and return all buffered inbound messages."""
|
||||
|
||||
@abstractmethod
|
||||
def mark_dead(self) -> None:
|
||||
"""Signal that the peer disconnected. Sets alive to False."""
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Release all resources. Idempotent."""
|
||||
@@ -0,0 +1,233 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ZMQ control-plane implementations for ECCPUConnector.
|
||||
|
||||
ZmqClientConnection — one DEALER socket to a single producer peer.
|
||||
ZmqClientTransport — pool of ZmqClientConnection objects (consumer side).
|
||||
ZmqServerTransport — ROUTER socket (producer side).
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import zmq
|
||||
from zmq.utils.monitor import recv_monitor_message
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.control.base import (
|
||||
ControlConnection,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import make_zmq_path, make_zmq_socket
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_HEARTBEAT_IVL_MS = 2000
|
||||
_HEARTBEAT_TIMEOUT_MS = 4000
|
||||
_HEARTBEAT_TTL_MS = 8000
|
||||
|
||||
|
||||
# ── Client side ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class ZmqClientConnection(ControlConnection):
|
||||
"""One DEALER socket to one producer peer.
|
||||
|
||||
Owns the socket and its optional monitor socket. NIXL state is NOT
|
||||
stored here — it belongs in ECSession above this layer.
|
||||
"""
|
||||
|
||||
dealer: zmq.Socket
|
||||
monitor: zmq.Socket | None
|
||||
_alive: bool = field(default=True, init=False)
|
||||
|
||||
@property
|
||||
def alive(self) -> bool:
|
||||
return self._alive
|
||||
|
||||
def send(self, msg: bytes) -> None:
|
||||
self.dealer.send_multipart([b"", msg])
|
||||
|
||||
def recv(self) -> list[bytes]:
|
||||
msgs: list[bytes] = []
|
||||
while True:
|
||||
try:
|
||||
frames = self.dealer.recv_multipart(flags=zmq.NOBLOCK)
|
||||
except zmq.Again:
|
||||
break
|
||||
except Exception:
|
||||
logger.exception("ec: DEALER recv failed")
|
||||
break
|
||||
if len(frames) != 2 or frames[0] != b"":
|
||||
logger.warning(
|
||||
"ec: dropped malformed frame envelope (%d frames)", len(frames)
|
||||
)
|
||||
continue
|
||||
msgs.append(frames[1])
|
||||
return msgs
|
||||
|
||||
def mark_dead(self) -> None:
|
||||
self._alive = False
|
||||
|
||||
def close(self) -> None:
|
||||
if self.monitor is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
self.dealer.disable_monitor()
|
||||
with contextlib.suppress(Exception):
|
||||
self.monitor.close(linger=0)
|
||||
with contextlib.suppress(Exception):
|
||||
self.dealer.close(linger=0)
|
||||
self._alive = False
|
||||
|
||||
|
||||
class ZmqClientTransport:
|
||||
"""Pool of ZmqClientConnection objects — one per producer peer.
|
||||
|
||||
Creates DEALER sockets lazily on first contact. Does not touch NIXL;
|
||||
NIXL registration is handled by ECCPUConsumer once an XferAck arrives.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._ctx = zmq.Context()
|
||||
self._connections: dict[PeerAddr, ZmqClientConnection] = {}
|
||||
|
||||
def connect(self, addr: PeerAddr) -> ZmqClientConnection:
|
||||
"""Return the existing connection for addr, or create a new one."""
|
||||
existing = self._connections.get(addr)
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
host, port = addr
|
||||
dealer = make_zmq_socket(
|
||||
ctx=self._ctx,
|
||||
path=make_zmq_path(scheme="tcp", host=host, port=port),
|
||||
socket_type=zmq.DEALER,
|
||||
bind=False,
|
||||
)
|
||||
dealer.setsockopt(zmq.HEARTBEAT_IVL, _HEARTBEAT_IVL_MS)
|
||||
dealer.setsockopt(zmq.HEARTBEAT_TIMEOUT, _HEARTBEAT_TIMEOUT_MS)
|
||||
dealer.setsockopt(zmq.HEARTBEAT_TTL, _HEARTBEAT_TTL_MS)
|
||||
|
||||
monitor_addr = f"inproc://ec-peer-mon-{host}-{port}"
|
||||
dealer.monitor(monitor_addr, zmq.EVENT_DISCONNECTED)
|
||||
monitor = self._ctx.socket(zmq.PAIR)
|
||||
monitor.connect(monitor_addr)
|
||||
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=monitor)
|
||||
self._connections[addr] = conn
|
||||
return conn
|
||||
|
||||
def poll(self) -> dict[PeerAddr, list[bytes]]:
|
||||
"""Non-blocking drain of messages from every peer's DEALER."""
|
||||
msgs: dict[PeerAddr, list[bytes]] = {}
|
||||
for addr, conn in list(self._connections.items()):
|
||||
conn_msgs = conn.recv()
|
||||
msgs[addr] = conn_msgs
|
||||
return msgs
|
||||
|
||||
def poll_dead(self) -> list[PeerAddr]:
|
||||
"""Non-blocking sweep of peer monitors; return addresses that died."""
|
||||
dead: list[PeerAddr] = []
|
||||
for addr, conn in list(self._connections.items()):
|
||||
if conn.monitor is None:
|
||||
continue
|
||||
try:
|
||||
while True:
|
||||
evt = recv_monitor_message(conn.monitor, flags=zmq.NOBLOCK)
|
||||
if evt["event"] == zmq.EVENT_DISCONNECTED:
|
||||
conn.mark_dead()
|
||||
dead.append(addr)
|
||||
break
|
||||
except zmq.Again:
|
||||
pass
|
||||
except Exception:
|
||||
logger.warning("ec: monitor poll failed for %s", addr, exc_info=True)
|
||||
return dead
|
||||
|
||||
def remove(self, addr: PeerAddr) -> ZmqClientConnection | None:
|
||||
"""Remove and close the connection for addr, returning it (or None)."""
|
||||
conn = self._connections.pop(addr, None)
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
return conn
|
||||
|
||||
def close(self) -> None:
|
||||
for conn in list(self._connections.values()):
|
||||
conn.close()
|
||||
self._connections.clear()
|
||||
try:
|
||||
self._ctx.destroy(linger=0)
|
||||
except Exception:
|
||||
logger.debug("ec: zmq context destroy failed", exc_info=True)
|
||||
|
||||
|
||||
# ── Server side ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class ZmqServerTransport:
|
||||
"""ROUTER socket for the producer side. Byte-level only, driven by
|
||||
ProducerSession.
|
||||
|
||||
poll(timeout_ms) waits up to timeout_ms for messages, then drains all that
|
||||
are available; timeout_ms=0 drains whatever has already arrived. send()
|
||||
routes a reply back to a specific peer by ZMQ identity.
|
||||
"""
|
||||
|
||||
def __init__(self, host: str, port: int) -> None:
|
||||
self._ctx = zmq.Context()
|
||||
self._router = make_zmq_socket(
|
||||
ctx=self._ctx,
|
||||
path=make_zmq_path(scheme="tcp", host=host, port=port),
|
||||
socket_type=zmq.ROUTER,
|
||||
bind=True,
|
||||
)
|
||||
self._poller = zmq.Poller()
|
||||
self._poller.register(self._router, zmq.POLLIN)
|
||||
|
||||
def poll(self, timeout_ms: int = 0) -> list[tuple[bytes, bytes]]:
|
||||
"""Wait up to timeout_ms for incoming messages, then drain all pending.
|
||||
|
||||
Returns a list of (identity, payload) pairs.
|
||||
"""
|
||||
if timeout_ms > 0:
|
||||
try:
|
||||
events = dict(self._poller.poll(timeout=timeout_ms))
|
||||
except zmq.ContextTerminated:
|
||||
return []
|
||||
except Exception:
|
||||
logger.exception("ec: router poller failed")
|
||||
return []
|
||||
if self._router not in events:
|
||||
return []
|
||||
|
||||
msgs: list[tuple[bytes, bytes]] = []
|
||||
while True:
|
||||
try:
|
||||
identity, _, payload = self._router.recv_multipart(flags=zmq.NOBLOCK)
|
||||
msgs.append((identity, payload))
|
||||
except zmq.Again:
|
||||
break
|
||||
except zmq.ContextTerminated:
|
||||
break
|
||||
except Exception:
|
||||
logger.exception("ec: router recv failed")
|
||||
break
|
||||
return msgs
|
||||
|
||||
def send(self, identity: bytes, payload: bytes) -> None:
|
||||
"""Send a reply payload to the peer identified by ZMQ identity."""
|
||||
try:
|
||||
self._router.send_multipart([identity, b"", payload])
|
||||
except Exception:
|
||||
logger.exception("ec: router send failed")
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
self._router.close(linger=0)
|
||||
except Exception:
|
||||
logger.debug("ec: router close failed", exc_info=True)
|
||||
try:
|
||||
self._ctx.destroy(linger=0)
|
||||
except Exception:
|
||||
logger.debug("ec: zmq context destroy failed", exc_info=True)
|
||||
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Abstract base class for the EC data-plane transport (NIXL/UCX)."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class DataTransport(ABC):
|
||||
"""Interface wrapping the NIXL data-plane for EC block transfers.
|
||||
|
||||
Peer lifecycle:
|
||||
add_remote_peer() — register a peer and store its dlist handle internally
|
||||
remove_remote_peer() — deregister and release the stored handle
|
||||
post_read() — issue a READ using the stored handle for agent_name
|
||||
|
||||
The dlist handle (prepped xfer descriptor list) is an implementation detail
|
||||
of the transport and is never exposed to callers.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_agent_metadata(self) -> bytes:
|
||||
"""Return this agent's serialized NIXL metadata blob."""
|
||||
|
||||
@abstractmethod
|
||||
def get_mem_descriptor(self) -> bytes:
|
||||
"""Return the msgpack-encoded block descriptor list for this agent."""
|
||||
|
||||
@abstractmethod
|
||||
def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str:
|
||||
"""Register a remote peer, prep its READ-source dlist, and return agent_name.
|
||||
|
||||
The dlist handle is stored internally and used by post_read().
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def remove_remote_peer(self, agent_name: str) -> None:
|
||||
"""Deregister a previously registered remote peer and release its handle."""
|
||||
|
||||
@abstractmethod
|
||||
def post_read(
|
||||
self,
|
||||
local_indices: list[int],
|
||||
agent_name: str,
|
||||
remote_indices: list[int],
|
||||
notif_msg: bytes,
|
||||
) -> Any:
|
||||
"""Issue a consumer-initiated READ using the stored dlist for agent_name.
|
||||
|
||||
Returns a transfer handle for use with check_xfer_state / release_xfer_handle.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_new_notifs(self) -> dict[str, list[bytes]]:
|
||||
"""Drain NIXL completion notifications addressed to this agent."""
|
||||
|
||||
@abstractmethod
|
||||
def check_xfer_state(self, handle: Any) -> str:
|
||||
"""Return the NIXL state string: 'DONE', 'PROC', or an error string."""
|
||||
|
||||
@abstractmethod
|
||||
def release_xfer_handle(self, handle: Any) -> None:
|
||||
"""Release a transfer handle returned by post_read."""
|
||||
|
||||
@abstractmethod
|
||||
def deregister(self) -> None:
|
||||
"""Deregister all memory and tear down the NIXL agent."""
|
||||
@@ -0,0 +1,162 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NIXL-backed data-plane transport for ECCPUConnector."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
|
||||
DataTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
|
||||
build_block_descs,
|
||||
deserialize_mem_descriptor,
|
||||
serialize_mem_descriptor,
|
||||
)
|
||||
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_NIXL_DRAM = "DRAM"
|
||||
|
||||
|
||||
class NixlDataTransport(DataTransport):
|
||||
"""Adapts NixlWrapper to the DataTransport interface.
|
||||
|
||||
Dlist handles (prepped xfer descriptor lists for remote peers) are stored
|
||||
internally keyed by agent_name. Callers never touch raw NIXL handles —
|
||||
post_read() resolves the handle from agent_name internally.
|
||||
|
||||
NixlWrapper is not thread-safe, and neither is this class. Every call site
|
||||
runs on the scheduler thread, which serializes the transport by
|
||||
construction; transfers are async inside NIXL and are drained from that
|
||||
same thread via get_new_notifs() / check_xfer_state().
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent_name: str,
|
||||
base_ptr: int,
|
||||
num_blocks: int,
|
||||
block_size_bytes: int,
|
||||
total_size_bytes: int,
|
||||
num_threads: int = 1,
|
||||
) -> None:
|
||||
if NixlWrapper is None or nixl_agent_config is None:
|
||||
raise RuntimeError(
|
||||
"ECCPUConnector requires NIXL; "
|
||||
"install the `nixl` package or set a different ec_connector."
|
||||
)
|
||||
# agent_name → remote dlist handle (from add_remote_peer, used by post_read)
|
||||
self._peer_handles: dict[str, int] = {}
|
||||
# None until the local region is registered, so deregister() is safe on
|
||||
# a transport whose registration failed part-way.
|
||||
self._reg_descs: Any = None
|
||||
|
||||
self._nixl = NixlWrapper(
|
||||
agent_name,
|
||||
nixl_agent_config(num_threads=num_threads, capture_telemetry=True),
|
||||
)
|
||||
self._register_local_memory(
|
||||
base_ptr, num_blocks, block_size_bytes, total_size_bytes
|
||||
)
|
||||
|
||||
def _register_local_memory(
|
||||
self,
|
||||
base_ptr: int,
|
||||
num_blocks: int,
|
||||
block_size_bytes: int,
|
||||
total_size_bytes: int,
|
||||
) -> None:
|
||||
"""Register the local region with NIXL and prep its transfer dlist.
|
||||
|
||||
Unwinds the registration if a later step fails: __init__ propagates the
|
||||
error, so the caller never receives an object it could deregister with.
|
||||
"""
|
||||
block_descs = build_block_descs(base_ptr, num_blocks, block_size_bytes)
|
||||
self._reg_descs = self._nixl.get_reg_descs(
|
||||
[(base_ptr, total_size_bytes, 0, "")], _NIXL_DRAM
|
||||
)
|
||||
self._nixl.register_memory(self._reg_descs, backends=["UCX"])
|
||||
try:
|
||||
xfer_descs = self._nixl.get_xfer_descs(block_descs, _NIXL_DRAM)
|
||||
self._local_xfer_handle = self._nixl.prep_xfer_dlist(
|
||||
"NIXL_INIT_AGENT", xfer_descs
|
||||
)
|
||||
self._agent_metadata = self._nixl.get_agent_metadata()
|
||||
self._mem_descriptor_bytes = serialize_mem_descriptor(block_descs)
|
||||
except Exception:
|
||||
self.deregister()
|
||||
raise
|
||||
|
||||
def get_agent_metadata(self) -> bytes:
|
||||
return self._agent_metadata
|
||||
|
||||
def get_mem_descriptor(self) -> bytes:
|
||||
return self._mem_descriptor_bytes
|
||||
|
||||
def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str:
|
||||
"""Register remote peer, prep its dlist, store handle internally.
|
||||
|
||||
Returns agent_name for use in post_read() and remove_remote_peer().
|
||||
"""
|
||||
agent_name = self._nixl.add_remote_agent(metadata)
|
||||
remote_blocks = deserialize_mem_descriptor(mem_descriptor)
|
||||
remote_xfer_descs = self._nixl.get_xfer_descs(remote_blocks, _NIXL_DRAM)
|
||||
self._peer_handles[agent_name] = self._nixl.prep_xfer_dlist(
|
||||
agent_name, remote_xfer_descs
|
||||
)
|
||||
return agent_name
|
||||
|
||||
def remove_remote_peer(self, agent_name: str) -> None:
|
||||
self._peer_handles.pop(agent_name, None)
|
||||
try:
|
||||
self._nixl.remove_remote_agent(agent_name)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"EC: remove_remote_peer failed for %s", agent_name, exc_info=True
|
||||
)
|
||||
|
||||
def post_read(
|
||||
self,
|
||||
local_indices: list[int],
|
||||
agent_name: str,
|
||||
remote_indices: list[int],
|
||||
notif_msg: bytes,
|
||||
) -> Any:
|
||||
if len(local_indices) != len(remote_indices):
|
||||
raise ValueError(
|
||||
f"EC: local/remote block count mismatch "
|
||||
f"({len(local_indices)} vs {len(remote_indices)})"
|
||||
)
|
||||
remote_handle = self._peer_handles[agent_name]
|
||||
handle = self._nixl.make_prepped_xfer(
|
||||
"READ",
|
||||
self._local_xfer_handle,
|
||||
local_indices,
|
||||
remote_handle,
|
||||
remote_indices,
|
||||
notif_msg=notif_msg,
|
||||
)
|
||||
self._nixl.transfer(handle)
|
||||
return handle
|
||||
|
||||
def get_new_notifs(self) -> dict[str, list[bytes]]:
|
||||
return self._nixl.get_new_notifs()
|
||||
|
||||
def check_xfer_state(self, handle: Any) -> str:
|
||||
return self._nixl.check_xfer_state(handle)
|
||||
|
||||
def release_xfer_handle(self, handle: Any) -> None:
|
||||
try:
|
||||
self._nixl.release_xfer_handle(handle)
|
||||
except Exception:
|
||||
logger.warning("EC: release_xfer_handle failed", exc_info=True)
|
||||
|
||||
def deregister(self) -> None:
|
||||
if self._reg_descs is None:
|
||||
return
|
||||
try:
|
||||
self._nixl.deregister_memory(self._reg_descs)
|
||||
except Exception:
|
||||
logger.warning("EC: deregister failed", exc_info=True)
|
||||
self._reg_descs = None
|
||||
@@ -0,0 +1,91 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Scheduler-side wire types for the ECCPUConnector.
|
||||
|
||||
Two peer-to-peer message types exchanged between consumer and producer over
|
||||
ZMQ (XferReq, XferAck) plus a peer-compatibility fingerprint
|
||||
(compute_ec_compatibility_hash) sent on every XferReq so a producer refuses
|
||||
to serve a mismatched peer.
|
||||
"""
|
||||
|
||||
import enum
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
import msgspec
|
||||
|
||||
# Bump when the on-wire shape (XferReq/XferAck) changes in a
|
||||
# backward-incompatible way. Peers reject mismatched versions.
|
||||
EC_CONNECTOR_VERSION: int = 1
|
||||
|
||||
|
||||
class XferStatus(enum.IntEnum):
|
||||
"""Outcome the producer reports for an XferReq."""
|
||||
|
||||
OK = 0
|
||||
NACK_MISSING = 1 # producer no longer holds the encoding
|
||||
NACK_INCOMPAT = 2 # peer-compatibility hash mismatch
|
||||
NACK_VERSION = 3 # connector wire-version mismatch
|
||||
NACK_INTERNAL = 4 # producer hit an unexpected error
|
||||
NACK_NOT_READY = 5 # announced, but its GPU->mmap save has not landed yet
|
||||
|
||||
|
||||
# The producer expects a retryable condition to clear on its own, so the
|
||||
# consumer re-requests the read rather than giving up on the encoding.
|
||||
RETRYABLE_NACKS = frozenset((XferStatus.NACK_NOT_READY,))
|
||||
# Ordinary outcomes rather than signs of a broken peer.
|
||||
EXPECTED_NACKS = RETRYABLE_NACKS | {XferStatus.NACK_MISSING}
|
||||
|
||||
|
||||
class XferReq( # type: ignore[call-arg]
|
||||
msgspec.Struct,
|
||||
tag="req",
|
||||
omit_defaults=True,
|
||||
):
|
||||
"""Consumer → Producer: I want to READ mm_hash; pin it and tell me where."""
|
||||
|
||||
mm_hash: str
|
||||
compatibility_hash: str
|
||||
# Consumer's session identity. Together with mm_hash it uniquely identifies
|
||||
# this transfer on both sides: the producer keys its active xfers by
|
||||
# (session_id, mm_hash), and the NIXL notif_msg carries the same pair.
|
||||
session_id: str = ""
|
||||
connector_version: int = EC_CONNECTOR_VERSION
|
||||
|
||||
|
||||
class XferAck( # type: ignore[call-arg]
|
||||
msgspec.Struct,
|
||||
tag="ack",
|
||||
omit_defaults=True,
|
||||
):
|
||||
"""Producer → Consumer: response to an XferReq.
|
||||
|
||||
On status == OK: producer has pinned source blocks and returns their
|
||||
indices plus fresh NIXL metadata for the consumer to register and READ.
|
||||
On any NACK_*: optional fields are empty; consumer falls back to local encode.
|
||||
"""
|
||||
|
||||
mm_hash: str
|
||||
status: XferStatus
|
||||
# Consumer's session_id echoed back from XferReq. Together with mm_hash it
|
||||
# identifies the exact transfer; the consumer uses both to build notif_msg.
|
||||
session_id: str = ""
|
||||
src_block_indices: list[int] = []
|
||||
agent_metadata: bytes = b""
|
||||
mem_descriptor: bytes = b""
|
||||
|
||||
|
||||
def compute_ec_compatibility_hash(
|
||||
vllm_version: str,
|
||||
model: str,
|
||||
dtype: str,
|
||||
block_size_bytes: int,
|
||||
) -> str:
|
||||
"""Peer-compatibility fingerprint over the four factors that determine
|
||||
byte-layout compatibility. Producer NACKs any XferReq whose hash differs.
|
||||
"""
|
||||
factors = json.dumps(
|
||||
[vllm_version, model, dtype, block_size_bytes],
|
||||
separators=(",", ":"),
|
||||
)
|
||||
return hashlib.sha256(factors.encode("utf-8")).hexdigest()
|
||||
@@ -7,16 +7,23 @@ Owns the mmap region and the embedding cache, and handles the producer
|
||||
for the ECCPUConnector.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
ECCPUWorkerMetadata,
|
||||
_get_encoder_cache_hidden_dim,
|
||||
create_ec_shared_region,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
PlaceholderMetadataResolver,
|
||||
collect_ec_item_metadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -40,6 +47,7 @@ class ECCPUScheduler:
|
||||
self._region = create_ec_shared_region(vllm_config)
|
||||
# Block allocator + LRU eviction policy for the shared region.
|
||||
self._cache = EmbeddingCache(self._region.num_blocks)
|
||||
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
|
||||
|
||||
# mm_hash → block IDs allocated this step for GPU→mmap saves.
|
||||
self._pending_saves: dict[str, list[int]] = {}
|
||||
@@ -63,6 +71,112 @@ class ECCPUScheduler:
|
||||
else 1
|
||||
)
|
||||
|
||||
self._ec_config = ec_config
|
||||
# NIXL p2p is an option of this connector, not engine-wide behavior, so
|
||||
# it lives in extra config. Extra config is not type coerced, so a value
|
||||
# that arrived as a JSON/CLI string is parsed, not truth-tested.
|
||||
raw_nixl = ec_config.get_from_extra_config("ec_enable_nixl", False)
|
||||
self._nixl_enabled = (
|
||||
raw_nixl
|
||||
if isinstance(raw_nixl, bool)
|
||||
else str(raw_nixl).strip().lower() in ("true", "1", "yes")
|
||||
)
|
||||
# NIXL fields default to None/empty so the gate-off path is untouched.
|
||||
self._data: Any = None
|
||||
self._compat_hash: str | None = None
|
||||
self._first_in_batch = True
|
||||
self._transport: Any = None
|
||||
self._producer_session: Any = None
|
||||
self._sessions: dict = {}
|
||||
self._in_flight: set[str] = set()
|
||||
self._tombstones: set[str] = set()
|
||||
self._step_completed: set[str] = set()
|
||||
self._peer_host: str | None = None
|
||||
self._peer_port: int | None = None
|
||||
# Model shape for size checks + compat hash; only set by
|
||||
# _setup_nixl (the gate-off path never touches model_config).
|
||||
self._dtype: torch.dtype | None = None
|
||||
self._hidden_dim: int = 0
|
||||
self._element_size: int = 0
|
||||
self._ack_timeout_s: float = 0.0
|
||||
if self._nixl_enabled:
|
||||
self._setup_nixl(vllm_config)
|
||||
|
||||
def _setup_nixl(self, vllm_config: "VllmConfig") -> None:
|
||||
# Lazy imports keep nixl/zmq off the gate-off path.
|
||||
from vllm import envs
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import (
|
||||
ZmqClientTransport,
|
||||
ZmqServerTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
|
||||
NixlDataTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
|
||||
compute_ec_compatibility_hash,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
|
||||
ProducerSession,
|
||||
)
|
||||
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
if NixlWrapper is None or nixl_agent_config is None:
|
||||
raise RuntimeError(
|
||||
"ec_enable_nixl requires NIXL; install the `nixl` package or "
|
||||
"remove ec_enable_nixl from ec_connector_extra_config."
|
||||
)
|
||||
engine_id = self._ec_config.engine_id
|
||||
assert engine_id is not None
|
||||
self._dtype = vllm_config.model_config.dtype
|
||||
self._hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
|
||||
self._element_size = torch.empty(0, dtype=self._dtype).element_size()
|
||||
# How long a consumer waits for an XferAck. The producer answers from
|
||||
# its scheduler step, so its reply latency scales with the encoder's
|
||||
# batch size: a deployment whose steps run longer must raise this.
|
||||
self._ack_timeout_s = float(
|
||||
self._ec_config.get_from_extra_config("consumer_ack_timeout_s", 2.0)
|
||||
)
|
||||
self._compat_hash = compute_ec_compatibility_hash(
|
||||
vllm_version=VLLM_VERSION,
|
||||
model=str(vllm_config.model_config.model),
|
||||
dtype=str(self._dtype),
|
||||
block_size_bytes=self._region.block_size_bytes,
|
||||
)
|
||||
if self._is_producer:
|
||||
self._peer_host = envs.VLLM_EC_SIDE_CHANNEL_HOST
|
||||
self._peer_port = envs.VLLM_EC_SIDE_CHANNEL_PORT
|
||||
|
||||
# Registering the region with NIXL and binding the control sockets are
|
||||
# the first side effects here. __init__ propagates a failure, so the
|
||||
# caller never receives a scheduler it could shut down: unwind through
|
||||
# the same teardown shutdown() uses.
|
||||
try:
|
||||
self._data = NixlDataTransport(
|
||||
agent_name=engine_id,
|
||||
base_ptr=self._region.blocks.data_ptr(),
|
||||
num_blocks=self._region.num_blocks,
|
||||
block_size_bytes=self._region.block_size_bytes,
|
||||
total_size_bytes=self._region.num_blocks
|
||||
* self._region.block_size_bytes,
|
||||
)
|
||||
if self._is_producer:
|
||||
assert self._peer_host is not None
|
||||
assert self._peer_port is not None
|
||||
self._producer_session = ProducerSession(
|
||||
transport=ZmqServerTransport(
|
||||
host=self._peer_host, port=self._peer_port
|
||||
),
|
||||
data=self._data,
|
||||
cache=self._cache,
|
||||
compat_hash=self._compat_hash,
|
||||
)
|
||||
if self._is_consumer:
|
||||
self._transport = ZmqClientTransport()
|
||||
except Exception:
|
||||
self._teardown_nixl()
|
||||
raise
|
||||
|
||||
def has_cache_item(self, identifier: str) -> bool:
|
||||
if not self._is_consumer:
|
||||
return False
|
||||
@@ -72,8 +186,202 @@ class ECCPUScheduler:
|
||||
def ensure_cache_available(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
if not self._nixl_enabled:
|
||||
return True # CPU offload never blocks.
|
||||
first = self._first_in_batch
|
||||
self._first_in_batch = False
|
||||
if not self._is_consumer:
|
||||
return True
|
||||
if first:
|
||||
self._poll_step()
|
||||
return self._nixl_consumer_admit(request, num_computed_tokens)
|
||||
|
||||
def _nixl_consumer_admit(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
"""Admit a request only once all its remote encodings are cached.
|
||||
|
||||
Returns True when every encoding the request needs is either already
|
||||
ready in the local cache or has no remote source. Any encoding whose
|
||||
NIXL READ is still in flight (or was just started this step) defers
|
||||
the request, which the scheduler re-presents on a later step.
|
||||
"""
|
||||
params: dict[str, dict[str, Any]] = (
|
||||
getattr(request, "ec_transfer_params", None) or {}
|
||||
)
|
||||
if not params:
|
||||
return True
|
||||
pending = False
|
||||
for feature in request.mm_features:
|
||||
pos = feature.mm_position
|
||||
if pos.offset + pos.length <= num_computed_tokens:
|
||||
continue
|
||||
mm_hash = feature.identifier
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is not None and entry.ready:
|
||||
# Local hit: upstream's update_state_after_alloc pins and
|
||||
# loads it through the same path as a natively cached entry.
|
||||
continue
|
||||
if mm_hash in self._in_flight:
|
||||
pending = True
|
||||
continue
|
||||
if mm_hash in self._step_completed:
|
||||
pending = True
|
||||
continue
|
||||
if mm_hash in self._tombstones:
|
||||
self._tombstones.discard(mm_hash)
|
||||
continue
|
||||
if entry is not None:
|
||||
# Present but not ready and not being fetched: its blocks are
|
||||
# held by a quarantined/settling DMA and cannot be reused.
|
||||
# Fall back to local compute this step; never re-alloc a
|
||||
# mm_hash already in the cache.
|
||||
continue
|
||||
info = params.get(mm_hash)
|
||||
# Absent entirely, or present with only placeholder metadata (no
|
||||
# cache entry existed on the producer side at request_finished
|
||||
# time): nothing to fetch, fall back to local compute.
|
||||
if not info or "peer_host" not in info:
|
||||
continue
|
||||
expected = pos.length * self._hidden_dim * self._element_size
|
||||
if int(info.get("size_bytes", -1)) != expected:
|
||||
logger.warning(
|
||||
"EC consumer: size mismatch mm_hash=%s; local encode", mm_hash
|
||||
)
|
||||
continue
|
||||
try:
|
||||
started = self._start_xfer(mm_hash, info, expected)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
|
||||
)
|
||||
continue
|
||||
if not started:
|
||||
continue
|
||||
self._in_flight.add(mm_hash)
|
||||
pending = True
|
||||
return not pending
|
||||
|
||||
def _start_xfer(
|
||||
self, mm_hash: str, info: "dict[str, Any]", size_bytes: int
|
||||
) -> bool:
|
||||
"""Allocate a not-ready cache entry and start a NIXL READ into it.
|
||||
|
||||
Returns True if the transfer was started. Returns False when the
|
||||
cache cannot accommodate the encoding, in which case the request
|
||||
falls back to local recomputation.
|
||||
"""
|
||||
import time
|
||||
from math import ceil
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
|
||||
ConsumerSession,
|
||||
)
|
||||
|
||||
n_blocks = max(1, ceil(size_bytes / self._region.block_size_bytes))
|
||||
entry = self._cache.alloc(mm_hash, n_blocks)
|
||||
if entry is None:
|
||||
logger.debug(
|
||||
"EC consumer: cache full for mm_hash=%s (%d blocks); local encode",
|
||||
mm_hash,
|
||||
n_blocks,
|
||||
)
|
||||
return False
|
||||
indices = list(entry.block_ids)
|
||||
addr = (info["peer_host"], int(info["peer_port"]))
|
||||
if addr not in self._sessions:
|
||||
zmq_conn = self._transport.connect(addr)
|
||||
assert self._compat_hash is not None
|
||||
self._sessions[addr] = ConsumerSession(
|
||||
addr=addr,
|
||||
zmq_conn=zmq_conn,
|
||||
transport=self._transport,
|
||||
data=self._data,
|
||||
compat_hash=self._compat_hash,
|
||||
)
|
||||
deadline = time.monotonic() + self._ack_timeout_s
|
||||
try:
|
||||
self._sessions[addr].start_xfer(mm_hash, indices, deadline)
|
||||
except Exception:
|
||||
self._cache.discard(mm_hash)
|
||||
raise
|
||||
logger.debug(
|
||||
"EC consumer: starting NIXL xfer mm_hash=%s from %s:%d blocks=%d",
|
||||
mm_hash,
|
||||
addr[0],
|
||||
addr[1],
|
||||
n_blocks,
|
||||
)
|
||||
return True
|
||||
|
||||
def _poll_step(self) -> None:
|
||||
import time
|
||||
|
||||
now = time.monotonic()
|
||||
all_messages = self._transport.poll()
|
||||
for addr, session in list(self._sessions.items()):
|
||||
session.poll(all_messages.get(addr, []), now)
|
||||
for addr in self._transport.poll_dead():
|
||||
self._on_peer_down(addr)
|
||||
for session in self._sessions.values():
|
||||
self._process_session_results(session)
|
||||
|
||||
def _process_session_results(self, session: Any) -> None:
|
||||
r = session.take_results()
|
||||
for mm_hash in r.completed:
|
||||
self._in_flight.discard(mm_hash)
|
||||
self._cache.mark_ready(mm_hash)
|
||||
self._step_completed.add(mm_hash)
|
||||
logger.debug("EC consumer: NIXL xfer complete mm_hash=%s", mm_hash)
|
||||
for mm_hash in r.tombstoned:
|
||||
self._in_flight.discard(mm_hash)
|
||||
self._cache.discard(mm_hash)
|
||||
self._tombstones.add(mm_hash)
|
||||
logger.debug("EC consumer: NIXL xfer failed mm_hash=%s", mm_hash)
|
||||
for mm_hash in r.quarantined:
|
||||
# DMA still running: keep the blocks reserved (entry stays
|
||||
# not-ready, hence non-evictable) until the xfer settles.
|
||||
self._in_flight.discard(mm_hash)
|
||||
self._tombstones.add(mm_hash)
|
||||
logger.debug(
|
||||
"EC consumer: NIXL xfer mm_hash=%s quarantined; DMA still running",
|
||||
mm_hash,
|
||||
)
|
||||
for mm_hash in r.retryable:
|
||||
# No tombstone: dropping every trace of the attempt is what lets
|
||||
# the next admit pass re-issue the read as if it were the first.
|
||||
self._in_flight.discard(mm_hash)
|
||||
self._cache.discard(mm_hash)
|
||||
logger.debug(
|
||||
"EC consumer: NIXL xfer mm_hash=%s retryable; will re-request",
|
||||
mm_hash,
|
||||
)
|
||||
for mm_hash, _block_indices in r.settled:
|
||||
self._cache.discard(mm_hash)
|
||||
logger.debug(
|
||||
"EC consumer: quarantined NIXL xfer mm_hash=%s settled", mm_hash
|
||||
)
|
||||
|
||||
def _on_peer_down(self, addr: Any) -> None:
|
||||
session = self._sessions.pop(addr, None)
|
||||
if session is None:
|
||||
return
|
||||
session.on_peer_down()
|
||||
self._process_session_results(session)
|
||||
session.close()
|
||||
logger.info("EC consumer: producer peer down addr=%s", addr)
|
||||
|
||||
def _promote_completed_reads(self) -> None:
|
||||
"""Drain the just-completed set built during ``_poll_step``.
|
||||
|
||||
Reads are marked ready in ``_process_session_results``, so a deferred
|
||||
request is re-admitted by ``ensure_cache_available`` and loaded through
|
||||
the same local path as a natively cached entry
|
||||
(``update_state_after_alloc`` -> ``_pending_loads``). No explicit
|
||||
pin/load bookkeeping is needed here beyond clearing the set.
|
||||
"""
|
||||
self._step_completed.clear()
|
||||
|
||||
def update_state_after_alloc(self, request: "Request", index: int) -> None:
|
||||
feature = request.mm_features[index]
|
||||
mm_hash = feature.identifier
|
||||
@@ -97,11 +405,17 @@ class ECCPUScheduler:
|
||||
) -> ECCPUConnectorMetadata:
|
||||
meta = ECCPUConnectorMetadata()
|
||||
if self._is_producer:
|
||||
if self._nixl_enabled and self._producer_session is not None:
|
||||
self._producer_session.poll_step()
|
||||
meta.saves = self._pending_saves
|
||||
self._pending_saves = {}
|
||||
if self._is_consumer:
|
||||
if self._nixl_enabled:
|
||||
self._promote_completed_reads()
|
||||
meta.loads = self._pending_loads
|
||||
self._pending_loads = {}
|
||||
if self._nixl_enabled:
|
||||
self._first_in_batch = True
|
||||
return meta
|
||||
|
||||
def update_connector_output(self, connector_output: "ECConnectorOutput") -> None:
|
||||
@@ -117,8 +431,15 @@ class ECCPUScheduler:
|
||||
return
|
||||
for mm_hash in meta.completed_saves:
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is not None and not entry.ready:
|
||||
if entry is None:
|
||||
logger.debug(
|
||||
"EC producer: worker reported completed save for unknown "
|
||||
"mm_hash=%s (already discarded/evicted?)",
|
||||
mm_hash,
|
||||
)
|
||||
elif not entry.ready:
|
||||
self._cache.mark_ready(mm_hash)
|
||||
logger.debug("EC producer: mm_hash=%s marked ready", mm_hash)
|
||||
for transfer_id in meta.completed_loads:
|
||||
pending = self._load_acks.get(transfer_id)
|
||||
if pending is None:
|
||||
@@ -131,14 +452,65 @@ class ECCPUScheduler:
|
||||
# as stale rather than releasing the pin a second time.
|
||||
del self._load_acks[transfer_id]
|
||||
self._cache.unpin(mm_hash)
|
||||
logger.debug("EC consumer: mm_hash=%s unpinned after load", mm_hash)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
"""True while any dispatched save or load has not been confirmed done.
|
||||
"""Keep the engine stepping so this connector's polls keep running.
|
||||
|
||||
Keeps the engine stepping so the worker's completion reports are polled
|
||||
even when no requests are otherwise runnable.
|
||||
With NIXL enabled the engine tick is the only driver of
|
||||
``ProducerSession.poll_step()``, and a producer's work arrives from a
|
||||
remote consumer, so nothing local would otherwise wake it. Without NIXL
|
||||
this only has to outlive dispatched saves and loads, so the engine can
|
||||
quiesce once the worker has confirmed them.
|
||||
"""
|
||||
return self._cache.has_held_entries()
|
||||
return self._nixl_enabled or self._cache.has_held_entries()
|
||||
|
||||
def request_finished(
|
||||
self, request: "Request"
|
||||
) -> tuple[bool, "dict[str, Any] | None"]:
|
||||
if not (self._nixl_enabled and self._is_producer):
|
||||
return False, None
|
||||
if not request.mm_features:
|
||||
return False, None
|
||||
|
||||
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
|
||||
|
||||
for feature in request.mm_features:
|
||||
mm_hash = feature.identifier
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is None:
|
||||
# Never saved, or evicted since. Publishing placeholder
|
||||
# metadata without an address invites an orchestrator to
|
||||
# rewrite the media into a reference to an encoding no
|
||||
# consumer can fetch, leaving the decoder nothing to embed.
|
||||
# Publish neither, so the media stays on the request.
|
||||
items[mm_hash]["metadata"] = {}
|
||||
logger.debug(
|
||||
"EC producer: mm_hash=%s absent at request_finished; "
|
||||
"announcing no metadata so the media is not rewritten away",
|
||||
mm_hash,
|
||||
)
|
||||
continue
|
||||
# Announce even if the save's GPU->mmap copy hasn't been
|
||||
# confirmed complete yet: a not-ready entry can't be evicted, so
|
||||
# it will still be here by the time a consumer's XferReq arrives.
|
||||
# A read arriving before the save lands is NACKed NACK_NOT_READY,
|
||||
# which the consumer retries on a later step rather than treating
|
||||
# as a miss.
|
||||
size_bytes = (
|
||||
feature.mm_position.length * self._hidden_dim * self._element_size
|
||||
)
|
||||
items[mm_hash].update(
|
||||
peer_host=self._peer_host,
|
||||
peer_port=self._peer_port,
|
||||
size_bytes=size_bytes,
|
||||
)
|
||||
logger.debug(
|
||||
"EC producer: announcing NIXL-readable encodings req_id=%s items=%s",
|
||||
request.request_id,
|
||||
items,
|
||||
)
|
||||
return False, items
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._pending_saves.clear()
|
||||
@@ -148,7 +520,45 @@ class ECCPUScheduler:
|
||||
self._is_producer = False
|
||||
self._is_consumer = False
|
||||
|
||||
if self._nixl_enabled:
|
||||
self._teardown_nixl()
|
||||
|
||||
try:
|
||||
self._region.cleanup()
|
||||
except Exception:
|
||||
logger.debug("ec: region cleanup failed", exc_info=True)
|
||||
|
||||
def _teardown_nixl(self) -> None:
|
||||
"""Close the control sockets and release the NIXL agent.
|
||||
|
||||
Guards each step and clears what it releases, so it is safe on a
|
||||
scheduler whose NIXL setup failed part-way and safe to call twice.
|
||||
The shared region is not touched — shutdown() owns that.
|
||||
"""
|
||||
if self._producer_session is not None:
|
||||
try:
|
||||
self._producer_session.close()
|
||||
except Exception:
|
||||
logger.debug("ec: producer session close failed", exc_info=True)
|
||||
self._producer_session = None
|
||||
|
||||
for session in list(self._sessions.values()):
|
||||
try:
|
||||
session.close()
|
||||
except Exception:
|
||||
logger.debug("ec: consumer session close failed", exc_info=True)
|
||||
self._sessions.clear()
|
||||
|
||||
if self._transport is not None:
|
||||
try:
|
||||
self._transport.close()
|
||||
except Exception:
|
||||
logger.debug("ec: client transport close failed", exc_info=True)
|
||||
self._transport = None
|
||||
|
||||
if self._data is not None:
|
||||
try:
|
||||
self._data.deregister()
|
||||
except Exception:
|
||||
logger.debug("ec: deregister failed", exc_info=True)
|
||||
self._data = None
|
||||
|
||||
@@ -122,6 +122,23 @@ class EmbeddingCache:
|
||||
self._evictable_block_count -= len(entry.block_ids)
|
||||
entry.pin()
|
||||
|
||||
def pin_if_ready(self, key: str) -> CacheEntry | None:
|
||||
"""Atomically pin *key* if present and ready; return its entry.
|
||||
|
||||
Returns None when the key is absent. An entry returned with `ready`
|
||||
False was not pinned: its save is still in flight, which the producer
|
||||
reports differently from a miss because such a read can be retried.
|
||||
"""
|
||||
with self._lock:
|
||||
entry = self._entries.get(key)
|
||||
if entry is None or not entry.ready:
|
||||
return entry
|
||||
if entry.evictable:
|
||||
del self._entries_free_list[key]
|
||||
self._evictable_block_count -= len(entry.block_ids)
|
||||
entry.pin()
|
||||
return entry
|
||||
|
||||
def unpin(self, key: str) -> None:
|
||||
"""Unpin an entry. Asserts currently pinned."""
|
||||
with self._lock:
|
||||
@@ -134,6 +151,21 @@ class EmbeddingCache:
|
||||
self._entries_free_list[key] = None
|
||||
self._evictable_block_count += len(entry.block_ids)
|
||||
|
||||
def discard(self, key: str) -> None:
|
||||
"""Remove a not-ready in-flight entry, returning its blocks to the pool.
|
||||
|
||||
Used when an in-flight fill (e.g. a NIXL READ) fails before the entry
|
||||
is marked ready. Asserts the entry is present and not ready — ready or
|
||||
pinned entries are reclaimed through eviction/unpin, not discard.
|
||||
"""
|
||||
with self._lock:
|
||||
entry = self._entries[key]
|
||||
assert entry._pin_count == -1, (
|
||||
f"EmbeddingCache: discard of ready/pinned entry {key!r}"
|
||||
)
|
||||
del self._entries[key]
|
||||
self._free_blocks.extend(entry.block_ids)
|
||||
|
||||
def _evict_until(self, n_blocks: int) -> None:
|
||||
"""Evict ready+unpinned entries FIFO until enough space. Lock held."""
|
||||
while len(self._free_blocks) < n_blocks:
|
||||
|
||||
@@ -0,0 +1,633 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Session objects for the ECCPUConnector.
|
||||
|
||||
Layer hierarchy: connector → session → transport (connection)
|
||||
|
||||
Two long-lived sessions:
|
||||
|
||||
ProducerSession — owns ZmqServerTransport. The connector calls poll_step()
|
||||
once per engine step, which drains the transport without blocking and
|
||||
feeds the raw bytes to poll(). poll() decodes XferReqs, grants or NACKs
|
||||
them, pins source blocks, sends XferAck replies, drains NIXL completion
|
||||
notifications, and sweeps expired xfers.
|
||||
|
||||
ConsumerSession — owns one ZmqClientConnection and the NIXL agent
|
||||
registration for that peer. The connector calls poll(messages, now)
|
||||
once per engine step with the raw bytes collected for this peer.
|
||||
poll() decodes XferAcks, dispatches them to ConsumerXfer objects,
|
||||
and advances all active xfer state machines.
|
||||
|
||||
Internal data classes:
|
||||
|
||||
ProducerXfer — one pinned read grant, keyed by session_id.
|
||||
ConsumerXfer — state machine for one NIXL READ, keyed by mm_hash.
|
||||
"""
|
||||
|
||||
import enum
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import (
|
||||
ZmqClientConnection,
|
||||
ZmqClientTransport,
|
||||
ZmqServerTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
|
||||
DataTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
|
||||
EC_CONNECTOR_VERSION,
|
||||
EXPECTED_NACKS,
|
||||
RETRYABLE_NACKS,
|
||||
XferAck,
|
||||
XferReq,
|
||||
XferStatus,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
PRODUCER_PIN_LEASE_S = 30.0
|
||||
_CONSUMER_READ_TIMEOUT_S = 20.0
|
||||
_CONSUMER_QUARANTINE_TIMEOUT_S = 60.0
|
||||
|
||||
|
||||
# ── XferState ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class XferState(enum.Enum):
|
||||
"""Lifecycle states of a single ConsumerXfer."""
|
||||
|
||||
WAITING_ACK = "waiting_ack" # XferReq sent, awaiting XferAck
|
||||
READING = "reading" # NIXL READ in flight
|
||||
DONE = "done" # READ completed; promote to local cache
|
||||
ACK_TIMEOUT = "ack_timeout" # XferAck never arrived; free blocks + tombstone
|
||||
READ_FAILED = "read_failed" # Unexpected NIXL state; free blocks + tombstone
|
||||
QUARANTINED = "quarantined" # Read timed out, NIXL unabortable; keep blocks
|
||||
SETTLED = "settled" # Quarantined DMA done; safe to free blocks
|
||||
|
||||
|
||||
# ── ProducerXfer ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProducerXfer:
|
||||
"""One pinned read grant on the producer side.
|
||||
|
||||
Pure data — lifecycle operations (pin/unpin) are performed by ProducerSession.
|
||||
Keyed by (consumer_session_id, mm_hash) in ProducerSession._active_xfers.
|
||||
"""
|
||||
|
||||
mm_hash: str
|
||||
block_indices: list[int]
|
||||
deadline: float
|
||||
|
||||
def is_expired(self) -> bool:
|
||||
return time.monotonic() > self.deadline
|
||||
|
||||
|
||||
# ── ConsumerXfer ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConsumerXfer:
|
||||
"""State machine for one NIXL READ on the consumer side.
|
||||
|
||||
transfer_handle is the in-flight NIXL handle returned by post_read() —
|
||||
used for check_xfer_state() / release_xfer_handle(). The dlist handle
|
||||
for the remote peer lives inside DataTransport, not here.
|
||||
"""
|
||||
|
||||
mm_hash: str
|
||||
block_indices: list[int]
|
||||
addr: PeerAddr
|
||||
deadline: float
|
||||
data: DataTransport
|
||||
consumer_session_id: str # session identity; combined with mm_hash for notif_msg
|
||||
|
||||
transfer_handle: Any | None = field(default=None)
|
||||
_quarantined: bool = field(default=False, init=False)
|
||||
|
||||
def handle_ack(self, ack: XferAck, agent_name: str) -> bool:
|
||||
"""Issue the NIXL READ on an OK XferAck. Returns False on any NACK."""
|
||||
if ack.status != XferStatus.OK:
|
||||
return False
|
||||
# notif_msg encodes both session_id and mm_hash so the producer can
|
||||
# do a direct (session_id, mm_hash) → ProducerXfer lookup on completion.
|
||||
notif_msg = f"{self.consumer_session_id}:{self.mm_hash}".encode()
|
||||
self.transfer_handle = self.data.post_read(
|
||||
self.block_indices,
|
||||
agent_name,
|
||||
ack.src_block_indices,
|
||||
notif_msg=notif_msg,
|
||||
)
|
||||
self.deadline = time.monotonic() + _CONSUMER_READ_TIMEOUT_S
|
||||
logger.debug(
|
||||
"EC consumer: NIXL READ posted mm_hash=%s agent=%s local_blocks=%d "
|
||||
"remote_blocks=%d",
|
||||
self.mm_hash,
|
||||
agent_name,
|
||||
len(self.block_indices),
|
||||
len(ack.src_block_indices),
|
||||
)
|
||||
return True
|
||||
|
||||
def poll(self, now: float) -> XferState:
|
||||
if self._quarantined:
|
||||
return self._poll_quarantined(now)
|
||||
if self.transfer_handle is None:
|
||||
return (
|
||||
XferState.ACK_TIMEOUT if now > self.deadline else XferState.WAITING_ACK
|
||||
)
|
||||
return self._poll_reading(now)
|
||||
|
||||
def _poll_reading(self, now: float) -> XferState:
|
||||
assert self.transfer_handle is not None
|
||||
try:
|
||||
state = self.data.check_xfer_state(self.transfer_handle)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"EC Connector: check_xfer_state failed for mm_hash=%s", self.mm_hash
|
||||
)
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
return XferState.READ_FAILED
|
||||
if state == "DONE":
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
logger.debug("EC consumer: NIXL READ completed mm_hash=%s", self.mm_hash)
|
||||
return XferState.DONE
|
||||
if state == "PROC":
|
||||
if now > self.deadline:
|
||||
self._quarantined = True
|
||||
self.deadline = now + _CONSUMER_QUARANTINE_TIMEOUT_S
|
||||
logger.warning(
|
||||
"EC Connector: READ for mm_hash=%s from %s:%d timed out; "
|
||||
"quarantining",
|
||||
self.mm_hash,
|
||||
self.addr[0],
|
||||
self.addr[1],
|
||||
)
|
||||
return XferState.QUARANTINED
|
||||
return XferState.READING
|
||||
logger.warning(
|
||||
"EC Connector: READ for mm_hash=%s unexpected NIXL state %r",
|
||||
self.mm_hash,
|
||||
state,
|
||||
)
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
return XferState.READ_FAILED
|
||||
|
||||
def _poll_quarantined(self, now: float) -> XferState:
|
||||
if self.transfer_handle is None:
|
||||
return XferState.SETTLED
|
||||
# If quarantine has been too long, assume NIXL is stuck and evict blocks.
|
||||
if now > self.deadline:
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
logger.warning(
|
||||
"EC Connector: quarantined mm_hash=%s from %s:%d timed out; "
|
||||
"evicting blocks",
|
||||
self.mm_hash,
|
||||
self.addr[0],
|
||||
self.addr[1],
|
||||
)
|
||||
return XferState.READ_FAILED
|
||||
try:
|
||||
state = self.data.check_xfer_state(self.transfer_handle)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"EC Connector: check_xfer_state failed for quarantined mm_hash=%s",
|
||||
self.mm_hash,
|
||||
)
|
||||
state = None
|
||||
if state == "PROC":
|
||||
return XferState.QUARANTINED
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
logger.debug(
|
||||
"EC Connector: quarantined mm_hash=%s settled (state=%s)",
|
||||
self.mm_hash,
|
||||
state,
|
||||
)
|
||||
return XferState.SETTLED
|
||||
|
||||
def cancel(self) -> None:
|
||||
"""Cancel a WAITING_ACK xfer (peer down). No tombstone — caller allows retry."""
|
||||
assert self.transfer_handle is None, "cancel() requires WAITING_ACK state"
|
||||
|
||||
def release(self) -> None:
|
||||
"""Force-release the transfer handle. Shutdown only."""
|
||||
if self.transfer_handle is not None:
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
|
||||
|
||||
# ── ProducerSession ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class ProducerSession:
|
||||
"""Long-lived producer session. Owns ZmqServerTransport.
|
||||
|
||||
poll_step() runs once per engine step: drains the transport without
|
||||
blocking, decodes XferReqs, grants or NACKs each one, drains NIXL completion
|
||||
notifications, and sweeps expired xfers.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
transport: ZmqServerTransport,
|
||||
data: DataTransport,
|
||||
cache: EmbeddingCache,
|
||||
compat_hash: str,
|
||||
) -> None:
|
||||
self._transport = transport
|
||||
self._data = data
|
||||
self._cache = cache # EmbeddingCache — single block owner
|
||||
self._compat_hash = compat_hash
|
||||
|
||||
# Key: "{consumer_session_id}:{mm_hash}" — matches the NIXL notif_msg exactly,
|
||||
# so completion notification lookup is a direct dict pop with no parsing.
|
||||
self._active_xfers: dict[str, ProducerXfer] = {}
|
||||
|
||||
self._req_decoder = msgspec.msgpack.Decoder(XferReq)
|
||||
self._req_encoder = msgspec.msgpack.Encoder()
|
||||
|
||||
def close(self) -> None:
|
||||
for xfer in self._active_xfers.values():
|
||||
self._cache.unpin(xfer.mm_hash) # release any still-held pins
|
||||
self._active_xfers.clear()
|
||||
self._transport.close()
|
||||
|
||||
def poll_step(self) -> None:
|
||||
"""Drain the transport without blocking and process what arrived."""
|
||||
self.poll(self._transport.poll(timeout_ms=0))
|
||||
|
||||
def poll(self, messages: list[tuple[bytes, bytes]]) -> None:
|
||||
"""Decode and process inbound XferReqs, drain NIXL notifs, sweep timeouts.
|
||||
|
||||
messages: (identity, payload) pairs from ZmqServerTransport.poll().
|
||||
"""
|
||||
for identity, payload in messages:
|
||||
try:
|
||||
req = self._req_decoder.decode(payload)
|
||||
except (msgspec.DecodeError, msgspec.ValidationError):
|
||||
logger.warning("EC Connector: dropped malformed XferReq")
|
||||
continue
|
||||
ack = self._grant_or_nack(req)
|
||||
self._transport.send(identity, self._req_encoder.encode(ack))
|
||||
|
||||
self._drain_notifs()
|
||||
self._sweep_timeouts()
|
||||
|
||||
def _grant_or_nack(self, req: XferReq) -> XferAck:
|
||||
if req.connector_version != EC_CONNECTOR_VERSION:
|
||||
logger.warning(
|
||||
"EC Connector: incompatible version for mm_hash=%s", req.mm_hash
|
||||
)
|
||||
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_VERSION)
|
||||
if req.compatibility_hash != self._compat_hash:
|
||||
logger.warning(
|
||||
"EC Connector: incompatible compat hash for mm_hash=%s", req.mm_hash
|
||||
)
|
||||
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INCOMPAT)
|
||||
if not req.session_id:
|
||||
# Grants are keyed by (session_id, mm_hash), so an omitted id from
|
||||
# one peer aliases another's: the first completion would unpin
|
||||
# blocks the other peer is still reading.
|
||||
logger.warning(
|
||||
"EC Connector: XferReq without session_id for mm_hash=%s", req.mm_hash
|
||||
)
|
||||
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INTERNAL)
|
||||
entry = self._cache.pin_if_ready(req.mm_hash)
|
||||
if entry is None or not entry.ready:
|
||||
status = (
|
||||
XferStatus.NACK_MISSING if entry is None else XferStatus.NACK_NOT_READY
|
||||
)
|
||||
logger.log(
|
||||
logging.INFO if status in EXPECTED_NACKS else logging.WARNING,
|
||||
"EC Connector: mm_hash=%s not serveable in local cache (%s); NACKing",
|
||||
req.mm_hash,
|
||||
status.name,
|
||||
)
|
||||
return XferAck(mm_hash=req.mm_hash, status=status)
|
||||
block_indices = list(entry.block_ids)
|
||||
key = f"{req.session_id}:{req.mm_hash}"
|
||||
if key in self._active_xfers:
|
||||
# This peer is re-requesting a grant it still holds, having timed
|
||||
# out waiting. Overwriting it drops the pin taken for it, which
|
||||
# neither a completion nor the sweep would ever release.
|
||||
self._cache.unpin(req.mm_hash)
|
||||
self._active_xfers[key] = ProducerXfer(
|
||||
mm_hash=req.mm_hash,
|
||||
block_indices=block_indices,
|
||||
deadline=time.monotonic() + PRODUCER_PIN_LEASE_S,
|
||||
)
|
||||
logger.debug(
|
||||
"EC producer: serving NIXL READ request mm_hash=%s key=%s blocks=%d",
|
||||
req.mm_hash,
|
||||
key,
|
||||
len(block_indices),
|
||||
)
|
||||
return XferAck(
|
||||
mm_hash=req.mm_hash,
|
||||
status=XferStatus.OK,
|
||||
session_id=req.session_id, # echo consumer's session_id back
|
||||
src_block_indices=block_indices,
|
||||
agent_metadata=self._data.get_agent_metadata(),
|
||||
mem_descriptor=self._data.get_mem_descriptor(),
|
||||
)
|
||||
|
||||
def _drain_notifs(self) -> None:
|
||||
try:
|
||||
notifs = self._data.get_new_notifs()
|
||||
except Exception:
|
||||
logger.exception("EC Connector: get_new_notifs failed")
|
||||
return
|
||||
for msgs in notifs.values():
|
||||
for msg in msgs:
|
||||
try:
|
||||
key = msg.decode("utf-8") # "{session_id}:{mm_hash}"
|
||||
except Exception:
|
||||
logger.exception("EC Connector: Dropped malformed NIXL notif")
|
||||
continue
|
||||
xfer = self._active_xfers.pop(key, None)
|
||||
if xfer is not None:
|
||||
self._cache.unpin(xfer.mm_hash)
|
||||
logger.debug(
|
||||
"EC producer: NIXL READ completed mm_hash=%s key=%s",
|
||||
xfer.mm_hash,
|
||||
key,
|
||||
)
|
||||
|
||||
def _sweep_timeouts(self) -> None:
|
||||
for key, xfer in list(self._active_xfers.items()):
|
||||
if xfer.is_expired():
|
||||
logger.warning(
|
||||
"EC producer: consumer did not complete NIXL READ "
|
||||
"mm_hash=%s key=%s within the pin lease; releasing "
|
||||
"pinned blocks",
|
||||
xfer.mm_hash,
|
||||
key,
|
||||
)
|
||||
self._cache.unpin(xfer.mm_hash)
|
||||
del self._active_xfers[key]
|
||||
|
||||
|
||||
# ── ConsumerSession ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConsumerSessionResults:
|
||||
"""Results from ConsumerSession.take_results()."""
|
||||
|
||||
completed: set[str] # promote to local cache
|
||||
tombstoned: set[str] # free blocks + prevent retry this step
|
||||
quarantined: set[str] # timed-out, DMA still running: prevent retry
|
||||
# but DO NOT free blocks — session holds them
|
||||
retryable: set[str] # free blocks, allow another attempt (not ready, peer down)
|
||||
settled: list[tuple[str, list[int]]] # (mm_hash, block_indices) quarantine cleared
|
||||
|
||||
|
||||
class ConsumerSession:
|
||||
"""Long-lived per-producer session on the consumer side.
|
||||
|
||||
Owns the ZMQ DEALER connection and the NIXL agent registration for one
|
||||
producer peer. The connector calls poll(messages, now) once per engine
|
||||
step with the raw bytes collected from this peer's DEALER socket.
|
||||
|
||||
poll() decodes XferAcks, dispatches them to the right ConsumerXfer,
|
||||
advances all active xfer state machines, and drains quarantined xfers.
|
||||
Results accumulate in internal sets; call take_results() to collect them.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
addr: PeerAddr,
|
||||
zmq_conn: ZmqClientConnection,
|
||||
transport: ZmqClientTransport,
|
||||
data: DataTransport,
|
||||
compat_hash: str,
|
||||
) -> None:
|
||||
self._addr = addr
|
||||
self._zmq = zmq_conn
|
||||
self._transport = transport
|
||||
self._data = data
|
||||
self._compat_hash = compat_hash
|
||||
self._session_id = str(
|
||||
uuid.uuid4()
|
||||
) # stable identity for (session_id:mm_hash) keys
|
||||
|
||||
self._nixl_agent_name: str | None = None
|
||||
self._nixl_metadata_bytes: bytes | None = None
|
||||
|
||||
self._xfers: dict[str, ConsumerXfer] = {} # mm_hash → xfer
|
||||
self._quarantined: list[ConsumerXfer] = [] # DMA pending, to poll
|
||||
self._newly_quarantined: set[str] = set() # quarantined since last take_results
|
||||
|
||||
self._completed: set[str] = set()
|
||||
self._tombstoned: set[str] = set()
|
||||
self._retryable: set[str] = set()
|
||||
self._settled: list[tuple[str, list[int]]] = []
|
||||
|
||||
self._encoder = msgspec.msgpack.Encoder()
|
||||
self._decoder = msgspec.msgpack.Decoder(XferAck)
|
||||
|
||||
@property
|
||||
def addr(self) -> PeerAddr:
|
||||
return self._addr
|
||||
|
||||
def start_xfer(
|
||||
self, mm_hash: str, block_indices: list[int], deadline: float
|
||||
) -> None:
|
||||
"""Create a ConsumerXfer and send the XferReq over the connection."""
|
||||
xfer = ConsumerXfer(
|
||||
mm_hash=mm_hash,
|
||||
block_indices=block_indices,
|
||||
addr=self._addr,
|
||||
deadline=deadline,
|
||||
data=self._data,
|
||||
consumer_session_id=self._session_id,
|
||||
)
|
||||
self._zmq.send(
|
||||
self._encoder.encode(
|
||||
XferReq(
|
||||
mm_hash=mm_hash,
|
||||
compatibility_hash=self._compat_hash,
|
||||
session_id=self._session_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
self._xfers[mm_hash] = xfer
|
||||
logger.debug(
|
||||
"EC Connector: XferReq sent mm_hash=%s to %s:%d",
|
||||
mm_hash,
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
)
|
||||
|
||||
def poll(self, messages: list[bytes], now: float) -> None:
|
||||
"""Decode inbound XferAcks, dispatch to xfers, and advance all state machines.
|
||||
|
||||
messages: raw bytes from ZmqClientTransport.poll() for this peer.
|
||||
Results accumulate until take_results() is called.
|
||||
"""
|
||||
for raw in messages:
|
||||
try:
|
||||
ack = self._decoder.decode(raw)
|
||||
except (msgspec.DecodeError, msgspec.ValidationError):
|
||||
logger.warning(
|
||||
"EC Connector: dropped malformed XferAck from %s:%d",
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
)
|
||||
continue
|
||||
self._dispatch_ack(ack)
|
||||
|
||||
for mm_hash, xfer in list(self._xfers.items()):
|
||||
state = xfer.poll(now)
|
||||
if state == XferState.DONE:
|
||||
self._completed.add(mm_hash)
|
||||
del self._xfers[mm_hash]
|
||||
elif state in (XferState.ACK_TIMEOUT, XferState.READ_FAILED):
|
||||
self._tombstoned.add(mm_hash)
|
||||
del self._xfers[mm_hash]
|
||||
elif state == XferState.QUARANTINED:
|
||||
# DMA may still be writing — caller must NOT free blocks.
|
||||
self._quarantined.append(xfer)
|
||||
self._newly_quarantined.add(mm_hash)
|
||||
del self._xfers[mm_hash]
|
||||
|
||||
self._drain_quarantine(now)
|
||||
|
||||
def _dispatch_ack(self, ack: XferAck) -> None:
|
||||
xfer = self._xfers.get(ack.mm_hash)
|
||||
if xfer is None or xfer.transfer_handle is not None:
|
||||
return # unknown mm_hash or duplicate ack
|
||||
|
||||
if ack.status != XferStatus.OK:
|
||||
logger.log(
|
||||
logging.INFO if ack.status in EXPECTED_NACKS else logging.WARNING,
|
||||
"EC Connector: NACK %s from %s:%d for mm_hash=%s",
|
||||
ack.status.name,
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
ack.mm_hash,
|
||||
)
|
||||
del self._xfers[ack.mm_hash]
|
||||
# The producer announced this encoding and its save is still in
|
||||
# flight, so another attempt on a later step can be granted. Every
|
||||
# other NACK is terminal for this mm_hash.
|
||||
if ack.status in RETRYABLE_NACKS:
|
||||
self._retryable.add(ack.mm_hash)
|
||||
else:
|
||||
self._tombstoned.add(ack.mm_hash)
|
||||
return
|
||||
|
||||
try:
|
||||
agent_name = self._ensure_registered(ack.agent_metadata, ack.mem_descriptor)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"EC Connector: failed to register peer %s:%d for mm_hash=%s",
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
ack.mm_hash,
|
||||
)
|
||||
del self._xfers[ack.mm_hash]
|
||||
self._tombstoned.add(ack.mm_hash)
|
||||
return
|
||||
|
||||
if not xfer.handle_ack(ack, agent_name):
|
||||
del self._xfers[ack.mm_hash]
|
||||
self._tombstoned.add(ack.mm_hash)
|
||||
|
||||
def _ensure_registered(self, metadata: bytes, mem_descriptor: bytes) -> str:
|
||||
"""Register or re-register the peer in DataTransport. Returns agent_name."""
|
||||
if self._nixl_agent_name is not None and self._nixl_metadata_bytes == metadata:
|
||||
return self._nixl_agent_name
|
||||
if self._nixl_agent_name is not None:
|
||||
logger.info(
|
||||
"EC Connector: producer %s:%d restarted; re-registering",
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
)
|
||||
self._data.remove_remote_peer(self._nixl_agent_name)
|
||||
agent_name = self._data.add_remote_peer(metadata, mem_descriptor)
|
||||
self._nixl_agent_name = agent_name
|
||||
self._nixl_metadata_bytes = metadata
|
||||
logger.debug(
|
||||
"EC Connector: registered peer %s:%d agent=%s",
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
agent_name,
|
||||
)
|
||||
return agent_name
|
||||
|
||||
def _drain_quarantine(self, now: float) -> None:
|
||||
still_pending: list[ConsumerXfer] = []
|
||||
for xfer in list(self._quarantined):
|
||||
state = xfer.poll(now)
|
||||
if state == XferState.SETTLED:
|
||||
self._settled.append((xfer.mm_hash, xfer.block_indices))
|
||||
else:
|
||||
still_pending.append(xfer)
|
||||
self._quarantined = still_pending
|
||||
|
||||
def on_peer_down(self) -> None:
|
||||
"""WAITING_ACK xfers → cancel (retry allowed); READING xfers → quarantine."""
|
||||
for mm_hash, xfer in list(self._xfers.items()):
|
||||
if xfer.transfer_handle is None:
|
||||
xfer.cancel()
|
||||
self._retryable.add(mm_hash)
|
||||
else:
|
||||
self._quarantined.append(xfer)
|
||||
self._newly_quarantined.add(mm_hash)
|
||||
self._xfers.clear()
|
||||
|
||||
def take_results(self) -> ConsumerSessionResults:
|
||||
"""Return and clear all accumulated results since the last call."""
|
||||
results = ConsumerSessionResults(
|
||||
completed=self._completed,
|
||||
tombstoned=self._tombstoned,
|
||||
quarantined=self._newly_quarantined,
|
||||
retryable=self._retryable,
|
||||
settled=self._settled,
|
||||
)
|
||||
self._completed = set()
|
||||
self._tombstoned = set()
|
||||
self._newly_quarantined = set()
|
||||
self._retryable = set()
|
||||
self._settled = []
|
||||
return results
|
||||
|
||||
def has_in_flight(self) -> bool:
|
||||
return bool(self._xfers) or bool(self._quarantined)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release all transfer handles, deregister NIXL peer, close connection.
|
||||
|
||||
Connection teardown is routed through the transport so the pooled
|
||||
connection is dropped in lock-step with the session — a session and its
|
||||
DEALER connection have a 1:1 lifecycle, so a later connect() to the same
|
||||
peer builds a fresh socket instead of handing back this dead one.
|
||||
"""
|
||||
for xfer in self._xfers.values():
|
||||
xfer.release()
|
||||
self._xfers.clear()
|
||||
for xfer in self._quarantined:
|
||||
xfer.release()
|
||||
self._quarantined.clear()
|
||||
if self._nixl_agent_name is not None:
|
||||
self._data.remove_remote_peer(self._nixl_agent_name)
|
||||
self._transport.remove(self._addr)
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Supporting utilities for the ECCPUConnector scheduler."""
|
||||
|
||||
import msgspec
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
# (host, port) pair identifying a remote peer's ZMQ side-channel endpoint.
|
||||
PeerAddr = tuple[str, int]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Msgpack (de)serialization for the list-of-(addr, size, device_id)
|
||||
# descriptor form that NIXL's `get_xfer_descs` expects.
|
||||
_MemDescList = list[tuple[int, int, int]]
|
||||
|
||||
_mem_desc_encoder = msgspec.msgpack.Encoder()
|
||||
_mem_desc_decoder = msgspec.msgpack.Decoder(_MemDescList)
|
||||
|
||||
|
||||
def serialize_mem_descriptor(descs: _MemDescList) -> bytes:
|
||||
return _mem_desc_encoder.encode(descs)
|
||||
|
||||
|
||||
def deserialize_mem_descriptor(payload: bytes) -> _MemDescList:
|
||||
return _mem_desc_decoder.decode(payload)
|
||||
|
||||
|
||||
def build_block_descs(
|
||||
base_ptr: int,
|
||||
num_blocks: int,
|
||||
block_size_bytes: int,
|
||||
device_id: int = 0,
|
||||
) -> _MemDescList:
|
||||
"""Per-block `(addr, size, device_id)` tuples for an `ECSharedRegion`.
|
||||
|
||||
Handed to `nixl.get_xfer_descs` to build a dlist where descriptor `i`
|
||||
addresses block `i` in the mmap. This makes a WRITE to block indices
|
||||
`[i1, i2, ...]` a simple `make_prepped_xfer(..., [i1, i2])` on both
|
||||
ends — no per-request address math.
|
||||
"""
|
||||
return [
|
||||
(base_ptr + i * block_size_bytes, block_size_bytes, device_id)
|
||||
for i in range(num_blocks)
|
||||
]
|
||||
@@ -55,7 +55,6 @@ def _coalesce_runs(
|
||||
split. `EmbeddingCache` hands out ascending ids, which is what makes runs
|
||||
findable at all.
|
||||
"""
|
||||
|
||||
ids = np.fromiter(block_ids, dtype=np.int64, count=len(block_ids))
|
||||
breaks = np.flatnonzero(np.diff(ids) != 1) + 1
|
||||
slots = np.concatenate(([0], breaks))
|
||||
|
||||
@@ -5,7 +5,6 @@ from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import safetensors
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
@@ -13,8 +12,11 @@ from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorMetadata,
|
||||
ECConnectorRole,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
PlaceholderMetadataResolver,
|
||||
collect_ec_item_metadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -52,8 +54,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
super().__init__(vllm_config=vllm_config, role=role)
|
||||
# req_id -> index
|
||||
self._mm_datas_need_loads: dict[str, int] = {}
|
||||
self._model_config = vllm_config.model_config
|
||||
self._metadata_fields_cache: dict[str, set[str]] = {}
|
||||
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
|
||||
transfer_config = vllm_config.ec_transfer_config
|
||||
if transfer_config is not None:
|
||||
self._storage_path = transfer_config.get_from_extra_config(
|
||||
@@ -169,36 +170,6 @@ class ECExampleConnector(ECConnectorBase):
|
||||
self._mm_datas_need_loads.clear()
|
||||
return meta
|
||||
|
||||
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
|
||||
"""Which processed keys this model needs published for `modality`.
|
||||
|
||||
Read from `MultiModalDataParser.embedding_fields`, the same declaration
|
||||
the consumer's parser requires, so the two cannot drift. An empty set
|
||||
means the modality cannot be delivered out of band, and the consumer
|
||||
will process the media itself.
|
||||
"""
|
||||
if modality in self._metadata_fields_cache:
|
||||
return self._metadata_fields_cache[modality]
|
||||
|
||||
fields: set[str] = set()
|
||||
try:
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
|
||||
fields = info.data_parser.placeholder_metadata_fields(modality)
|
||||
except Exception:
|
||||
# Reporting nothing is a safe degradation: the consumer falls back to
|
||||
# processing the media itself.
|
||||
logger.warning(
|
||||
"Could not determine the placeholder metadata fields for "
|
||||
"modality %s; the consumer will preprocess the media itself.",
|
||||
modality,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
self._metadata_fields_cache[modality] = fields
|
||||
return fields
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
@@ -214,28 +185,10 @@ class ECExampleConnector(ECConnectorBase):
|
||||
if not self.is_producer:
|
||||
return False, None
|
||||
|
||||
items = []
|
||||
for feature in request.mm_features:
|
||||
metadata = {}
|
||||
# `data` is None for items served from the processor cache, in which
|
||||
# case the metadata is unavailable here and the consumer has to fall
|
||||
# back to processing the media itself.
|
||||
if feature.data is not None:
|
||||
wanted = self._placeholder_metadata_fields(feature.modality)
|
||||
for key, value in feature.data.get_data().items():
|
||||
if key not in wanted:
|
||||
continue
|
||||
if isinstance(value, torch.Tensor):
|
||||
metadata[key] = value.tolist()
|
||||
elif is_list_of(value, (int, float)):
|
||||
# Some metadata (e.g. Qwen3-VL video timestamps) is
|
||||
# produced as a plain list rather than a tensor.
|
||||
metadata[key] = value
|
||||
items.append({"mm_hash": feature.identifier, **metadata})
|
||||
|
||||
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
|
||||
if not items:
|
||||
return False, None
|
||||
return False, {"ec_items": items}
|
||||
return False, items
|
||||
|
||||
# ==============================
|
||||
# Helper functions
|
||||
|
||||
@@ -2,8 +2,88 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""EC connector helper utilities."""
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.v1.outputs import ECConnectorOutput, ModelRunnerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class PlaceholderMetadataResolver:
|
||||
"""Resolves which processed keys a model needs published per modality.
|
||||
|
||||
Reads `MultiModalDataParser.embedding_fields`, the same declaration the
|
||||
consumer's parser requires, so the two cannot drift. An empty set means
|
||||
the modality cannot be delivered out of band, and the consumer will
|
||||
process the media itself.
|
||||
"""
|
||||
|
||||
def __init__(self, model_config: "ModelConfig") -> None:
|
||||
self._model_config = model_config
|
||||
self._cache: dict[str, set[str]] = {}
|
||||
|
||||
def fields_for(self, modality: str) -> set[str]:
|
||||
if modality in self._cache:
|
||||
return self._cache[modality]
|
||||
|
||||
fields: set[str] = set()
|
||||
try:
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
|
||||
fields = info.data_parser.placeholder_metadata_fields(modality)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Could not determine the placeholder metadata fields for "
|
||||
"modality %s; the consumer will preprocess the media itself.",
|
||||
modality,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
self._cache[modality] = fields
|
||||
return fields
|
||||
|
||||
|
||||
def collect_ec_item_metadata(
|
||||
mm_features: "list[MultiModalFeatureSpec]",
|
||||
resolver: PlaceholderMetadataResolver,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Build one `ec_transfer_params` entry per feature for `request_finished()`.
|
||||
|
||||
Keyed by mm_hash, each entry carries a `metadata` dict with whatever
|
||||
placeholder fields `resolver` says this model needs published for its
|
||||
modality, so a consumer can skip the image transform. `data` is None for
|
||||
items served from the processor cache, in which case the metadata is
|
||||
unavailable here and the consumer has to fall back to processing the
|
||||
media itself. A connector that also has transfer coordinates to report
|
||||
(e.g. NIXL peer_host/peer_port/size_bytes) merges those in alongside
|
||||
`metadata`, not into it.
|
||||
"""
|
||||
items: dict[str, dict[str, Any]] = {}
|
||||
for feature in mm_features:
|
||||
metadata: dict[str, Any] = {}
|
||||
if feature.data is not None:
|
||||
wanted = resolver.fields_for(feature.modality)
|
||||
for key, value in feature.data.get_data().items():
|
||||
if key not in wanted:
|
||||
continue
|
||||
if isinstance(value, torch.Tensor):
|
||||
metadata[key] = value.tolist()
|
||||
elif is_list_of(value, (int, float)):
|
||||
# Some metadata (e.g. Qwen3-VL video timestamps) is
|
||||
# produced as a plain list rather than a tensor.
|
||||
metadata[key] = value
|
||||
items[feature.identifier] = {"metadata": metadata}
|
||||
return items
|
||||
|
||||
|
||||
class ECOutputAggregator:
|
||||
"""Merge every worker's EC connector output onto the single
|
||||
|
||||
Reference in New Issue
Block a user