[Mini-VLLM] Remove orphan subsystems: snapshot, CPU workers, dead decoders

- Delete the snapshot package, CLI command, and its launch tests
- Delete CPU workers, cpu_communicator, and the hw_agnostic layer package
- Delete the EC CPU connector, keeping the EC base glue intact
- Delete dead composite/triton_flashinfer backends and third_party/flashmla
- Delete dead video decoders, video-bound tests, and orphan utils
- Remove the deep_gemm_mega_moe option and dead allreduce fusion pair

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 09:27:06 +07:00
co-authored by Hinata
parent 2b544b7ba0
commit bd70a61c20
88 changed files with 4 additions and 20247 deletions
@@ -1,339 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from typing import Any
import torch
from torch.distributed import ProcessGroup
from vllm.distributed.utils import pickle
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.platforms.interface import CpuArchEnum
from .base_device_communicator import DeviceCommunicatorBase
logger = init_logger(__name__)
class CpuCommunicator(DeviceCommunicatorBase):
def __init__(
self,
cpu_group: ProcessGroup,
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
use_all2all: bool = False,
):
super().__init__(
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
)
self.dist_module = torch.distributed
if (
(
current_platform.get_cpu_architecture() == CpuArchEnum.X86
or current_platform.get_cpu_architecture() == CpuArchEnum.ARM
or current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC
or current_platform.get_cpu_architecture() == CpuArchEnum.S390X
)
and hasattr(torch.ops._C, "init_shm_manager")
and (unique_name.startswith("tp") or unique_name.startswith("pp"))
and self._all_group_ranks_share_shm_group_name()
):
self.dist_module = _CPUSHMDistributed(self)
elif unique_name.startswith("tp") or unique_name.startswith("pp"):
logger.info(
"CPU SHM communicator disabled for group %s: ranks do not share "
"the same SHM group name, falling back to torch.distributed.",
unique_name,
)
# send/recv tensor_dict is only supported through the SHM communicator backend
self.supports_tensor_dict = isinstance(self.dist_module, _CPUSHMDistributed)
if self.use_all2all:
if self.all2all_backend not in (
"naive",
"allgather_reducescatter",
): # type: ignore[has-type]
logger.warning(
"`%s` all2all manager is not supported on CPU. "
"Falling back to `allgather_reducescatter` manager.",
self.all2all_backend, # type: ignore[has-type]
)
from .all2all import AgRsAll2AllManager
self.all2all_manager = AgRsAll2AllManager(self.cpu_group)
logger.info("Using allgather_reducescatter all2all manager.")
def _all_group_ranks_share_shm_group_name(self) -> bool:
"""CPUSHM requires all ranks in this group to agree on one SHM group name.
This is a lightweight consistency check for VLLM_DIST_IDENT/name inputs.
"""
local_name = _CPUSHMDistributed.make_group_name(self)
names: list[str] = [""] * self.world_size
torch.distributed.all_gather_object(
names,
local_name,
group=self.device_group,
)
return len(set(names)) == 1
def all_reduce(self, input_):
self.dist_module.all_reduce(input_, group=self.device_group)
return input_
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> torch.Tensor | None:
"""NOTE: We assume that the input tensor is on the same device across
all the ranks.
NOTE: `dst` is the local rank of the destination rank.
"""
world_size = self.world_size
assert -input_.dim() <= dim < input_.dim(), (
f"Invalid dim ({dim}) for input tensor with shape {input_.size()}"
)
if dim < 0:
# Convert negative dim to positive.
dim += input_.dim()
# Allocate output tensor.
if self.rank_in_group == dst:
gather_list = [torch.empty_like(input_) for _ in range(world_size)]
else:
gather_list = None
# Gather.
self.dist_module.gather(
input_, gather_list, dst=self.ranks[dst], group=self.device_group
)
if self.rank_in_group == dst:
output_tensor = torch.cat(gather_list, dim=dim)
else:
output_tensor = None
return output_tensor
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
if dim < 0:
# Convert negative dim to positive.
dim += input_.dim()
input_size = input_.size()
# NOTE: we have to use concat-style all-gather here,
# stack-style all-gather has compatibility issues with
# torch.compile . see https://github.com/pytorch/pytorch/issues/138795
output_size = (input_size[0] * self.world_size,) + input_size[1:]
# Allocate output tensor.
output_tensor = torch.empty(
output_size, dtype=input_.dtype, device=input_.device
)
# All-gather.
self.dist_module.all_gather_into_tensor(
output_tensor, input_, group=self.device_group
)
# Reshape
output_tensor = output_tensor.reshape((self.world_size,) + input_size)
output_tensor = output_tensor.movedim(0, dim)
output_tensor = output_tensor.reshape(
input_size[:dim]
+ (self.world_size * input_size[dim],)
+ input_size[dim + 1 :]
)
return output_tensor
def send_tensor_dict(
self,
tensor_dict: dict[str, torch.Tensor | Any],
dst: int,
) -> None:
if not self.supports_tensor_dict:
raise NotImplementedError(
"CpuCommunicator does not support tensor dict fastpath with "
"torch.distributed backend."
)
return self.dist_module.send_tensor_dict(tensor_dict, dst)
def recv_tensor_dict(
self,
src: int,
) -> dict[str, torch.Tensor | Any]:
if not self.supports_tensor_dict:
raise NotImplementedError(
"CpuCommunicator does not support tensor dict fastpath with "
"torch.distributed backend."
)
return self.dist_module.recv_tensor_dict(src)
def dispatch_router_logits(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
is_sequence_parallel: bool = False,
extra_tensors: list[torch.Tensor] | None = None,
) -> (
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch_router_logits(
hidden_states,
router_logits,
is_sequence_parallel,
extra_tensors,
)
def dispatch(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
is_sequence_parallel: bool = False,
extra_tensors: list[torch.Tensor] | None = None,
) -> (
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch(
hidden_states,
topk_weights,
topk_ids,
is_sequence_parallel,
extra_tensors=extra_tensors,
)
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.combine(
hidden_states,
is_sequence_parallel,
)
class _CPUSHMDistributed:
def __init__(self, communicator: CpuCommunicator):
self.communicator = communicator
self.group_name = self.make_group_name(communicator)
self.handle = self._init_cpu_shm()
@staticmethod
def make_group_name(communicator: CpuCommunicator) -> str:
instance_identifier = os.environ["VLLM_DIST_IDENT"]
unique_name = communicator.unique_name
instance_identifier = f"{instance_identifier}-{unique_name}"
group_ranks = [str(rank) for rank in communicator.ranks]
shm_group_identifier = f"[{'-'.join(group_ranks)}]"
return f"{instance_identifier}-{shm_group_identifier}-cpushm"
def _init_cpu_shm(self) -> int:
thread_num_tensor = torch.tensor(
[torch.get_num_threads()],
dtype=torch.int64,
)
torch.distributed.all_reduce(
thread_num_tensor,
op=torch.distributed.ReduceOp.MIN,
group=self.communicator.device_group,
)
thread_num = thread_num_tensor.item()
handle = torch.ops._C.init_shm_manager(
self.group_name,
self.communicator.world_size,
self.communicator.rank,
thread_num,
)
torch.distributed.barrier(self.communicator.device_group)
torch.ops._C.join_shm_manager(
handle,
self.group_name,
)
torch.distributed.barrier(self.communicator.device_group)
return handle
def all_reduce(
self, input: torch.Tensor, group: ProcessGroup | None = None
) -> None:
torch.ops._C.shm_allreduce(self.handle, input)
def gather(
self,
input: torch.Tensor,
gather_list: list[torch.Tensor] | None,
dst: int = -1,
group: ProcessGroup | None = None,
) -> None:
# Note: different from the torch gather, here we use local dst rank.
torch.ops._C.shm_gather(
self.handle,
input,
gather_list,
torch.distributed.get_group_rank(group, dst),
)
def all_gather_into_tensor(
self,
output: torch.Tensor,
input: torch.Tensor,
group: ProcessGroup | None = None,
) -> None:
torch.ops._C.shm_all_gather(self.handle, input, output)
def send_tensor_dict(
self,
tensor_dict: dict[str, torch.Tensor | Any],
dst: int,
) -> None:
key_list = list(tensor_dict.keys())
value_list = list(tensor_dict.values())
size_list = []
for v in value_list:
if not isinstance(v, torch.Tensor):
raise RuntimeError("CpuCommunicator only supports sending tensors.")
size_list.append(v.size())
key_size_tensor = torch.frombuffer(
pickle.dumps([key_list, size_list]), dtype=torch.uint8
)
value_list.append(key_size_tensor)
torch.ops._C.shm_send_tensor_list(self.handle, value_list, dst)
return None
def recv_tensor_dict(
self,
src: int,
) -> dict[str, torch.Tensor | Any]:
tensor_list = torch.ops._C.shm_recv_tensor_list(self.handle, src)
value_list: list[torch.Tensor] = tensor_list[:-1]
key_size_tensor = tensor_list[-1]
key_size = pickle.loads(key_size_tensor.numpy().tobytes())
key_list = key_size[0]
size_list = key_size[1]
assert len(key_list) == len(size_list)
assert len(key_list) == len(value_list)
tensor_dict: dict[str, torch.Tensor] = {}
for key, size, t in zip(key_list, size_list, value_list):
tensor_dict[key] = t.view(size)
return tensor_dict
@@ -1,98 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared types for the ECCPUConnector scheduler and worker delegates."""
from dataclasses import dataclass, field
from typing import TYPE_CHECKING
import torch
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
ECConnectorWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
@dataclass
class ECCPUConnectorMetadata(ECConnectorMetadata):
"""Per-step scheduler → worker payload for the ECCPUConnector.
Populated by `ECCPUScheduler.build_connector_meta`; consumed by
`ECCPUWorker` via the mixin's `bind_connector_metadata`.
"""
# Producer role: mm_hashes the scheduler has just allocated CPU
# blocks for this step; the worker's save_caches copies
# encoder_cache[mm_hash] → mmap at these block IDs.
saves: dict[str, list[int]] = field(default_factory=dict)
# Consumer role: mm_hashes whose bytes are available in the local mmap,
# mapped to (transfer_id, block_ids, shape); the worker's start_load_caches
# copies mmap[block_ids] → GPU encoder_cache and reports the transfer_id
# once its copy lands. The id is minted per dispatch, so two loads of the
# same mm_hash in different steps are distinct and a late report from the
# earlier one cannot release the later one's pin.
loads: dict[str, tuple[int, list[int], tuple[int, int]]] = field(
default_factory=dict
)
@dataclass
class ECCPUWorkerMetadata(ECConnectorWorkerMetadata):
"""Per-step worker → scheduler payload for the ECCPUConnector.
Reports the GPU copies that completed this step: saved mm_hashes become
safe to mark ready, and loaded transfers become safe to unpin once every
participating rank has reported them. Built by
`ECCPUWorker.build_connector_worker_meta`; consumed by
`ECCPUScheduler.update_connector_output`.
Loads are reported by transfer id rather than mm_hash because every rank
copies the same blocks: `aggregate` concatenates, so an id appears once
per reporting rank and the scheduler can count participants off the list.
"""
completed_saves: list[str] = field(default_factory=list)
completed_loads: list[int] = field(default_factory=list)
def aggregate(self, other: ECConnectorWorkerMetadata) -> ECConnectorWorkerMetadata:
assert isinstance(other, ECCPUWorkerMetadata)
self.completed_saves.extend(other.completed_saves)
self.completed_loads.extend(other.completed_loads)
return self
def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion:
"""Build the EC mmap region from `vllm_config`.
Both `ECCPUScheduler` and `ECCPUWorker` call this to get the same
shared region (same engine_id, same block_size_bytes).
"""
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None, "ec_transfer_config required to build region"
dp_index = vllm_config.parallel_config.data_parallel_index
engine_id = f"{vllm_config.instance_id}_dp{dp_index}"
dtype = vllm_config.model_config.dtype
hidden_dim = vllm_config.model_config.get_inputs_embeds_size()
element_size = torch.empty(0, dtype=dtype).element_size()
block_size_bytes = hidden_dim * element_size
cpu_bytes = ec_config.ec_connector_extra_config.get("ec_cpu_bytes")
if not cpu_bytes:
raise ValueError("ec_cpu_bytes must be specified in ec_connector_extra_config")
cpu_bytes = int(cpu_bytes)
num_blocks = cpu_bytes // block_size_bytes
return ECSharedRegion(
engine_id=engine_id,
num_blocks=num_blocks,
block_size_bytes=block_size_bytes,
)
@@ -1,153 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ECCPUConnector — CPU encoder-cache offloading.
A role-routed shell: one instance per process. The scheduler delegate owns the
mmap region and offload bookkeeping; the worker delegate owns the mmap view and
GPU copy plumbing. An ec_both instance reuses encoder outputs it has already
offloaded to CPU instead of recomputing them.
"""
from typing import TYPE_CHECKING, Any
import torch
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorBase,
ECConnectorRole,
ECConnectorWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ECConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
class ECCPUConnector(ECConnectorBase):
"""EC connector that offloads encoder cache to a shared CPU mmap region."""
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole) -> None:
super().__init__(vllm_config=vllm_config, role=role)
if not vllm_config.use_v2_model_runner:
raise ValueError(
"ECCPUConnector requires the V2 model runner. "
"Set VLLM_USE_V2_MODEL_RUNNER=1."
)
self.connector_worker = None
self.connector_scheduler = None
if role == ECConnectorRole.WORKER:
self.connector_worker = self._make_worker(vllm_config)
elif role == ECConnectorRole.SCHEDULER:
self.connector_scheduler = self._make_scheduler(vllm_config)
else:
raise ValueError(f"Unknown ECConnectorRole: {role}")
# Construction seams.
def _make_worker(self, vllm_config: "VllmConfig"):
# Deferred import: the worker module touches torch/CUDA at import time
# via the region, so keep that cost off the scheduler path.
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import (
ECCPUWorker,
)
return ECCPUWorker(vllm_config)
def _make_scheduler(self, vllm_config: "VllmConfig"):
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import (
ECCPUScheduler,
)
return ECCPUScheduler(vllm_config)
# Worker-side forwarders.
def start_load_caches(
self, encoder_cache: dict[str, torch.Tensor], **kwargs
) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, ECCPUConnectorMetadata)
self.connector_worker.start_load_caches(
encoder_cache, connector_metadata=metadata
)
def save_caches(
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs
) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, ECCPUConnectorMetadata)
self.connector_worker.save_caches(
encoder_cache, mm_hash, connector_metadata=metadata
)
# Scheduler-side forwarders.
def has_cache_item(self, identifier: str) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.has_cache_item(identifier)
def ensure_cache_available(
self, request: "Request", num_computed_tokens: int
) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.ensure_cache_available(
request, num_computed_tokens
)
def take_unavailable_requests(self) -> set[str]:
assert self.connector_scheduler is not None
return self.connector_scheduler.take_unavailable_requests()
def update_state_after_alloc(self, request: "Request", index: int) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.update_state_after_alloc(request, index)
def build_connector_meta(
self, scheduler_output: "SchedulerOutput"
) -> ECCPUConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def request_finished(
self, request: "Request"
) -> tuple[bool, "dict[str, Any] | None"]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
if self.connector_worker is not None:
self.connector_worker.flush_saves()
return None, None
def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None:
if self.connector_worker is not None:
return self.connector_worker.build_connector_worker_meta()
return None
def update_connector_output(self, connector_output: "ECConnectorOutput") -> None:
if self.connector_scheduler is not None:
self.connector_scheduler.update_connector_output(connector_output)
def has_pending_push_work(self) -> bool:
if self.connector_scheduler is not None:
return self.connector_scheduler.has_pending_push_work()
return False
# Shared.
def shutdown(self) -> None:
if self.connector_scheduler is not None:
self.connector_scheduler.shutdown()
if self.connector_worker is not None:
self.connector_worker.shutdown()
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Abstract base classes for the EC control plane.
ControlConnection wraps a single peer channel (one DEALER socket).
"""
from abc import ABC, abstractmethod
class ControlConnection(ABC):
"""Bidirectional message channel to a single remote peer.
All I/O is non-blocking: send enqueues without blocking; recv drains
all buffered inbound messages since the last call.
"""
@property
@abstractmethod
def alive(self) -> bool:
"""True if the connection is still usable."""
@abstractmethod
def send(self, msg: bytes) -> None:
"""Enqueue msg for delivery. Must not block."""
@abstractmethod
def recv(self) -> list[bytes]:
"""Drain and return all buffered inbound messages."""
@abstractmethod
def mark_dead(self) -> None:
"""Signal that the peer disconnected. Sets alive to False."""
@abstractmethod
def close(self) -> None:
"""Release all resources. Idempotent."""
@@ -1,233 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ZMQ control-plane implementations for ECCPUConnector.
ZmqClientConnection — one DEALER socket to a single producer peer.
ZmqClientTransport — pool of ZmqClientConnection objects (consumer side).
ZmqServerTransport — ROUTER socket (producer side).
"""
import contextlib
from dataclasses import dataclass, field
import zmq
from zmq.utils.monitor import recv_monitor_message
from vllm.distributed.ec_transfer.ec_connector.cpu.control.base import (
ControlConnection,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr
from vllm.logger import init_logger
from vllm.utils.network_utils import make_zmq_path, make_zmq_socket
logger = init_logger(__name__)
_HEARTBEAT_IVL_MS = 2000
_HEARTBEAT_TIMEOUT_MS = 4000
_HEARTBEAT_TTL_MS = 8000
# ── Client side ───────────────────────────────────────────────────────────────
@dataclass
class ZmqClientConnection(ControlConnection):
"""One DEALER socket to one producer peer.
Owns the socket and its optional monitor socket. NIXL state is NOT
stored here — it belongs in ECSession above this layer.
"""
dealer: zmq.Socket
monitor: zmq.Socket | None
_alive: bool = field(default=True, init=False)
@property
def alive(self) -> bool:
return self._alive
def send(self, msg: bytes) -> None:
self.dealer.send_multipart([b"", msg])
def recv(self) -> list[bytes]:
msgs: list[bytes] = []
while True:
try:
frames = self.dealer.recv_multipart(flags=zmq.NOBLOCK)
except zmq.Again:
break
except Exception:
logger.exception("ec: DEALER recv failed")
break
if len(frames) != 2 or frames[0] != b"":
logger.warning(
"ec: dropped malformed frame envelope (%d frames)", len(frames)
)
continue
msgs.append(frames[1])
return msgs
def mark_dead(self) -> None:
self._alive = False
def close(self) -> None:
if self.monitor is not None:
with contextlib.suppress(Exception):
self.dealer.disable_monitor()
with contextlib.suppress(Exception):
self.monitor.close(linger=0)
with contextlib.suppress(Exception):
self.dealer.close(linger=0)
self._alive = False
class ZmqClientTransport:
"""Pool of ZmqClientConnection objects — one per producer peer.
Creates DEALER sockets lazily on first contact. Does not touch NIXL;
NIXL registration is handled by ECCPUConsumer once an XferAck arrives.
"""
def __init__(self) -> None:
self._ctx = zmq.Context()
self._connections: dict[PeerAddr, ZmqClientConnection] = {}
def connect(self, addr: PeerAddr) -> ZmqClientConnection:
"""Return the existing connection for addr, or create a new one."""
existing = self._connections.get(addr)
if existing is not None:
return existing
host, port = addr
dealer = make_zmq_socket(
ctx=self._ctx,
path=make_zmq_path(scheme="tcp", host=host, port=port),
socket_type=zmq.DEALER,
bind=False,
)
dealer.setsockopt(zmq.HEARTBEAT_IVL, _HEARTBEAT_IVL_MS)
dealer.setsockopt(zmq.HEARTBEAT_TIMEOUT, _HEARTBEAT_TIMEOUT_MS)
dealer.setsockopt(zmq.HEARTBEAT_TTL, _HEARTBEAT_TTL_MS)
monitor_addr = f"inproc://ec-peer-mon-{host}-{port}"
dealer.monitor(monitor_addr, zmq.EVENT_DISCONNECTED)
monitor = self._ctx.socket(zmq.PAIR)
monitor.connect(monitor_addr)
conn = ZmqClientConnection(dealer=dealer, monitor=monitor)
self._connections[addr] = conn
return conn
def poll(self) -> dict[PeerAddr, list[bytes]]:
"""Non-blocking drain of messages from every peer's DEALER."""
msgs: dict[PeerAddr, list[bytes]] = {}
for addr, conn in list(self._connections.items()):
conn_msgs = conn.recv()
msgs[addr] = conn_msgs
return msgs
def poll_dead(self) -> list[PeerAddr]:
"""Non-blocking sweep of peer monitors; return addresses that died."""
dead: list[PeerAddr] = []
for addr, conn in list(self._connections.items()):
if conn.monitor is None:
continue
try:
while True:
evt = recv_monitor_message(conn.monitor, flags=zmq.NOBLOCK)
if evt["event"] == zmq.EVENT_DISCONNECTED:
conn.mark_dead()
dead.append(addr)
break
except zmq.Again:
pass
except Exception:
logger.warning("ec: monitor poll failed for %s", addr, exc_info=True)
return dead
def remove(self, addr: PeerAddr) -> ZmqClientConnection | None:
"""Remove and close the connection for addr, returning it (or None)."""
conn = self._connections.pop(addr, None)
if conn is not None:
conn.close()
return conn
def close(self) -> None:
for conn in list(self._connections.values()):
conn.close()
self._connections.clear()
try:
self._ctx.destroy(linger=0)
except Exception:
logger.debug("ec: zmq context destroy failed", exc_info=True)
# ── Server side ───────────────────────────────────────────────────────────────
class ZmqServerTransport:
"""ROUTER socket for the producer side. Byte-level only, driven by
ProducerSession.
poll(timeout_ms) waits up to timeout_ms for messages, then drains all that
are available; timeout_ms=0 drains whatever has already arrived. send()
routes a reply back to a specific peer by ZMQ identity.
"""
def __init__(self, host: str, port: int) -> None:
self._ctx = zmq.Context()
self._router = make_zmq_socket(
ctx=self._ctx,
path=make_zmq_path(scheme="tcp", host=host, port=port),
socket_type=zmq.ROUTER,
bind=True,
)
self._poller = zmq.Poller()
self._poller.register(self._router, zmq.POLLIN)
def poll(self, timeout_ms: int = 0) -> list[tuple[bytes, bytes]]:
"""Wait up to timeout_ms for incoming messages, then drain all pending.
Returns a list of (identity, payload) pairs.
"""
if timeout_ms > 0:
try:
events = dict(self._poller.poll(timeout=timeout_ms))
except zmq.ContextTerminated:
return []
except Exception:
logger.exception("ec: router poller failed")
return []
if self._router not in events:
return []
msgs: list[tuple[bytes, bytes]] = []
while True:
try:
identity, _, payload = self._router.recv_multipart(flags=zmq.NOBLOCK)
msgs.append((identity, payload))
except zmq.Again:
break
except zmq.ContextTerminated:
break
except Exception:
logger.exception("ec: router recv failed")
break
return msgs
def send(self, identity: bytes, payload: bytes) -> None:
"""Send a reply payload to the peer identified by ZMQ identity."""
try:
self._router.send_multipart([identity, b"", payload])
except Exception:
logger.exception("ec: router send failed")
def close(self) -> None:
try:
self._router.close(linger=0)
except Exception:
logger.debug("ec: router close failed", exc_info=True)
try:
self._ctx.destroy(linger=0)
except Exception:
logger.debug("ec: zmq context destroy failed", exc_info=True)
@@ -1,67 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Abstract base class for the EC data-plane transport (NIXL/UCX)."""
from abc import ABC, abstractmethod
from typing import Any
class DataTransport(ABC):
"""Interface wrapping the NIXL data-plane for EC block transfers.
Peer lifecycle:
add_remote_peer() — register a peer and store its dlist handle internally
remove_remote_peer() — deregister and release the stored handle
post_read() — issue a READ using the stored handle for agent_name
The dlist handle (prepped xfer descriptor list) is an implementation detail
of the transport and is never exposed to callers.
"""
@abstractmethod
def get_agent_metadata(self) -> bytes:
"""Return this agent's serialized NIXL metadata blob."""
@abstractmethod
def get_mem_descriptor(self) -> bytes:
"""Return the msgpack-encoded block descriptor list for this agent."""
@abstractmethod
def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str:
"""Register a remote peer, prep its READ-source dlist, and return agent_name.
The dlist handle is stored internally and used by post_read().
"""
@abstractmethod
def remove_remote_peer(self, agent_name: str) -> None:
"""Deregister a previously registered remote peer and release its handle."""
@abstractmethod
def post_read(
self,
local_indices: list[int],
agent_name: str,
remote_indices: list[int],
notif_msg: bytes,
) -> Any:
"""Issue a consumer-initiated READ using the stored dlist for agent_name.
Returns a transfer handle for use with check_xfer_state / release_xfer_handle.
"""
@abstractmethod
def get_new_notifs(self) -> dict[str, list[bytes]]:
"""Drain NIXL completion notifications addressed to this agent."""
@abstractmethod
def check_xfer_state(self, handle: Any) -> str:
"""Return the NIXL state string: 'DONE', 'PROC', or an error string."""
@abstractmethod
def release_xfer_handle(self, handle: Any) -> None:
"""Release a transfer handle returned by post_read."""
@abstractmethod
def deregister(self) -> None:
"""Deregister all memory and tear down the NIXL agent."""
@@ -1,162 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NIXL-backed data-plane transport for ECCPUConnector."""
from typing import Any
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
DataTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
build_block_descs,
deserialize_mem_descriptor,
serialize_mem_descriptor,
)
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
from vllm.logger import init_logger
logger = init_logger(__name__)
_NIXL_DRAM = "DRAM"
class NixlDataTransport(DataTransport):
"""Adapts NixlWrapper to the DataTransport interface.
Dlist handles (prepped xfer descriptor lists for remote peers) are stored
internally keyed by agent_name. Callers never touch raw NIXL handles —
post_read() resolves the handle from agent_name internally.
NixlWrapper is not thread-safe, and neither is this class. Every call site
runs on the scheduler thread, which serializes the transport by
construction; transfers are async inside NIXL and are drained from that
same thread via get_new_notifs() / check_xfer_state().
"""
def __init__(
self,
agent_name: str,
base_ptr: int,
num_blocks: int,
block_size_bytes: int,
total_size_bytes: int,
num_threads: int = 1,
) -> None:
if NixlWrapper is None or nixl_agent_config is None:
raise RuntimeError(
"ECCPUConnector requires NIXL; "
"install the `nixl` package or set a different ec_connector."
)
# agent_name → remote dlist handle (from add_remote_peer, used by post_read)
self._peer_handles: dict[str, int] = {}
# None until the local region is registered, so deregister() is safe on
# a transport whose registration failed part-way.
self._reg_descs: Any = None
self._nixl = NixlWrapper(
agent_name,
nixl_agent_config(num_threads=num_threads, capture_telemetry=True),
)
self._register_local_memory(
base_ptr, num_blocks, block_size_bytes, total_size_bytes
)
def _register_local_memory(
self,
base_ptr: int,
num_blocks: int,
block_size_bytes: int,
total_size_bytes: int,
) -> None:
"""Register the local region with NIXL and prep its transfer dlist.
Unwinds the registration if a later step fails: __init__ propagates the
error, so the caller never receives an object it could deregister with.
"""
block_descs = build_block_descs(base_ptr, num_blocks, block_size_bytes)
self._reg_descs = self._nixl.get_reg_descs(
[(base_ptr, total_size_bytes, 0, "")], _NIXL_DRAM
)
self._nixl.register_memory(self._reg_descs, backends=["UCX"])
try:
xfer_descs = self._nixl.get_xfer_descs(block_descs, _NIXL_DRAM)
self._local_xfer_handle = self._nixl.prep_xfer_dlist(
"NIXL_INIT_AGENT", xfer_descs
)
self._agent_metadata = self._nixl.get_agent_metadata()
self._mem_descriptor_bytes = serialize_mem_descriptor(block_descs)
except Exception:
self.deregister()
raise
def get_agent_metadata(self) -> bytes:
return self._agent_metadata
def get_mem_descriptor(self) -> bytes:
return self._mem_descriptor_bytes
def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str:
"""Register remote peer, prep its dlist, store handle internally.
Returns agent_name for use in post_read() and remove_remote_peer().
"""
agent_name = self._nixl.add_remote_agent(metadata)
remote_blocks = deserialize_mem_descriptor(mem_descriptor)
remote_xfer_descs = self._nixl.get_xfer_descs(remote_blocks, _NIXL_DRAM)
self._peer_handles[agent_name] = self._nixl.prep_xfer_dlist(
agent_name, remote_xfer_descs
)
return agent_name
def remove_remote_peer(self, agent_name: str) -> None:
self._peer_handles.pop(agent_name, None)
try:
self._nixl.remove_remote_agent(agent_name)
except Exception:
logger.warning(
"EC: remove_remote_peer failed for %s", agent_name, exc_info=True
)
def post_read(
self,
local_indices: list[int],
agent_name: str,
remote_indices: list[int],
notif_msg: bytes,
) -> Any:
if len(local_indices) != len(remote_indices):
raise ValueError(
f"EC: local/remote block count mismatch "
f"({len(local_indices)} vs {len(remote_indices)})"
)
remote_handle = self._peer_handles[agent_name]
handle = self._nixl.make_prepped_xfer(
"READ",
self._local_xfer_handle,
local_indices,
remote_handle,
remote_indices,
notif_msg=notif_msg,
)
self._nixl.transfer(handle)
return handle
def get_new_notifs(self) -> dict[str, list[bytes]]:
return self._nixl.get_new_notifs()
def check_xfer_state(self, handle: Any) -> str:
return self._nixl.check_xfer_state(handle)
def release_xfer_handle(self, handle: Any) -> None:
try:
self._nixl.release_xfer_handle(handle)
except Exception:
logger.warning("EC: release_xfer_handle failed", exc_info=True)
def deregister(self) -> None:
if self._reg_descs is None:
return
try:
self._nixl.deregister_memory(self._reg_descs)
except Exception:
logger.warning("EC: deregister failed", exc_info=True)
self._reg_descs = None
@@ -1,193 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Lightweight mmap-backed shared memory region for encoder cache (EC) data.
Modeled after SharedOffloadRegion (vllm/v1/kv_offload/cpu/) but simplified
for EC: flat shared layout, no multi-tensor cursor, no block_size_factor.
"""
import errno
import mmap
import os
import time
from collections.abc import Callable
import numpy as np
import torch
from vllm.logger import init_logger
logger = init_logger(__name__)
# MADV_POPULATE_WRITE was added in Linux 5.14 (value 23).
_MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23)
def _madvise_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None:
mmap_obj.madvise(_MADV_POPULATE_WRITE, offset, length)
def _fallback_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None:
# Touch one byte per page via a read-modify-write so existing bytes are
# preserved — a peer worker may have already written EC data into this
# shared mmap by the time we run on a kernel without MADV_POPULATE_WRITE.
arr = np.frombuffer(mmap_obj, dtype=np.uint8)
arr[offset : offset + length : mmap.PAGESIZE] |= 0
def _get_populate_write_fn(
mmap_obj: mmap.mmap,
) -> Callable[[mmap.mmap, int, int], None]:
"""Select the pre-faulting method once for this mmap."""
try:
_madvise_populate_write(mmap_obj, 0, mmap.PAGESIZE)
except OSError as e:
if e.errno != errno.EINVAL:
raise
logger.warning(
"MADV_POPULATE_WRITE is not supported; falling back to per-page "
"writes for EC mmap pre-population. Startup may be slower."
)
return _fallback_populate_write
return _madvise_populate_write
def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0):
"""Spin-wait until the file reaches expected_size (creator truncated it)."""
deadline = time.monotonic() + timeout
while True:
if os.fstat(fd).st_size >= expected_size:
return
if time.monotonic() > deadline:
raise TimeoutError(
f"Timed out waiting for EC mmap file to reach {expected_size} bytes"
)
time.sleep(0.005)
class ECSharedRegion:
"""Flat mmap-backed memory region shared across TP workers for
encoder cache blocks.
Layout: (num_blocks, block_size_bytes) — contiguous, no per-worker
interleaving. All workers map the same file and see identical data.
File path: /dev/shm/vllm_ec_{engine_id}.mmap
This class owns only the shared memory substrate (mmap lifecycle, the
`blocks` view, CUDA host registration). Block allocation and eviction
are tracked by `EmbeddingCache` in the scheduler process.
"""
def __init__(
self,
engine_id: str,
num_blocks: int,
block_size_bytes: int,
) -> None:
self.num_blocks = num_blocks
self.block_size_bytes = block_size_bytes
total_size_bytes = num_blocks * block_size_bytes
# Path in /dev/shm (tmpfs); unique per engine instance.
self._mmap_path = f"/dev/shm/vllm_ec_{engine_id}.mmap"
# True for the process that created the file (responsible for unlink).
self._is_creator = False
# True after successful cudaHostRegister (cleanup must unregister).
self._is_pinned = False
# File descriptor for the shared memory backing file.
try:
self._fd: int | None = os.open(
self._mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600
)
os.ftruncate(self._fd, total_size_bytes)
self._is_creator = True
logger.info(
"Created EC mmap file %s (%.2f MiB)",
self._mmap_path,
total_size_bytes / (1 << 20),
)
except FileExistsError:
self._fd = os.open(self._mmap_path, os.O_RDWR)
try:
_wait_for_file_size(self._fd, total_size_bytes)
except Exception:
os.close(self._fd)
self._fd = None
raise
logger.info("Opened existing EC mmap file %s", self._mmap_path)
# MAP_SHARED mmap over _fd; all processes see the same pages.
self._mmap_obj: mmap.mmap | None = mmap.mmap(
self._fd,
total_size_bytes,
flags=mmap.MAP_SHARED,
prot=mmap.PROT_READ | mmap.PROT_WRITE,
)
if self._is_creator:
populate_write_fn = _get_populate_write_fn(self._mmap_obj)
populate_write_fn(self._mmap_obj, 0, total_size_bytes)
# (num_blocks, block_size_bytes) int8 tensor over the mmap buffer.
self.blocks: torch.Tensor = torch.frombuffer(
memoryview(self._mmap_obj), dtype=torch.int8
).view(num_blocks, block_size_bytes)
# Cached for cudaHostRegister/Unregister and pointer math.
self._blocks_ptr: int = self.blocks.data_ptr()
self._blocks_nbytes: int = self.blocks.nbytes
def pin_memory(self) -> None:
"""Register the entire mmap as CUDA pinned memory for fast DMA.
Each TP worker process owns its own virtual address for the same
shared physical pages, so every process must register independently.
No-op when CUDA is not available or already pinned.
"""
if self._is_pinned or not torch.cuda.is_available():
return
result = torch.cuda.cudart().cudaHostRegister(
self._blocks_ptr, self._blocks_nbytes, 0
)
if result.value != 0:
logger.warning(
"cudaHostRegister failed (code=%d) — "
"transfers will still work but may be slower (unpinned DMA)",
result.value,
)
else:
logger.debug("cudaHostRegister %.2f MB", self._blocks_nbytes / 1e6)
self._is_pinned = True
def cleanup(self) -> None:
"""Tear down the region. Lifecycle method; no concurrent access."""
logger.info("Starting ECSharedRegion cleanup...")
if self._is_creator:
try:
os.unlink(self._mmap_path)
logger.info("Removed EC mmap file %s", self._mmap_path)
except Exception:
logger.warning(
"Failed to unlink path %s", self._mmap_path, exc_info=True
)
self._is_creator = False
if self._is_pinned:
result = torch.cuda.cudart().cudaHostUnregister(self._blocks_ptr)
if result.value != 0:
logger.warning("cudaHostUnregister failed (code=%d)", result)
self._is_pinned = False
if hasattr(self, "blocks"):
del self.blocks
if self._mmap_obj:
try:
self._mmap_obj.close()
except Exception:
logger.warning("Failed to close mmap_obj", exc_info=True)
self._mmap_obj = None
if self._fd is not None:
try:
os.close(self._fd)
except Exception:
logger.warning("Failed to close fd %s", self._fd, exc_info=True)
self._fd = None
@@ -1,91 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scheduler-side wire types for the ECCPUConnector.
Two peer-to-peer message types exchanged between consumer and producer over
ZMQ (XferReq, XferAck) plus a peer-compatibility fingerprint
(compute_ec_compatibility_hash) sent on every XferReq so a producer refuses
to serve a mismatched peer.
"""
import enum
import hashlib
import json
import msgspec
# Bump when the wire format or tensor/block layout changes in a
# backward-incompatible way. Peers reject mismatched versions.
EC_CONNECTOR_VERSION: int = 2
class XferStatus(enum.IntEnum):
"""Outcome the producer reports for an XferReq."""
OK = 0
NACK_MISSING = 1 # producer no longer holds the encoding
NACK_INCOMPAT = 2 # peer-compatibility hash mismatch
NACK_VERSION = 3 # connector wire-version mismatch
NACK_INTERNAL = 4 # producer hit an unexpected error
NACK_NOT_READY = 5 # announced, but its GPU->mmap save has not landed yet
# The producer expects a retryable condition to clear on its own, so the
# consumer re-requests the read rather than giving up on the encoding.
RETRYABLE_NACKS = frozenset((XferStatus.NACK_NOT_READY,))
# Ordinary outcomes rather than signs of a broken peer.
EXPECTED_NACKS = RETRYABLE_NACKS | {XferStatus.NACK_MISSING}
class XferReq( # type: ignore[call-arg]
msgspec.Struct,
tag="req",
omit_defaults=True,
):
"""Consumer → Producer: I want to READ mm_hash; pin it and tell me where."""
mm_hash: str
compatibility_hash: str
# Consumer's session identity. Together with mm_hash it uniquely identifies
# this transfer on both sides: the producer keys its active xfers by
# (session_id, mm_hash), and the NIXL notif_msg carries the same pair.
session_id: str = ""
connector_version: int = EC_CONNECTOR_VERSION
class XferAck( # type: ignore[call-arg]
msgspec.Struct,
tag="ack",
omit_defaults=True,
):
"""Producer → Consumer: response to an XferReq.
On status == OK: producer has pinned source blocks and returns their
indices plus fresh NIXL metadata for the consumer to register and READ.
On any NACK_*: optional fields are empty; consumer falls back to local encode.
"""
mm_hash: str
status: XferStatus
# Consumer's session_id echoed back from XferReq. Together with mm_hash it
# identifies the exact transfer; the consumer uses both to build notif_msg.
session_id: str = ""
src_block_indices: list[int] = []
agent_metadata: bytes = b""
mem_descriptor: bytes = b""
def compute_ec_compatibility_hash(
vllm_version: str,
model: str,
dtype: str,
block_size_bytes: int,
) -> str:
"""Peer-compatibility fingerprint over the four factors that determine
byte-layout compatibility. Producer NACKs any XferReq whose hash differs.
"""
factors = json.dumps(
[vllm_version, model, dtype, block_size_bytes],
separators=(",", ":"),
)
return hashlib.sha256(factors.encode("utf-8")).hexdigest()
@@ -1,700 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ECCPUScheduler — CPU offload scheduler delegate.
Owns the mmap region and the embedding cache, and handles the producer
(GPU->CPU offload) and consumer (CPU->GPU reload) scheduler-side logic
for the ECCPUConnector.
"""
import time
from collections.abc import Collection
from typing import TYPE_CHECKING, Any
import torch
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
ECCPUWorkerMetadata,
create_ec_shared_region,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
get_encoder_output_width,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ECConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
# Total remote wait budget per request item, including all retries.
_ADMIT_DEFER_TIMEOUT_S = 60.0
class _RemoteUnavailable(Exception):
"""An item cannot be fetched within its request's remote wait budget."""
class ECCPUScheduler:
"""Scheduler delegate for the ECCPUConnector."""
def __init__(self, vllm_config: "VllmConfig") -> None:
self._vllm_config = vllm_config
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None
self._is_producer: bool = ec_config.is_ec_producer
self._is_consumer: bool = ec_config.is_ec_consumer
self._region = create_ec_shared_region(vllm_config)
# Block allocator + LRU eviction policy for the shared region.
self._cache = EmbeddingCache(self._region.num_blocks)
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
# mm_hash → block IDs allocated this step for GPU→mmap saves.
self._pending_saves: dict[str, list[int]] = {}
# mm_hash → (transfer_id, block IDs, shape) to load from mmap→GPU.
self._pending_loads: dict[str, tuple[int, list[int], tuple[int, int]]] = {}
# Dispatched loads awaiting completion reports, keyed by transfer id:
# transfer_id → (mm_hash, reports still outstanding). The pin taken at
# dispatch is released only once the count reaches zero.
self._load_acks: dict[int, tuple[str, int]] = {}
self._next_transfer_id = 0
pc = vllm_config.parallel_config
# Reports to expect per load. Only pipeline stage 0 runs the EC
# connector, so the participants are the tp × pcp ranks of that stage.
# Other executors deliver a single rank's output, and expecting reports
# that cannot arrive would hold the pin forever.
self._expected_load_reports = (
pc.tensor_parallel_size * pc.prefill_context_parallel_size
if pc.distributed_executor_backend == "mp"
else 1
)
self._ec_config = ec_config
# NIXL p2p is an option of this connector, not engine-wide behavior, so
# it lives in extra config. Extra config is not type coerced, so a value
# that arrived as a JSON/CLI string is parsed, not truth-tested.
raw_nixl = ec_config.get_from_extra_config("ec_enable_nixl", False)
self._nixl_enabled = (
raw_nixl
if isinstance(raw_nixl, bool)
else str(raw_nixl).strip().lower() in ("true", "1", "yes")
)
# NIXL fields default to None/empty so the gate-off path is untouched.
self._data: Any = None
self._compat_hash: str | None = None
self._first_in_batch = True
self._transport: Any = None
self._producer_session: Any = None
self._sessions: dict = {}
self._in_flight: set[str] = set()
self._tombstones: set[str] = set()
self._step_completed: set[str] = set()
# Per-request clocks survive retries and are cleared on completion.
self._deferred_since: dict[tuple[str, str], float] = {}
# Requests needing a remote encoding that will not arrive. Drained by
# take_unavailable_requests(); the scheduler aborts them.
self._unrecoverable: set[str] = set()
self._peer_host: str | None = None
self._peer_port: int | None = None
dtype = vllm_config.model_config.dtype
assert isinstance(dtype, torch.dtype)
self._dtype = dtype
self._ack_timeout_s: float = 0.0
if self._nixl_enabled:
try:
self._setup_nixl(vllm_config)
except Exception:
self.shutdown()
raise
def _setup_nixl(self, vllm_config: "VllmConfig") -> None:
# Lazy imports keep nixl/zmq off the gate-off path.
from vllm import envs
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import (
ZmqClientTransport,
ZmqServerTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
NixlDataTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
compute_ec_compatibility_hash,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
PRODUCER_PIN_LEASE_S,
ProducerSession,
)
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
from vllm.version import __version__ as VLLM_VERSION
if NixlWrapper is None or nixl_agent_config is None:
raise RuntimeError(
"ec_enable_nixl requires NIXL; install the `nixl` package or "
"remove ec_enable_nixl from ec_connector_extra_config."
)
engine_id = self._ec_config.engine_id
assert engine_id is not None
# How long a consumer waits for an XferAck. The producer answers from
# its scheduler step, so its reply latency scales with the encoder's
# batch size: a deployment whose steps run longer must raise this.
self._ack_timeout_s = float(
self._ec_config.get_from_extra_config(
"consumer_ack_timeout_s", PRODUCER_PIN_LEASE_S
)
)
self._compat_hash = compute_ec_compatibility_hash(
vllm_version=VLLM_VERSION,
model=str(vllm_config.model_config.model),
dtype=str(self._dtype),
block_size_bytes=self._region.block_size_bytes,
)
if self._is_producer:
self._peer_host = envs.VLLM_EC_SIDE_CHANNEL_HOST
self._peer_port = envs.VLLM_EC_SIDE_CHANNEL_PORT
self._data = NixlDataTransport(
agent_name=engine_id,
base_ptr=self._region.blocks.data_ptr(),
num_blocks=self._region.num_blocks,
block_size_bytes=self._region.block_size_bytes,
total_size_bytes=self._region.num_blocks * self._region.block_size_bytes,
)
if self._is_producer:
assert self._peer_host is not None
assert self._peer_port is not None
self._producer_session = ProducerSession(
transport=ZmqServerTransport(
host=self._peer_host, port=self._peer_port
),
data=self._data,
cache=self._cache,
compat_hash=self._compat_hash,
)
if self._is_consumer:
self._transport = ZmqClientTransport()
def has_cache_item(self, identifier: str) -> bool:
if not self._is_consumer:
return False
entry = self._cache.get(identifier)
return entry is not None and entry.ready
def ensure_cache_available(
self,
request: "Request",
num_computed_tokens: int,
local_cache_hashes: Collection[str] | None = None,
) -> bool:
if not self._nixl_enabled:
return True # CPU offload never blocks.
first = self._first_in_batch
self._first_in_batch = False
if not self._is_consumer:
return True
if first:
self._poll_step()
return self._nixl_consumer_admit(request, num_computed_tokens)
def _nixl_consumer_admit(
self, request: "Request", num_computed_tokens: int
) -> bool:
"""Admit a request once its required remote encodings are ready.
Each item has a per-request wait budget that survives transfer retries.
Unavailable items fall back to local input; without it, the request is
reported by take_unavailable_requests() for the scheduler to abort.
Returns:
True if no item needs to wait for a remote encoding. False if any
item is pending or the request must be aborted.
"""
if not request.ec_transfer_params:
return True
now = time.monotonic()
pending = False
for feature in request.mm_features:
try:
if not self._admit_item(request, feature, num_computed_tokens, now):
pending = True
except _RemoteUnavailable as error:
if not self._fail_or_fallback(request, feature, str(error)):
return False
return not pending
def _admit_item(
self,
request: "Request",
feature: "MultiModalFeatureSpec",
num_computed_tokens: int,
now: float,
) -> bool:
"""Check one encoding, starting a remote read if needed.
Returns:
True if already computed, cached, or not remotely sourced. False
while waiting for a read or cache space, including across retries.
Raises:
_RemoteUnavailable: The remote source is invalid, the read failed,
or the item's total wait budget expired.
"""
pos = feature.mm_position
mm_hash = feature.identifier
if pos.offset + pos.length <= num_computed_tokens:
self._deferred_since.pop((request.request_id, mm_hash), None)
return True
announced = (request.ec_transfer_params or {}).get(mm_hash)
# Without a producer address there is nothing to fetch: the request
# carries what the model needs and the encoder runs locally.
# `ec_transfer_params` reaches us from the request, so its shape is
# checked rather than assumed.
remote: dict[str, Any] | None = (
announced
if isinstance(announced, dict) and "peer_host" in announced
else None
)
entry = self._cache.get(mm_hash)
if entry is not None and entry.ready:
# Local hit: upstream's update_state_after_alloc pins and
# loads it through the same path as a natively cached entry.
self._deferred_since.pop((request.request_id, mm_hash), None)
return True
if remote is None:
return True
since = self._deferred_since.setdefault((request.request_id, mm_hash), now)
if now - since > _ADMIT_DEFER_TIMEOUT_S:
raise _RemoteUnavailable(f"the remote wait exceeded {now - since:.0f}s")
if mm_hash in self._in_flight or mm_hash in self._step_completed:
return False
if mm_hash in self._tombstones:
self._tombstones.discard(mm_hash)
raise _RemoteUnavailable("the remote read failed")
if entry is not None:
# Present but not ready and not being fetched: its blocks are
# held by a quarantined/settling DMA and cannot be reused.
return False
try:
size = int(remote["size_bytes"])
shape = tuple(int(dim) for dim in remote["shape"])
except (KeyError, TypeError, ValueError) as error:
raise _RemoteUnavailable("the announced size was unusable") from error
# The width is the producer's to report; the row count is not.
num_embeds = pos.get_num_embeds()
if (
len(shape) != 2
or shape[0] != num_embeds
or size != self._num_blocks(shape) * self._region.block_size_bytes
):
logger.warning(
"EC consumer: size mismatch mm_hash=%s announced=%d shape=%s "
"expected %d rows",
mm_hash,
size,
shape,
num_embeds,
)
raise _RemoteUnavailable("the announced size was wrong")
try:
started = self._start_xfer(mm_hash, remote, size, (shape[0], shape[1]))
except Exception as error:
logger.exception(
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
)
raise _RemoteUnavailable("the read could not be started") from error
if started:
self._in_flight.add(mm_hash)
return False
def _fail_or_fallback(
self, request: "Request", feature: "MultiModalFeatureSpec", why: str
) -> bool:
"""Resolve an unavailable remote encoding and clear its wait budget.
Returns:
True if local media or embeddings are available, removing this
item's remote announcement so subsequent steps do not retry it.
False if only placeholder metadata (or no input) is available,
recording the request for take_unavailable_requests() to drain.
"""
mm_hash = feature.identifier
self._deferred_since.pop((request.request_id, mm_hash), None)
data = feature.data
metadata = (
self._metadata_resolver.fields_for(feature.modality) if data else set()
)
if data and any(
value.data is not None for key, value in data.items() if key not in metadata
):
# Passthrough embeddings bypass the processor cache. SHM address
# items therefore refer to media that the worker can encode.
request.ec_transfer_params = dict(request.ec_transfer_params or {})
request.ec_transfer_params.pop(mm_hash, None)
logger.warning(
"EC consumer: request %s mm_hash=%s: %s; using local input",
request.request_id,
mm_hash,
why,
)
return True
self._unrecoverable.add(request.request_id)
logger.error(
"EC consumer: request %s needs remote encoding mm_hash=%s but %s. "
"No local model input is available; failing the request.",
request.request_id,
mm_hash,
why,
)
return False
def take_unavailable_requests(self) -> set[str]:
"""Return and clear IDs of requests that cannot obtain their encodings.
The scheduler must abort these requests rather than leave them waiting.
Requests that can fall back to local input are not included, and each
recorded failure is returned only once.
"""
if not self._unrecoverable:
return set()
failed = self._unrecoverable
self._unrecoverable = set()
return failed
def _start_xfer(
self,
mm_hash: str,
info: "dict[str, Any]",
size_bytes: int,
shape: tuple[int, int],
) -> bool:
"""Allocate a not-ready cache entry and start a NIXL READ into it.
Returns True if the transfer was started. Returns False when the
cache cannot accommodate the encoding; admission will retry it.
"""
from math import ceil
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
ConsumerSession,
)
n_blocks = max(1, ceil(size_bytes / self._region.block_size_bytes))
entry = self._cache.alloc(mm_hash, n_blocks, shape)
if entry is None:
logger.debug(
"EC consumer: cache full for mm_hash=%s (%d blocks); deferring",
mm_hash,
n_blocks,
)
return False
indices = list(entry.block_ids)
addr = (info["peer_host"], int(info["peer_port"]))
if addr not in self._sessions:
zmq_conn = self._transport.connect(addr)
assert self._compat_hash is not None
self._sessions[addr] = ConsumerSession(
addr=addr,
zmq_conn=zmq_conn,
transport=self._transport,
data=self._data,
compat_hash=self._compat_hash,
)
deadline = time.monotonic() + self._ack_timeout_s
try:
self._sessions[addr].start_xfer(mm_hash, indices, deadline)
except Exception:
self._cache.discard(mm_hash)
raise
logger.debug(
"EC consumer: starting NIXL xfer mm_hash=%s from %s:%d blocks=%d",
mm_hash,
addr[0],
addr[1],
n_blocks,
)
return True
def _poll_step(self) -> None:
now = time.monotonic()
all_messages = self._transport.poll()
for addr, session in list(self._sessions.items()):
session.poll(all_messages.get(addr, []), now)
for addr in self._transport.poll_dead():
self._on_peer_down(addr)
for session in self._sessions.values():
self._process_session_results(session)
def _process_session_results(self, session: Any) -> None:
r = session.take_results()
for mm_hash in r.completed:
self._in_flight.discard(mm_hash)
self._cache.mark_ready(mm_hash)
self._step_completed.add(mm_hash)
logger.debug("EC consumer: NIXL xfer complete mm_hash=%s", mm_hash)
for mm_hash in r.tombstoned:
self._in_flight.discard(mm_hash)
self._cache.discard(mm_hash)
self._tombstones.add(mm_hash)
logger.debug("EC consumer: NIXL xfer failed mm_hash=%s", mm_hash)
for mm_hash in r.quarantined:
# DMA still running: keep the blocks reserved (entry stays
# not-ready, hence non-evictable) until the xfer settles.
self._in_flight.discard(mm_hash)
self._tombstones.add(mm_hash)
logger.debug(
"EC consumer: NIXL xfer mm_hash=%s quarantined; DMA still running",
mm_hash,
)
for mm_hash in r.retryable:
# Release this attempt's blocks, retaining the request wait budget.
self._in_flight.discard(mm_hash)
self._cache.discard(mm_hash)
logger.debug(
"EC consumer: NIXL xfer mm_hash=%s retryable; will re-request",
mm_hash,
)
for mm_hash, _block_indices in r.settled:
self._cache.discard(mm_hash)
logger.debug(
"EC consumer: quarantined NIXL xfer mm_hash=%s settled", mm_hash
)
def _on_peer_down(self, addr: Any) -> None:
session = self._sessions.pop(addr, None)
if session is None:
return
session.on_peer_down()
self._process_session_results(session)
session.close()
logger.info("EC consumer: producer peer down addr=%s", addr)
def _promote_completed_reads(self) -> None:
"""Drain the just-completed set built during ``_poll_step``.
Reads are marked ready in ``_process_session_results``, so a deferred
request is re-admitted by ``ensure_cache_available`` and loaded through
the same local path as a natively cached entry
(``update_state_after_alloc`` -> ``_pending_loads``). No explicit
pin/load bookkeeping is needed here beyond clearing the set.
"""
self._step_completed.clear()
def _num_blocks(self, shape: tuple[int, ...]) -> int:
size_bytes = shape[0] * shape[1] * self._dtype.itemsize
block_size = self._region.block_size_bytes
return (size_bytes + block_size - 1) // block_size
def update_state_after_alloc(self, request: "Request", index: int) -> None:
feature = request.mm_features[index]
mm_hash = feature.identifier
if self._is_producer and self._cache.get(mm_hash) is None:
shape = (
feature.mm_position.get_num_embeds(),
get_encoder_output_width(self._vllm_config, feature.modality),
)
entry = self._cache.alloc(mm_hash, self._num_blocks(shape), shape)
if entry is not None:
self._pending_saves[mm_hash] = list(entry.block_ids)
if self._is_consumer and mm_hash not in self._pending_loads:
entry = self._cache.get(mm_hash)
if entry is not None and entry.ready:
assert entry.shape is not None
self._cache.pin(mm_hash)
transfer_id = self._next_transfer_id
self._next_transfer_id += 1
self._pending_loads[mm_hash] = (
transfer_id,
list(entry.block_ids),
entry.shape,
)
self._load_acks[transfer_id] = (mm_hash, self._expected_load_reports)
def build_connector_meta(
self, scheduler_output: "SchedulerOutput"
) -> ECCPUConnectorMetadata:
meta = ECCPUConnectorMetadata()
if self._is_producer:
if self._nixl_enabled and self._producer_session is not None:
self._producer_session.poll_step()
meta.saves = self._pending_saves
self._pending_saves = {}
if self._is_consumer:
if self._nixl_enabled:
self._promote_completed_reads()
meta.loads = self._pending_loads
self._pending_loads = {}
if self._nixl_enabled:
self._first_in_batch = True
return meta
def update_connector_output(self, connector_output: "ECConnectorOutput") -> None:
"""Apply the worker's memcpy-completion report to the cache.
Completed saves become safe to mark ready. A load is unpinned once
every participating rank has reported its transfer id; reports for a
transfer that has already been released, or for one this scheduler
never dispatched, are ignored.
"""
meta = connector_output.ec_connector_worker_meta
if not isinstance(meta, ECCPUWorkerMetadata):
return
for mm_hash in meta.completed_saves:
entry = self._cache.get(mm_hash)
if entry is None:
logger.debug(
"EC producer: worker reported completed save for unknown "
"mm_hash=%s (already discarded/evicted?)",
mm_hash,
)
elif not entry.ready:
self._cache.mark_ready(mm_hash)
logger.debug("EC producer: mm_hash=%s marked ready", mm_hash)
for transfer_id in meta.completed_loads:
pending = self._load_acks.get(transfer_id)
if pending is None:
continue
mm_hash, outstanding = pending
if outstanding > 1:
self._load_acks[transfer_id] = (mm_hash, outstanding - 1)
continue
# Drop the entry before unpinning so a replayed report is treated
# as stale rather than releasing the pin a second time.
del self._load_acks[transfer_id]
self._cache.unpin(mm_hash)
logger.debug("EC consumer: mm_hash=%s unpinned after load", mm_hash)
def has_pending_push_work(self) -> bool:
"""Keep the engine stepping so this connector's polls keep running.
With NIXL enabled the engine tick is the only driver of
``ProducerSession.poll_step()``, and a producer's work arrives from a
remote consumer, so nothing local would otherwise wake it. Without NIXL
this only has to outlive dispatched saves and loads, so the engine can
quiesce once the worker has confirmed them.
"""
return self._nixl_enabled or self._cache.has_held_entries()
def request_finished(
self, request: "Request"
) -> tuple[bool, "dict[str, Any] | None"]:
for feature in request.mm_features:
self._deferred_since.pop((request.request_id, feature.identifier), None)
if not (self._nixl_enabled and self._is_producer):
return False, None
if not request.mm_features:
return False, None
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
for feature in request.mm_features:
mm_hash = feature.identifier
entry = self._cache.get(mm_hash)
if entry is None:
# Never saved, or evicted since. Publishing placeholder
# metadata without an address invites an orchestrator to
# rewrite the media into a reference to an encoding no
# consumer can fetch, leaving the decoder nothing to embed.
# Publish neither, so the media stays on the request.
items[mm_hash]["metadata"] = {}
logger.debug(
"EC producer: mm_hash=%s absent at request_finished; "
"announcing no metadata so the media is not rewritten away",
mm_hash,
)
continue
# Announce even if the save's GPU->mmap copy hasn't been
# confirmed complete yet: a not-ready entry can't be evicted, so
# it will still be here by the time a consumer's XferReq arrives.
# A read arriving before the save lands is NACKed NACK_NOT_READY,
# which the consumer retries on a later step rather than treating
# as a miss.
assert entry.shape is not None
size_bytes = len(entry.block_ids) * self._region.block_size_bytes
items[mm_hash].update(
peer_host=self._peer_host,
peer_port=self._peer_port,
size_bytes=size_bytes,
shape=list(entry.shape),
)
logger.debug(
"EC producer: announcing NIXL-readable encodings req_id=%s items=%s",
request.request_id,
items,
)
return False, items
def shutdown(self) -> None:
self._pending_saves.clear()
self._pending_loads.clear()
self._load_acks.clear()
self._deferred_since.clear()
self._is_producer = False
self._is_consumer = False
if self._nixl_enabled:
self._teardown_nixl()
try:
self._region.cleanup()
except Exception:
logger.debug("ec: region cleanup failed", exc_info=True)
def _teardown_nixl(self) -> None:
"""Close the control sockets and release the NIXL agent.
Guards each step and clears what it releases, so it is safe on a
scheduler whose NIXL setup failed part-way and safe to call twice.
The shared region is not touched — shutdown() owns that.
"""
if self._producer_session is not None:
try:
self._producer_session.close()
except Exception:
logger.debug("ec: producer session close failed", exc_info=True)
self._producer_session = None
for session in list(self._sessions.values()):
try:
session.close()
except Exception:
logger.debug("ec: consumer session close failed", exc_info=True)
self._sessions.clear()
if self._transport is not None:
try:
self._transport.close()
except Exception:
logger.debug("ec: client transport close failed", exc_info=True)
self._transport = None
if self._data is not None:
try:
self._data.deregister()
except Exception:
logger.debug("ec: deregister failed", exc_info=True)
self._data = None
@@ -1,184 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""EmbeddingCache — named-entry block cache with FIFO eviction.
Manages a fixed pool of block IDs keyed by content identity (mm_hash).
Entries transition through: not-ready → ready (evictable) → pinned.
Eviction targets ready + unpinned entries in FIFO order.
All public methods are thread-safe.
"""
import threading
from collections import OrderedDict
class CacheEntry:
"""A single cache entry. Read `.block_ids`, `.shape` and `.ready` freely;
mutations only through EmbeddingCache methods.
`shape` is the encoder output the blocks hold, so a load can restore it
without knowing how wide this model's encoder output is.
"""
__slots__ = ("block_ids", "shape", "_pin_count")
def __init__(
self, block_ids: tuple[int, ...], shape: tuple[int, int] | None = None
) -> None:
self.block_ids = block_ids
self.shape = shape
self._pin_count = -1 # not ready
@property
def ready(self) -> bool:
return self._pin_count >= 0
@property
def evictable(self) -> bool:
return self._pin_count == 0
def mark_ready(self):
self._pin_count = 0
def pin(self):
self._pin_count += 1
class EmbeddingCache:
"""Fixed-size block cache with FIFO eviction.
Entries are keyed by content identity (e.g. mm_hash). Blocks are
allocated from a free-set; when space is needed, ready + unpinned
entries are evicted oldest-first.
The caller is responsible for deciding *when* to call ``mark_ready``
(e.g. after enough engine steps have elapsed for the worker to have
completed the write).
"""
def __init__(self, num_blocks: int) -> None:
self._num_blocks = num_blocks
self._lock = threading.Lock()
# Available block IDs, LIFO stack.
self._free_blocks: list[int] = list(range(num_blocks))
# Key → live entry (all states: not-ready, ready, pinned).
self._entries: dict[str, CacheEntry] = {}
# Ready + unpinned entries in insertion order (FIFO eviction).
self._entries_free_list: OrderedDict[str, None] = OrderedDict()
# Total blocks reclaimable via eviction (fail-fast check).
self._evictable_block_count: int = 0
def get(self, key: str) -> CacheEntry | None:
"""Return the entry for *key*, or None if not present."""
with self._lock:
return self._entries.get(key)
def has_held_entries(self) -> bool:
"""True if any entry is not-ready or pinned (an in-flight transfer).
Held entries are exactly those absent from the eviction free list, so
they represent saves awaiting mark_ready or loads awaiting unpin.
"""
with self._lock:
return len(self._entries) != len(self._entries_free_list)
def alloc(
self, key: str, n_blocks: int, shape: tuple[int, int] | None = None
) -> CacheEntry | None:
"""Allocate *n_blocks* for *key*, evicting as needed.
The entry starts not-ready. Returns None if there is not enough
space even after evicting all evictable entries.
"""
with self._lock:
assert key not in self._entries, (
f"EmbeddingCache: duplicate alloc for {key!r}"
)
assert n_blocks <= self._num_blocks, (
f"EmbeddingCache: {n_blocks} blocks requested but capacity "
f"is {self._num_blocks}"
)
if len(self._free_blocks) + self._evictable_block_count < n_blocks:
return None
self._evict_until(n_blocks)
# Canonical ascending order: keeps a freed entry's blocks contiguous
# in the free list when it is evicted, and lets consumers detect
# which of an entry's blocks are adjacent.
block_ids = tuple(sorted(self._free_blocks.pop() for _ in range(n_blocks)))
entry = CacheEntry(block_ids, shape)
self._entries[key] = entry
return entry
def mark_ready(self, key: str) -> None:
"""Mark an entry as ready (data is CPU-visible)."""
with self._lock:
entry = self._entries[key]
assert entry._pin_count == -1, (
f"EmbeddingCache: mark_ready on already-ready entry {key!r}"
)
entry.mark_ready()
self._entries_free_list[key] = None
self._evictable_block_count += len(entry.block_ids)
def pin(self, key: str) -> None:
"""Pin an entry (prevent eviction)."""
with self._lock:
entry = self._entries[key]
assert entry.ready, f"EmbeddingCache: pin of not-ready entry {key!r}"
if entry.evictable:
del self._entries_free_list[key]
self._evictable_block_count -= len(entry.block_ids)
entry.pin()
def pin_if_ready(self, key: str) -> CacheEntry | None:
"""Atomically pin *key* if present and ready; return its entry.
Returns None when the key is absent. An entry returned with `ready`
False was not pinned: its save is still in flight, which the producer
reports differently from a miss because such a read can be retried.
"""
with self._lock:
entry = self._entries.get(key)
if entry is None or not entry.ready:
return entry
if entry.evictable:
del self._entries_free_list[key]
self._evictable_block_count -= len(entry.block_ids)
entry.pin()
return entry
def unpin(self, key: str) -> None:
"""Unpin an entry. Asserts currently pinned."""
with self._lock:
entry = self._entries[key]
assert entry._pin_count > 0, (
f"EmbeddingCache: unpin of unpinned entry {key!r}"
)
entry._pin_count -= 1
if entry._pin_count == 0:
self._entries_free_list[key] = None
self._evictable_block_count += len(entry.block_ids)
def discard(self, key: str) -> None:
"""Remove a not-ready in-flight entry, returning its blocks to the pool.
Used when an in-flight fill (e.g. a NIXL READ) fails before the entry
is marked ready. Asserts the entry is present and not ready — ready or
pinned entries are reclaimed through eviction/unpin, not discard.
"""
with self._lock:
entry = self._entries[key]
assert entry._pin_count == -1, (
f"EmbeddingCache: discard of ready/pinned entry {key!r}"
)
del self._entries[key]
self._free_blocks.extend(entry.block_ids)
def _evict_until(self, n_blocks: int) -> None:
"""Evict ready+unpinned entries FIFO until enough space. Lock held."""
while len(self._free_blocks) < n_blocks:
key, _ = self._entries_free_list.popitem(last=False)
entry = self._entries.pop(key)
self._evictable_block_count -= len(entry.block_ids)
self._free_blocks.extend(entry.block_ids)
@@ -1,644 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Session objects for the ECCPUConnector.
Layer hierarchy: connector → session → transport (connection)
Two long-lived sessions:
ProducerSession — owns ZmqServerTransport. The connector calls poll_step()
once per engine step, which drains the transport without blocking and
feeds the raw bytes to poll(). poll() decodes XferReqs, grants or NACKs
them, pins source blocks, sends XferAck replies, drains NIXL completion
notifications, and sweeps expired xfers.
ConsumerSession — owns one ZmqClientConnection and the NIXL agent
registration for that peer. The connector calls poll(messages, now)
once per engine step with the raw bytes collected for this peer.
poll() decodes XferAcks, dispatches them to ConsumerXfer objects,
and advances all active xfer state machines.
Internal data classes:
ProducerXfer — one pinned read grant, keyed by session_id.
ConsumerXfer — state machine for one NIXL READ, keyed by mm_hash.
"""
import enum
import logging
import time
import uuid
from dataclasses import dataclass, field
from typing import Any
import msgspec
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import (
ZmqClientConnection,
ZmqClientTransport,
ZmqServerTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
DataTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
EC_CONNECTOR_VERSION,
EXPECTED_NACKS,
RETRYABLE_NACKS,
XferAck,
XferReq,
XferStatus,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr
from vllm.logger import init_logger
logger = init_logger(__name__)
PRODUCER_PIN_LEASE_S = 30.0
_CONSUMER_READ_TIMEOUT_S = 20.0
_CONSUMER_QUARANTINE_TIMEOUT_S = 60.0
# ── XferState ─────────────────────────────────────────────────────────────────
class XferState(enum.Enum):
"""Lifecycle states of a single ConsumerXfer."""
WAITING_ACK = "waiting_ack" # XferReq sent, awaiting XferAck
READING = "reading" # NIXL READ in flight
DONE = "done" # READ completed; promote to local cache
ACK_TIMEOUT = "ack_timeout" # XferAck never arrived; free blocks + retry
READ_FAILED = "read_failed" # Unexpected NIXL state; free blocks + tombstone
QUARANTINED = "quarantined" # Read timed out, NIXL unabortable; keep blocks
SETTLED = "settled" # Quarantined DMA done; safe to free blocks
# ── ProducerXfer ──────────────────────────────────────────────────────────────
@dataclass
class ProducerXfer:
"""One pinned read grant on the producer side.
Pure data — lifecycle operations (pin/unpin) are performed by ProducerSession.
Keyed by (consumer_session_id, mm_hash) in ProducerSession._active_xfers.
"""
mm_hash: str
block_indices: list[int]
deadline: float
def is_expired(self) -> bool:
return time.monotonic() > self.deadline
# ── ConsumerXfer ──────────────────────────────────────────────────────────────
@dataclass
class ConsumerXfer:
"""State machine for one NIXL READ on the consumer side.
transfer_handle is the in-flight NIXL handle returned by post_read() —
used for check_xfer_state() / release_xfer_handle(). The dlist handle
for the remote peer lives inside DataTransport, not here.
"""
mm_hash: str
block_indices: list[int]
addr: PeerAddr
deadline: float
data: DataTransport
consumer_session_id: str # session identity; combined with mm_hash for notif_msg
transfer_handle: Any | None = field(default=None)
_quarantined: bool = field(default=False, init=False)
def handle_ack(self, ack: XferAck, agent_name: str) -> bool:
"""Issue the NIXL READ on an OK XferAck. Returns False on any NACK."""
if ack.status != XferStatus.OK:
return False
# notif_msg encodes both session_id and mm_hash so the producer can
# do a direct (session_id, mm_hash) → ProducerXfer lookup on completion.
notif_msg = f"{self.consumer_session_id}:{self.mm_hash}".encode()
self.transfer_handle = self.data.post_read(
self.block_indices,
agent_name,
ack.src_block_indices,
notif_msg=notif_msg,
)
self.deadline = time.monotonic() + _CONSUMER_READ_TIMEOUT_S
logger.debug(
"EC consumer: NIXL READ posted mm_hash=%s agent=%s local_blocks=%d "
"remote_blocks=%d",
self.mm_hash,
agent_name,
len(self.block_indices),
len(ack.src_block_indices),
)
return True
def poll(self, now: float) -> XferState:
if self._quarantined:
return self._poll_quarantined(now)
if self.transfer_handle is None:
return (
XferState.ACK_TIMEOUT if now > self.deadline else XferState.WAITING_ACK
)
return self._poll_reading(now)
def _poll_reading(self, now: float) -> XferState:
assert self.transfer_handle is not None
try:
state = self.data.check_xfer_state(self.transfer_handle)
except Exception:
logger.exception(
"EC Connector: check_xfer_state failed for mm_hash=%s", self.mm_hash
)
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
return XferState.READ_FAILED
if state == "DONE":
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
logger.debug("EC consumer: NIXL READ completed mm_hash=%s", self.mm_hash)
return XferState.DONE
if state == "PROC":
if now > self.deadline:
self._quarantined = True
self.deadline = now + _CONSUMER_QUARANTINE_TIMEOUT_S
logger.warning(
"EC Connector: READ for mm_hash=%s from %s:%d timed out; "
"quarantining",
self.mm_hash,
self.addr[0],
self.addr[1],
)
return XferState.QUARANTINED
return XferState.READING
logger.warning(
"EC Connector: READ for mm_hash=%s unexpected NIXL state %r",
self.mm_hash,
state,
)
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
return XferState.READ_FAILED
def _poll_quarantined(self, now: float) -> XferState:
if self.transfer_handle is None:
return XferState.SETTLED
# If quarantine has been too long, assume NIXL is stuck and evict blocks.
if now > self.deadline:
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
logger.warning(
"EC Connector: quarantined mm_hash=%s from %s:%d timed out; "
"evicting blocks",
self.mm_hash,
self.addr[0],
self.addr[1],
)
return XferState.READ_FAILED
try:
state = self.data.check_xfer_state(self.transfer_handle)
except Exception:
logger.exception(
"EC Connector: check_xfer_state failed for quarantined mm_hash=%s",
self.mm_hash,
)
state = None
if state == "PROC":
return XferState.QUARANTINED
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
logger.debug(
"EC Connector: quarantined mm_hash=%s settled (state=%s)",
self.mm_hash,
state,
)
return XferState.SETTLED
def cancel(self) -> None:
"""Cancel a WAITING_ACK xfer (peer down). No tombstone — caller allows retry."""
assert self.transfer_handle is None, "cancel() requires WAITING_ACK state"
def release(self) -> None:
"""Force-release the transfer handle. Shutdown only."""
if self.transfer_handle is not None:
self.data.release_xfer_handle(self.transfer_handle)
self.transfer_handle = None
# ── ProducerSession ───────────────────────────────────────────────────────────
class ProducerSession:
"""Long-lived producer session. Owns ZmqServerTransport.
poll_step() runs once per engine step: drains the transport without
blocking, decodes XferReqs, grants or NACKs each one, drains NIXL completion
notifications, and sweeps expired xfers.
"""
def __init__(
self,
transport: ZmqServerTransport,
data: DataTransport,
cache: EmbeddingCache,
compat_hash: str,
) -> None:
self._transport = transport
self._data = data
self._cache = cache # EmbeddingCache — single block owner
self._compat_hash = compat_hash
# Key: "{consumer_session_id}:{mm_hash}" — matches the NIXL notif_msg exactly,
# so completion notification lookup is a direct dict pop with no parsing.
self._active_xfers: dict[str, ProducerXfer] = {}
self._req_decoder = msgspec.msgpack.Decoder(XferReq)
self._req_encoder = msgspec.msgpack.Encoder()
def close(self) -> None:
for xfer in self._active_xfers.values():
self._cache.unpin(xfer.mm_hash) # release any still-held pins
self._active_xfers.clear()
self._transport.close()
def poll_step(self) -> None:
"""Drain the transport without blocking and process what arrived."""
self.poll(self._transport.poll(timeout_ms=0))
def poll(self, messages: list[tuple[bytes, bytes]]) -> None:
"""Decode and process inbound XferReqs, drain NIXL notifs, sweep timeouts.
messages: (identity, payload) pairs from ZmqServerTransport.poll().
"""
for identity, payload in messages:
try:
req = self._req_decoder.decode(payload)
except (msgspec.DecodeError, msgspec.ValidationError):
logger.warning("EC Connector: dropped malformed XferReq")
continue
ack = self._grant_or_nack(req)
self._transport.send(identity, self._req_encoder.encode(ack))
self._drain_notifs()
self._sweep_timeouts()
def _grant_or_nack(self, req: XferReq) -> XferAck:
if req.connector_version != EC_CONNECTOR_VERSION:
logger.warning(
"EC Connector: incompatible version for mm_hash=%s", req.mm_hash
)
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_VERSION)
if req.compatibility_hash != self._compat_hash:
logger.warning(
"EC Connector: incompatible compat hash for mm_hash=%s", req.mm_hash
)
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INCOMPAT)
if not req.session_id:
# Grants are keyed by (session_id, mm_hash), so an omitted id from
# one peer aliases another's: the first completion would unpin
# blocks the other peer is still reading.
logger.warning(
"EC Connector: XferReq without session_id for mm_hash=%s", req.mm_hash
)
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INTERNAL)
entry = self._cache.pin_if_ready(req.mm_hash)
if entry is None or not entry.ready:
status = (
XferStatus.NACK_MISSING if entry is None else XferStatus.NACK_NOT_READY
)
logger.log(
logging.INFO if status in EXPECTED_NACKS else logging.WARNING,
"EC Connector: mm_hash=%s not serveable in local cache (%s); NACKing",
req.mm_hash,
status.name,
)
return XferAck(mm_hash=req.mm_hash, status=status)
block_indices = list(entry.block_ids)
key = f"{req.session_id}:{req.mm_hash}"
if key in self._active_xfers:
# This peer is re-requesting a grant it still holds, having timed
# out waiting. Overwriting it drops the pin taken for it, which
# neither a completion nor the sweep would ever release.
self._cache.unpin(req.mm_hash)
self._active_xfers[key] = ProducerXfer(
mm_hash=req.mm_hash,
block_indices=block_indices,
deadline=time.monotonic() + PRODUCER_PIN_LEASE_S,
)
logger.debug(
"EC producer: serving NIXL READ request mm_hash=%s key=%s blocks=%d",
req.mm_hash,
key,
len(block_indices),
)
return XferAck(
mm_hash=req.mm_hash,
status=XferStatus.OK,
session_id=req.session_id, # echo consumer's session_id back
src_block_indices=block_indices,
agent_metadata=self._data.get_agent_metadata(),
mem_descriptor=self._data.get_mem_descriptor(),
)
def _drain_notifs(self) -> None:
try:
notifs = self._data.get_new_notifs()
except Exception:
logger.exception("EC Connector: get_new_notifs failed")
return
for msgs in notifs.values():
for msg in msgs:
try:
key = msg.decode("utf-8") # "{session_id}:{mm_hash}"
except Exception:
logger.exception("EC Connector: Dropped malformed NIXL notif")
continue
xfer = self._active_xfers.pop(key, None)
if xfer is not None:
self._cache.unpin(xfer.mm_hash)
logger.debug(
"EC producer: NIXL READ completed mm_hash=%s key=%s",
xfer.mm_hash,
key,
)
def _sweep_timeouts(self) -> None:
for key, xfer in list(self._active_xfers.items()):
if xfer.is_expired():
logger.warning(
"EC producer: consumer did not complete NIXL READ "
"mm_hash=%s key=%s within the pin lease; releasing "
"pinned blocks",
xfer.mm_hash,
key,
)
self._cache.unpin(xfer.mm_hash)
del self._active_xfers[key]
# ── ConsumerSession ───────────────────────────────────────────────────────────
@dataclass
class ConsumerSessionResults:
"""Results from ConsumerSession.take_results()."""
completed: set[str] # promote to local cache
tombstoned: set[str] # free blocks + prevent retry this step
quarantined: set[str] # timed-out, DMA still running: prevent retry
# but DO NOT free blocks — session holds them
retryable: set[str] # free blocks, allow another attempt (not ready, peer down)
settled: list[tuple[str, list[int]]] # (mm_hash, block_indices) quarantine cleared
class ConsumerSession:
"""Long-lived per-producer session on the consumer side.
Owns the ZMQ DEALER connection and the NIXL agent registration for one
producer peer. The connector calls poll(messages, now) once per engine
step with the raw bytes collected from this peer's DEALER socket.
poll() decodes XferAcks, dispatches them to the right ConsumerXfer,
advances all active xfer state machines, and drains quarantined xfers.
Results accumulate in internal sets; call take_results() to collect them.
"""
def __init__(
self,
addr: PeerAddr,
zmq_conn: ZmqClientConnection,
transport: ZmqClientTransport,
data: DataTransport,
compat_hash: str,
) -> None:
self._addr = addr
self._zmq = zmq_conn
self._transport = transport
self._data = data
self._compat_hash = compat_hash
self._session_id = str(
uuid.uuid4()
) # stable identity for (session_id:mm_hash) keys
self._nixl_agent_name: str | None = None
self._nixl_metadata_bytes: bytes | None = None
self._xfers: dict[str, ConsumerXfer] = {} # mm_hash → xfer
self._quarantined: list[ConsumerXfer] = [] # DMA pending, to poll
self._newly_quarantined: set[str] = set() # quarantined since last take_results
self._completed: set[str] = set()
self._tombstoned: set[str] = set()
self._retryable: set[str] = set()
self._settled: list[tuple[str, list[int]]] = []
self._encoder = msgspec.msgpack.Encoder()
self._decoder = msgspec.msgpack.Decoder(XferAck)
@property
def addr(self) -> PeerAddr:
return self._addr
def start_xfer(
self, mm_hash: str, block_indices: list[int], deadline: float
) -> None:
"""Create a ConsumerXfer and send the XferReq over the connection."""
xfer = ConsumerXfer(
mm_hash=mm_hash,
block_indices=block_indices,
addr=self._addr,
deadline=deadline,
data=self._data,
consumer_session_id=self._session_id,
)
self._zmq.send(
self._encoder.encode(
XferReq(
mm_hash=mm_hash,
compatibility_hash=self._compat_hash,
session_id=self._session_id,
)
)
)
self._xfers[mm_hash] = xfer
logger.debug(
"EC Connector: XferReq sent mm_hash=%s to %s:%d",
mm_hash,
self._addr[0],
self._addr[1],
)
def poll(self, messages: list[bytes], now: float) -> None:
"""Decode inbound XferAcks, dispatch to xfers, and advance all state machines.
messages: raw bytes from ZmqClientTransport.poll() for this peer.
Results accumulate until take_results() is called.
"""
for raw in messages:
try:
ack = self._decoder.decode(raw)
except (msgspec.DecodeError, msgspec.ValidationError):
logger.warning(
"EC Connector: dropped malformed XferAck from %s:%d",
self._addr[0],
self._addr[1],
)
continue
self._dispatch_ack(ack)
for mm_hash, xfer in list(self._xfers.items()):
state = xfer.poll(now)
if state == XferState.DONE:
self._completed.add(mm_hash)
del self._xfers[mm_hash]
elif state == XferState.ACK_TIMEOUT:
# The scheduler bounds retries with a per-request deadline.
logger.warning(
"EC consumer: no XferAck for mm_hash=%s from %s:%d in time; "
"will re-request",
mm_hash,
self._addr[0],
self._addr[1],
)
self._retryable.add(mm_hash)
del self._xfers[mm_hash]
elif state == XferState.READ_FAILED:
self._tombstoned.add(mm_hash)
del self._xfers[mm_hash]
elif state == XferState.QUARANTINED:
# DMA may still be writing — caller must NOT free blocks.
self._quarantined.append(xfer)
self._newly_quarantined.add(mm_hash)
del self._xfers[mm_hash]
self._drain_quarantine(now)
def _dispatch_ack(self, ack: XferAck) -> None:
xfer = self._xfers.get(ack.mm_hash)
if xfer is None or xfer.transfer_handle is not None:
return # unknown mm_hash or duplicate ack
if ack.status != XferStatus.OK:
logger.log(
logging.INFO if ack.status in EXPECTED_NACKS else logging.WARNING,
"EC Connector: NACK %s from %s:%d for mm_hash=%s",
ack.status.name,
self._addr[0],
self._addr[1],
ack.mm_hash,
)
del self._xfers[ack.mm_hash]
# The producer announced this encoding and its save is still in
# flight, so another attempt on a later step can be granted. Every
# other NACK is terminal for this mm_hash.
if ack.status in RETRYABLE_NACKS:
self._retryable.add(ack.mm_hash)
else:
self._tombstoned.add(ack.mm_hash)
return
try:
agent_name = self._ensure_registered(ack.agent_metadata, ack.mem_descriptor)
except Exception:
logger.exception(
"EC Connector: failed to register peer %s:%d for mm_hash=%s",
self._addr[0],
self._addr[1],
ack.mm_hash,
)
del self._xfers[ack.mm_hash]
self._tombstoned.add(ack.mm_hash)
return
if not xfer.handle_ack(ack, agent_name):
del self._xfers[ack.mm_hash]
self._tombstoned.add(ack.mm_hash)
def _ensure_registered(self, metadata: bytes, mem_descriptor: bytes) -> str:
"""Register or re-register the peer in DataTransport. Returns agent_name."""
if self._nixl_agent_name is not None and self._nixl_metadata_bytes == metadata:
return self._nixl_agent_name
if self._nixl_agent_name is not None:
logger.info(
"EC Connector: producer %s:%d restarted; re-registering",
self._addr[0],
self._addr[1],
)
self._data.remove_remote_peer(self._nixl_agent_name)
agent_name = self._data.add_remote_peer(metadata, mem_descriptor)
self._nixl_agent_name = agent_name
self._nixl_metadata_bytes = metadata
logger.debug(
"EC Connector: registered peer %s:%d agent=%s",
self._addr[0],
self._addr[1],
agent_name,
)
return agent_name
def _drain_quarantine(self, now: float) -> None:
still_pending: list[ConsumerXfer] = []
for xfer in list(self._quarantined):
state = xfer.poll(now)
if state == XferState.SETTLED:
self._settled.append((xfer.mm_hash, xfer.block_indices))
else:
still_pending.append(xfer)
self._quarantined = still_pending
def on_peer_down(self) -> None:
"""WAITING_ACK xfers → cancel (retry allowed); READING xfers → quarantine."""
for mm_hash, xfer in list(self._xfers.items()):
if xfer.transfer_handle is None:
xfer.cancel()
self._retryable.add(mm_hash)
else:
self._quarantined.append(xfer)
self._newly_quarantined.add(mm_hash)
self._xfers.clear()
def take_results(self) -> ConsumerSessionResults:
"""Return and clear all accumulated results since the last call."""
results = ConsumerSessionResults(
completed=self._completed,
tombstoned=self._tombstoned,
quarantined=self._newly_quarantined,
retryable=self._retryable,
settled=self._settled,
)
self._completed = set()
self._tombstoned = set()
self._newly_quarantined = set()
self._retryable = set()
self._settled = []
return results
def has_in_flight(self) -> bool:
return bool(self._xfers) or bool(self._quarantined)
def close(self) -> None:
"""Release all transfer handles, deregister NIXL peer, close connection.
Connection teardown is routed through the transport so the pooled
connection is dropped in lock-step with the session — a session and its
DEALER connection have a 1:1 lifecycle, so a later connect() to the same
peer builds a fresh socket instead of handing back this dead one.
"""
for xfer in self._xfers.values():
xfer.release()
self._xfers.clear()
for xfer in self._quarantined:
xfer.release()
self._quarantined.clear()
if self._nixl_agent_name is not None:
self._data.remove_remote_peer(self._nixl_agent_name)
self._transport.remove(self._addr)
@@ -1,47 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Supporting utilities for the ECCPUConnector scheduler."""
import msgspec
from vllm.logger import init_logger
# (host, port) pair identifying a remote peer's ZMQ side-channel endpoint.
PeerAddr = tuple[str, int]
logger = init_logger(__name__)
# Msgpack (de)serialization for the list-of-(addr, size, device_id)
# descriptor form that NIXL's `get_xfer_descs` expects.
_MemDescList = list[tuple[int, int, int]]
_mem_desc_encoder = msgspec.msgpack.Encoder()
_mem_desc_decoder = msgspec.msgpack.Decoder(_MemDescList)
def serialize_mem_descriptor(descs: _MemDescList) -> bytes:
return _mem_desc_encoder.encode(descs)
def deserialize_mem_descriptor(payload: bytes) -> _MemDescList:
return _mem_desc_decoder.decode(payload)
def build_block_descs(
base_ptr: int,
num_blocks: int,
block_size_bytes: int,
device_id: int = 0,
) -> _MemDescList:
"""Per-block `(addr, size, device_id)` tuples for an `ECSharedRegion`.
Handed to `nixl.get_xfer_descs` to build a dlist where descriptor `i`
addresses block `i` in the mmap. This makes a WRITE to block indices
`[i1, i2, ...]` a simple `make_prepped_xfer(..., [i1, i2])` on both
ends — no per-request address math.
"""
return [
(base_ptr + i * block_size_bytes, block_size_bytes, device_id)
for i in range(num_blocks)
]
@@ -1,455 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Worker-side of the ECCPUConnector.
Thin, stateless across steps: opens the shared mmap region and uses the
per-step connector metadata (`ECCPUConnectorMetadata`) to decide which
blocks to copy in each direction.
"""
from collections import deque
from collections.abc import Sequence
from dataclasses import dataclass
from enum import Enum, auto
from itertools import chain
from typing import TYPE_CHECKING
import numpy as np
import torch
from vllm._custom_ops import swap_blocks_batch
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
ECCPUWorkerMetadata,
create_ec_shared_region,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.worker.descriptor_buffers import (
DescriptorBufferPool,
DescriptorBuffers,
)
from vllm.distributed.parallel_state import (
get_pcp_group,
get_tensor_model_parallel_rank,
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.platform_utils import is_pin_memory_available
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
class ECCPUTransferDirection(Enum):
HOST_TO_DEVICE = auto()
DEVICE_TO_HOST = auto()
def _coalesce_runs(
block_ids: Sequence[int],
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Split block ids into runs that are consecutive in the region.
Returns `(slots, first_blocks, num_blocks)`, one element per run, where
`slots` is the run's offset in the flat block sequence. Per-descriptor
overhead dominates these copies, so describing a run with one descriptor
rather than one per block is worth an order of magnitude.
Both callers derive the non-region side of a run from its slot, so that
side advances in lockstep with the region and only the region has to be
split. `EmbeddingCache` hands out ascending ids, which is what makes runs
findable at all.
"""
ids = np.fromiter(block_ids, dtype=np.int64, count=len(block_ids))
breaks = np.flatnonzero(np.diff(ids) != 1) + 1
slots = np.concatenate(([0], breaks))
ends = np.concatenate((breaks, [ids.size]))
return slots, ids[slots], ends - slots
@dataclass
class Transfer:
"""A batched copy in flight on a dedicated GPU stream.
Its descriptor buffers and stream are held until `end_event` fires; only
then are the buffers and stream safe to recycle and the `completions` safe
to report. `start_event`/`end_event` bracket the copy so its elapsed time
can be measured on completion.
`completions` is what the scheduler is told about this copy: mm_hashes for
saves, transfer ids for loads.
"""
start_event: torch.Event
end_event: torch.Event
completions: list[str] | list[int]
bufs: DescriptorBuffers
stream: torch.Stream
num_bytes: int
class ECCPUWorker:
"""Worker-side delegate for the ECCPUConnector.
- Producer role: copies `encoder_cache[mm_hash]` → `mmap[block_ids]`
for each entry in `metadata.saves`. Descriptor buffers are filled
directly in `save_caches`; the actual DMA is issued as a single
batched call in `flush_saves`.
- Consumer role: copies `mmap[block_ids]` → `encoder_cache[mm_hash]`
for all entries in `metadata.loads` via a single `swap_blocks_batch`
call on a dedicated stream.
- On `ec_both` nodes both paths run back-to-back in a single step.
Every batched copy runs on a stream drawn from a pool so it overlaps the
compute stream, bracketed by a start/end event pair. The stream, events,
and descriptor buffers are recycled once the end event fires.
Both paths move memory between the two streams, so each buffer is registered
against the stream that is not its own: save sources against the save stream,
load destinations against the compute stream. Without that, the caching
allocator would be free to reissue memory a copy or a queued read still
needs, since it only tracks the stream a buffer was allocated on.
"""
def __init__(self, vllm_config: "VllmConfig") -> None:
self._region = create_ec_shared_region(vllm_config)
# Model dtype; used to reinterpret raw int8 blocks on load.
self._dtype = vllm_config.model_config.dtype
self._pin_shared_region()
# All TP/PCP ranks hold identical encoder output. Only one rank
# per mmap needs to write — saves host memory bandwidth.
# DCP is a subdivision of TP, so tp_rank==0 covers it.
self._is_save_rank = (
get_tensor_model_parallel_rank() == 0 and get_pcp_group().rank_in_group == 0
)
# Descriptor buffer pool (recycled across steps, shared by both paths).
self._buf_pool = DescriptorBufferPool()
# Active save buffer being filled during save_caches calls this step.
self._save_bufs: DescriptorBuffers | None = None
self._save_count: int = 0
# Bytes accumulated into the active save buffer this step, for logging.
self._save_bytes: int = 0
# mm_hashes accumulated into the active save buffer this step.
self._save_mm_hashes: list[str] = []
# Stream the active save buffer's copy will run on, held from the moment
# the buffer opens so `save_caches` can register its sources against it.
self._save_stream: torch.Stream | None = None
# Batched copies whose end event has not yet fired. Buffers and stream
# stay held until the event fires; the completions are reported to the
# scheduler only then. Separate per direction because saves and loads
# can be in flight concurrently.
self._inflight_saves: deque[Transfer] = deque()
self._inflight_loads: deque[Transfer] = deque()
# Streams and timing events recycled across steps to avoid per-step
# create/destroy churn. Shared by both directions.
self._stream_pool: list[torch.Stream] = []
self._event_pool: list[torch.Event] = []
def _acquire_stream(self) -> torch.Stream:
if self._stream_pool:
return self._stream_pool.pop()
return current_platform.Stream()
def _acquire_event(self) -> torch.Event:
return (
self._event_pool.pop()
if self._event_pool
else torch.Event(enable_timing=True)
)
def _collect_finished(self, inflight: deque[Transfer], direction: str) -> list:
"""Pop transfers whose end event has fired, recycle their stream,
events, and buffers, and return their completions.
The front check is conservative: a transfer is reported only once its
own end event fires, so completions are always genuinely done, and each
is reported exactly once because the transfer is popped as it is
reported. A later transfer that happens to finish first simply waits
behind the front and is reported on a subsequent poll.
"""
done: list = []
while inflight and inflight[0].end_event.query():
transfer = inflight.popleft()
done.extend(transfer.completions)
elapsed_ms = transfer.start_event.elapsed_time(transfer.end_event)
logger.debug(
"EC %s: %d entr%s (%d bytes) took %.3f ms",
direction,
len(transfer.completions),
"y" if len(transfer.completions) == 1 else "ies",
transfer.num_bytes,
elapsed_ms,
)
self._buf_pool.release(transfer.bufs)
self._stream_pool.append(transfer.stream)
self._event_pool.append(transfer.start_event)
self._event_pool.append(transfer.end_event)
return done
def _pin_shared_region(self) -> None:
"""Prepare the shared region for device transfers."""
if is_pin_memory_available():
self._region.pin_memory()
def _submit_transfer(
self,
bufs: DescriptorBuffers,
count: int,
direction: ECCPUTransferDirection,
) -> None:
"""Submit a transfer on the current backend stream."""
src_ptrs = bufs.src_ptrs[:count]
dst_ptrs = bufs.dst_ptrs[:count]
sizes = bufs.sizes[:count]
if direction == ECCPUTransferDirection.HOST_TO_DEVICE:
swap_blocks_batch(
src_ptrs,
dst_ptrs,
sizes,
is_src_access_order_any=True,
)
else:
swap_blocks_batch(src_ptrs, dst_ptrs, sizes)
def _shutdown_transfer_backend(self) -> None:
"""Release backend-specific transfer resources."""
def save_caches(
self,
encoder_cache: dict[str, torch.Tensor],
mm_hash: str,
connector_metadata: ECCPUConnectorMetadata,
) -> None:
"""Fill descriptor buffers directly for batched flush.
Registers `encoder_cache[mm_hash]` against the stream the copy will run
on, so the caching allocator will not hand that memory to another
allocation before the copy has read it. The descriptor buffers hold raw
addresses, and the caller is free to evict the entry as soon as this
step's saves are dispatched.
"""
if not self._is_save_rank:
return
block_ids = connector_metadata.saves.get(mm_hash)
if block_ids is None:
return
src = encoder_cache[mm_hash]
total_bytes = src.numel() * src.element_size()
block_size = self._region.block_size_bytes
allocated_bytes = len(block_ids) * block_size
assert total_bytes <= allocated_bytes, (
f"EC: encoder output exceeds allocated blocks for mm_hash={mm_hash}: "
f"{total_bytes} bytes but only {allocated_bytes} allocated "
f"({len(block_ids)} blocks × {block_size}). "
f"shape={list(src.shape)}"
)
if self._save_bufs is None:
total = sum(len(v) for v in connector_metadata.saves.values())
self._save_bufs = self._buf_pool.acquire(total)
self._save_stream = self._acquire_stream()
self._save_mm_hashes = []
assert self._save_count + len(block_ids) <= self._save_bufs.src_ptrs.numel()
bufs = self._save_bufs
assert self._save_stream is not None
src.record_stream(self._save_stream)
# Device addresses are unsigned and can have the high bit set (XPU USM),
# so they are added up in uint64; signed arithmetic rejects them.
src_base = np.uint64(src.view(-1).view(torch.uint8).data_ptr())
dst_base = self._region.blocks.data_ptr()
# Copy only the tensor bytes, excluding any block padding.
n_used = -(-total_bytes // block_size)
if n_used:
slots, first_blocks, num_blocks = _coalesce_runs(block_ids[:n_used])
run_bytes = num_blocks * block_size
# The output can stop partway into the final run.
run_bytes[-1] = total_bytes - slots[-1] * block_size
bufs.add_copies(
self._save_count,
src_base + slots.astype(np.uint64) * np.uint64(block_size),
dst_base + first_blocks * block_size,
run_bytes,
)
self._save_count += slots.size
self._save_bytes += total_bytes
self._save_mm_hashes.append(mm_hash)
def flush_saves(self) -> None:
"""Flush all accumulated saves in a single swap_blocks_batch call.
Runs the copy on the stream the batch acquired when it opened, gated
behind the compute stream that produced the encoder outputs, brackets it
with start/end events, and enqueues the batch as in-flight. The stream,
events, and descriptor buffers are recycled once the end event fires (see
`_collect_finished`), which is also when the saved mm_hashes become
safe to mark ready.
"""
if self._save_count == 0:
return
bufs = self._save_bufs
stream = self._save_stream
assert bufs is not None and stream is not None
n = self._save_count
num_bytes = self._save_bytes
# Gate the GPU→CPU copy behind the compute stream: it reads the encoder
# outputs the model just produced there.
stream.wait_stream(current_platform.current_stream())
start_event = self._acquire_event()
end_event = self._acquire_event()
with current_platform.stream(stream):
start_event.record(stream)
self._submit_transfer(
bufs,
n,
ECCPUTransferDirection.DEVICE_TO_HOST,
)
end_event.record(stream)
self._inflight_saves.append(
Transfer(
start_event=start_event,
end_event=end_event,
completions=self._save_mm_hashes,
bufs=bufs,
stream=stream,
num_bytes=num_bytes,
)
)
self._save_bufs = None
self._save_stream = None
self._save_count = 0
self._save_bytes = 0
self._save_mm_hashes = []
def start_load_caches(
self,
encoder_cache: dict[str, torch.Tensor],
connector_metadata: ECCPUConnectorMetadata,
) -> None:
"""Consumer path: single batched copy of all loads from mmap→GPU.
The destination is registered against the compute stream that consumes
it, so the caching allocator will not hand that memory to a later load
while reads of it are still queued there.
"""
if not connector_metadata.loads:
return
block_size = self._region.block_size_bytes
blocks = self._region.blocks
dtype = self._dtype
device_type = current_platform.device_type
src_base = blocks.data_ptr()
# Copy every dispatched load. The scheduler routes an input here only
# after missing the GPU encoder cache, so a resident hit is not expected;
# copying unconditionally means every participating rank reports each
# transfer exactly once, which is what the scheduler counts to release
# the pin.
# Ordered by first block so entries that neighbour each other in the
# region land next to each other in the destination and coalesce into
# one descriptor. Every loop below consumes this same order.
load_items = sorted(
connector_metadata.loads.items(), key=lambda kv: kv[1][1][0]
)
total_blocks = sum(len(block_ids) for _, (_, block_ids, _) in load_items)
stream = self._acquire_stream()
compute_stream = current_platform.current_stream()
start_event = self._acquire_event()
end_event = self._acquire_event()
with current_platform.stream(stream):
# Single contiguous destination buffer for all loads.
dst_buf = torch.empty(
total_blocks, block_size, dtype=torch.int8, device=device_type
)
dst_buf.record_stream(compute_stream)
# See save_caches: device addresses are added up in uint64.
dst_buf_base = np.uint64(dst_buf.data_ptr())
bufs = self._buf_pool.acquire(total_blocks)
slots, first_blocks, num_blocks = _coalesce_runs(
list(chain.from_iterable(ids for _, (_, ids, _) in load_items))
)
bufs.add_copies(
0,
src_base + first_blocks * block_size,
dst_buf_base + slots.astype(np.uint64) * np.uint64(block_size),
num_blocks * block_size,
)
op_idx = slots.size
start_event.record(stream)
self._submit_transfer(
bufs,
op_idx,
ECCPUTransferDirection.HOST_TO_DEVICE,
)
end_event.record(stream)
self._inflight_loads.append(
Transfer(
start_event=start_event,
end_event=end_event,
completions=[tid for _, (tid, _, _) in load_items],
bufs=bufs,
stream=stream,
num_bytes=total_blocks * block_size,
)
)
# Slice contiguous buffer into per-hash views.
offset = 0
for mm_hash, (_, block_ids, shape) in load_items:
n = len(block_ids)
flat = dst_buf[offset : offset + n].view(dtype).reshape(-1)
encoder_cache[mm_hash] = flat[: shape[0] * shape[1]].view(shape)
offset += n
current_platform.current_stream().wait_stream(stream)
def build_connector_worker_meta(self) -> ECCPUWorkerMetadata | None:
"""Report the GPU copies that completed on this rank this step: saved
mm_hashes and loaded transfer ids.
Returns None when nothing finished, so the scheduler sees no payload.
"""
completed_saves = self._collect_finished(self._inflight_saves, "save")
completed_loads = self._collect_finished(self._inflight_loads, "load")
if not completed_saves and not completed_loads:
return None
return ECCPUWorkerMetadata(
completed_saves=completed_saves,
completed_loads=completed_loads,
)
def shutdown(self) -> None:
for transfer in (*self._inflight_saves, *self._inflight_loads):
transfer.end_event.synchronize()
self._shutdown_transfer_backend()
self._save_bufs = None
self._save_stream = None
self._save_count = 0
self._save_bytes = 0
self._save_mm_hashes = []
self._inflight_saves.clear()
self._inflight_loads.clear()
self._stream_pool.clear()
self._event_pool.clear()
try:
self._region.cleanup()
except Exception:
logger.debug("EC: worker region cleanup failed", exc_info=True)
@@ -1,90 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reusable pool of (src_ptrs, dst_ptrs, sizes) tensor triples.
Used by ECCPUWorker to batch swap_blocks_batch descriptors without
per-step allocation overhead.
"""
from typing import NamedTuple
import numpy as np
import torch
from vllm.platforms import current_platform
# CUDA/ROCm cache_kernels.cu requires int64 pointers; the XPU DMA engine
# requires uint64 (see vllm._custom_ops.swap_blocks_batch).
_PTR_DTYPE = torch.uint64 if current_platform.is_xpu() else torch.int64
class DescriptorBuffers(NamedTuple):
src_ptrs: torch.Tensor
dst_ptrs: torch.Tensor
sizes: torch.Tensor
# Numpy aliases of the three tensors, written via add_copies().
src_np: np.ndarray
dst_np: np.ndarray
sizes_np: np.ndarray
def add_copies(
self,
first: int,
sources: np.ndarray,
destinations: np.ndarray,
byte_counts: np.ndarray,
) -> None:
"""Store a batch of copies in these buffers, the first at *first*.
One copy is one source address, one destination address, and a number
of bytes. `swap_blocks_batch` performs every copy held in the buffers
in a single call, so a caller fills in all of a step's copies before
asking for any of them to run, and passes *first* to say where its own
copies begin.
Whole batches go in at once because storing a single number into a
torch tensor costs microseconds. Adding copies one at a time can take
longer than performing them.
TODO(torch>=2.14): write the tensors directly and drop these numpy
aliases once the minimum supported torch is 2.14. They exist only
because torch's setitem unpacks the value as a signed long long before
2.14 (pytorch#191458), rejecting XPU USM addresses >= 2**63, while the
two's-complement rewrite those versions accept is in turn rejected by
uint64 from 2.14 on. Numpy casts against the array's own type and so
works on either. Byte counts are small enough to need none of this
care, and use their alias only to match the addresses.
"""
last = first + len(sources)
self.src_np[first:last] = sources
self.dst_np[first:last] = destinations
self.sizes_np[first:last] = byte_counts
class DescriptorBufferPool:
"""Pool of descriptor buffer triples for swap_blocks_batch.
Each buffer is a `DescriptorBuffers` namedtuple of three 1-D tensors
(dtype `_PTR_DTYPE`, platform-dependent) of equal length, paired with
numpy aliases used to fill them. Buffers are recycled across steps; if a
returned buffer is too small it is discarded and a fresh one allocated.
"""
def __init__(self) -> None:
# LIFO stack of idle buffer triples.
self._pool: list[DescriptorBuffers] = []
def acquire(self, n: int) -> DescriptorBuffers:
"""Get a buffer triple with capacity >= *n*."""
if self._pool:
bufs = self._pool.pop()
if bufs.src_ptrs.numel() >= n:
return bufs
src, dst, sizes = (torch.empty(n, dtype=_PTR_DTYPE) for _ in range(3))
return DescriptorBuffers(
src, dst, sizes, src.numpy(), dst.numpy(), sizes.numpy()
)
def release(self, bufs: DescriptorBuffers) -> None:
"""Return a buffer triple to the pool for reuse."""
self._pool.append(bufs)
@@ -1,231 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import safetensors
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorBase,
ECConnectorMetadata,
ECConnectorRole,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.logger import init_logger
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.v1.core.sched.output import SchedulerOutput
if TYPE_CHECKING:
from vllm.v1.request import Request
logger = init_logger(__name__)
@dataclass
class MMMeta:
mm_hash: str
num_token: int
@staticmethod
def make_meta(mm_hash, num_token) -> "MMMeta":
return MMMeta(mm_hash=mm_hash, num_token=num_token)
@dataclass
class ECExampleConnectorMetadata(ECConnectorMetadata):
mm_datas: list[MMMeta]
def __init__(self):
self.mm_datas = []
def add_mm_data(self, mm_data: MMMeta):
self.mm_datas.append(mm_data)
class ECExampleConnector(ECConnectorBase):
# NOTE: This is Simple debug implementation of the EC connector.
# It save / load the EC cache to / from the disk.
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole):
super().__init__(vllm_config=vllm_config, role=role)
# req_id -> index
self._mm_datas_need_loads: dict[str, int] = {}
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
transfer_config = vllm_config.ec_transfer_config
if transfer_config is not None:
self._storage_path = transfer_config.get_from_extra_config(
"shared_storage_path", "/tmp"
)
logger.debug(transfer_config)
logger.debug("Shared storage path is %s", self._storage_path)
else:
raise ValueError("ec_transfer_config must be set for ECConnectorBase")
def start_load_caches(self, encoder_cache, **kwargs) -> None:
"""Start loading the cache from the connector into vLLM's encoder cache.
This method loads the encoder cache based on metadata provided by the scheduler.
It is called before `_gather_mm_embeddings` for the EC Connector. For EC,
the `encoder_cache` and `mm_hash` are stored in `kwargs`.
Args:
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
data hashes (`mm_hash`) to encoder cache tensors.
kwargs (dict): Additional keyword arguments for the connector.
"""
from vllm.platforms import current_platform
# Get the metadata
metadata: ECConnectorMetadata = self._get_connector_metadata()
assert isinstance(metadata, ECExampleConnectorMetadata)
assert encoder_cache is not None
if metadata is None:
logger.warning(
"In connector.start_load_caches, but the connector metadata is None"
)
return
# A bare "cuda" makes safetensors load onto cuda:0. Pin the load to the
# selected device so tensor-parallel workers load the cache locally.
device = current_platform.device_type
if current_platform.is_cuda_alike():
device = f"{device}:{current_platform.current_device()}"
# Load the EC for each mm data
for mm_data in metadata.mm_datas:
if mm_data.mm_hash in encoder_cache:
continue
filename = self._generate_filename_debug(mm_data.mm_hash)
with gpu_sync_allowed():
ec_cache = safetensors.torch.load_file(filename, device=device)[
"ec_cache"
]
encoder_cache[mm_data.mm_hash] = ec_cache
logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash)
def save_caches(self, encoder_cache, mm_hash, **kwargs) -> None:
"""Save the encoder cache to the connector.
This method saves the encoder cache from the worker's local storage
to shared storage or another external connector.
Args:
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
data hashes (`mm_hash`) to encoder cache tensors.
mm_hash (str): The hash of the multimodal data whose cache is being saved.
kwargs (dict): Additional keyword arguments for the connector.
"""
# Return if it is PD Instance
if not self.is_producer:
return
filename = self._generate_filename_debug(mm_hash)
ec_cache = encoder_cache[mm_hash]
with gpu_sync_allowed():
tensors = {"ec_cache": ec_cache.detach().cpu()}
safetensors.torch.save_file(tensors, filename)
logger.debug("Save cache successful for mm_hash %s", mm_hash)
def has_cache_item(
self,
identifier: str,
) -> bool:
"""Check if cache exist externally for the media.
Args:
identifier (str): the identifier of the media.
Returns:
Bool indicate that media exists in cache or not
"""
return self._found_match_for_mm_data(identifier)
def update_state_after_alloc(
self,
request: "Request",
index: int,
) -> None:
"""Update ECConnector state after encoder cache allocation."""
mm_hash = request.mm_features[index].identifier
# Only load cache if it is consumer and cache exists
if not self.is_consumer or not self.has_cache_item(mm_hash):
return
num_encoder_token = request.get_num_encoder_embeds(index)
self._mm_datas_need_loads[mm_hash] = num_encoder_token
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> ECConnectorMetadata:
"""Build the connector metadata for this step.
This function should NOT modify any fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
This only build for load mm_data only
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
meta = ECExampleConnectorMetadata()
for mm_hash, num_encoder_token in self._mm_datas_need_loads.items():
meta.add_mm_data(MMMeta.make_meta(mm_hash, num_encoder_token))
self._mm_datas_need_loads.clear()
return meta
def request_finished(
self,
request: "Request",
) -> tuple[bool, dict[str, Any] | None]:
"""Report each item's cache key and grid so a consumer can skip the
image transform.
A consumer only needs the grid to size the prompt's placeholder range;
the embedding itself arrives through this connector. Reporting the grid
the producer actually computed keeps the two sides in agreement without
the caller re-deriving it from the raw media.
"""
if not self.is_producer:
return False, None
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
if not items:
return False, None
return False, items
# ==============================
# Helper functions
# ==============================
def _found_match_for_mm_data(self, mm_hash) -> bool:
"""Check if the cache is hit for the request."""
filename = self._generate_filename_debug(mm_hash, create_folder=False)
return os.path.exists(filename)
def _generate_foldername_debug(
self,
mm_hash: str,
create_folder: bool = True, # <- now defaults to True
) -> str:
"""Return the folder in which the cache for this mm_hash lives.
If `create_folder` is True (default) the directory is created
recursively the first time it is needed.
"""
foldername = os.path.join(self._storage_path, mm_hash)
if create_folder:
os.makedirs(foldername, exist_ok=True)
return foldername
def _generate_filename_debug(
self,
mm_hash: str,
create_folder: bool = True,
) -> str:
"""Return the full path of the safetensors file for this mm_hash.
If `create_folder` is True (default) the parent directory is created.
"""
foldername = self._generate_foldername_debug(mm_hash, create_folder)
return os.path.join(foldername, "encoder_cache.safetensors")