[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,9 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Internal building blocks for the Mooncake encoder-cache connector.
The public connector remains in :mod:`mooncake_ec_connector`. This package
separates configuration, control-plane messaging, transfer state, registered
memory, and worker/scheduler orchestration so that each component has one
owner and can be tested independently.
"""
@@ -1,155 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING
from vllm.utils.network_utils import make_zmq_path
if TYPE_CHECKING:
from vllm.config import VllmConfig
# Scheduler and Worker must agree on when a remote reservation becomes stale.
_RESERVATION_TTL_SECONDS = 300
def _positive_int(name: str, value: object) -> int:
message = f"ECMooncakeConnector requires {name} to be a positive integer."
if isinstance(value, bool):
raise ValueError(message)
if isinstance(value, int):
result = value
elif isinstance(value, float) and math.isfinite(value) and value.is_integer():
result = int(value)
elif isinstance(value, str):
try:
result = int(value)
except ValueError as error:
raise ValueError(message) from error
else:
raise ValueError(message)
if result <= 0:
raise ValueError(f"ECMooncakeConnector requires {name} > 0.")
return result
def _positive_float(name: str, value: object) -> float:
message = f"ECMooncakeConnector requires {name} > 0."
if isinstance(value, bool):
raise ValueError(message)
try:
result = float(value) # type: ignore[arg-type]
except (TypeError, ValueError) as error:
raise ValueError(message) from error
if not math.isfinite(result) or result <= 0:
raise ValueError(message)
return result
@dataclass(frozen=True)
class MooncakeECConfig:
"""Validated settings shared by the Scheduler and Worker roles.
``control_port`` is the first TP-shard port after the DP offset;
``control_addr`` targets that shard, which advertises the full topology.
``control_host`` is the address this instance both advertises and binds,
so reaching a Consumer means being routed to it rather than finding it on
every interface.
"""
is_producer: bool
is_consumer: bool
protocol: str
buffer_device: str
control_host: str
control_port: int
control_addr: str
control_timeout_ms: int
push_wait_timeout_s: float
pool_size: int
cross_encoder_cache: bool = False
store_max_pending_items: int = 32
store_max_pending_bytes: int = 2 * 1024**3
store_read_buffer_bytes: int = 128 * 1024**2
@classmethod
def from_vllm_config(cls, vllm_config: VllmConfig) -> MooncakeECConfig:
parallel_config = vllm_config.parallel_config
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None
if ec_config.is_ec_producer:
if parallel_config.tensor_parallel_size > 1:
raise ValueError(
"ECMooncakeConnector producers require tensor_parallel_size=1."
)
if parallel_config.pipeline_parallel_size > 1:
raise ValueError(
"ECMooncakeConnector producers do not support pipeline parallelism."
)
if parallel_config.data_parallel_size > 1:
raise ValueError(
"ECMooncakeConnector producers require data_parallel_size=1."
)
registered_buffer_size = _positive_int(
"ec_buffer_size", ec_config.ec_buffer_size
)
get = ec_config.get_from_extra_config
shared_reuse = get("cross_encoder_cache", False)
if not isinstance(shared_reuse, bool):
raise ValueError("cross_encoder_cache must be a boolean")
if shared_reuse:
if not ec_config.is_encode_only:
raise ValueError(
"cross_encoder_cache requires an Encoder-only producer"
)
if not vllm_config.use_v2_model_runner:
raise ValueError("cross_encoder_cache requires Model Runner V2")
if vllm_config.lora_config is not None:
raise ValueError("cross_encoder_cache does not support LoRA")
mm_config = vllm_config.model_config.multimodal_config
if mm_config is not None and mm_config.mm_processor_cache_gb == 0:
raise ValueError(
"cross_encoder_cache requires mm_processor_cache_gb > 0 "
"to preserve content identifiers across Encoders and restarts"
)
control_port = int(ec_config.ec_port) + (
parallel_config.data_parallel_index * parallel_config.tensor_parallel_size
)
highest_port = control_port + parallel_config.tensor_parallel_size - 1
if not 1 <= control_port <= highest_port <= 65535:
raise ValueError("ECMooncakeConnector ec_port must be in 1..65535.")
return cls(
is_producer=ec_config.is_ec_producer,
is_consumer=ec_config.is_ec_consumer,
protocol=str(get("mooncake_protocol", "rdma")),
buffer_device=str(ec_config.ec_buffer_device or "cuda").lower(),
control_host=str(ec_config.ec_ip),
control_port=control_port,
control_addr=make_zmq_path("tcp", ec_config.ec_ip, control_port),
control_timeout_ms=max(
1,
math.ceil(
_positive_float("control_timeout_s", get("control_timeout_s", 30))
* 1000
),
),
push_wait_timeout_s=_positive_float(
"push_wait_timeout_s", get("push_wait_timeout_s", 60)
),
pool_size=registered_buffer_size,
cross_encoder_cache=shared_reuse,
store_max_pending_items=_positive_int(
"store_max_pending_items", get("store_max_pending_items", 32)
),
store_max_pending_bytes=_positive_int(
"store_max_pending_bytes", get("store_max_pending_bytes", 2 * 1024**3)
),
store_read_buffer_bytes=_positive_int(
"store_read_buffer_bytes", get("store_read_buffer_bytes", 128 * 1024**2)
),
)
@@ -1,460 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ZMQ control plane for Mooncake encoder-cache reservations and events.
Tensor bytes never travel through this module. It coordinates destination
reservations, completion/cancellation, TP-shard discovery, and readiness
notifications while :mod:`transfer` owns the Mooncake data plane.
"""
from __future__ import annotations
import threading
import time
from collections import deque
from collections.abc import Callable
from concurrent.futures import Executor, Future
from typing import Any
import torch
import zmq
from vllm.logger import init_logger
from vllm.utils.network_utils import make_zmq_path
logger = init_logger(__name__)
_MAX_PENDING_EVENTS = 4096
_RESERVATION_REAP_INTERVAL_SECONDS = 1
ControlRequest = dict[str, Any]
ControlResponse = dict[str, Any]
class ControlClient:
"""Send control requests through reusable, thread-local REQ sockets."""
def __init__(self, timeout_ms: int) -> None:
self._context = zmq.Context()
self._timeout_ms = timeout_ms
self._local = threading.local()
self._topologies: dict[str, list[str]] = {}
self._closed = False
def _sockets(self) -> dict[str, zmq.Socket]:
sockets = getattr(self._local, "sockets", None)
if sockets is None:
sockets = {}
self._local.sockets = sockets
return sockets
def _discard(self, addr: str) -> None:
socket = self._sockets().pop(addr, None)
if socket is not None:
socket.close(linger=0)
def _exchange(self, addr: str, payload: ControlRequest) -> ControlResponse:
sockets = self._sockets()
socket = sockets.get(addr)
if socket is None:
socket = self._context.socket(zmq.REQ)
socket.setsockopt(zmq.IPV6, 1)
socket.setsockopt(zmq.RCVTIMEO, self._timeout_ms)
socket.setsockopt(zmq.SNDTIMEO, self._timeout_ms)
socket.setsockopt(zmq.LINGER, 0)
socket.connect(addr)
sockets[addr] = socket
try:
socket.send_json(payload)
response = socket.recv_json()
except Exception:
self._discard(addr)
raise
assert isinstance(response, dict)
return response
def request(self, addr: str, payload: ControlRequest) -> Any:
response = self._exchange(addr, payload)
if not response.get("ok"):
raise RuntimeError(response.get("error", "EC control request failed"))
return response.get("result")
def discover_shards(self, base_addr: str) -> list[str] | None:
if base_addr in self._topologies:
return self._topologies[base_addr]
try:
reply = self.request(base_addr, {"op": "peers"})
ports = reply.get("ports") if isinstance(reply, dict) else None
if not isinstance(ports, list) or not ports:
raise ValueError("invalid or empty peer list")
prefix = base_addr.rsplit(":", 1)[0]
shards = [f"{prefix}:{int(port)}" for port in ports]
except Exception:
logger.warning(
"EC Mooncake consumer at %s did not report its shards",
base_addr,
exc_info=True,
)
return None
self._topologies[base_addr] = shards
return shards
def close(self) -> None:
if self._closed:
return
self._closed = True
self._context.destroy(linger=0)
def make_cancel_request(
transfer_id: str,
reservation_id: str = "",
*,
abandon: bool = False,
refresh: bool = False,
) -> ControlRequest:
request: ControlRequest = {
"op": "cancel",
"transfer_id": transfer_id,
"reservation_id": reservation_id,
}
if abandon:
request["abandon"] = True
if refresh:
request["refresh"] = True
return request
class EventInbox:
"""Receive Consumer readiness events without blocking the Scheduler.
Subscribing needs one blocking control request per shard, so it runs on
`executor` when one is given: an unreachable Consumer must cost the
Scheduler nothing, not one control timeout per drain.
"""
def __init__(self, client: ControlClient, executor: Executor | None = None) -> None:
self._client = client
self._executor = executor
self._discovery: Future[list[str] | None] | None = None
self._context: zmq.Context | None = None
self._socket: zmq.Socket | None = None
self._closed = False
self.shard_count = 1
def _endpoints(self, base_addr: str) -> list[str] | None:
"""Ask every shard where it publishes readiness events.
Blocking: called on `self._executor` unless there is none.
"""
shards = self._client.discover_shards(base_addr)
if shards is None:
return None
endpoints = []
for addr in shards:
try:
event_port = self._client.request(addr, {"op": "event_port"})
address, _ = addr.rsplit(":", 1)
endpoints.append(f"{address}:{int(event_port)}")
except Exception:
logger.warning(
"EC Mooncake could not subscribe to the event channel of "
"consumer shard %s; retrying the complete topology later.",
addr,
)
return None
return endpoints
def _connect(self, base_addr: str) -> None:
if self._socket is not None or self._closed:
return
if self._executor is None:
self._install(self._endpoints(base_addr))
return
if self._discovery is None:
self._discovery = self._executor.submit(self._endpoints, base_addr)
return
if not self._discovery.done():
return
discovery, self._discovery = self._discovery, None
try:
self._install(discovery.result())
except Exception:
logger.warning(
"EC Mooncake event-channel discovery for %s failed; retrying.",
base_addr,
exc_info=True,
)
def _install(self, endpoints: list[str] | None) -> None:
"""Adopt a discovered topology.
Runs on the caller's thread so the socket is only ever touched there.
"""
if not endpoints or self._closed:
return
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.setsockopt(zmq.IPV6, 1)
for endpoint in endpoints:
socket.connect(endpoint)
self._context = context
self._socket = socket
self.shard_count = len(endpoints)
def drain(self, base_addr: str) -> list[dict[str, Any]]:
self._connect(base_addr)
if self._socket is None:
return []
events = []
while True:
try:
events.append(self._socket.recv_json(flags=zmq.DONTWAIT))
except zmq.Again:
return events
except zmq.ZMQError:
logger.warning("EC Mooncake event channel failed", exc_info=True)
return events
except (ValueError, UnicodeError):
# The event channel is a plain PULL socket: an undecodable
# frame must cost one frame, not the engine.
logger.warning(
"Discarding an undecodable EC Mooncake event.", exc_info=True
)
def close(self) -> None:
if self._closed:
return
self._closed = True
if self._discovery is not None:
self._discovery.cancel()
self._discovery = None
if self._socket is not None:
self._socket.close(linger=0)
self._socket = None
if self._context is not None:
self._context.term()
class ConsumerControlServer:
"""Expose Consumer reservations and readiness events over ZMQ.
One server runs on every receiving TP rank. The REP channel handles
reservation operations, while a PUSH channel publishes newly ready items
to the Scheduler.
"""
def __init__(
self,
host: str,
port: int,
reserve: Callable[[dict[str, Any]], dict[str, Any]],
status: Callable[[str], dict[str, Any] | None],
complete: Callable[[str, str], tuple[bool, bool]],
cancel: Callable[[str, str, bool, bool], bool],
reap: Callable[[], int],
peer_ports: list[int] | None = None,
device: torch.device | None = None,
drain_ready: Callable[[], list[str]] = lambda: [],
) -> None:
self.host = host
self.port = port
self.peer_ports = peer_ports or [port]
self._device = device
self.event_port: int | None = None
self._reserve = reserve
self._status = status
self._complete = complete
self._cancel = cancel
self._reap = reap
self._drain_ready = drain_ready
self._stop = threading.Event()
self._started = threading.Event()
self._thread: threading.Thread | None = None
self._startup_error: Exception | None = None
def start(self) -> None:
def loop() -> None:
if self._device is not None and self._device.type == "cuda":
torch.accelerator.set_device_index(self._device.index or 0)
context = zmq.Context()
socket = context.socket(zmq.REP)
event_socket = context.socket(zmq.PUSH)
socket.setsockopt(zmq.IPV6, 1)
event_socket.setsockopt(zmq.IPV6, 1)
pending_events: deque[dict[str, Any]] = deque()
def queue_event(event: dict[str, Any]) -> None:
event["shard"] = self.port
if len(pending_events) >= _MAX_PENDING_EVENTS:
dropped = pending_events.popleft()
logger.warning(
"EC Mooncake event backlog full on port %d; dropping "
"readiness for transfer_id=%s",
self.port,
dropped.get("transfer_id"),
)
pending_events.append(event)
def queue_ready(transfer_id: str) -> None:
status = self._status(transfer_id)
if status is not None:
queue_event({"transfer_id": transfer_id, **status})
last_reap_at = time.monotonic()
socket.setsockopt(zmq.RCVTIMEO, 100)
try:
socket.bind(make_zmq_path("tcp", self.host, self.port))
event_socket.bind(make_zmq_path("tcp", self.host, 0))
self.event_port = int(
event_socket.getsockopt(zmq.LAST_ENDPOINT)
.decode()
.rsplit(":", 1)[1]
)
except Exception as e:
self._startup_error = e
self._started.set()
socket.close(linger=0)
event_socket.close(linger=0)
context.term()
return
self._started.set()
try:
while not self._stop.is_set():
while pending_events:
try:
event_socket.send_json(
pending_events[0], flags=zmq.DONTWAIT
)
except zmq.Again:
break
pending_events.popleft()
now = time.monotonic()
if now - last_reap_at >= _RESERVATION_REAP_INTERVAL_SECONDS:
self._reap()
last_reap_at = now
try:
request = socket.recv_json()
except zmq.Again:
continue
except Exception:
# The frame arrived but did not decode. REP still owes a
# reply, so answer before returning to the loop.
logger.exception(
"EC Mooncake control channel on port %d received an "
"undecodable request",
self.port,
)
socket.send_json(
{"ok": False, "error": "malformed control request"}
)
continue
try:
op = request.get("op")
result: Any = None
if op in ("reserve", "reserve_batch"):
items = (
request["items"] if op == "reserve_batch" else [request]
)
results = []
for item in items:
try:
reserved = self._reserve(item)
results.append({"ok": True, "result": reserved})
if reserved.get("ready"):
queue_ready(str(item["transfer_id"]))
except Exception as exc:
results.append({"ok": False, "error": str(exc)})
if op == "reserve_batch":
result = {"items": results}
elif not results[0]["ok"]:
raise RuntimeError(results[0]["error"])
else:
result = results[0]["result"]
elif op == "status":
result = self._status(str(request["transfer_id"]))
elif op == "event_port":
result = self.event_port
elif op == "peers":
result = {"ports": self.peer_ports}
elif op in ("complete", "complete_batch"):
items = (
request["items"]
if op == "complete_batch"
else [request]
)
completions = []
for item in items:
transfer_id = str(item["transfer_id"])
accepted, became_ready = self._complete(
transfer_id, str(item["reservation_id"])
)
for ready_id in self._drain_ready():
queue_ready(ready_id)
completions.append(
{
"completed": accepted,
"became_ready": became_ready,
}
)
if not became_ready:
continue
queue_ready(transfer_id)
result = (
{"items": completions}
if op == "complete_batch"
else completions[0]
)
elif op == "cancel":
result = {
"cancelled": self._cancel(
str(request["transfer_id"]),
str(request.get("reservation_id", "")),
bool(request.get("abandon", False)),
bool(request.get("refresh", False)),
)
}
else:
raise ValueError(f"unknown control op: {op!r}")
socket.send_json({"ok": True, "result": result})
except Exception as e:
socket.send_json({"ok": False, "error": str(e)})
except Exception:
# `finally` closes the sockets and the thread ends, so without
# this every later reserve against this shard would surface
# only as a control timeout with nothing to attribute it to.
logger.exception(
"EC Mooncake control channel on port %d stopped serving",
self.port,
)
raise
finally:
socket.close(linger=0)
event_socket.close(linger=0)
context.term()
self._thread = threading.Thread(
target=loop, name="ec-mooncake-control", daemon=True
)
self._thread.start()
if not self._started.wait(timeout=5):
raise RuntimeError("EC Mooncake control channel failed to start")
if self._startup_error is not None:
raise RuntimeError("EC Mooncake control channel failed to bind") from (
self._startup_error
)
logger.info(
"EC Mooncake control channel listening on tcp://%s:%d (events tcp://%s:%d)",
self.host,
self.port,
self.host,
self.event_port,
)
def close(self) -> None:
if self._thread is None:
return
self._stop.set()
self._thread.join()
self._thread = None
@@ -1,504 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Registered-memory allocation and residency for Mooncake transfers.
The Producer pool stages source tensors in a registered slab. The Consumer
pool owns destination allocations from reservation through publication,
resident reuse, CUDA-safe retirement, and pressure-driven reclamation.
"""
from __future__ import annotations
import bisect
import threading
from collections import OrderedDict
from collections.abc import Callable
from dataclasses import dataclass
from typing import Generic, TypeVar
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
MooncakeTransfer,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
_T = TypeVar("_T")
@dataclass
class MemoryAllocation:
"""Describe a tensor view carved from the Consumer receive slab."""
offset: int
size: int
tensor: torch.Tensor
@dataclass
class _ResidentEntry(Generic[_T]):
"""Store one resident value and its ownership accounting."""
value: _T
pinned: bool = True
leases: int = 0
@dataclass
class ResidentLease(Generic[_T]):
"""Represent a borrow of one resident entry."""
key: str
_entry: _ResidentEntry[_T]
_active: bool = True
@property
def value(self) -> _T:
return self._entry.value
class ContiguousAllocator:
"""Own a registered slab and its aligned regions; callers serialize access."""
def __init__(self, capacity: int, alignment: int = 256):
self.alignment = alignment
self._capacity = capacity
self._free = [(0, capacity)]
self.tensor: torch.Tensor | None = None
self._disabled = False
def prepare(self, device: torch.device, transfer: MooncakeTransfer) -> None:
if self.tensor is not None or self._disabled:
return
try:
tensor = torch.empty(self._capacity, dtype=torch.uint8, device=device)
ret = transfer.register_memory(tensor)
if ret != 0:
raise RuntimeError(f"Mooncake returned {ret}")
except (RuntimeError, torch.OutOfMemoryError) as error:
self._disabled = True
logger.warning("Could not initialize the EC registered buffer: %s", error)
return
self.tensor = tensor
self._free = [(0, tensor.nbytes)]
logger.info("Registered %d-byte buffer for Mooncake EC", tensor.nbytes)
def view(
self, offset: int, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
) -> torch.Tensor:
assert self.tensor is not None
return self.tensor.narrow(0, offset, nbytes).view(dtype).view(shape)
def close(self, transfer: MooncakeTransfer) -> bool:
if self.tensor is not None and not transfer.unregister_memory(self.tensor):
return False
self.tensor = None
self._free.clear()
return True
def allocate(self, nbytes: int) -> tuple[int, int] | None:
size = (nbytes + self.alignment - 1) // self.alignment * self.alignment
for index, (offset, available) in enumerate(self._free):
if size > available:
continue
if size == available:
self._free.pop(index)
else:
self._free[index] = (offset + size, available - size)
return offset, size
return None
def free(self, offset: int, size: int) -> None:
index = bisect.bisect_left(self._free, (offset, size))
self._free.insert(index, (offset, size))
if index + 1 < len(self._free):
next_offset, next_size = self._free[index + 1]
if offset + size == next_offset:
self._free[index] = (offset, size + next_size)
self._free.pop(index + 1)
if index > 0:
previous_offset, previous_size = self._free[index - 1]
current_offset, current_size = self._free[index]
if previous_offset + previous_size == current_offset:
self._free[index - 1] = (
previous_offset,
previous_size + current_size,
)
self._free.pop(index)
class ResidentPool(Generic[_T]):
"""Track resident values, active leases, and LRU eviction eligibility."""
def __init__(self):
self._entries: dict[str, _ResidentEntry[_T]] = {}
self._evictable: OrderedDict[str, None] = OrderedDict()
def referenced(self) -> list[str]:
return [key for key, entry in self._entries.items() if entry.pinned]
def get(self, key: str) -> _T | None:
entry = self._entries.get(key)
return entry.value if entry is not None else None
def insert(
self,
key: str,
value: _T,
) -> _T | None:
"""Pin an entry and return a displaced value that has no owners."""
previous = self._entries.get(key)
entry = _ResidentEntry(value)
self._entries[key] = entry
self._evictable.pop(key, None)
if previous is not None and previous.leases == 0:
return previous.value
return None
def pin(self, key: str) -> _T | None:
entry = self._entries.get(key)
if entry is None:
return None
self._evictable.pop(key, None)
entry.pinned = True
return entry.value
def acquire(self, key: str) -> ResidentLease[_T] | None:
entry = self._entries.get(key)
if entry is None:
return None
entry.leases += 1
self._evictable.pop(key, None)
return ResidentLease(key, entry)
def release(self, lease: ResidentLease[_T]) -> _T | None:
if not lease._active:
return None
lease._active = False
entry = lease._entry
entry.leases -= 1
current = self._entries.get(lease.key)
if current is not entry:
if entry.leases == 0:
return entry.value
return None
if not entry.pinned and entry.leases == 0:
self._evictable[lease.key] = None
return None
def consume(self, lease: ResidentLease[_T]) -> tuple[_T, _T | None]:
current = self._entries[lease.key]
current.pinned = True
self._evictable.pop(lease.key, None)
released = self.release(lease)
return current.value, released
def retire(self, key: str) -> None:
if key not in self._entries:
return
entry = self._entries[key]
entry.pinned = False
if entry.leases == 0:
self._evictable[key] = None
def evict_lru(self, evict: Callable[[str, _T], bool]) -> str | None:
for key in list(self._evictable):
entry = self._entries[key]
if not evict(key, entry.value):
continue
self._evictable.pop(key, None)
del self._entries[key]
return key
return None
def clear(self) -> None:
self._entries.clear()
self._evictable.clear()
@dataclass
class StagedSources:
"""Own Producer tensor views and their staging-slab regions."""
tensors: list[torch.Tensor]
regions: list[tuple[int, int]]
class ProducerMemoryPool:
"""Own the Producer staging slab and regions carved from it."""
def __init__(self, capacity: int, transfer: MooncakeTransfer) -> None:
self._transfer = transfer
self._allocator = ContiguousAllocator(capacity)
self._lock = threading.Lock()
self._local = threading.local()
@property
def tensor(self) -> torch.Tensor | None:
return self._allocator.tensor
def _free_regions(self, regions: list[tuple[int, int]]) -> None:
for offset, size in regions:
self._allocator.free(offset, size)
def stage(self, tensors: list[torch.Tensor]) -> StagedSources | None:
"""Copy tensors into one registered slab, or return None for fallback."""
if not tensors:
return StagedSources([], [])
allocator = self._allocator
staged: list[torch.Tensor] = []
regions: list[tuple[int, int]] = []
with self._lock:
allocator.prepare(tensors[0].device, self._transfer)
pool = allocator.tensor
if pool is None:
return None
for tensor in tensors:
region = allocator.allocate(tensor.nbytes)
if region is None:
self._free_regions(regions)
return None
regions.append(region)
staged.append(
allocator.view(
region[0], tensor.nbytes, tuple(tensor.shape), tensor.dtype
)
)
if pool.device.type == "cuda":
stream = getattr(self._local, "stream", None)
if stream is None:
stream = torch.cuda.Stream(device=pool.device)
self._local.stream = stream
with torch.cuda.stream(stream):
for destination, source in zip(staged, tensors):
destination.copy_(source, non_blocking=True)
stream.synchronize()
else:
for destination, source in zip(staged, tensors):
destination.copy_(source)
return StagedSources(staged, regions)
def release(self, staged: StagedSources) -> None:
if not staged.regions:
return
with self._lock:
self._free_regions(staged.regions)
def close(self) -> None:
with self._lock:
self._allocator.close(self._transfer)
class ConsumerMemoryPool:
"""Own the registered receive slab and resident allocation lifecycle."""
def __init__(
self,
capacity: int,
transfer: MooncakeTransfer,
) -> None:
self._transfer = transfer
self._allocator = ContiguousAllocator(capacity)
self._residents: ResidentPool[MemoryAllocation] = ResidentPool()
self._retire_events: dict[str, torch.Event] = {}
self._pending_frees: list[tuple[torch.Event, MemoryAllocation]] = []
self._reclaimed: set[str] = set()
self.lock = threading.RLock()
@property
def tensor(self) -> torch.Tensor | None:
return self._allocator.tensor
def prepare(
self,
device: torch.device,
) -> None:
with self.lock:
self._allocator.prepare(device, self._transfer)
def _free(self, allocation: MemoryAllocation) -> None:
self._allocator.free(allocation.offset, allocation.size)
def free(self, allocation: MemoryAllocation) -> None:
with self.lock:
self._free(allocation)
def _defer_or_free(
self, allocation: MemoryAllocation, event: torch.Event | None
) -> None:
if event is None or event.query():
self._free(allocation)
else:
self._pending_frees.append((event, allocation))
def _poll_frees_locked(self) -> None:
pending = []
for event, allocation in self._pending_frees:
if event.query():
self._free(allocation)
else:
pending.append((event, allocation))
self._pending_frees = pending
def _reclaim_locked(self, nbytes: int) -> tuple[int, int] | None:
def evict(mm_hash: str, allocation: MemoryAllocation) -> bool:
event = self._retire_events.pop(mm_hash, None)
self._defer_or_free(allocation, event)
self._reclaimed.add(mm_hash)
return True
while self._residents.evict_lru(evict) is not None:
# Eviction defers any free whose CUDA event is still pending, so
# those bytes reach the allocator only once the event is polled.
self._poll_frees_locked()
region = self._allocator.allocate(nbytes)
if region is not None:
return region
self._poll_frees_locked()
return self._allocator.allocate(nbytes)
def _make_allocation(
self,
region: tuple[int, int],
nbytes: int,
shape: tuple[int, ...],
dtype: torch.dtype,
) -> MemoryAllocation:
offset, size = region
tensor = self._allocator.view(offset, nbytes, shape, dtype)
return MemoryAllocation(offset, size, tensor)
def try_allocate(
self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
) -> MemoryAllocation | None:
with self.lock:
allocator = self._allocator
assert allocator.tensor is not None
self._poll_frees_locked()
region = allocator.allocate(nbytes)
if region is None:
return None
return self._make_allocation(region, nbytes, shape, dtype)
def reclaim_and_allocate(
self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
) -> MemoryAllocation | None:
with self.lock:
region = self._reclaim_locked(nbytes)
if region is None:
return None
return self._make_allocation(region, nbytes, shape, dtype)
def acquire_cached(
self, mm_hash: str, shape: tuple[int, ...], dtype: torch.dtype
) -> ResidentLease[MemoryAllocation] | None:
with self.lock:
allocation = self._residents.get(mm_hash)
if allocation is None:
return None
if (
tuple(allocation.tensor.shape) != shape
or allocation.tensor.dtype != dtype
):
raise ValueError("conflicting cached tensor for mm_hash")
return self._residents.acquire(mm_hash)
def take_resident(
self, mm_hash: str, shape: tuple[int, ...], dtype_name: str
) -> torch.Tensor | None:
with self.lock:
allocation = self._residents.get(mm_hash)
if allocation is None:
return None
tensor = allocation.tensor
if (
tuple(tensor.shape) != shape
or str(tensor.dtype).split(".")[-1] != dtype_name
):
return None
self._residents.pin(mm_hash)
self._retire_events.pop(mm_hash, None)
return tensor
def _record_release_event(self) -> torch.Event | None:
pool = self.tensor
if pool is None or pool.device.type != "cuda":
return None
event = torch.Event()
event.record(torch.accelerator.current_stream(pool.device))
return event
def release_cached(self, lease: ResidentLease[MemoryAllocation]) -> None:
with self.lock:
released = self._residents.release(lease)
if released is not None:
self._defer_or_free(released, self._record_release_event())
def publish(
self,
mm_hash: str,
allocation: MemoryAllocation,
lease: ResidentLease[MemoryAllocation] | None = None,
*,
pin: bool = True,
) -> MemoryAllocation:
with self.lock:
if lease is not None:
canonical, released = self._residents.consume(lease)
self._retire_events.pop(mm_hash, None)
if released is not None:
self._defer_or_free(released, self._record_release_event())
return canonical
previous = self._residents.get(mm_hash)
displaced = self._residents.insert(mm_hash, allocation)
if not pin:
self._residents.retire(mm_hash)
event = None
if previous is not None and previous is not allocation:
event = self._retire_events.pop(mm_hash, None)
if displaced is None or displaced is allocation:
return allocation
if event is None:
event = self._record_release_event()
self._defer_or_free(displaced, event)
return allocation
def retire_stale(
self,
encoder_cache: dict[str, torch.Tensor],
reserved_hashes: set[str] | None = None,
*,
freed: list[str] | None = None,
) -> None:
if self.tensor is None:
return
with self.lock:
for mm_hash in self._residents.referenced() if freed is None else freed:
allocation = self._residents.get(mm_hash)
if allocation is None:
continue
if encoder_cache.get(mm_hash) is allocation.tensor:
continue
if reserved_hashes and mm_hash in reserved_hashes:
continue
event = self._record_release_event()
if event is not None:
self._retire_events[mm_hash] = event
self._residents.retire(mm_hash)
self._poll_frees_locked()
def drain_reclaimed(self) -> set[str]:
with self.lock:
reclaimed = self._reclaimed
self._reclaimed = set()
return reclaimed
def close(self) -> None:
with self.lock:
if not self._allocator.close(self._transfer):
return
self._residents.clear()
self._retire_events.clear()
self._pending_frees.clear()
@@ -1,79 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Metadata exchanged by the Mooncake encoder-cache connector."""
from __future__ import annotations
from dataclasses import dataclass, field
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
ECConnectorWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
TensorSpec,
)
@dataclass
class ECMooncakeLoadSpec:
"""Describe a remote reservation or resident allocation to load."""
mm_hash: str
nbytes: int
shape: tuple[int, ...]
dtype: str
transfer_id: str
# The consumer pool still holds this item, so the load is a local handoff:
# no transfer, no producer.
local: bool = False
@dataclass
class ECMooncakePushSpec:
"""Describe a destination reservation prepared before a tensor is ready."""
mm_hash: str
nbytes: int
shape: tuple[int, ...]
dtype: str
consumer_zmq: str
transfer_id: str
request_id: str = ""
@dataclass
class ECMooncakeConnectorMetadata(ECConnectorMetadata):
"""Worker operations emitted for one Scheduler step."""
loads: list[ECMooncakeLoadSpec] = field(default_factory=list)
pushes: list[ECMooncakePushSpec] = field(default_factory=list)
freed: list[str] | None = None
store_candidates: dict[str, TensorSpec] = field(default_factory=dict)
@dataclass
class ECMooncakeWorkerMetadata(ECConnectorWorkerMetadata):
"""Completion state reported from Workers to the Scheduler."""
loaded: set[str] = field(default_factory=set)
failed_loads: set[str] = field(default_factory=set)
# Items the receive pool dropped under pressure. The scheduler assumes an
# evicted item stays resident until told otherwise.
reclaimed: set[str] = field(default_factory=set)
pending_saves: bool = False
failed_saves: set[str] = field(default_factory=set)
def aggregate(self, other: ECConnectorWorkerMetadata) -> ECMooncakeWorkerMetadata:
assert isinstance(other, ECMooncakeWorkerMetadata)
return ECMooncakeWorkerMetadata(
# Every tensor-parallel rank gathers the embedding from its own
# cache, so an item counts as loaded only where all of them have
# it; one rank falling short must fail the load rather than leave
# the scheduler believing it is ready.
loaded=self.loaded & other.loaded,
failed_loads=self.failed_loads | other.failed_loads,
reclaimed=self.reclaimed | other.reclaimed,
pending_saves=self.pending_saves or other.pending_saves,
failed_saves=self.failed_saves | other.failed_saves,
)
@@ -1,418 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Producer-side push lifecycle, futures, and source-tensor ownership.
This module records when a push may advance or release its source. Worker
orchestration performs the actual control exchanges and Mooncake writes.
"""
from __future__ import annotations
import threading
from collections import OrderedDict
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import suppress
from dataclasses import dataclass, field
from enum import Enum, auto
from typing import Any
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakePushSpec,
)
def _same_destination(
existing: ECMooncakePushSpec, incoming: ECMooncakePushSpec
) -> bool:
"""Whether two specs describe one push, ignoring which request asked.
The same encoding requested twice shares a transfer id legitimately; only
a different payload or destination is a collision.
"""
return (
existing.mm_hash == incoming.mm_hash
and existing.nbytes == incoming.nbytes
and existing.shape == incoming.shape
and existing.dtype == incoming.dtype
and existing.consumer_zmq == incoming.consumer_zmq
)
class ProducerPushState(Enum):
"""Lifecycle of one Producer push from reservation to terminal state."""
# The reservation reply and source tensor may arrive in either order.
WAITING_INPUTS = auto()
WRITING = auto()
NOTIFYING = auto()
DONE = auto()
CANCEL_PENDING = auto()
CANCELLED = auto()
FAILED = auto()
@dataclass
class ProducerPushRecord:
"""Collect all asynchronous state for one Producer push."""
spec: ECMooncakePushSpec
state: ProducerPushState
reservation_future: Future[list[dict[str, Any]]]
reservations: list[dict[str, Any]] = field(default_factory=list)
shard_futures: list[Future[Any]] = field(default_factory=list)
source_tensor: torch.Tensor | None = None
source_event: torch.Event | None = None
batch_future: Future[None] | None = None
error: str | None = None
_ALLOWED_TRANSITIONS = {
ProducerPushState.WAITING_INPUTS: {
ProducerPushState.WRITING,
ProducerPushState.CANCEL_PENDING,
ProducerPushState.FAILED,
},
ProducerPushState.WRITING: {
ProducerPushState.NOTIFYING,
ProducerPushState.FAILED,
},
ProducerPushState.NOTIFYING: {
ProducerPushState.DONE,
ProducerPushState.FAILED,
},
ProducerPushState.CANCEL_PENDING: {
ProducerPushState.CANCELLED,
ProducerPushState.FAILED,
},
ProducerPushState.DONE: set(),
ProducerPushState.CANCELLED: set(),
ProducerPushState.FAILED: set(),
}
_TERMINAL_STATES = {
ProducerPushState.DONE,
ProducerPushState.CANCELLED,
ProducerPushState.FAILED,
}
_TERMINAL_LIMIT = 1 << 16
class ProducerPushManager:
"""Own Producer push records, transitions, and source tensor leases.
Terminal records are retained for duplicate-metadata idempotency. Separate
ordered indexes keep hot polling paths from scanning those tombstones.
"""
def __init__(self, wake: Callable[[], None] = lambda: None) -> None:
self._records: OrderedDict[str, ProducerPushRecord] = OrderedDict()
self._active_ids: OrderedDict[str, None] = OrderedDict()
self._reapable_terminal_ids: OrderedDict[str, None] = OrderedDict()
self._unreported_ids: OrderedDict[str, None] = OrderedDict()
self._batch_ids: OrderedDict[str, None] = OrderedDict()
self._source_waiters: dict[str, OrderedDict[str, None]] = {}
self._lock = threading.RLock()
self._wake = wake
def reserve(
self,
spec: ECMooncakePushSpec,
submit: Callable[[], Future[list[dict[str, Any]]]],
) -> tuple[ProducerPushRecord, bool]:
with self._lock:
existing = self._records.get(spec.transfer_id)
if existing is not None:
if not _same_destination(existing.spec, spec):
raise ValueError(
f"Conflicting EC destination for transfer_id={spec.transfer_id}"
)
return existing, False
record = ProducerPushRecord(
spec=spec,
state=ProducerPushState.WAITING_INPUTS,
reservation_future=submit(),
)
self._records[spec.transfer_id] = record
self._active_ids[spec.transfer_id] = None
self._source_waiters.setdefault(spec.mm_hash, OrderedDict())[
spec.transfer_id
] = None
record.reservation_future.add_done_callback(
lambda future: self._reservation_done(record, future)
)
return record, True
def bind_source(
self,
mm_hash: str,
tensor: torch.Tensor,
ready_event: torch.Event | None,
) -> None:
with self._lock:
waiters = self._source_waiters.pop(mm_hash, OrderedDict())
for transfer_id in waiters:
record = self._records[transfer_id]
if (
record.source_tensor is not None
or record.state is not ProducerPushState.WAITING_INPUTS
):
continue
record.source_tensor = tensor
record.source_event = ready_event
self._wake()
def submit_batches(
self,
executor: ThreadPoolExecutor,
run_batch: Callable[[list[ProducerPushRecord]], None],
*,
wait: bool = False,
) -> bool:
pending_event = False
with self._lock:
grouped: dict[str, list[ProducerPushRecord]] = {}
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if (
record.source_tensor is not None
and record.batch_future is None
and record.state is ProducerPushState.WAITING_INPUTS
):
if not record.reservation_future.done():
continue
if record.source_event is not None:
if wait:
record.source_event.synchronize()
elif not record.source_event.query():
pending_event = True
continue
grouped.setdefault(record.spec.consumer_zmq, []).append(record)
batches = list(grouped.values())
for records in batches:
future = executor.submit(run_batch, records)
for record in records:
record.batch_future = future
self._batch_ids[record.spec.transfer_id] = None
return pending_event
def resolve_reservations(self, record: ProducerPushRecord) -> list[dict[str, Any]]:
results = record.reservation_future.result()
with self._lock:
record.reservations = results
return list(results)
def finish_reservations(
self,
outcomes: list[tuple[ProducerPushRecord, list[dict[str, Any]] | Exception]],
) -> None:
with self._lock:
for record, result in outcomes:
if isinstance(result, Exception):
record.reservation_future.set_exception(result)
else:
record.reservation_future.set_result(result)
def _reservation_done(
self,
record: ProducerPushRecord,
future: Future[list[dict[str, Any]]],
) -> None:
try:
future.result()
except Exception as exc:
with self._lock:
self._set_error(record, exc)
if (
record.state is ProducerPushState.WAITING_INPUTS
and record.source_tensor is None
):
self._transition(record, ProducerPushState.FAILED)
finally:
self._wake()
def settle_all(self, records: list[ProducerPushRecord]) -> None:
for record in records:
with suppress(Exception):
record.reservation_future.result()
def replace_reservations(
self,
record: ProducerPushRecord,
reservations: list[dict[str, Any]],
) -> None:
with self._lock:
record.reservations = reservations
def track_shard_futures(
self,
records: list[ProducerPushRecord],
futures: list[Future[Any]],
) -> None:
with self._lock:
for record in records:
record.shard_futures.extend(futures)
def begin_writing(self, record: ProducerPushRecord) -> None:
with self._lock:
if record.source_tensor is None:
raise RuntimeError(
f"Producer push {record.spec.transfer_id!r} has no source tensor"
)
self._transition(record, ProducerPushState.WRITING)
def begin_notifying(self, records: list[ProducerPushRecord]) -> None:
with self._lock:
for record in records:
self._transition(record, ProducerPushState.NOTIFYING)
def complete(self, records: list[ProducerPushRecord]) -> None:
with self._lock:
self._check_sources_releasable(records)
for record in records:
self._transition(record, ProducerPushState.DONE)
self._release_source(record)
def fail(self, records: list[ProducerPushRecord], error: Exception) -> None:
with self._lock:
self._check_sources_releasable(records)
for record in records:
if record.state not in _TERMINAL_STATES:
self._set_error(record, error)
self._transition(record, ProducerPushState.FAILED)
self._release_source(record)
def cancel_requests(self, request_ids: set[str] | None) -> list[ProducerPushRecord]:
"""Cancel source-less waiters for requests, or all waiters at shutdown."""
cancelled = []
with self._lock:
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if (
(
request_ids is not None
and record.spec.request_id not in request_ids
)
or record.source_tensor is not None
or record.batch_future is not None
):
continue
if record.state is not ProducerPushState.WAITING_INPUTS:
continue
self._transition(record, ProducerPushState.CANCEL_PENDING)
cancelled.append(record)
return cancelled
def finish_cancel(self, record: ProducerPushRecord) -> None:
with self._lock:
if record.state is ProducerPushState.CANCEL_PENDING:
self._transition(record, ProducerPushState.CANCELLED)
def submit_cancel(
self,
record: ProducerPushRecord,
executor: ThreadPoolExecutor,
run_cancel: Callable[[ProducerPushRecord], None],
) -> None:
with self._lock:
future = executor.submit(run_cancel, record)
record.batch_future = future
transfer_id = record.spec.transfer_id
self._batch_ids[transfer_id] = None
if record.state in _TERMINAL_STATES:
self._reapable_terminal_ids.pop(transfer_id, None)
def poll(self) -> list[tuple[str, str]]:
failures = []
with self._lock:
for transfer_id in list(self._batch_ids):
record = self._records[transfer_id]
future = record.batch_future
assert future is not None
if not future.done():
continue
try:
future.result()
except Exception as exc:
self._set_error(record, exc)
self._batch_ids.pop(transfer_id)
self._mark_reapable(record)
for transfer_id in list(self._unreported_ids):
if transfer_id in self._batch_ids:
continue
record = self._records[transfer_id]
assert record.error is not None
failures.append((record.spec.mm_hash, record.error))
self._unreported_ids.pop(transfer_id)
self._mark_reapable(record)
self._reap_terminals()
return failures
@property
def pending(self) -> bool:
with self._lock:
return bool(self._active_ids or self._batch_ids)
def _release_source(self, record: ProducerPushRecord) -> None:
record.source_tensor = None
record.source_event = None
def _set_error(self, record: ProducerPushRecord, error: BaseException) -> None:
if record.error is None:
record.error = str(error)
if record.state in _TERMINAL_STATES:
transfer_id = record.spec.transfer_id
self._unreported_ids[transfer_id] = None
self._reapable_terminal_ids.pop(transfer_id, None)
def _reap_terminals(self) -> None:
while len(self._reapable_terminal_ids) > _TERMINAL_LIMIT:
transfer_id, _ = self._reapable_terminal_ids.popitem(last=False)
self._records.pop(transfer_id)
def _mark_reapable(self, record: ProducerPushRecord) -> None:
transfer_id = record.spec.transfer_id
if (
record.state in _TERMINAL_STATES
and transfer_id not in self._batch_ids
and transfer_id not in self._unreported_ids
):
self._reapable_terminal_ids[transfer_id] = None
def _drop_source_waiter(self, record: ProducerPushRecord) -> None:
waiters = self._source_waiters.get(record.spec.mm_hash)
if waiters is None:
return
waiters.pop(record.spec.transfer_id, None)
if not waiters:
self._source_waiters.pop(record.spec.mm_hash)
@staticmethod
def _check_sources_releasable(records: list[ProducerPushRecord]) -> None:
for record in records:
futures = [record.reservation_future, *record.shard_futures]
if record.source_tensor is not None and not all(
future.done() for future in futures
):
raise RuntimeError(
f"Producer push {record.spec.transfer_id!r} released its "
"source too early"
)
def _transition(self, record: ProducerPushRecord, state: ProducerPushState) -> None:
if state not in _ALLOWED_TRANSITIONS[record.state]:
raise RuntimeError(
f"Cannot transition producer push {record.spec.transfer_id!r} from "
f"{record.state.name} to {state.name}"
)
record.state = state
if state is not ProducerPushState.WAITING_INPUTS:
self._drop_source_waiter(record)
if state in _TERMINAL_STATES:
transfer_id = record.spec.transfer_id
self._active_ids.pop(transfer_id, None)
if record.error is not None:
self._unreported_ids[transfer_id] = None
self._mark_reapable(record)
@@ -1,487 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Consumer-side reservation lifecycle and destination-memory ownership.
Reservations make remote writes idempotent and ensure cancellation or expiry
cannot free a destination while Mooncake may still be writing into it.
"""
from __future__ import annotations
import threading
import time
import uuid
from collections import OrderedDict
from dataclasses import dataclass
from enum import Enum, auto
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import (
ConsumerMemoryPool,
MemoryAllocation,
ResidentLease,
)
class ConsumerReservationState(Enum):
"""Lifecycle of one Consumer destination reservation."""
WRITING = auto()
READY = auto()
CANCEL_PENDING = auto()
EXPIRE_PENDING = auto()
CANCELLED = auto()
EXPIRED = auto()
_ALLOWED_TRANSITIONS = {
ConsumerReservationState.WRITING: {
ConsumerReservationState.READY,
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
ConsumerReservationState.CANCELLED,
},
ConsumerReservationState.READY: {
ConsumerReservationState.CANCELLED,
ConsumerReservationState.EXPIRED,
},
ConsumerReservationState.CANCEL_PENDING: {ConsumerReservationState.CANCELLED},
ConsumerReservationState.EXPIRE_PENDING: {
ConsumerReservationState.CANCELLED,
ConsumerReservationState.EXPIRED,
},
ConsumerReservationState.CANCELLED: set(),
ConsumerReservationState.EXPIRED: set(),
}
_ACTIVE_STATES = {
ConsumerReservationState.WRITING,
ConsumerReservationState.READY,
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
}
_DEFERRED_STATES = {
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
}
_WRITER_OWNED_STATES = {
ConsumerReservationState.WRITING,
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
}
@dataclass
class ConsumerReservation:
"""Own the identity and destination allocation of one remote push."""
transfer_id: str
mm_hash: str
reservation_id: str
state: ConsumerReservationState
shape: tuple[int, ...] = ()
dtype: str = ""
allocation: MemoryAllocation | None = None
lease: ResidentLease[MemoryAllocation] | None = None
expires_at: float = 0
writer_id: str = ""
class ConsumerReservationManager:
"""Own reservation transitions and destination allocation releases.
A remote writer owns a ``WRITING`` allocation. Cancellation and expiry are
therefore deferred until completion, unless refresh explicitly abandons
that writer before replacing its reservation.
"""
def __init__(
self,
memory: ConsumerMemoryPool,
lease_ttl: float,
tombstone_limit: int,
) -> None:
self._memory = memory
self._lease_ttl = lease_ttl
self._tombstone_limit = tombstone_limit
self._records: dict[str, ConsumerReservation] = {}
self._active_ids: dict[str, None] = {}
self._tombstones: OrderedDict[str, None] = OrderedDict()
self._writers: dict[str, ConsumerReservation] = {}
self._followers: dict[str, dict[str, None]] = {}
self._ready: list[str] = []
self._condition = threading.Condition(memory.lock)
self._shutting_down = False
def reserve(
self,
transfer_id: str,
mm_hash: str,
nbytes: int,
shape: tuple[int, ...],
dtype_name: str,
dtype: torch.dtype,
) -> tuple[ConsumerReservation | None, bool]:
with self._condition:
if self._shutting_down:
raise RuntimeError("Consumer reservation manager is shutting down")
existing = self._records.get(transfer_id)
if (
existing is not None
and existing.state is ConsumerReservationState.CANCELLED
):
return existing, False
if (
existing is not None
and existing.state is ConsumerReservationState.EXPIRED
):
self._remove(transfer_id)
existing = None
if (
existing is not None
and existing.state is ConsumerReservationState.EXPIRE_PENDING
):
raise RuntimeError(
f"Transfer {transfer_id!r} still has an active writer"
)
if existing is not None:
if (
existing.mm_hash != mm_hash
or existing.shape != shape
or existing.dtype != dtype_name
):
raise ValueError("conflicting reservation for transfer_id")
if existing.state not in _ACTIVE_STATES:
raise RuntimeError(
f"Cannot reserve transfer {transfer_id!r} in "
f"{existing.state.name}"
)
if existing.state is ConsumerReservationState.WRITING:
existing.expires_at = time.monotonic() + self._lease_ttl
return existing, False
lease = self._memory.acquire_cached(mm_hash, shape, dtype)
now = time.monotonic()
if lease is not None:
record = ConsumerReservation(
transfer_id,
mm_hash,
uuid.uuid4().hex,
ConsumerReservationState.READY,
shape,
dtype_name,
lease.value,
lease,
now + self._lease_ttl,
)
self._insert(record)
return record, False
writer = self._writers.get(mm_hash)
if writer is not None and writer.state is ConsumerReservationState.WRITING:
if writer.shape != shape or writer.dtype != dtype_name:
raise ValueError("conflicting in-flight tensor for mm_hash")
record = ConsumerReservation(
transfer_id,
mm_hash,
uuid.uuid4().hex,
ConsumerReservationState.WRITING,
shape,
dtype_name,
writer.allocation,
expires_at=now + self._lease_ttl,
writer_id=writer.transfer_id,
)
self._insert(record)
self._followers.setdefault(writer.transfer_id, {})[transfer_id] = None
return record, False
allocation = self._memory.try_allocate(nbytes, shape, dtype)
if allocation is None:
self._expire_locked(time.monotonic())
allocation = self._memory.try_allocate(nbytes, shape, dtype)
if allocation is None:
allocation = self._memory.reclaim_and_allocate(nbytes, shape, dtype)
if allocation is None:
return None, False
record = ConsumerReservation(
transfer_id,
mm_hash,
uuid.uuid4().hex,
ConsumerReservationState.WRITING,
shape,
dtype_name,
allocation,
None,
now + self._lease_ttl,
)
self._insert(record)
self._writers[mm_hash] = record
return record, True
def status(self, transfer_id: str) -> ConsumerReservation | None:
with self._memory.lock:
record = self._records.get(transfer_id)
if record is None or record.state not in _ACTIVE_STATES:
return None
return record
def complete(self, transfer_id: str, reservation_id: str) -> tuple[bool, bool]:
with self._condition:
try:
record = self._records.get(transfer_id)
if record is None or record.reservation_id != reservation_id:
return False, False
if record.state is ConsumerReservationState.READY:
return True, False
if record.writer_id:
return False, False
if record.state in _WRITER_OWNED_STATES:
self._publish_followers(record)
if record.state in _DEFERRED_STATES:
terminal = (
ConsumerReservationState.CANCELLED
if record.state is ConsumerReservationState.CANCEL_PENDING
else ConsumerReservationState.EXPIRED
)
self._terminate(record, terminal)
return True, False
if record.state is not ConsumerReservationState.WRITING:
return False, False
self._transition(record, ConsumerReservationState.READY)
record.expires_at = time.monotonic() + self._lease_ttl
return True, True
finally:
self._condition.notify_all()
def _publish_followers(self, writer: ConsumerReservation) -> None:
if self._writers.get(writer.mm_hash) is writer:
self._writers.pop(writer.mm_hash)
followers = self._followers.pop(writer.transfer_id, {})
if not followers:
return
assert writer.allocation is not None
allocation = self._memory.publish(writer.mm_hash, writer.allocation, pin=False)
for record in [writer, *(self._records[key] for key in followers)]:
record.allocation = allocation
record.lease = self._memory.acquire_cached(
record.mm_hash, record.shape, allocation.tensor.dtype
)
assert record.lease is not None
if record is not writer:
record.writer_id = ""
self._transition(record, ConsumerReservationState.READY)
record.expires_at = time.monotonic() + self._lease_ttl
self._ready.append(record.transfer_id)
def drain_ready(self) -> list[str]:
with self._memory.lock:
ready, self._ready = self._ready, []
return ready
def begin_shutdown(self) -> None:
"""Stop new reservations and cancel everything without a remote writer."""
with self._condition:
if self._shutting_down:
return
self._shutting_down = True
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if record.writer_id or record.state is ConsumerReservationState.READY:
self._terminate(record, ConsumerReservationState.CANCELLED)
elif record.state is ConsumerReservationState.WRITING:
self._defer(record, ConsumerReservationState.CANCEL_PENDING)
self._condition.notify_all()
def wait_for_writers(self, timeout: float) -> bool:
"""Wait until no reservation is still owned by a remote writer."""
def writers_finished() -> bool:
return not any(
self._records[transfer_id].state in _WRITER_OWNED_STATES
for transfer_id in self._active_ids
)
with self._condition:
return self._condition.wait_for(writers_finished, timeout=max(0, timeout))
def cancel(
self,
transfer_id: str,
reservation_id: str,
abandon: bool = False,
refresh: bool = False,
) -> bool:
with self._memory.lock:
record = self._records.get(transfer_id)
if (
record is not None
and reservation_id
and record.reservation_id != reservation_id
):
return False
if record is None:
record = ConsumerReservation(
transfer_id,
"",
"",
ConsumerReservationState.CANCELLED,
)
self._insert(record)
self._set_tombstone_deadline(record)
self._reap_tombstones(time.monotonic())
return True
if record.state is ConsumerReservationState.CANCELLED:
self._set_tombstone_deadline(record)
self._reap_tombstones(time.monotonic())
return True
if refresh:
if not abandon or record.state not in {
ConsumerReservationState.WRITING,
ConsumerReservationState.EXPIRE_PENDING,
}:
return False
if record.state is ConsumerReservationState.WRITING:
self._transition(record, ConsumerReservationState.EXPIRE_PENDING)
self._terminate(record, ConsumerReservationState.EXPIRED)
self._condition.notify_all()
self._reap_tombstones(time.monotonic())
return True
if record.writer_id:
self._terminate(record, ConsumerReservationState.CANCELLED)
return True
if record.state in _DEFERRED_STATES and not abandon:
return True
if record.state is ConsumerReservationState.WRITING and not abandon:
self._defer(record, ConsumerReservationState.CANCEL_PENDING)
return True
if record.state not in _ACTIVE_STATES:
return False
self._terminate(record, ConsumerReservationState.CANCELLED)
if self._shutting_down:
self._condition.notify_all()
self._reap_tombstones(time.monotonic())
return True
def take(self, transfer_id: str, mm_hash: str) -> MemoryAllocation:
with self._memory.lock:
record = self._records.get(transfer_id)
if (
record is None
or record.state is not ConsumerReservationState.READY
or record.mm_hash != mm_hash
or record.allocation is None
):
raise RuntimeError(
f"Pushed EC tensor is not ready for mm_hash={mm_hash}"
)
allocation = self._memory.publish(mm_hash, record.allocation, record.lease)
record.allocation = None
record.lease = None
self._remove(transfer_id)
return allocation
def expire(self) -> int:
with self._memory.lock:
return self._expire_locked(time.monotonic())
def retire_stale(
self, encoder_cache: dict[str, torch.Tensor], freed: list[str] | None = None
) -> None:
with self._memory.lock:
self._memory.retire_stale(encoder_cache, freed=freed)
def _expire_locked(self, now: float) -> int:
expired = 0
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if record.expires_at > now:
continue
if record.state is ConsumerReservationState.READY:
self._terminate(record, ConsumerReservationState.EXPIRED)
expired += 1
elif record.state is ConsumerReservationState.WRITING:
if record.writer_id:
self._terminate(record, ConsumerReservationState.CANCELLED)
else:
self._defer(record, ConsumerReservationState.EXPIRE_PENDING)
self._reap_tombstones(now)
return expired
def _defer(
self, record: ConsumerReservation, state: ConsumerReservationState
) -> None:
"""Keep the destination until the writer completes or abandons it."""
self._transition(record, state)
record.expires_at = float("inf")
def _terminate(
self, record: ConsumerReservation, state: ConsumerReservationState
) -> None:
self._transition(record, state)
self._release(record)
self._set_tombstone_deadline(record)
def _release(self, record: ConsumerReservation) -> None:
if record.writer_id:
followers = self._followers.get(record.writer_id, {})
followers.pop(record.transfer_id, None)
record.allocation = None
return
if self._writers.get(record.mm_hash) is record:
self._writers.pop(record.mm_hash)
for transfer_id in list(self._followers.pop(record.transfer_id, {})):
self._terminate(
self._records[transfer_id], ConsumerReservationState.CANCELLED
)
allocation = record.allocation
if allocation is None:
return
if record.lease is not None:
self._memory.release_cached(record.lease)
else:
self._memory.free(allocation)
record.allocation = None
record.lease = None
def _set_tombstone_deadline(self, record: ConsumerReservation) -> None:
record.expires_at = time.monotonic() + self._lease_ttl
self._tombstones[record.transfer_id] = None
self._tombstones.move_to_end(record.transfer_id)
def _reap_tombstones(self, now: float) -> int:
dropped = 0
while self._tombstones:
transfer_id = next(iter(self._tombstones))
record = self._records[transfer_id]
if (
record.expires_at > now
and len(self._tombstones) <= self._tombstone_limit
):
break
self._remove(transfer_id)
dropped += 1
return dropped
def _insert(self, record: ConsumerReservation) -> None:
self._records[record.transfer_id] = record
if record.state in _ACTIVE_STATES:
self._active_ids[record.transfer_id] = None
def _remove(self, transfer_id: str) -> None:
self._records.pop(transfer_id, None)
self._active_ids.pop(transfer_id, None)
self._tombstones.pop(transfer_id, None)
def _transition(
self, record: ConsumerReservation, state: ConsumerReservationState
) -> None:
if state not in _ALLOWED_TRANSITIONS[record.state]:
raise RuntimeError(
f"Cannot transition {record.transfer_id!r} from "
f"{record.state.name} to {state.name}"
)
record.state = state
if state in _ACTIVE_STATES:
self._active_ids[record.transfer_id] = None
else:
self._active_ids.pop(record.transfer_id, None)
@@ -1,561 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scheduler-side planning and observation for Mooncake cache transfers.
The Scheduler prepares Producer reservations, consumes Consumer readiness
events, emits per-step Worker metadata, and converts Worker results back into
request availability without owning tensor memory or running data transfers.
"""
from __future__ import annotations
import math
import time
from collections import OrderedDict
from concurrent.futures import Future, ThreadPoolExecutor
from typing import TYPE_CHECKING, Any
import torch
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
get_encoder_output_width,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
_RESERVATION_TTL_SECONDS,
MooncakeECConfig,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.control import (
ControlClient,
EventInbox,
make_cancel_request,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeConnectorMetadata,
ECMooncakeLoadSpec,
ECMooncakePushSpec,
ECMooncakeWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.state import (
SchedulerTransferState,
SchedulerTransferTable,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
ensure_mooncake_available,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
TensorSpec,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ECConnectorOutput
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
# Unmatched `ec_items` entries are a protocol desync, not a per-request
# event: warn on the first few, then only once in a while.
_MAX_UNRESOLVED_TRANSFER_ID_WARNINGS = 5
_DRAIN_MIN_INTERVAL = 0.005
_MAX_PENDING_EVENTS = 4096
_MAX_TERMINAL_TRANSFER_RECORDS = 1 << 16
_CANCEL_ATTEMPTS = 2
_CONTROL_WORKERS = 8
class ECMooncakeScheduler:
"""Coordinate Mooncake transfers from the vLLM Scheduler process."""
def __init__(self, vllm_config: VllmConfig) -> None:
ensure_mooncake_available()
config = MooncakeECConfig.from_vllm_config(vllm_config)
self._store_candidates: list[tuple[str, str, TensorSpec]] = []
self._cross_encoder_cache = config.cross_encoder_cache
self._is_producer = config.is_producer
self._is_consumer = config.is_consumer
self._control_addr = config.control_addr
self._push_wait_timeout = config.push_wait_timeout_s
self._vllm_config = vllm_config
self._model_config = vllm_config.model_config
self._control_client = ControlClient(config.control_timeout_ms)
self._control_executor = ThreadPoolExecutor(
max_workers=_CONTROL_WORKERS,
thread_name_prefix="ec-mooncake-control",
)
self._event_inbox = EventInbox(self._control_client, self._control_executor)
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
self._unresolved_transfer_ids = 0
self._drain_pending = True
self._drained_at = 0.0
self._pending_cancels: dict[str, Future[Any]] = {}
self._transfers = SchedulerTransferTable(
config.pool_size, _RESERVATION_TTL_SECONDS
)
self._scheduler_pending_work = False
self._pushes_to_prepare: dict[str, ECMooncakePushSpec] = {}
self._prepared_push_transfer_ids: set[str] = set()
self._event_ready_shards: OrderedDict[str, set[int]] = OrderedDict()
# Mirror of the engine's encoder cache, maintained from the alloc and
# free notifications the Scheduler already sends. Tracking it here
# keeps `ensure_cache_available` on the upstream two-argument shape.
self._local_cache: set[str] = set()
self._failed_saves: set[str] = set()
def _cancel_remote(self, consumer_zmq: str, transfer_id: str) -> bool:
pending = None
for _ in range(_CANCEL_ATTEMPTS):
pending = self._control_client.discover_shards(consumer_zmq)
if pending is not None:
break
if pending is None:
raise RuntimeError(
f"Could not discover every EC consumer shard at {consumer_zmq}"
)
cancelled = False
error: BaseException | None = None
for _ in range(_CANCEL_ATTEMPTS):
failed = []
for addr in pending:
try:
result = self._control_client.request(
addr,
make_cancel_request(transfer_id),
)
except Exception as exc:
if error is None:
error = exc
failed.append(addr)
continue
cancelled |= isinstance(result, dict) and bool(result.get("cancelled"))
if not failed:
return cancelled
pending = failed
if error is not None:
raise error
return cancelled
def _note_awaiting_push(
self,
mm_hash: str,
transfer_id: str,
request_id: str,
now: float,
) -> None:
record = self._transfers.wait_for_event(
transfer_id,
request_id,
mm_hash,
now + self._push_wait_timeout,
)
if record is None:
# The id names another encoding; the request is already failed.
return
if record.state is not SchedulerTransferState.WAITING_EVENT:
return
assert record.deadline is not None
if now < record.deadline:
return
elapsed = now - record.deadline + self._push_wait_timeout
self._transfers.mark_unavailable(transfer_id, now)
logger.warning(
"EC Mooncake waited %.1fs for a push of mm_hash=%s "
"(transfer_id=%s) that never arrived; "
"requests needing it fail with a retryable error.",
elapsed,
mm_hash,
transfer_id,
)
def take_unavailable_requests(self) -> set[str]:
failed, self._failed_saves = self._failed_saves, set()
return failed | self._transfers.take_unavailable_requests()
def _poll_pending_cancels(self) -> None:
pending = {}
for transfer_id, future in self._pending_cancels.items():
if not future.done():
pending[transfer_id] = future
continue
try:
future.result()
except Exception:
logger.warning(
"EC Mooncake reservation cancellation failed", exc_info=True
)
self._pending_cancels = pending
def _note_shard_ready(self, data: dict[str, Any]) -> bool:
shard_count = self._event_inbox.shard_count
if shard_count <= 1:
return True
transfer_id = str(data["transfer_id"])
record = self._transfers.get(transfer_id)
if (
record is not None
and record.state is not SchedulerTransferState.WAITING_EVENT
):
return False
shard = data.get("shard")
shards = self._event_ready_shards.setdefault(transfer_id, set())
self._event_ready_shards.move_to_end(transfer_id)
shards.add(int(shard) if shard is not None else len(shards))
if len(shards) < shard_count:
while len(self._event_ready_shards) > _MAX_PENDING_EVENTS:
self._event_ready_shards.popitem(last=False)
return False
self._event_ready_shards.pop(transfer_id, None)
return True
def _forget_shard_readiness(self, transfer_id: str) -> None:
self._event_ready_shards.pop(transfer_id, None)
def _store_pushed_spec(self, data: dict[str, Any]) -> bool:
transfer_id = str(data["transfer_id"])
identifier = str(data["mm_hash"])
_, accepted = self._transfers.observe_ready(
ECMooncakeLoadSpec(
mm_hash=identifier,
nbytes=int(data["nbytes"]),
shape=tuple(int(value) for value in data["shape"]),
dtype=str(data["dtype"]),
transfer_id=transfer_id,
),
time.monotonic() + _RESERVATION_TTL_SECONDS,
)
return accepted
def _queue_cancel(
self,
transfer_id: str,
mm_hash: str = "",
request_id: str = "",
) -> None:
if not self._transfers.cancel(
transfer_id,
time.monotonic(),
mm_hash=mm_hash,
request_id=request_id,
):
return
self._forget_shard_readiness(transfer_id)
self._pending_cancels[transfer_id] = self._control_executor.submit(
self._cancel_remote,
self._control_addr,
transfer_id,
)
def _expire_transfers(self) -> None:
now = time.monotonic()
expired = self._transfers.expire(now, _MAX_TERMINAL_TRANSFER_RECORDS)
for record in expired:
self._queue_cancel(record.transfer_id)
def _drain_push_notifications(self) -> None:
now = time.monotonic()
if not self._drain_pending and now - self._drained_at < _DRAIN_MIN_INTERVAL:
return
self._drain_pending = False
self._drained_at = now
self._poll_pending_cancels()
self._expire_transfers()
events = self._event_inbox.drain(self._control_addr)
for data in events:
try:
if not isinstance(data, dict):
raise TypeError("EC readiness event must be an object")
if not data.get("ready"):
continue
self._accept_ready_event(data)
except (KeyError, TypeError, ValueError):
# Readiness events cross a plain PULL socket, so a malformed
# one costs that event and not the engine.
logger.warning(
"Discarding a malformed EC readiness event.", exc_info=True
)
def _accept_ready_event(self, data: dict[str, Any]) -> None:
transfer_id = str(data["transfer_id"])
record = self._transfers.get(transfer_id)
if record is not None and record.state in {
SchedulerTransferState.CANCELLED,
SchedulerTransferState.UNAVAILABLE,
SchedulerTransferState.EXPIRED,
SchedulerTransferState.FAILED,
}:
return
if not self._note_shard_ready(data):
return
self._store_pushed_spec(data)
def has_cache_item(self, identifier: str) -> bool:
if not self._is_consumer:
return False
self._drain_push_notifications()
return self._transfers.has_state(
identifier,
(
SchedulerTransferState.READY,
SchedulerTransferState.RESIDENT,
SchedulerTransferState.AVAILABLE,
),
)
def _warn_unresolved_transfer_id(
self, request: Any, index: int, where: str
) -> None:
"""Report an `ec_items` entry that cannot be matched to a feature.
A caller that cannot resolve a transfer id has to fall back to a
locally invented one or skip its bookkeeping entirely, and either way
the two sides of a transfer stop agreeing on its name. That desyncs
silently -- reservations are never released and only surface minutes
later as a full buffer pool -- so say it out loud the first time.
`mm_hash` in `ec_items` must be a value some engine reported, never one
the caller derived itself: `mm_features[i].identifier` folds in the
engine's media_io_kwargs and mm_processor_kwargs, so a media uuid alone
no longer equals it. Omit the field to match by position instead.
"""
self._unresolved_transfer_ids += 1
count = self._unresolved_transfer_ids
if count > _MAX_UNRESOLVED_TRANSFER_ID_WARNINGS and count % 1000:
return
params = getattr(request, "ec_transfer_params", None)
items = (params or {}).get("ec_items") or []
logger.warning(
"EC Mooncake could not resolve a transfer id at %s for req=%s "
"index=%d: feature identifier=%s does not match any of the %d "
"ec_items %s (ec_transfer_params present=%s). Occurrence %d. "
"ec_items[].mm_hash must echo an engine-reported identifier, or "
"be omitted so the entry is matched by position.",
where,
request.request_id,
index,
request.mm_features[index].identifier[:16],
len(items),
[str(item.get("mm_hash"))[:16] for item in items[:4]],
params is not None,
count,
)
@staticmethod
def _request_transfer_id(request: Any, index: int) -> str | None:
params = getattr(request, "ec_transfer_params", None) or {}
items = params.get("ec_items") or []
mm_hash = request.mm_features[index].identifier
if index < len(items):
item = items[index]
if item.get("mm_hash") in (None, mm_hash) and item.get("transfer_id"):
return str(item["transfer_id"])
for item in items:
if item.get("mm_hash") == mm_hash and item.get("transfer_id"):
return str(item["transfer_id"])
return None
def ensure_cache_available(self, request: Any, num_computed_tokens: int) -> bool:
if self._is_producer:
for index, feature in enumerate(request.mm_features):
if (
feature.mm_position.offset + feature.mm_position.length
> num_computed_tokens
):
self._prepare_push_spec(request, index)
if not self._is_consumer:
return True
self._drain_push_notifications()
# One timestamp for the whole decision, so the deadlines it hands out
# cannot disagree between features.
now = time.monotonic()
all_ready = True
for index, feature in enumerate(request.mm_features):
if (
feature.mm_position.offset + feature.mm_position.length
<= num_computed_tokens
):
continue
mm_hash = feature.identifier
transfer_id = self._request_transfer_id(request, index)
if transfer_id is not None:
self._transfers.touch_available(
transfer_id, now + _RESERVATION_TTL_SECONDS
)
if mm_hash in self._local_cache:
continue
if self._transfers.has_state(mm_hash, (SchedulerTransferState.READY,)):
continue
if self._transfers.has_state(mm_hash, (SchedulerTransferState.LOADING,)):
all_ready = False
continue
record = self._transfers.begin_load(
mm_hash,
transfer_id,
request.request_id,
now + self._push_wait_timeout,
)
if record is not None:
self._scheduler_pending_work = True
all_ready = False
else:
waiting_id = transfer_id or f"{request.request_id}:{index}"
self._note_awaiting_push(mm_hash, waiting_id, request.request_id, now)
all_ready = False
return all_ready
def _prepare_push_spec(self, request: Any, index: int) -> None:
params = getattr(request, "ec_transfer_params", None) or {}
consumer_zmq = params.get("consumer_zmq")
mm_hash = request.mm_features[index].identifier
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
# Still push: after the proxy has rewritten the item to embeds the
# consumer has no media left to fall back on, so a nameless push
# beats none. But the consumer knows this transfer by the id it
# sent, not by the one invented here, so its cancel will never
# reach the reservation this push is about to take.
if consumer_zmq:
self._warn_unresolved_transfer_id(request, index, "push prepare")
transfer_id = f"{request.request_id}:{index}"
if not consumer_zmq or transfer_id in self._prepared_push_transfer_ids:
return
output = self._encoder_output_spec(request, index)
self._pushes_to_prepare[transfer_id] = ECMooncakePushSpec(
mm_hash=mm_hash,
nbytes=output.nbytes,
shape=output.shape,
dtype=output.dtype.removeprefix("torch."),
consumer_zmq=str(consumer_zmq),
transfer_id=transfer_id,
request_id=request.request_id,
)
self._prepared_push_transfer_ids.add(transfer_id)
def _encoder_output_spec(self, request: Any, index: int) -> TensorSpec:
dtype = self._model_config.dtype
assert isinstance(dtype, torch.dtype)
hidden_dim = get_encoder_output_width(
self._vllm_config, request.mm_features[index].modality
)
shape = (request.get_num_encoder_embeds(index), hidden_dim)
return TensorSpec(shape, str(dtype), math.prod(shape) * dtype.itemsize)
def update_state_after_alloc(self, request: Any, index: int) -> None:
self._local_cache.add(request.mm_features[index].identifier)
if self._is_producer:
self._prepare_push_spec(request, index)
if (
self._cross_encoder_cache
and request.mm_features[index].modality == "image"
):
self._store_candidates.append(
(
request.request_id,
request.mm_features[index].identifier,
self._encoder_output_spec(request, index),
)
)
def update_state_after_free(self, request: Any, index: int) -> None:
if not self._is_consumer:
return
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
self._warn_unresolved_transfer_id(request, index, "encoder-cache free")
return
self._queue_cancel(
transfer_id,
mm_hash=request.mm_features[index].identifier,
request_id=request.request_id,
)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> ECConnectorMetadata:
for mm_hash in scheduler_output.free_encoder_mm_hashes:
self._local_cache.discard(mm_hash)
self._transfers.release_ready(mm_hash, time.monotonic())
for transfer_id in self._transfers.drain_orphaned():
self._queue_cancel(transfer_id)
meta = ECMooncakeConnectorMetadata(
freed=scheduler_output.free_encoder_mm_hashes
)
preempted = scheduler_output.preempted_req_ids or set()
meta.store_candidates = {
identifier: output
for request_id, identifier, output in self._store_candidates
if request_id not in preempted
}
self._store_candidates.clear()
for push_spec in self._pushes_to_prepare.values():
meta.pushes.append(push_spec)
self._pushes_to_prepare.clear()
for record in self._transfers.take_loads_to_dispatch():
assert record.spec is not None
meta.loads.append(record.spec)
self._poll_pending_cancels()
self._drain_pending = True
return meta
def update_connector_output(self, connector_output: ECConnectorOutput) -> None:
meta = connector_output.ec_connector_worker_meta
if not isinstance(meta, ECMooncakeWorkerMetadata):
return
for mm_hash in meta.loaded:
self._transfers.complete_load(mm_hash)
for mm_hash in meta.failed_loads:
self._transfers.fail_load(mm_hash, time.monotonic())
for mm_hash in meta.reclaimed:
self._transfers.reclaim(mm_hash, time.monotonic())
self._scheduler_pending_work = meta.pending_saves
self._failed_saves.update(meta.failed_saves)
def has_pending_push_work(self) -> bool:
return self._scheduler_pending_work
def request_finished(self, request: Any) -> tuple[bool, dict[str, Any] | None]:
if self._is_consumer:
for index in range(len(request.mm_features)):
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
self._warn_unresolved_transfer_id(request, index, "request finish")
continue
self._queue_cancel(
transfer_id,
mm_hash=request.mm_features[index].identifier,
request_id=request.request_id,
)
if self._is_producer and self._prepared_push_transfer_ids:
for index in range(len(request.mm_features)):
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
transfer_id = f"{request.request_id}:{index}"
self._prepared_push_transfer_ids.discard(transfer_id)
if not self._is_producer:
return False, None
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
for index, feature in enumerate(request.mm_features):
transfer_id = self._request_transfer_id(request, index)
if transfer_id is not None:
items[feature.identifier]["transfer_id"] = transfer_id
if not items:
return False, None
return False, items
def close(self) -> None:
self._control_executor.shutdown(wait=True, cancel_futures=True)
self._control_client.close()
self._event_inbox.close()
@@ -1,448 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scheduler-owned lifecycle and indexes for Consumer-bound transfers."""
from __future__ import annotations
from collections import OrderedDict, deque
from collections.abc import Iterable
from dataclasses import dataclass, field, replace
from enum import Enum, auto
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeLoadSpec,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
class SchedulerTransferState(Enum):
"""Lifecycle states visible to the Scheduler.
The states cover waiting for a Consumer event, dispatching a Worker load,
retaining a locally reusable tensor, and terminal failure conditions.
"""
WAITING_EVENT = auto()
AVAILABLE = auto()
LOADING = auto()
READY = auto()
RESIDENT = auto()
UNAVAILABLE = auto()
EXPIRED = auto()
FAILED = auto()
CANCELLED = auto()
@dataclass
class SchedulerTransfer:
"""Track one transfer as observed by the Scheduler."""
transfer_id: str
request_id: str
mm_hash: str
state: SchedulerTransferState
spec: ECMooncakeLoadSpec | None
deadline: float | None
notified_requests: set[str] = field(default_factory=set, repr=False)
_TERMINAL_STATES = {
SchedulerTransferState.UNAVAILABLE,
SchedulerTransferState.EXPIRED,
SchedulerTransferState.FAILED,
SchedulerTransferState.CANCELLED,
}
class SchedulerTransferTable:
"""Own Scheduler transfer state, lookup indexes, and dispatch queues.
The Scheduler is single-threaded, so direct transitions are sufficient;
the Producer and Consumer managers retain stricter transition matrices
because callbacks and control requests can race there.
"""
def __init__(self, resident_capacity: int, tombstone_ttl: float) -> None:
self._resident_capacity = resident_capacity
self._tombstone_ttl = tombstone_ttl
self._records: OrderedDict[str, SchedulerTransfer] = OrderedDict()
self._active_ids: dict[str, None] = {}
self._terminal_ids: OrderedDict[str, None] = OrderedDict()
self._resident_ids: OrderedDict[str, None] = OrderedDict()
self._resident_bytes = 0
self._hash_index: dict[str, deque[str]] = {}
self._loads_to_dispatch: OrderedDict[str, None] = OrderedDict()
self._unavailable_requests: set[str] = set()
# Records expired straight out of READY/RESIDENT: the consumer worker
# still holds their push reservation, and nothing else tells it to let
# go, so the destination buffer would sit pinned until the lease TTL.
self._orphaned: list[str] = []
def get(self, transfer_id: str) -> SchedulerTransfer | None:
return self._records.get(transfer_id)
def drain_orphaned(self) -> list[str]:
"""Transfer IDs expired out of READY/RESIDENT since the last drain.
The caller must release the consumer worker's reservation for each:
`cancel()` still accepts an EXPIRED record, so the usual cancel path
applies.
"""
orphaned = self._orphaned
self._orphaned = []
return orphaned
def records_for_hash(
self,
mm_hash: str,
states: Iterable[SchedulerTransferState],
) -> list[SchedulerTransfer]:
wanted = set(states)
return [
record
for transfer_id in self._hash_index.get(mm_hash, ())
if (record := self._records.get(transfer_id)) is not None
and record.state in wanted
]
def first_for_hash(
self,
mm_hash: str,
states: Iterable[SchedulerTransferState],
) -> SchedulerTransfer | None:
states = tuple(states)
if len(states) == 1:
wanted_state = states[0]
for transfer_id in self._hash_index.get(mm_hash, ()):
record = self._records.get(transfer_id)
if record is not None and record.state is wanted_state:
return record
return None
wanted_states = set(states)
for transfer_id in self._hash_index.get(mm_hash, ()):
record = self._records.get(transfer_id)
if record is not None and record.state in wanted_states:
return record
return None
def has_state(self, mm_hash: str, states: Iterable[SchedulerTransferState]) -> bool:
return self.first_for_hash(mm_hash, states) is not None
def wait_for_event(
self,
transfer_id: str,
request_id: str,
mm_hash: str,
deadline: float,
) -> SchedulerTransfer | None:
"""Track a request waiting for a push.
Returns None when `transfer_id` already names a different encoding.
The id comes from the request, so a collision has to fail that one
request; re-issuing it re-runs the encode under a fresh id.
"""
record = self._records.get(transfer_id)
if record is None:
record = SchedulerTransfer(
transfer_id=transfer_id,
request_id=request_id,
mm_hash=mm_hash,
state=SchedulerTransferState.WAITING_EVENT,
spec=None,
deadline=deadline,
)
self._insert(record)
elif not self._identity_matches(record, mm_hash):
self._refuse_colliding_id(record, mm_hash, request_id)
return None
else:
if not record.request_id:
record.request_id = request_id
if record.state in _TERMINAL_STATES and request_id:
self._notify_unavailable(record, request_id)
return record
def observe_ready(
self, spec: ECMooncakeLoadSpec, deadline: float
) -> tuple[SchedulerTransfer, bool]:
transfer_id = spec.transfer_id
record = self._records.get(transfer_id)
if record is None:
record = SchedulerTransfer(
transfer_id=transfer_id,
request_id="",
mm_hash=spec.mm_hash,
state=SchedulerTransferState.WAITING_EVENT,
spec=None,
deadline=None,
)
self._insert(record)
elif not self._identity_matches(record, spec.mm_hash):
self._refuse_colliding_id(record, spec.mm_hash, record.request_id)
return record, False
if record.state is not SchedulerTransferState.WAITING_EVENT:
return record, False
record.spec = spec
record.deadline = deadline
self._transition(record, SchedulerTransferState.AVAILABLE)
return record, True
def touch_available(self, transfer_id: str, deadline: float) -> None:
record = self._records.get(transfer_id)
if record is not None and record.state is SchedulerTransferState.AVAILABLE:
record.deadline = deadline
def begin_load(
self,
mm_hash: str,
transfer_id: str | None = None,
request_id: str = "",
deadline: float | None = None,
) -> SchedulerTransfer | None:
record = self._records.get(transfer_id) if transfer_id else None
if record is not None and (
record.mm_hash != mm_hash
or record.state is not SchedulerTransferState.AVAILABLE
):
record = None
if record is None:
record = self.first_for_hash(
mm_hash,
(
SchedulerTransferState.AVAILABLE,
SchedulerTransferState.RESIDENT,
),
)
if record is None or record.spec is None:
return None
if not record.request_id:
record.request_id = request_id
# A dispatched load that never reports back would otherwise hold this
# hash in LOADING for good, deferring every later request for it.
record.deadline = deadline
self._transition(record, SchedulerTransferState.LOADING)
self._loads_to_dispatch[record.transfer_id] = None
return record
def take_loads_to_dispatch(self) -> list[SchedulerTransfer]:
records = [
record
for transfer_id in self._loads_to_dispatch
if (record := self._records.get(transfer_id)) is not None
and record.state is SchedulerTransferState.LOADING
]
self._loads_to_dispatch.clear()
return records
def complete_load(self, mm_hash: str) -> bool:
record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,))
if record is None:
return self.has_state(mm_hash, (SchedulerTransferState.READY,))
self._transition(record, SchedulerTransferState.READY)
return True
def fail_load(self, mm_hash: str, now: float) -> bool:
record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,))
if record is None:
return self.has_state(mm_hash, (SchedulerTransferState.FAILED,))
self._transition(record, SchedulerTransferState.FAILED, now=now)
return True
def release_ready(self, mm_hash: str, now: float) -> None:
ready = self.records_for_hash(mm_hash, (SchedulerTransferState.READY,))
if ready:
canonical = ready[-1]
for record in self.records_for_hash(
mm_hash,
(SchedulerTransferState.READY, SchedulerTransferState.RESIDENT),
):
if record is not canonical:
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
if canonical.spec is None:
self._transition(canonical, SchedulerTransferState.EXPIRED, now=now)
else:
canonical.spec = replace(canonical.spec, local=True)
self._transition(canonical, SchedulerTransferState.RESIDENT)
self._evict_residents(now)
def reclaim(self, mm_hash: str, now: float) -> None:
for record in self.records_for_hash(
mm_hash,
(SchedulerTransferState.READY, SchedulerTransferState.RESIDENT),
):
if record.state is SchedulerTransferState.READY:
record.spec = None
else:
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
def mark_unavailable(self, transfer_id: str, now: float) -> None:
record = self._records[transfer_id]
self._transition(record, SchedulerTransferState.UNAVAILABLE, now=now)
if record.request_id:
self._notify_unavailable(record, record.request_id)
def cancel(
self,
transfer_id: str,
now: float,
mm_hash: str = "",
request_id: str = "",
) -> bool:
record = self._records.get(transfer_id)
if (
record is not None
and mm_hash
and not self._identity_matches(record, mm_hash)
):
return False
if record is None:
record = SchedulerTransfer(
transfer_id=transfer_id,
request_id=request_id,
mm_hash=mm_hash,
state=SchedulerTransferState.WAITING_EVENT,
spec=None,
deadline=None,
)
self._insert(record)
if record.state in {
SchedulerTransferState.CANCELLED,
SchedulerTransferState.READY,
SchedulerTransferState.RESIDENT,
SchedulerTransferState.FAILED,
}:
return False
self._transition(record, SchedulerTransferState.CANCELLED, now=now)
self._loads_to_dispatch.pop(transfer_id, None)
return True
def expire(self, now: float, terminal_limit: int) -> list[SchedulerTransfer]:
if terminal_limit < 0:
raise ValueError("terminal_limit must be non-negative")
expired = []
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if record.deadline is None or record.deadline > now:
continue
if record.state is SchedulerTransferState.AVAILABLE:
self._transition(
record,
SchedulerTransferState.EXPIRED,
now=now,
)
expired.append(record)
elif record.state is SchedulerTransferState.LOADING:
self._transition(
record,
SchedulerTransferState.UNAVAILABLE,
now=now,
)
if record.request_id:
self._notify_unavailable(record, record.request_id)
expired.append(record)
while self._terminal_ids:
transfer_id = next(iter(self._terminal_ids))
record = self._records[transfer_id]
if (
record.deadline is not None
and record.deadline > now
and len(self._terminal_ids) <= terminal_limit
):
break
self._remove(transfer_id)
return expired
def take_unavailable_requests(self) -> set[str]:
unavailable = self._unavailable_requests
self._unavailable_requests = set()
return unavailable
def _notify_unavailable(self, record: SchedulerTransfer, request_id: str) -> None:
if request_id not in record.notified_requests:
record.notified_requests.add(request_id)
self._unavailable_requests.add(request_id)
def _insert(self, record: SchedulerTransfer) -> None:
self._records[record.transfer_id] = record
self._active_ids[record.transfer_id] = None
if record.mm_hash:
self._hash_index.setdefault(record.mm_hash, deque()).append(
record.transfer_id
)
@staticmethod
def _identity_matches(record: SchedulerTransfer, mm_hash: str) -> bool:
return not record.mm_hash or record.mm_hash == mm_hash
def _refuse_colliding_id(
self, record: SchedulerTransfer, mm_hash: str, request_id: str
) -> None:
"""Report a transfer id that already names another encoding.
Transfer ids arrive on the request, so a collision is reachable from
outside and must not reach the engine as an exception.
"""
logger.warning(
"EC Mooncake transfer_id=%s already names mm_hash=%s; refusing "
"mm_hash=%s for request %s",
record.transfer_id,
record.mm_hash[:16],
mm_hash[:16],
request_id or "<unknown>",
)
if request_id:
self._unavailable_requests.add(request_id)
def _transition(
self,
record: SchedulerTransfer,
state: SchedulerTransferState,
now: float | None = None,
) -> None:
if record.state is SchedulerTransferState.RESIDENT:
self._resident_ids.pop(record.transfer_id, None)
if record.spec is not None:
self._resident_bytes -= record.spec.nbytes
if state is SchedulerTransferState.RESIDENT:
self._resident_ids[record.transfer_id] = None
if record.spec is not None:
self._resident_bytes += record.spec.nbytes
if state in _TERMINAL_STATES:
if now is None:
raise ValueError("Terminal transition requires a timestamp")
# Keep a bounded tombstone so late events and repeat request IDs
# remain idempotent instead of reviving a finished transfer.
record.deadline = now + self._tombstone_ttl
self._active_ids.pop(record.transfer_id, None)
self._terminal_ids[record.transfer_id] = None
self._terminal_ids.move_to_end(record.transfer_id)
if state is SchedulerTransferState.EXPIRED and record.state in (
SchedulerTransferState.READY,
SchedulerTransferState.RESIDENT,
):
self._orphaned.append(record.transfer_id)
record.state = state
self._records.move_to_end(record.transfer_id)
def _evict_residents(self, now: float) -> None:
while self._resident_bytes > self._resident_capacity and self._resident_ids:
record = self._records[next(iter(self._resident_ids))]
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
def _remove(self, transfer_id: str) -> None:
record = self._records.pop(transfer_id, None)
self._active_ids.pop(transfer_id, None)
self._terminal_ids.pop(transfer_id, None)
if transfer_id in self._resident_ids:
self._resident_ids.pop(transfer_id)
if record is not None and record.spec is not None:
self._resident_bytes -= record.spec.nbytes
self._loads_to_dispatch.pop(transfer_id, None)
if record is None or not record.mm_hash:
return
transfer_ids = self._hash_index[record.mm_hash]
transfer_ids.remove(transfer_id)
if not transfer_ids:
self._hash_index.pop(record.mm_hash)
@@ -1,209 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake data-plane engine and memory-registration ownership.
The wrapper isolates optional-engine initialization, session addressing,
synchronous batched writes, and the lifetime of transient source registrations.
"""
from __future__ import annotations
import threading
from dataclasses import dataclass
import torch
from vllm.logger import init_logger
logger = init_logger(__name__)
_MOONCAKE_IMPORT_ERROR: ImportError | None = None
try:
from mooncake.engine import TransferEngine
except ImportError as error:
TransferEngine = None # type: ignore[misc, assignment]
_MOONCAKE_IMPORT_ERROR = error
def ensure_mooncake_available() -> None:
if _MOONCAKE_IMPORT_ERROR is not None:
raise ImportError(
"Install mooncake-transfer-engine to use ECMooncakeConnector."
) from _MOONCAKE_IMPORT_ERROR
@dataclass
class _SourceRegistration:
"""Retain one transient source range while batches reference it."""
tensor: torch.Tensor
nbytes: int
users: int = 1
class MooncakeTransfer:
"""Own a lazy Mooncake engine and transient memory registrations."""
def __init__(self, hostname: str, protocol: str) -> None:
self._hostname = hostname
self._protocol = protocol
self._engine: TransferEngine | None = None
self._engine_lock = threading.Lock()
self._source_registrations: dict[int, _SourceRegistration] = {}
self._pending_unregister: dict[int, torch.Tensor] = {}
self._registration_lock = threading.Lock()
self._closed = False
def _ensure_engine(self) -> TransferEngine:
if self._engine is not None:
return self._engine
with self._engine_lock:
if self._engine is not None:
return self._engine
engine = TransferEngine()
ret = engine.initialize(self._hostname, "P2PHANDSHAKE", self._protocol, "")
if ret != 0:
raise RuntimeError("Mooncake TransferEngine initialization failed.")
self._engine = engine
logger.info(
"ECMooncakeConnector TransferEngine ready at %s:%d",
self._hostname,
engine.get_rpc_port(),
)
return self._engine
def ensure_ready(self) -> None:
self._ensure_engine()
def local_session(self) -> str:
engine = self._ensure_engine()
return f"{self._hostname}:{engine.get_rpc_port()}"
def register_memory(self, tensor: torch.Tensor) -> int:
return self._ensure_engine().batch_register_memory(
[tensor.data_ptr()], [tensor.nbytes]
)
def unregister_memory(self, tensor: torch.Tensor) -> bool:
engine = self._ensure_engine()
address = tensor.data_ptr()
ret = engine.unregister_memory(address)
with self._registration_lock:
if ret != 0:
logger.error(
"Mooncake EC memory unregistration failed for address %d: %d",
address,
ret,
)
self._pending_unregister[address] = tensor
return False
self._pending_unregister.pop(address, None)
return True
@staticmethod
def _source_range(tensor: torch.Tensor) -> tuple[int, int]:
# Encoder batches commonly split one storage into sibling tensor views.
# Registering each view's exact bytes avoids overlapping memory regions.
return tensor.data_ptr(), tensor.nbytes
def acquire_sources(self, tensors: list[torch.Tensor]) -> list[int]:
ranges: dict[int, tuple[int, torch.Tensor]] = {}
for tensor in tensors:
address, nbytes = self._source_range(tensor)
ranges.setdefault(address, (nbytes, tensor))
engine = self._ensure_engine()
acquired: list[int] = []
new_addresses: list[int] = []
new_lengths: list[int] = []
with self._registration_lock:
for address, (nbytes, tensor) in ranges.items():
entry = self._source_registrations.get(address)
if entry is not None:
if entry.nbytes != nbytes:
raise RuntimeError(
"Mooncake EC source storage changed size while registered"
)
entry.users += 1
acquired.append(address)
continue
new_addresses.append(address)
new_lengths.append(nbytes)
self._source_registrations[address] = _SourceRegistration(
tensor=tensor,
nbytes=nbytes,
)
acquired.append(address)
if new_addresses:
ret = engine.batch_register_memory(new_addresses, new_lengths)
if ret != 0:
for address in acquired:
entry = self._source_registrations[address]
entry.users -= 1
if entry.users == 0:
del self._source_registrations[address]
raise RuntimeError("Mooncake EC source registration failed")
return acquired
def release_sources(self, addresses: list[int]) -> bool:
if not addresses:
return True
with self._registration_lock:
unused = []
for address in addresses:
entry = self._source_registrations.get(address)
if entry is None:
continue
entry.users -= 1
if entry.users == 0:
unused.append(address)
if not unused:
return True
ret = self._ensure_engine().batch_unregister_memory(unused)
if ret != 0:
logger.warning(
"Keeping %d EC source tensors registered after Mooncake "
"unregistration failure",
len(unused),
)
return False
for address in unused:
del self._source_registrations[address]
self._pending_unregister.pop(address, None)
return True
def write(
self,
session: str,
sources: list[int],
destinations: list[int],
lengths: list[int],
) -> None:
"""Write one synchronous batch, returning only at terminal status."""
ret = self._ensure_engine().batch_transfer_sync_write(
session, sources, destinations, lengths
)
if ret != 0:
raise RuntimeError(
f"Mooncake EC push to {session} failed with status {ret}"
)
def close(self) -> None:
if self._closed:
return
self._closed = True
engine = self._engine
if engine is None:
return
with self._registration_lock:
addresses = list(self._source_registrations)
addresses.extend(self._pending_unregister)
if not addresses:
return
ret = engine.batch_unregister_memory(list(dict.fromkeys(addresses)))
if ret != 0:
logger.error("Mooncake EC batch memory unregistration failed: %d", ret)
return
self._source_registrations.clear()
self._pending_unregister.clear()
@@ -1,988 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Worker-side orchestration of Mooncake control, memory, and data planes.
Consumer Workers expose rank-local reservations and publish received tensors.
Producer Workers reserve every destination shard, bind computed sources, run
batched Mooncake writes, and report asynchronous completion to the Scheduler.
"""
from __future__ import annotations
import math
import threading
import time
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from functools import partial
from typing import TYPE_CHECKING, Any, TypeVar, cast
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
_RESERVATION_TTL_SECONDS,
MooncakeECConfig,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.control import (
ConsumerControlServer,
ControlClient,
make_cancel_request,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import (
ConsumerMemoryPool,
ProducerMemoryPool,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeConnectorMetadata,
ECMooncakePushSpec,
ECMooncakeWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.producer import (
ProducerPushManager,
ProducerPushRecord,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.reservation import (
ConsumerReservationManager,
ConsumerReservationState,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
MooncakeTransfer,
ensure_mooncake_available,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
build_embedding_namespace,
)
from vllm.logger import init_logger
from vllm.utils.network_utils import get_ip
logger = init_logger(__name__)
_T = TypeVar("_T")
if TYPE_CHECKING:
from vllm.config import VllmConfig
from ..mooncake_store_embedding.backend import (
MooncakeEmbeddingStoreBackend,
)
_RESERVATION_REFRESH_SECONDS = _RESERVATION_TTL_SECONDS / 2
_MAX_CANCELLED_TRANSFER_IDS = 1 << 16
_CANCEL_ATTEMPTS = 2
_TRANSFER_WORKERS = 4
_CONTROL_WORKERS = 8
class _FanoutError(RuntimeError):
"""Retain shard outcomes after every started task settles."""
def __init__(self, error: BaseException, results: list[Any | None]) -> None:
self.results = results
super().__init__(str(error))
class ECMooncakeWorker:
"""Orchestrate consumer reservations and producer push batches.
Consumers may use TP, PP, and DP. Only the first PP stage receives encoder
outputs, and each TP rank exposes a consecutive control port and receives
the same source concurrently. Producers remain unsharded and unreplicated.
With DP, the caller must route both halves of a request to the same replica
and pass that replica's control address to the producer.
"""
def __init__(self, vllm_config: VllmConfig) -> None:
ensure_mooncake_available()
config = MooncakeECConfig.from_vllm_config(vllm_config)
self._store_config = (
(
build_embedding_namespace(vllm_config),
config.store_max_pending_items,
config.store_max_pending_bytes,
config.store_read_buffer_bytes,
)
if config.cross_encoder_cache
else None
)
self._output_store: MooncakeEmbeddingStoreBackend | None = None
self.is_producer = config.is_producer
self.is_consumer = config.is_consumer
self._buffer_device = config.buffer_device
self._control_host = config.control_host
self._control_port = config.control_port
self._transfer = MooncakeTransfer(get_ip(), config.protocol)
self._consumer_memory = ConsumerMemoryPool(
config.pool_size,
self._transfer,
)
self._reservations = ConsumerReservationManager(
self._consumer_memory,
_RESERVATION_TTL_SECONDS,
_MAX_CANCELLED_TRANSFER_IDS,
)
self._consumer_rank_resolved = False
self._is_receiving_rank = True
self._tp_rank = 0
self._tp_size = 1
self._control_server: ConsumerControlServer | None = None
# Worker producer
self._producer_memory = ProducerMemoryPool(
config.pool_size,
self._transfer,
)
self._control_client = ControlClient(config.control_timeout_ms)
self._shutdown_drain_timeout_s = config.control_timeout_ms / 1000
self._io_executor = ThreadPoolExecutor(
max_workers=_TRANSFER_WORKERS,
thread_name_prefix="ec-mooncake-transfer",
)
self._control_executor = ThreadPoolExecutor(
max_workers=_CONTROL_WORKERS,
thread_name_prefix="ec-mooncake-control",
)
self._shard_pool: ThreadPoolExecutor | None = None
self._shard_pool_lock = threading.Lock()
self._push_ready = threading.Event()
self._producer_pushes = ProducerPushManager(self._push_ready.set)
self._dispatch_stop = threading.Event()
self._dispatcher: threading.Thread | None = None
self._failed_saves: set[str] = set()
self._collecting_sources = False
self._completed_loads: set[str] = set()
self._failed_loads: set[str] = set()
self._shutdown = False
if self.is_producer:
self._dispatcher = threading.Thread(
target=self._dispatch_pushes, name="ec-mooncake-ready", daemon=True
)
self._dispatcher.start()
def _dispatch_pushes(self) -> None:
pending_event = False
while not self._dispatch_stop.is_set():
self._push_ready.wait(timeout=0.001 if pending_event else None)
self._push_ready.clear()
if self._dispatch_stop.is_set():
break
if self._collecting_sources:
pending_event = False
continue
pending_event = self._producer_pushes.submit_batches(
self._io_executor, self._push_batch
)
def _resolve_consumer_rank(self) -> None:
"""Place this worker in the consumer receive topology."""
if self._consumer_rank_resolved:
return
self._consumer_rank_resolved = True
try:
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
tp_group = get_tp_group()
self._tp_rank = tp_group.rank_in_group
self._tp_size = tp_group.world_size
self._is_receiving_rank = get_pp_group().is_first_rank
except AssertionError:
# Groups are only absent outside a distributed run, where this
# worker is the whole consumer.
self._tp_rank = 0
self._tp_size = 1
self._is_receiving_rank = True
def start_services(self) -> None:
if self._store_config is not None and self._output_store is None:
from ..mooncake_store_embedding.backend import (
MooncakeEmbeddingStoreBackend,
)
from ..mooncake_store_embedding.store_client import (
create_mooncake_embedding_store_client,
)
namespace, max_items, max_bytes, read_bytes = self._store_config
self._output_store = MooncakeEmbeddingStoreBackend(
create_mooncake_embedding_store_client(read_buffer_bytes=read_bytes),
namespace,
max_pending_items=max_items,
max_pending_bytes=max_bytes,
)
if not self.is_consumer or self._control_server is not None:
return
self._resolve_consumer_rank()
if not self._is_receiving_rank:
# Later pipeline stages hold no encoder outputs, so they need
# neither a receive pool nor a control channel.
return
self._consumer_memory.prepare(torch.device(self._buffer_device))
consumer_pool = self._consumer_memory.tensor
if consumer_pool is None:
raise RuntimeError(
"Mooncake push mode requires a registered consumer buffer pool."
)
base_port = self._control_port
self._control_server = ConsumerControlServer(
# The reservation channel hands out registered-memory addresses
# and takes cancellations, so it listens where the Consumer
# advertises itself (`ec_ip`), not on every interface.
self._control_host,
base_port + self._tp_rank,
self._reserve_push_destination,
self._push_status,
self._reservations.complete,
self._reservations.cancel,
self._reservations.expire,
peer_ports=[base_port + rank for rank in range(self._tp_size)],
device=consumer_pool.device,
drain_ready=self._reservations.drain_ready,
)
try:
self._control_server.start()
except Exception:
self._control_server.close()
self._control_server = None
raise
def _reserve_push_destination(self, payload: dict[str, Any]) -> dict[str, Any]:
transfer_id = str(payload["transfer_id"])
mm_hash = str(payload["mm_hash"])
nbytes = int(payload["nbytes"])
shape = tuple(int(value) for value in payload["shape"])
dtype_name = str(payload["dtype"])
dtype = getattr(torch, dtype_name, None)
if not isinstance(dtype, torch.dtype):
raise ValueError(f"Unsupported torch dtype string: {dtype_name!r}")
expected_nbytes = math.prod(shape) * dtype.itemsize
if expected_nbytes != nbytes:
raise ValueError("shape and dtype do not match nbytes")
self._reservations.expire()
reservation, should_write = self._reservations.reserve(
transfer_id, mm_hash, nbytes, shape, dtype_name, dtype
)
if reservation is None:
raise RuntimeError("EC consumer buffer pool is full")
if reservation.state in {
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.CANCELLED,
}:
return {
"reservation_id": "",
"dst_session": "",
"dst_ptr": 0,
"nbytes": nbytes,
"write": False,
"ready": False,
"cancelled": True,
}
assert reservation.allocation is not None
return {
"reservation_id": reservation.reservation_id,
"dst_session": self._transfer.local_session(),
"dst_ptr": reservation.allocation.tensor.data_ptr(),
"nbytes": reservation.allocation.tensor.nbytes,
"write": should_write,
"ready": reservation.state is ConsumerReservationState.READY,
"cached": reservation.lease is not None,
}
def _push_status(self, transfer_id: str) -> dict[str, Any] | None:
reservation = self._reservations.status(transfer_id)
if reservation is None:
return None
assert reservation.allocation is not None
return {
"mm_hash": reservation.mm_hash,
"ready": reservation.state is ConsumerReservationState.READY,
"reservation_id": reservation.reservation_id,
"nbytes": reservation.allocation.tensor.nbytes,
"shape": list(reservation.shape),
"dtype": reservation.dtype,
}
def _shard_executor(self) -> ThreadPoolExecutor:
"""Use a separate pool so nested shard fan-out cannot deadlock."""
with self._shard_pool_lock:
if self._shard_pool is None:
self._shard_pool = ThreadPoolExecutor(
max_workers=32, thread_name_prefix="ec-mooncake-shard"
)
return self._shard_pool
def _reserve_one(self, addr: str, spec: ECMooncakePushSpec) -> dict[str, Any]:
result = self._control_client.request(
addr,
{
"op": "reserve",
"transfer_id": spec.transfer_id,
"mm_hash": spec.mm_hash,
"nbytes": spec.nbytes,
"shape": list(spec.shape),
"dtype": spec.dtype,
},
)
if not isinstance(result, dict):
raise RuntimeError("Invalid EC reservation response")
result["_received_at"] = time.monotonic()
result["addr"] = addr
return result
def _run_fanout(
self,
tasks: list[Callable[[], _T]],
on_submit: Callable[[int, Future[_T]], None] | None = None,
) -> list[_T]:
"""Run every started shard task and retain partial results on failure.
Waiting for all submitted tasks is what makes source-memory release and
partial-reservation cleanup safe after one shard fails.
"""
if not tasks:
return []
futures: list[tuple[int, Future[_T]]] = []
results: list[_T | None] = [None] * len(tasks)
error: BaseException | None = None
for index, task in enumerate(tasks[1:], 1):
try:
future = self._shard_executor().submit(task)
except Exception as exc:
error = exc
break
futures.append((index, future))
if on_submit is not None:
on_submit(index, future)
if error is None:
try:
results[0] = tasks[0]()
except Exception as exc:
error = exc
for index, future in futures:
try:
results[index] = future.result()
except Exception as exc:
if error is None:
error = exc
if error is not None:
raise _FanoutError(error, results)
return cast(list[_T], results)
def _cancel_reservations(
self,
spec: ECMooncakePushSpec,
reservations: list[dict[str, Any]],
*,
refresh: bool = False,
record: ProducerPushRecord | None = None,
) -> None:
reservations = [
shard for shard in reservations if not shard.get("cancelled", False)
]
if not reservations:
return
def cancel(shard: dict[str, Any]) -> dict[str, Any]:
addr = shard.get("addr")
if not isinstance(addr, str) or not addr:
raise RuntimeError(
"EC reservation is missing a confirmed shard address"
)
result = self._control_client.request(
addr,
make_cancel_request(
spec.transfer_id,
str(shard.get("reservation_id", "")),
abandon=True,
refresh=refresh,
),
)
if not isinstance(result, dict) or not result.get("cancelled"):
raise RuntimeError(
f"Could not cancel EC reservation for mm_hash={spec.mm_hash}"
)
return shard
def track(_index: int, future: Future[dict[str, Any]]) -> None:
if record is not None:
self._producer_pushes.track_shard_futures([record], [future])
self._run_fanout([partial(cancel, shard) for shard in reservations], track)
def _retry_cancel_reservations(
self,
spec: ECMooncakePushSpec,
reservations: list[dict[str, Any]],
*,
record: ProducerPushRecord | None = None,
) -> None:
pending = [shard for shard in reservations if not shard.get("cancelled", False)]
error: _FanoutError | None = None
for _ in range(_CANCEL_ATTEMPTS):
try:
self._cancel_reservations(spec, pending, record=record)
except _FanoutError as exc:
error = exc
pending = [
shard
for index, shard in enumerate(pending)
if exc.results[index] is None
]
continue
return
assert error is not None
raise error
def _reserve_remote(self, spec: ECMooncakePushSpec) -> list[dict[str, Any]]:
"""Reserve a destination on every shard of the consumer."""
result = self._reserve_remote_many([spec])[0]
if isinstance(result, Exception):
raise result
return result
def _refresh_remote_reservations(
self,
spec: ECMooncakePushSpec,
reservations: list[dict[str, Any]],
record: ProducerPushRecord | None = None,
) -> list[dict[str, Any]]:
stale = [
shard
for shard in reservations
if not shard.get("ready", False)
and not shard.get("cached", False)
and not shard.get("cancelled", False)
]
try:
self._cancel_reservations(spec, stale, refresh=True, record=record)
except _FanoutError as exc:
pending = [
shard for index, shard in enumerate(stale) if exc.results[index] is None
]
try:
self._retry_cancel_reservations(spec, pending, record=record)
except _FanoutError as cleanup_error:
raise exc from cleanup_error
raise
return self._reserve_remote(spec)
@staticmethod
def _validate_push_source(push: ProducerPushRecord) -> None:
tensor = push.source_tensor
assert tensor is not None
spec = push.spec
if tuple(tensor.shape) != tuple(spec.shape):
raise ValueError(f"EC source shape mismatch for mm_hash={spec.mm_hash}")
if str(tensor.dtype).split(".")[-1] != spec.dtype:
raise ValueError(f"EC source dtype mismatch for mm_hash={spec.mm_hash}")
if not tensor.is_contiguous():
raise ValueError(f"EC source must be contiguous for mm_hash={spec.mm_hash}")
if tensor.nbytes != spec.nbytes:
raise ValueError(f"EC source size mismatch for mm_hash={spec.mm_hash}")
def start_save_caches(
self,
metadata: ECMooncakeConnectorMetadata,
encoder_cache: dict[str, torch.Tensor] | None = None,
**kwargs: Any,
) -> None:
self._collecting_sources = True
new: dict[str, list[ProducerPushRecord]] = {}
for spec in metadata.pushes:
try:
record, created = self._producer_pushes.reserve(spec, Future)
if created:
new.setdefault(spec.consumer_zmq, []).append(record)
except ValueError:
logger.warning("Rejected conflicting EC push", exc_info=True)
self._failed_saves.add(spec.request_id)
for records in new.values():
future = self._control_executor.submit(self._reserve_batch, records)
future.add_done_callback(partial(self._reservation_batch_done, records))
if not isinstance(encoder_cache, dict):
return
for mm_hash in dict.fromkeys(spec.mm_hash for spec in metadata.pushes):
tensor = encoder_cache.get(mm_hash)
if tensor is not None:
self._bind_push_source(tensor, mm_hash)
if self._output_store is not None:
loaded = self._output_store.resolve_inputs(
metadata.store_candidates, encoder_cache, self._buffer_device
)
# Loaded entries precede the runner's new-output snapshot, so they
# need explicit P2P source binding here.
for mm_hash in loaded:
self._bind_push_source(encoder_cache[mm_hash], mm_hash)
def _reserve_batch(self, records: list[ProducerPushRecord]) -> None:
"""Resolve independent item futures with one reserve RPC per shard."""
if len(records) == 1:
record = records[0]
try:
record.reservation_future.set_result(self._reserve_remote(record.spec))
except Exception as exc:
record.reservation_future.set_exception(exc)
return
outcomes = self._reserve_remote_many([record.spec for record in records])
self._producer_pushes.finish_reservations(list(zip(records, outcomes)))
def _reserve_remote_many(
self, specs: list[ECMooncakePushSpec]
) -> list[list[dict[str, Any]] | Exception]:
shards = None
for _ in range(_CANCEL_ATTEMPTS):
shards = self._control_client.discover_shards(specs[0].consumer_zmq)
if shards is not None:
break
if shards is None:
raise RuntimeError(
f"Could not discover every EC consumer shard at {specs[0].consumer_zmq}"
)
def reserve(addr: str) -> list[dict[str, Any]]:
if len(specs) == 1:
return [{"ok": True, "result": self._reserve_one(addr, specs[0])}]
response = self._control_client.request(
addr,
{
"op": "reserve_batch",
"items": [
{
"transfer_id": spec.transfer_id,
"mm_hash": spec.mm_hash,
"nbytes": spec.nbytes,
"shape": list(spec.shape),
"dtype": spec.dtype,
}
for spec in specs
],
},
)
items = response["items"]
if len(items) != len(specs):
raise RuntimeError("Malformed EC reservation batch response")
for item in items:
if item.get("ok"):
item["result"]["_received_at"] = time.monotonic()
return items
fanout_error = None
results: list[Any]
try:
results = self._run_fanout([partial(reserve, addr) for addr in shards])
except _FanoutError as exc:
results = exc.results
fanout_error = exc
outcomes: list[list[dict[str, Any]] | Exception] = []
for index, spec in enumerate(specs):
reservations: list[dict[str, Any]] = []
error = None
for addr, items in zip(shards, results):
item = items[index] if items is not None else {}
if item.get("ok"):
reservations.append({**item["result"], "addr": addr})
else:
error = fanout_error or RuntimeError(item["error"])
reservations.append({"addr": addr, "reservation_id": ""})
if error is None:
outcomes.append(reservations)
else:
failure = _FanoutError(error, list(reservations))
try:
self._retry_cancel_reservations(spec, reservations)
except Exception as cleanup_error:
failure.__cause__ = cleanup_error
logger.exception("Failed to release partial EC reservation batch")
outcomes.append(failure)
return outcomes
@staticmethod
def _reservation_batch_done(
records: list[ProducerPushRecord], future: Future[None]
) -> None:
error = future.exception()
if error is not None:
for record in records:
if not record.reservation_future.done():
record.reservation_future.set_exception(error)
def start_load_caches(
self,
metadata: ECMooncakeConnectorMetadata,
encoder_cache: dict[str, torch.Tensor],
**kwargs: Any,
) -> None:
self._resolve_consumer_rank()
if not self._is_receiving_rank:
# Later pipeline stages never gather multimodal embeddings.
return
self._transfer.ensure_ready()
if self._buffer_device == "cuda" and not torch.accelerator.is_available():
raise RuntimeError(
"ECMooncakeConnector requires CUDA for ec_buffer_device=cuda"
)
self._reservations.retire_stale(encoder_cache, metadata.freed)
for spec in metadata.loads:
if spec.mm_hash in encoder_cache:
if not spec.local:
# The spec's id is one shard's; cancel by transfer.
self._reservations.cancel(spec.transfer_id, "")
self._completed_loads.add(spec.mm_hash)
continue
if spec.local:
tensor = self._consumer_memory.take_resident(
spec.mm_hash, tuple(spec.shape), spec.dtype
)
else:
try:
allocation = self._reservations.take(spec.transfer_id, spec.mm_hash)
tensor = allocation.tensor
except RuntimeError as e:
logger.warning("EC Mooncake pushed load failed: %s", e)
tensor = None
if tensor is None:
self._failed_loads.add(spec.mm_hash)
else:
encoder_cache[spec.mm_hash] = tensor
self._completed_loads.add(spec.mm_hash)
def _push_batch(self, pushes: list[ProducerPushRecord]) -> None:
started_at = time.monotonic()
ready: list[tuple[ProducerPushRecord, dict[str, Any]]] = []
written_pushes: dict[str, ProducerPushRecord] = {}
failure: Exception | None = None
try:
for push in pushes:
self._validate_push_source(push)
reservations = self._producer_pushes.resolve_reservations(push)
stale = [
index
for index, shard in enumerate(reservations)
if not shard.get("ready", False)
and not shard.get("cancelled", False)
and time.monotonic() - float(shard.get("_received_at", started_at))
>= _RESERVATION_REFRESH_SECONDS
]
if stale:
reservations = self._refresh_remote_reservations(
push.spec, reservations, push
)
self._producer_pushes.replace_reservations(push, reservations)
self._producer_pushes.begin_writing(push)
writable = [
shard
for shard in reservations
if not shard.get("cached", False)
and not shard.get("cancelled", False)
and shard.get("write", True)
]
source = push.source_tensor
assert source is not None
for shard in writable:
if int(shard["nbytes"]) != source.nbytes:
raise RuntimeError(
"Reserved EC size does not match tensor for "
f"mm_hash={push.spec.mm_hash}"
)
ready.append((push, shard))
written_pushes.setdefault(push.spec.transfer_id, push)
if ready:
# Stage each source once, then write it to every destination.
source_index = {
push.spec.transfer_id: index
for index, push in enumerate(written_pushes.values())
}
tensors = [
cast(torch.Tensor, push.source_tensor)
for push in written_pushes.values()
if push.source_tensor is not None
]
lengths = [tensor.nbytes for tensor in tensors]
staged = self._producer_memory.stage(tensors)
registered_sources: list[int] = []
if staged is not None:
sources = staged.tensors
else:
sources = tensors
registered_sources = self._transfer.acquire_sources(tensors)
addresses = [tensor.data_ptr() for tensor in sources]
try:
by_session: dict[str, list[tuple[int, int]]] = {}
session_records: dict[str, dict[str, ProducerPushRecord]] = {}
for push, shard in ready:
session = str(shard["dst_session"])
by_session.setdefault(session, []).append(
(source_index[push.spec.transfer_id], int(shard["dst_ptr"]))
)
session_records.setdefault(session, {})[
push.spec.transfer_id
] = push
def write(session: str, items: list[tuple[int, int]]) -> None:
self._transfer.write(
session,
[addresses[index] for index, _ in items],
[dst for _, dst in items],
[lengths[index] for index, _ in items],
)
sessions = list(by_session.items())
# Write shards concurrently to avoid serial TP latency.
def track_write(index: int, future: Future[None]) -> None:
session = sessions[index][0]
self._producer_pushes.track_shard_futures(
list(session_records[session].values()), [future]
)
writes: list[Callable[[], None]] = [
partial(write, *session) for session in sessions
]
self._run_fanout(writes, track_write)
finally:
if staged is not None:
self._producer_memory.release(staged)
self._transfer.release_sources(registered_sources)
self._producer_pushes.begin_notifying(pushes)
self._notify_completions(ready)
self._producer_pushes.complete(pushes)
except Exception as exc:
# Report asynchronously; raising here would fail EngineCore.
failure = exc
logger.exception(
"EC Mooncake push batch failed for mm_hashes=%s",
[push.spec.mm_hash for push in pushes],
)
self._producer_pushes.settle_all(pushes)
self._abandon_pushes(pushes)
finally:
if failure is not None:
self._producer_pushes.fail(pushes, failure)
def _notify_completions(
self, notifications: list[tuple[ProducerPushRecord, dict[str, Any]]]
) -> None:
"""Tell the consumer, in one message per destination, what landed."""
if not notifications:
return
by_destination: dict[str, list[tuple[ProducerPushRecord, dict[str, Any]]]] = {}
for push, reservation in notifications:
by_destination.setdefault(
str(reservation.get("addr", push.spec.consumer_zmq)), []
).append((push, reservation))
destinations = list(by_destination.items())
def notify(
consumer_zmq: str,
items: list[tuple[ProducerPushRecord, dict[str, Any]]],
) -> None:
result = self._control_client.request(
consumer_zmq,
{
"op": "complete_batch",
"items": [
{
"transfer_id": push.spec.transfer_id,
"reservation_id": reservation["reservation_id"],
}
for push, reservation in items
],
},
)
completions = result.get("items", []) if isinstance(result, dict) else []
if len(completions) != len(items):
raise RuntimeError("Malformed EC completion response")
for (push, _), completion in zip(items, completions):
if not completion.get("completed"):
raise RuntimeError(
f"Unknown EC reservation for mm_hash={push.spec.mm_hash}"
)
def track(index: int, future: Future[None]) -> None:
records = {
push.spec.transfer_id: push for push, _ in destinations[index][1]
}
self._producer_pushes.track_shard_futures(list(records.values()), [future])
self._run_fanout(
[
partial(notify, destination, items)
for destination, items in destinations
],
track,
)
@staticmethod
def _known_reservations(record: ProducerPushRecord) -> list[dict[str, Any]]:
if record.reservations:
return list(record.reservations)
try:
return list(record.reservation_future.result())
except _FanoutError as exc:
return [result for result in exc.results if isinstance(result, dict)]
except Exception:
return []
def _abandon_pushes(self, pushes: list[ProducerPushRecord]) -> None:
"""Release the consumer-side reservations of a batch that failed."""
for push in pushes:
shards = self._known_reservations(push)
if not shards:
# No confirmed topology means there is no safe address to
# cancel. The original push failure remains observable via
# ProducerPushManager.fail below.
continue
try:
self._retry_cancel_reservations(push.spec, shards, record=push)
except _FanoutError:
logger.exception(
"Failed to abandon EC reservations for transfer_id=%s",
push.spec.transfer_id,
)
def _flush_pending_pushes(self) -> None:
self._producer_pushes.submit_batches(
self._io_executor,
self._push_batch,
)
def _bind_push_source(self, tensor: torch.Tensor, mm_hash: str) -> None:
ready_event = None
if tensor.device.type == "cuda":
ready_event = torch.Event()
ready_event.record(torch.accelerator.current_stream(tensor.device))
self._producer_pushes.bind_source(mm_hash, tensor, ready_event)
def _cancel_orphaned_reservation(self, record: ProducerPushRecord) -> None:
resolution_error: Exception | None = None
try:
reservations = self._producer_pushes.resolve_reservations(record)
except Exception as exc:
resolution_error = exc
reservations = self._known_reservations(record)
reservations = [
shard for shard in reservations if not shard.get("cancelled", False)
]
if not reservations:
if resolution_error is not None:
self._producer_pushes.fail([record], resolution_error)
else:
self._producer_pushes.finish_cancel(record)
return
try:
self._retry_cancel_reservations(record.spec, reservations, record=record)
except Exception as cleanup_error:
if resolution_error is not None:
combined_error = RuntimeError(
f"EC reservation resolution failed ({resolution_error}); "
f"cleanup also failed ({cleanup_error})"
)
combined_error.__cause__ = resolution_error
cleanup_error = combined_error
self._producer_pushes.fail([record], cleanup_error)
return
if resolution_error is not None:
self._producer_pushes.fail([record], resolution_error)
else:
self._producer_pushes.finish_cancel(record)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
if not self.is_producer:
return None, None
if self._output_store is not None:
self._output_store.reap()
for record in self._producer_pushes.cancel_requests(finished_req_ids):
self._producer_pushes.submit_cancel(
record,
self._io_executor,
self._cancel_orphaned_reservation,
)
return None, None
def save_caches(
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs: Any
) -> None:
if not self.is_producer:
return
tensor = encoder_cache[mm_hash]
self._bind_push_source(tensor, mm_hash)
if self._output_store is not None:
self._output_store.save_output(mm_hash, tensor)
def build_connector_worker_meta(self) -> ECMooncakeWorkerMetadata | None:
if self.is_consumer and not self._is_receiving_rank:
# `loaded` is intersected across reporting ranks, so a stage that
# never loads must not report at all rather than report nothing.
return None
self._collecting_sources = False
self._push_ready.set()
self._flush_pending_pushes()
failures = self._producer_pushes.poll()
for mm_hash, error in failures:
logger.error(
"EC Mooncake async save failed for mm_hash=%s: %s",
mm_hash,
error,
)
reclaimed = self._consumer_memory.drain_reclaimed()
meta = ECMooncakeWorkerMetadata(
loaded=self._completed_loads,
failed_loads=self._failed_loads,
reclaimed=reclaimed,
pending_saves=self._producer_pushes.pending,
failed_saves=self._failed_saves,
)
self._failed_saves = set()
self._completed_loads = set()
self._failed_loads = set()
return meta
def close(self) -> None:
if self._shutdown:
return
self._shutdown = True
dispatcher = getattr(self, "_dispatcher", None)
if dispatcher is not None:
self._dispatch_stop.set()
self._push_ready.set()
dispatcher.join()
if self._control_server is not None:
self._reservations.begin_shutdown()
self._flush_pending_pushes()
for record in self._producer_pushes.cancel_requests(None):
self._producer_pushes.submit_cancel(
record,
self._io_executor,
self._cancel_orphaned_reservation,
)
self._control_executor.shutdown(wait=True)
self._producer_pushes.submit_batches(
self._io_executor, self._push_batch, wait=True
)
self._io_executor.shutdown(wait=True)
if self._shard_pool is not None:
self._shard_pool.shutdown(wait=True)
# Every producer-side thread that could hold a control socket is stopped.
self._control_client.close()
drained = True
if self._control_server is not None:
drained = self._reservations.wait_for_writers(
self._shutdown_drain_timeout_s
)
self._control_server.close()
if drained:
self._consumer_memory.close()
else:
logger.error(
"Timed out waiting for Mooncake EC writers; keeping the consumer "
"receive pool registered"
)
self._producer_memory.close()
self._transfer.close()
if self._output_store is not None:
self._output_store.shutdown()
self._output_store = None
@@ -1,3 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake Store support for cross-Encoder output reuse."""
@@ -1,214 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Encoder-side Mooncake Store resolver and publisher."""
from __future__ import annotations
import threading
import traceback
from collections.abc import Mapping
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
import torch
from vllm.logger import init_logger
from .data import TensorSpec, make_embedding_key
from .store_client import (
EmbeddingStoreError,
EmbeddingStoreOperationError,
MooncakeEmbeddingStoreClient,
)
logger = init_logger(__name__)
@dataclass
class _StoreSave:
tensor: torch.Tensor | None
ready_event: torch.Event | None
storage_ptr: int
def release(self) -> None:
self.tensor = None
self.ready_event = None
class MooncakeEmbeddingStoreBackend:
"""Resolve immutable encoder outputs and publish misses asynchronously."""
def __init__(
self,
store_client: MooncakeEmbeddingStoreClient,
namespace: str,
*,
max_pending_items: int,
max_pending_bytes: int,
) -> None:
self.store_client = store_client
self.namespace = namespace
self.max_pending_items = max_pending_items
self.max_pending_bytes = max_pending_bytes
self._executor = ThreadPoolExecutor(
max_workers=1,
thread_name_prefix="ec-mooncake-store",
)
self._pending: dict[str, tuple[Future[None], _StoreSave]] = {}
self._pending_storages: dict[int, tuple[torch.UntypedStorage, int]] = {}
self._pending_lock = threading.Lock()
self._pending_bytes = 0
self._step_candidates: set[str] = set()
self._closed = False
self._fatal_error: BaseException | None = None
def save_output(self, identifier: str, tensor: torch.Tensor) -> None:
if identifier in self._step_candidates:
self.reap()
self._enqueue_save(identifier, tensor)
def resolve_inputs(
self,
expected_tensors: Mapping[str, TensorSpec],
encoder_cache: dict[str, torch.Tensor],
device: torch.device | str,
) -> set[str]:
"""Load outputs before encoding; unresolved items retain normal computation."""
self.reap()
self._step_candidates = set(expected_tensors)
candidates = [key for key in expected_tensors if key not in encoder_cache]
if not candidates:
return set()
pool_keys = [
make_embedding_key(self.namespace, identifier) for identifier in candidates
]
try:
exists = self.store_client.batch_exists(pool_keys)
except (EmbeddingStoreOperationError, OSError):
logger.warning(
"Mooncake embedding Store lookup failed; falling back to encoder",
exc_info=True,
)
return set()
hits = {
pool_key: expected_tensors[identifier]
for identifier, pool_key, hit in zip(
candidates, pool_keys, exists, strict=True
)
if hit
}
try:
tensors = self.store_client.load_tensors(hits, device)
except (EmbeddingStoreOperationError, OSError):
logger.warning(
"Mooncake embedding Store GET failed; falling back to encoder",
exc_info=True,
)
return set()
loaded: set[str] = set()
for identifier, pool_key in zip(candidates, pool_keys, strict=True):
if pool_key in tensors:
encoder_cache[identifier] = tensors[pool_key]
loaded.add(identifier)
return loaded
def _enqueue_save(self, identifier: str, tensor: torch.Tensor) -> bool:
"""Admit contiguous outputs, charging each retained storage once."""
if not tensor.is_contiguous():
return False
storage = tensor.untyped_storage()
storage_ptr = storage.data_ptr()
with self._pending_lock:
if self._fatal_error is not None:
raise EmbeddingStoreError(
"Store publisher has failed"
) from self._fatal_error
if self._closed or identifier in self._pending:
return False
_, ref_count = self._pending_storages.get(storage_ptr, (storage, 0))
budget_bytes = 0 if ref_count else storage.nbytes()
if (
len(self._pending) >= self.max_pending_items
or self._pending_bytes + budget_bytes > self.max_pending_bytes
):
return False
# Only admitted work creates an event. Runtime errors propagate.
save = _StoreSave(
tensor,
_record_tensor_ready_event(tensor),
storage_ptr,
)
future = self._executor.submit(
self._save, make_embedding_key(self.namespace, identifier), save
)
self._pending[identifier] = (future, save)
# Keep the storage alive until reaping so its address cannot be reused.
self._pending_storages[storage_ptr] = (storage, ref_count + 1)
self._pending_bytes += budget_bytes
return True
def _save(self, pool_key: str, save: _StoreSave) -> None:
try:
if self.store_client.exists(pool_key):
return
tensor = save.tensor
assert tensor is not None
if save.ready_event is not None:
save.ready_event.synchronize()
self.store_client.put_tensor(pool_key, tensor)
except BaseException as error:
# Returned client frames can otherwise keep tensors via a Future's
# traceback. Uncertain I/O owners remain retained by the client.
traceback.clear_frames(error.__traceback__)
raise
finally:
tensor = None
save.release()
def reap(self) -> None:
with self._pending_lock:
for identifier, (future, save) in list(self._pending.items()):
# Observe once; a racing completion is handled on the next poll.
if not future.done():
continue
try:
future.result() # Already done: never waits on Store I/O.
except (EmbeddingStoreOperationError, OSError) as error:
logger.warning("Store PUT failed for %s: %s", identifier, error)
# This failure is consumed; do not retain publisher/caller frames.
error.__traceback__ = None
except BaseException as error:
# The client may still own unsafe native-I/O buffers. Keep
# their charge and fatal result until worker termination.
if self._fatal_error is None:
self._fatal_error = error
continue
del self._pending[identifier]
storage, ref_count = self._pending_storages[save.storage_ptr]
if ref_count == 1:
del self._pending_storages[save.storage_ptr]
self._pending_bytes -= storage.nbytes()
else:
self._pending_storages[save.storage_ptr] = (storage, ref_count - 1)
fatal = self._fatal_error
if fatal is not None:
raise fatal
def shutdown(self) -> None:
with self._pending_lock:
if self._closed:
return
self._closed = True
self._executor.shutdown(wait=True, cancel_futures=False)
self.reap()
self.store_client.close()
def _record_tensor_ready_event(tensor: torch.Tensor) -> torch.Event | None:
if tensor.device.type != "cuda":
return None
event = torch.Event()
event.record(torch.accelerator.current_stream(tensor.device))
return event
@@ -1,51 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Content identities and output contracts for the shared Encoder Store."""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from vllm.config import VllmConfig
EMBEDDING_CACHE_KEY_VERSION = "v3"
def make_embedding_key(namespace: str, identifier: str) -> str:
return f"{namespace}@id:{identifier}"
@dataclass(frozen=True)
class TensorSpec:
"""Contiguous output contract used by the Encoder and Store codec."""
shape: tuple[int, ...]
dtype: str
nbytes: int
def build_embedding_namespace(vllm_config: VllmConfig) -> str:
model_config = vllm_config.model_config
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None
extra_config = ec_config.ec_connector_extra_config
multimodal_config = model_config.multimodal_config
encoder_hash = (
multimodal_config.compute_hash()
if multimodal_config is not None
else "encoder:default"
)
cache_prefix = str(extra_config.get("embedding_cache_prefix", ""))
model_name = str(extra_config.get("embedding_model_identity", model_config.model))
prefix = f"{cache_prefix}@" if cache_prefix else ""
return (
f"{prefix}embedding"
f"@model:{model_name}"
f"@revision:{model_config.revision or 'default'}"
f"@encoder:{encoder_hash}"
f"@dtype:{model_config.dtype}"
f"@protocol:{EMBEDDING_CACHE_KEY_VERSION}"
)
@@ -1,463 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Thin Mooncake Store client for embedding objects.
Originally adapted from vLLM PR #47302 (0d1f71f5d36c). The v3 key namespace
uses a compact header for complete 2D embeddings, not that PR's tensor format.
"""
from __future__ import annotations
import ctypes
import struct
import threading
from collections.abc import Callable, Iterator, Mapping
from contextlib import contextmanager
from enum import IntEnum
from typing import Any
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
TensorSpec,
)
from vllm.distributed.mooncake_store import MooncakeStoreConfig, setup_mooncake_store
from vllm.logger import init_logger
from vllm.utils.network_utils import get_ip
logger = init_logger(__name__)
# A failed transfer can still use registered memory after the binding returns.
# Poisoned clients retain their owners until process exit, including if normal
# worker teardown drops the connector. New operations are rejected.
_UNSAFE_CLIENTS: list[Any] = []
# Values from Mooncake v0.3.12 / v0.3.12.post1 types.h (ErrorCode).
# Python exposes integer returns, not this enum.
class _MooncakeErrorCode(IntEnum):
NO_AVAILABLE_HANDLE = -200
INVALID_PARAMS = -600
INVALID_REPLICA = -702
REPLICA_IS_NOT_READY = -703
OBJECT_NOT_FOUND = -704
OBJECT_ALREADY_EXISTS = -705
LEASE_EXPIRED = -707
RPC_FAIL = -900
RPC_TIMEOUT = -901
TENANT_QUOTA_EXCEEDED = -1700
# For RAM batch reads and single-replica writes, these rejections occur
# before submission or after completed I/O. Transfer timeouts can leave I/O
# in flight; all other failures retain registered buffers and poison the client.
_SAFE_IO_REJECTIONS = frozenset(
{
_MooncakeErrorCode.NO_AVAILABLE_HANDLE,
_MooncakeErrorCode.REPLICA_IS_NOT_READY,
_MooncakeErrorCode.OBJECT_NOT_FOUND,
_MooncakeErrorCode.OBJECT_ALREADY_EXISTS,
_MooncakeErrorCode.RPC_FAIL,
_MooncakeErrorCode.RPC_TIMEOUT,
}
)
# Batch GET validates capacity/replicas before I/O and checks leases afterwards.
# PUT rejects tenant quotas before allocating replicas or submitting transfers.
_SAFE_GET_REJECTIONS = _SAFE_IO_REJECTIONS | {
_MooncakeErrorCode.INVALID_PARAMS,
_MooncakeErrorCode.INVALID_REPLICA,
_MooncakeErrorCode.LEASE_EXPIRED,
}
_SAFE_PUT_REJECTIONS = _SAFE_IO_REJECTIONS | {_MooncakeErrorCode.TENANT_QUOTA_EXCEEDED}
_MOONCAKE_TENSOR_OBJECT_MAGIC = 0x4D4F4F4E
# Magic, dtype, token count, hidden dimension. Format versioning is in the key.
_MOONCAKE_TENSOR_HEADER = struct.Struct("<IIQQ")
_MOONCAKE_DTYPE_TO_TORCH_DTYPE = {
0: "torch.float32",
11: "torch.float16",
12: "torch.bfloat16",
}
_TORCH_DTYPE_TO_MOONCAKE_DTYPE = {
value: key for key, value in _MOONCAKE_DTYPE_TO_TORCH_DTYPE.items()
}
def create_mooncake_embedding_store_client(
read_buffer_bytes: int = 128 * 1024**2,
) -> MooncakeEmbeddingStoreClient:
try:
from mooncake.store import ( # type: ignore
MooncakeDistributedStore,
ObjectDataType,
ReplicateConfig,
)
except ImportError as e:
raise ImportError(
"Install mooncake with Store support to enable cross_encoder_cache."
) from e
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils
config = MooncakeStoreConfig.load_from_config()
if config.enable_offload:
raise ValueError(
"cross_encoder_cache supports a RAM Store; disable enable_offload"
)
store = MooncakeDistributedStore()
local_ip = get_ip()
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
setup_mooncake_store(store, config, local_hostname)
logger.info(
"Initialized embedding Mooncake store mode=%s global_segment_size=%d "
"local_buffer_size=%d",
config.mode,
config.global_segment_size,
config.local_buffer_size,
)
replicate_config = ReplicateConfig()
replicate_config.data_type = ObjectDataType.TENSOR
return MooncakeEmbeddingStoreClient(
store, replicate_config=replicate_config, read_buffer_bytes=read_buffer_bytes
)
class EmbeddingStoreError(RuntimeError):
"""A Store failure that must propagate to the worker."""
class EmbeddingStoreOperationError(EmbeddingStoreError):
"""A rejected or completed operation that permits normal Encoder fallback."""
def _batch_get_completed_safely(results: Any, capacities: list[int]) -> bool:
return (
isinstance(results, list)
and len(results) == len(capacities)
and all(
type(result) is int
and (0 <= result <= capacity or result in _SAFE_GET_REJECTIONS)
for result, capacity in zip(results, capacities, strict=True)
)
)
class MooncakeEmbeddingStoreClient:
"""Wraps Mooncake object and buffer APIs used by embedding transfer."""
def __init__(
self,
store: Any,
replicate_config: Any | None = None,
*,
read_buffer_bytes: int = 128 * 1024**2,
):
self._lifetime_lock = threading.Lock()
self._unsafe_owners: list[Any] = []
self._poisoned = False
self.store = store
self.replicate_config = replicate_config
self._read_buffer_bytes = read_buffer_bytes
self._read_buffer: torch.Tensor | None = None
self._put_header: ctypes.Array[ctypes.c_char] | None = None
def _check_healthy(self) -> None:
if self._poisoned:
raise EmbeddingStoreError(
"Mooncake I/O completion is unconfirmed; worker must exit"
)
def _poison(self, owners: list[Any]) -> None:
with self._lifetime_lock:
self._unsafe_owners.extend(owners)
if not self._poisoned:
self._poisoned = True
_UNSAFE_CLIENTS.append(self)
@contextmanager
def _registered_io(
self,
buffers: list[tuple[Any, int, int]],
) -> Iterator[Callable[[int], None]]:
self._check_healthy()
registered = []
submitted = False
confirmed = False
def confirm(result: int) -> None:
nonlocal confirmed
if type(result) is not int or (
result < 0 and result not in _SAFE_PUT_REJECTIONS
):
raise EmbeddingStoreError(
"Mooncake I/O completion is unconfirmed; retaining buffers"
)
confirmed = True
try:
for owner, addr, size in buffers:
ret = self.store.register_buffer(addr, size)
if ret != 0:
raise EmbeddingStoreOperationError(
f"Failed to register embedding buffer: {ret}"
)
registered.append(addr)
submitted = True
yield confirm
except BaseException as error:
if submitted and not confirmed:
self._poison([owner for owner, _, _ in buffers])
raise EmbeddingStoreError(
"Mooncake I/O completion is unconfirmed; retaining buffers"
) from error
raise
finally:
if not submitted or confirmed:
try:
for addr in reversed(registered):
self._unregister_buffer(addr)
except BaseException:
self._poison([owner for owner, _, _ in buffers])
raise
def close(self) -> None:
"""Close the supported Mooncake binding after all I/O is drained."""
self._check_healthy()
if self._read_buffer is not None:
try:
self._unregister_buffer(self._read_buffer.data_ptr())
except BaseException:
self._poison([self._read_buffer])
raise
self._read_buffer = None
if self._put_header is not None:
try:
self._unregister_buffer(ctypes.addressof(self._put_header))
except BaseException:
self._poison([self._put_header])
raise
self._put_header = None
ret = self.store.close()
if ret != 0:
raise EmbeddingStoreError(
f"failed to close embedding Mooncake Store: {ret}"
)
def exists(self, pool_key: str) -> bool:
return self.batch_exists([pool_key])[0]
def batch_exists(self, keys: list[str]) -> list[bool]:
self._check_healthy()
if not keys:
return []
states = self.store.batch_is_exist(keys)
if len(states) != len(keys):
raise RuntimeError(
"Mooncake Store returned an unexpected number of lookup results"
)
return [state == 1 for state in states]
def load_tensors(
self, expected: Mapping[str, TensorSpec], device: torch.device | str
) -> dict[str, torch.Tensor]:
"""Synchronously load runner inputs into independently owned tensors."""
self._check_healthy()
device = torch.device(device)
loaded: dict[str, torch.Tensor] = {}
chunk: dict[str, TensorSpec] = {}
chunk_bytes = 0
for key, spec in expected.items():
size = _MOONCAKE_TENSOR_HEADER.size + spec.nbytes
if size > self._read_buffer_bytes:
logger.warning(
"Skipping Store GET for %s: object bytes=%d exceed read buffer=%d",
key,
size,
self._read_buffer_bytes,
)
continue
if chunk_bytes + size > self._read_buffer_bytes:
loaded.update(self._load_chunk(chunk, device))
chunk = {}
chunk_bytes = 0
chunk[key] = spec
chunk_bytes += size
if chunk:
loaded.update(self._load_chunk(chunk, device))
return loaded
def _load_chunk(
self, expected: dict[str, TensorSpec], device: torch.device
) -> dict[str, torch.Tensor]:
self._check_healthy()
if self._read_buffer is None:
buffer = torch.empty(
self._read_buffer_bytes,
dtype=torch.uint8,
device="cpu",
pin_memory=device.type == "cuda",
)
ret = self.store.register_buffer(buffer.data_ptr(), buffer.nbytes)
if ret != 0:
raise EmbeddingStoreOperationError(
f"Failed to register embedding read buffer: {ret}"
)
self._read_buffer = buffer
buffer = self._read_buffer
base = buffer.data_ptr()
sizes = [
_MOONCAKE_TENSOR_HEADER.size + spec.nbytes for spec in expected.values()
]
offsets: list[int] = []
end = 0
for size in sizes:
offsets.append(end)
end += size
try:
results = self.store.batch_get_into(
list(expected), [base + offset for offset in offsets], sizes
)
if not _batch_get_completed_safely(results, sizes):
raise EmbeddingStoreError("Unsafe batch GET results")
except BaseException as error:
self._poison([buffer])
raise EmbeddingStoreError(
"Mooncake I/O completion is unconfirmed; retaining read buffer"
) from error
valid: list[tuple[str, TensorSpec, int]] = []
for (key, spec), offset, size, result in zip(
expected.items(), offsets, sizes, results, strict=True
):
if result != size:
logger.warning(
"Store GET skipped for %s: result=%d, expected bytes=%d",
key,
result,
size,
)
continue
try:
_validate_mooncake_tensor_metadata(
key,
ctypes.string_at(base + offset, _MOONCAKE_TENSOR_HEADER.size),
spec,
)
except EmbeddingStoreOperationError as error:
logger.warning("Store GET skipped for %s: %s", key, error)
continue
valid.append((key, spec, offset + _MOONCAKE_TENSOR_HEADER.size))
if not valid:
return {}
# Allocate every destination before submitting any asynchronous copy.
targets = {
key: torch.empty(
spec.shape,
dtype=getattr(torch, spec.dtype.removeprefix("torch.")),
device=device,
)
for key, spec, _ in valid
}
ready = torch.Event() if device.type == "cuda" else None
try:
for key, spec, offset in valid:
targets[key].view(torch.uint8).view(-1).copy_(
buffer[offset : offset + spec.nbytes],
non_blocking=ready is not None,
)
if ready is not None:
ready.record(torch.accelerator.current_stream(device))
ready.synchronize()
except BaseException as error:
if ready is not None:
self._poison([buffer, *targets.values(), ready])
raise EmbeddingStoreError(
"Embedding copy completion is unconfirmed; retaining buffers"
) from error
raise
return targets
def put_tensor(self, pool_key: str, tensor: torch.Tensor) -> None:
self._check_healthy()
if not tensor.is_contiguous():
raise EmbeddingStoreOperationError("embedding tensor must be contiguous")
metadata = _encode_mooncake_tensor_metadata(tensor)
# The single publisher reuses this header only after PUT completion.
if self._put_header is None:
header = ctypes.create_string_buffer(len(metadata))
ret = self.store.register_buffer(ctypes.addressof(header), len(metadata))
if ret != 0:
raise EmbeddingStoreOperationError(
f"Failed to register embedding header: {ret}"
)
self._put_header = header
header_ptr = ctypes.addressof(self._put_header)
ctypes.memmove(header_ptr, metadata, len(metadata))
data_size = tensor.numel() * tensor.element_size()
buffers = [
(tensor, tensor.data_ptr(), data_size),
]
with self._registered_io(buffers) as confirm:
[result] = self.store.batch_put_from_multi_buffers(
[pool_key],
[[header_ptr, tensor.data_ptr()]],
[[len(metadata), data_size]],
self.replicate_config,
)
confirm(result)
if result < 0:
raise EmbeddingStoreOperationError(
f"failed to put embedding tensor for {pool_key}"
)
def _unregister_buffer(self, addr: int) -> None:
try:
ret = self.store.unregister_buffer(addr)
except Exception as error:
raise EmbeddingStoreError(
f"could not confirm buffer unregistration addr={addr:#x}"
) from error
if ret != 0:
raise EmbeddingStoreError(
f"failed to unregister embedding buffer addr={addr:#x}: {ret}"
)
def _validate_mooncake_tensor_metadata(
pool_key: str,
metadata: bytes,
expected: TensorSpec,
) -> None:
magic, dtype, num_tokens, hidden_dim = _MOONCAKE_TENSOR_HEADER.unpack(metadata)
if magic != _MOONCAKE_TENSOR_OBJECT_MAGIC:
raise EmbeddingStoreOperationError(
f"invalid Mooncake tensor metadata header for {pool_key}"
)
if _MOONCAKE_DTYPE_TO_TORCH_DTYPE.get(dtype) != expected.dtype:
raise EmbeddingStoreOperationError(
f"unexpected Mooncake tensor dtype for {pool_key}: {dtype}"
)
if (num_tokens, hidden_dim) != expected.shape:
raise EmbeddingStoreOperationError(
f"embedding tensor shape mismatch for {pool_key}: expected={expected.shape}"
)
def _encode_mooncake_tensor_metadata(tensor: torch.Tensor) -> bytes:
dtype = str(tensor.dtype)
if dtype not in _TORCH_DTYPE_TO_MOONCAKE_DTYPE:
raise EmbeddingStoreOperationError(
f"unsupported embedding tensor dtype: {dtype}"
)
if tensor.ndim != 2:
raise EmbeddingStoreOperationError(
f"embedding tensor must be 2D: shape={tuple(tensor.shape)}"
)
return _MOONCAKE_TENSOR_HEADER.pack(
_MOONCAKE_TENSOR_OBJECT_MAGIC,
_TORCH_DTYPE_TO_MOONCAKE_DTYPE[dtype],
*tensor.shape,
)