[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,296 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.distributed as dist
from torch.distributed import ProcessGroup
import vllm.envs as envs
from vllm.logger import init_logger
from .base_device_communicator import DeviceCommunicatorBase
logger = init_logger(__name__)
class XpuCommunicator(DeviceCommunicatorBase):
def __init__(
self,
cpu_group: ProcessGroup,
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
use_all2all: bool = False,
):
super().__init__(
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
)
self.ca_comm: None = None
if self.use_all2all:
if self.all2all_backend in ("naive", "allgather_reducescatter"):
from .all2all import AgRsAll2AllManager
self.all2all_manager = AgRsAll2AllManager(self.cpu_group)
logger.info("Using AgRs manager on XPU device.")
else: # type: ignore[has-type]
logger.warning(
"`%s` all2all manager is not supported on XPU. "
"Falling back to AgRs manager for XPU, "
"which is the Default backend",
self.all2all_backend, # type: ignore[has-type]
)
from .all2all import AgRsAll2AllManager
self.all2all_manager = AgRsAll2AllManager(self.cpu_group)
logger.info("Using AgRs manager on XPU device.")
def _fixed_rank_sum(self, input_: torch.Tensor) -> torch.Tensor:
flat_input = input_.reshape(-1)
gathered = torch.empty(
(self.world_size, flat_input.numel()),
dtype=input_.dtype,
device=input_.device,
)
dist.all_gather_single(gathered.view(-1), flat_input, group=self.device_group)
output = gathered[0].clone()
for rank in range(1, self.world_size):
output.add_(gathered[rank])
return output.view(input_.shape)
def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
if envs.VLLM_BATCH_INVARIANT:
# XCCL changes its reduction order with message size. Gather first
# and accumulate in rank order, independent of the token batch.
return self._fixed_rank_sum(input_)
output = input_.clone()
dist.all_reduce(output, group=self.device_group)
return output
def reduce_scatter(self, input_: torch.Tensor, dim: int = -1):
world_size = self.world_size
if dim < 0:
# Convert negative dim to positive.
dim += input_.dim()
# Note: This will produce an incorrect answer if we don't make
# the input_tensor contiguous. Possible bug in reduce_scatter_tensor?
input_tensor = input_.movedim(0, dim).contiguous()
assert input_tensor.shape[0] % world_size == 0
chunk_size = input_tensor.shape[0] // world_size
output_shape = (chunk_size,) + input_tensor.shape[1:]
output = torch.empty(
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
)
if envs.VLLM_BATCH_INVARIANT:
reduced = self._fixed_rank_sum(input_tensor)
start = self.rank_in_group * chunk_size
output.copy_(reduced.narrow(0, start, chunk_size))
else:
dist.reduce_scatter_tensor(output, input_tensor, group=self.device_group)
# Reshape before returning
return output.movedim(0, dim).contiguous()
def reduce_scatterv(
self, input_: torch.Tensor, dim: int = -1, sizes: list[int] | None = None
):
world_size = self.world_size
if dim < 0:
# Convert negative dim to positive.
dim += input_.dim()
# Note: This will produce an incorrect answer if we don't make
# the input_tensor contiguous. Possible bug in reduce_scatter_tensor?
input_tensor = input_.movedim(0, dim).contiguous()
if sizes is not None:
assert len(sizes) == world_size
assert input_tensor.shape[0] == sum(sizes)
chunk_size = sizes[self.rank_in_group]
else:
assert input_tensor.shape[0] % world_size == 0
chunk_size = input_tensor.shape[0] // world_size
output_shape = (chunk_size,) + input_tensor.shape[1:]
output = torch.empty(
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
)
if envs.VLLM_BATCH_INVARIANT:
reduced = self._fixed_rank_sum(input_tensor)
start = (
self.rank_in_group * chunk_size
if sizes is None
else sum(sizes[: self.rank_in_group])
)
output.copy_(reduced.narrow(0, start, chunk_size))
elif sizes is None or sizes.count(sizes[0]) == len(sizes):
dist.reduce_scatter_tensor(output, input_tensor, group=self.device_group)
elif torch.xpu.device_count() < self.world_size:
# uneven reduce_scatter is not supported when the
# process cannot see every device in the group (
# e.g. external-LBDP).
max_size = max(sizes)
padded = input_tensor.new_zeros(
(world_size * max_size,) + input_tensor.shape[1:]
)
for rank, split in enumerate(input_tensor.split(sizes, dim=0)):
padded[rank * max_size : rank * max_size + sizes[rank]] = split
padded_output = input_tensor.new_empty((max_size,) + input_tensor.shape[1:])
dist.reduce_scatter_tensor(padded_output, padded, group=self.device_group)
output = padded_output[:chunk_size]
else:
input_splits = list(input_tensor.split(sizes, dim=0))
dist.reduce_scatter(output, input_splits, group=self.device_group)
# Reshape before returning
return output.movedim(0, dim).contiguous()
def all_gatherv(
self,
input_: torch.Tensor | list[torch.Tensor],
dim: int = 0,
sizes: list[int] | None = None,
):
if dim != 0:
raise NotImplementedError("only dim 0 all-gatherv is supported")
world_size = self.world_size
# 'sizes' is not needed if all inputs in the same group have the same
# shape
if sizes is not None and all(s == sizes[0] for s in sizes):
sizes = None
def _all_gather_single(input_: torch.Tensor, sizes: list[int] | None = None):
input_size = input_.size()
if sizes is not None:
assert len(sizes) == world_size
assert input_.shape[dim] == sizes[self.rank_in_group], (
f"{input_.shape[dim]} != {sizes[self.rank_in_group]}"
)
output_size = (sum(sizes),) + input_size[1:]
else:
output_size = (input_size[0] * world_size,) + input_size[1:]
# Allocate output tensor.
output_tensor = torch.empty(
output_size, dtype=input_.dtype, device=input_.device
)
if sizes is not None:
all_gather_list = []
for size in sizes:
all_gather_list.append(
torch.empty(
(size,) + input_.shape[1:],
dtype=input_.dtype,
device=input_.device,
)
)
dist.all_gather(all_gather_list, input_, group=self.device_group)
output_tensor = torch.cat(all_gather_list, dim=0)
else:
dist.all_gather([output_tensor], input_, group=self.device_group)
return output_tensor
if isinstance(input_, torch.Tensor):
return _all_gather_single(input_, sizes)
output_list = []
for inp in input_:
output_list.append(_all_gather_single(inp, sizes=sizes))
return output_list
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> torch.Tensor | None:
assert -input_.dim() <= dim < input_.dim(), (
f"Invalid dim ({dim}) for input tensor with shape {input_.size()}"
)
if dim < 0:
# Convert negative dim to positive.
dim += input_.dim()
# For xpu path, gather doesn't work properly together with ray
# cluster so we use all_gather instead for now.
input_size = input_.size()
# Allocate output tensor.
output_tensor = torch.empty(
(self.world_size,) + input_size, dtype=input_.dtype, device=input_.device
)
# All-gather.
dist.all_gather_single(output_tensor, input_, group=self.device_group)
if self.rank_in_group == dst:
# Reshape
output_tensor = output_tensor.movedim(0, dim)
output_tensor = output_tensor.reshape(
input_size[:dim]
+ (self.world_size * input_size[dim],)
+ input_size[dim + 1 :]
)
else:
output_tensor = None
return output_tensor
def broadcast(self, input_: torch.Tensor, src: int = 0) -> None:
dist.broadcast(input_, src=src, group=self.device_group)
def dispatch_router_logits(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
is_sequence_parallel: bool = False,
extra_tensors: list[torch.Tensor] | None = None,
) -> (
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch_router_logits(
hidden_states,
router_logits,
is_sequence_parallel,
extra_tensors,
)
def dispatch(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
is_sequence_parallel: bool = False,
extra_tensors: list[torch.Tensor] | None = None,
) -> (
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch(
hidden_states,
topk_weights,
topk_ids,
is_sequence_parallel,
extra_tensors=extra_tensors,
)
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.combine(
hidden_states,
is_sequence_parallel,
)
@@ -1,9 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Internal building blocks for the Mooncake encoder-cache connector.
The public connector remains in :mod:`mooncake_ec_connector`. This package
separates configuration, control-plane messaging, transfer state, registered
memory, and worker/scheduler orchestration so that each component has one
owner and can be tested independently.
"""
@@ -1,155 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import TYPE_CHECKING
from vllm.utils.network_utils import make_zmq_path
if TYPE_CHECKING:
from vllm.config import VllmConfig
# Scheduler and Worker must agree on when a remote reservation becomes stale.
_RESERVATION_TTL_SECONDS = 300
def _positive_int(name: str, value: object) -> int:
message = f"ECMooncakeConnector requires {name} to be a positive integer."
if isinstance(value, bool):
raise ValueError(message)
if isinstance(value, int):
result = value
elif isinstance(value, float) and math.isfinite(value) and value.is_integer():
result = int(value)
elif isinstance(value, str):
try:
result = int(value)
except ValueError as error:
raise ValueError(message) from error
else:
raise ValueError(message)
if result <= 0:
raise ValueError(f"ECMooncakeConnector requires {name} > 0.")
return result
def _positive_float(name: str, value: object) -> float:
message = f"ECMooncakeConnector requires {name} > 0."
if isinstance(value, bool):
raise ValueError(message)
try:
result = float(value) # type: ignore[arg-type]
except (TypeError, ValueError) as error:
raise ValueError(message) from error
if not math.isfinite(result) or result <= 0:
raise ValueError(message)
return result
@dataclass(frozen=True)
class MooncakeECConfig:
"""Validated settings shared by the Scheduler and Worker roles.
``control_port`` is the first TP-shard port after the DP offset;
``control_addr`` targets that shard, which advertises the full topology.
``control_host`` is the address this instance both advertises and binds,
so reaching a Consumer means being routed to it rather than finding it on
every interface.
"""
is_producer: bool
is_consumer: bool
protocol: str
buffer_device: str
control_host: str
control_port: int
control_addr: str
control_timeout_ms: int
push_wait_timeout_s: float
pool_size: int
cross_encoder_cache: bool = False
store_max_pending_items: int = 32
store_max_pending_bytes: int = 2 * 1024**3
store_read_buffer_bytes: int = 128 * 1024**2
@classmethod
def from_vllm_config(cls, vllm_config: VllmConfig) -> MooncakeECConfig:
parallel_config = vllm_config.parallel_config
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None
if ec_config.is_ec_producer:
if parallel_config.tensor_parallel_size > 1:
raise ValueError(
"ECMooncakeConnector producers require tensor_parallel_size=1."
)
if parallel_config.pipeline_parallel_size > 1:
raise ValueError(
"ECMooncakeConnector producers do not support pipeline parallelism."
)
if parallel_config.data_parallel_size > 1:
raise ValueError(
"ECMooncakeConnector producers require data_parallel_size=1."
)
registered_buffer_size = _positive_int(
"ec_buffer_size", ec_config.ec_buffer_size
)
get = ec_config.get_from_extra_config
shared_reuse = get("cross_encoder_cache", False)
if not isinstance(shared_reuse, bool):
raise ValueError("cross_encoder_cache must be a boolean")
if shared_reuse:
if not ec_config.is_encode_only:
raise ValueError(
"cross_encoder_cache requires an Encoder-only producer"
)
if not vllm_config.use_v2_model_runner:
raise ValueError("cross_encoder_cache requires Model Runner V2")
if vllm_config.lora_config is not None:
raise ValueError("cross_encoder_cache does not support LoRA")
mm_config = vllm_config.model_config.multimodal_config
if mm_config is not None and mm_config.mm_processor_cache_gb == 0:
raise ValueError(
"cross_encoder_cache requires mm_processor_cache_gb > 0 "
"to preserve content identifiers across Encoders and restarts"
)
control_port = int(ec_config.ec_port) + (
parallel_config.data_parallel_index * parallel_config.tensor_parallel_size
)
highest_port = control_port + parallel_config.tensor_parallel_size - 1
if not 1 <= control_port <= highest_port <= 65535:
raise ValueError("ECMooncakeConnector ec_port must be in 1..65535.")
return cls(
is_producer=ec_config.is_ec_producer,
is_consumer=ec_config.is_ec_consumer,
protocol=str(get("mooncake_protocol", "rdma")),
buffer_device=str(ec_config.ec_buffer_device or "cuda").lower(),
control_host=str(ec_config.ec_ip),
control_port=control_port,
control_addr=make_zmq_path("tcp", ec_config.ec_ip, control_port),
control_timeout_ms=max(
1,
math.ceil(
_positive_float("control_timeout_s", get("control_timeout_s", 30))
* 1000
),
),
push_wait_timeout_s=_positive_float(
"push_wait_timeout_s", get("push_wait_timeout_s", 60)
),
pool_size=registered_buffer_size,
cross_encoder_cache=shared_reuse,
store_max_pending_items=_positive_int(
"store_max_pending_items", get("store_max_pending_items", 32)
),
store_max_pending_bytes=_positive_int(
"store_max_pending_bytes", get("store_max_pending_bytes", 2 * 1024**3)
),
store_read_buffer_bytes=_positive_int(
"store_read_buffer_bytes", get("store_read_buffer_bytes", 128 * 1024**2)
),
)
@@ -1,460 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ZMQ control plane for Mooncake encoder-cache reservations and events.
Tensor bytes never travel through this module. It coordinates destination
reservations, completion/cancellation, TP-shard discovery, and readiness
notifications while :mod:`transfer` owns the Mooncake data plane.
"""
from __future__ import annotations
import threading
import time
from collections import deque
from collections.abc import Callable
from concurrent.futures import Executor, Future
from typing import Any
import torch
import zmq
from vllm.logger import init_logger
from vllm.utils.network_utils import make_zmq_path
logger = init_logger(__name__)
_MAX_PENDING_EVENTS = 4096
_RESERVATION_REAP_INTERVAL_SECONDS = 1
ControlRequest = dict[str, Any]
ControlResponse = dict[str, Any]
class ControlClient:
"""Send control requests through reusable, thread-local REQ sockets."""
def __init__(self, timeout_ms: int) -> None:
self._context = zmq.Context()
self._timeout_ms = timeout_ms
self._local = threading.local()
self._topologies: dict[str, list[str]] = {}
self._closed = False
def _sockets(self) -> dict[str, zmq.Socket]:
sockets = getattr(self._local, "sockets", None)
if sockets is None:
sockets = {}
self._local.sockets = sockets
return sockets
def _discard(self, addr: str) -> None:
socket = self._sockets().pop(addr, None)
if socket is not None:
socket.close(linger=0)
def _exchange(self, addr: str, payload: ControlRequest) -> ControlResponse:
sockets = self._sockets()
socket = sockets.get(addr)
if socket is None:
socket = self._context.socket(zmq.REQ)
socket.setsockopt(zmq.IPV6, 1)
socket.setsockopt(zmq.RCVTIMEO, self._timeout_ms)
socket.setsockopt(zmq.SNDTIMEO, self._timeout_ms)
socket.setsockopt(zmq.LINGER, 0)
socket.connect(addr)
sockets[addr] = socket
try:
socket.send_json(payload)
response = socket.recv_json()
except Exception:
self._discard(addr)
raise
assert isinstance(response, dict)
return response
def request(self, addr: str, payload: ControlRequest) -> Any:
response = self._exchange(addr, payload)
if not response.get("ok"):
raise RuntimeError(response.get("error", "EC control request failed"))
return response.get("result")
def discover_shards(self, base_addr: str) -> list[str] | None:
if base_addr in self._topologies:
return self._topologies[base_addr]
try:
reply = self.request(base_addr, {"op": "peers"})
ports = reply.get("ports") if isinstance(reply, dict) else None
if not isinstance(ports, list) or not ports:
raise ValueError("invalid or empty peer list")
prefix = base_addr.rsplit(":", 1)[0]
shards = [f"{prefix}:{int(port)}" for port in ports]
except Exception:
logger.warning(
"EC Mooncake consumer at %s did not report its shards",
base_addr,
exc_info=True,
)
return None
self._topologies[base_addr] = shards
return shards
def close(self) -> None:
if self._closed:
return
self._closed = True
self._context.destroy(linger=0)
def make_cancel_request(
transfer_id: str,
reservation_id: str = "",
*,
abandon: bool = False,
refresh: bool = False,
) -> ControlRequest:
request: ControlRequest = {
"op": "cancel",
"transfer_id": transfer_id,
"reservation_id": reservation_id,
}
if abandon:
request["abandon"] = True
if refresh:
request["refresh"] = True
return request
class EventInbox:
"""Receive Consumer readiness events without blocking the Scheduler.
Subscribing needs one blocking control request per shard, so it runs on
`executor` when one is given: an unreachable Consumer must cost the
Scheduler nothing, not one control timeout per drain.
"""
def __init__(self, client: ControlClient, executor: Executor | None = None) -> None:
self._client = client
self._executor = executor
self._discovery: Future[list[str] | None] | None = None
self._context: zmq.Context | None = None
self._socket: zmq.Socket | None = None
self._closed = False
self.shard_count = 1
def _endpoints(self, base_addr: str) -> list[str] | None:
"""Ask every shard where it publishes readiness events.
Blocking: called on `self._executor` unless there is none.
"""
shards = self._client.discover_shards(base_addr)
if shards is None:
return None
endpoints = []
for addr in shards:
try:
event_port = self._client.request(addr, {"op": "event_port"})
address, _ = addr.rsplit(":", 1)
endpoints.append(f"{address}:{int(event_port)}")
except Exception:
logger.warning(
"EC Mooncake could not subscribe to the event channel of "
"consumer shard %s; retrying the complete topology later.",
addr,
)
return None
return endpoints
def _connect(self, base_addr: str) -> None:
if self._socket is not None or self._closed:
return
if self._executor is None:
self._install(self._endpoints(base_addr))
return
if self._discovery is None:
self._discovery = self._executor.submit(self._endpoints, base_addr)
return
if not self._discovery.done():
return
discovery, self._discovery = self._discovery, None
try:
self._install(discovery.result())
except Exception:
logger.warning(
"EC Mooncake event-channel discovery for %s failed; retrying.",
base_addr,
exc_info=True,
)
def _install(self, endpoints: list[str] | None) -> None:
"""Adopt a discovered topology.
Runs on the caller's thread so the socket is only ever touched there.
"""
if not endpoints or self._closed:
return
context = zmq.Context()
socket = context.socket(zmq.PULL)
socket.setsockopt(zmq.IPV6, 1)
for endpoint in endpoints:
socket.connect(endpoint)
self._context = context
self._socket = socket
self.shard_count = len(endpoints)
def drain(self, base_addr: str) -> list[dict[str, Any]]:
self._connect(base_addr)
if self._socket is None:
return []
events = []
while True:
try:
events.append(self._socket.recv_json(flags=zmq.DONTWAIT))
except zmq.Again:
return events
except zmq.ZMQError:
logger.warning("EC Mooncake event channel failed", exc_info=True)
return events
except (ValueError, UnicodeError):
# The event channel is a plain PULL socket: an undecodable
# frame must cost one frame, not the engine.
logger.warning(
"Discarding an undecodable EC Mooncake event.", exc_info=True
)
def close(self) -> None:
if self._closed:
return
self._closed = True
if self._discovery is not None:
self._discovery.cancel()
self._discovery = None
if self._socket is not None:
self._socket.close(linger=0)
self._socket = None
if self._context is not None:
self._context.term()
class ConsumerControlServer:
"""Expose Consumer reservations and readiness events over ZMQ.
One server runs on every receiving TP rank. The REP channel handles
reservation operations, while a PUSH channel publishes newly ready items
to the Scheduler.
"""
def __init__(
self,
host: str,
port: int,
reserve: Callable[[dict[str, Any]], dict[str, Any]],
status: Callable[[str], dict[str, Any] | None],
complete: Callable[[str, str], tuple[bool, bool]],
cancel: Callable[[str, str, bool, bool], bool],
reap: Callable[[], int],
peer_ports: list[int] | None = None,
device: torch.device | None = None,
drain_ready: Callable[[], list[str]] = lambda: [],
) -> None:
self.host = host
self.port = port
self.peer_ports = peer_ports or [port]
self._device = device
self.event_port: int | None = None
self._reserve = reserve
self._status = status
self._complete = complete
self._cancel = cancel
self._reap = reap
self._drain_ready = drain_ready
self._stop = threading.Event()
self._started = threading.Event()
self._thread: threading.Thread | None = None
self._startup_error: Exception | None = None
def start(self) -> None:
def loop() -> None:
if self._device is not None and self._device.type == "cuda":
torch.accelerator.set_device_index(self._device.index or 0)
context = zmq.Context()
socket = context.socket(zmq.REP)
event_socket = context.socket(zmq.PUSH)
socket.setsockopt(zmq.IPV6, 1)
event_socket.setsockopt(zmq.IPV6, 1)
pending_events: deque[dict[str, Any]] = deque()
def queue_event(event: dict[str, Any]) -> None:
event["shard"] = self.port
if len(pending_events) >= _MAX_PENDING_EVENTS:
dropped = pending_events.popleft()
logger.warning(
"EC Mooncake event backlog full on port %d; dropping "
"readiness for transfer_id=%s",
self.port,
dropped.get("transfer_id"),
)
pending_events.append(event)
def queue_ready(transfer_id: str) -> None:
status = self._status(transfer_id)
if status is not None:
queue_event({"transfer_id": transfer_id, **status})
last_reap_at = time.monotonic()
socket.setsockopt(zmq.RCVTIMEO, 100)
try:
socket.bind(make_zmq_path("tcp", self.host, self.port))
event_socket.bind(make_zmq_path("tcp", self.host, 0))
self.event_port = int(
event_socket.getsockopt(zmq.LAST_ENDPOINT)
.decode()
.rsplit(":", 1)[1]
)
except Exception as e:
self._startup_error = e
self._started.set()
socket.close(linger=0)
event_socket.close(linger=0)
context.term()
return
self._started.set()
try:
while not self._stop.is_set():
while pending_events:
try:
event_socket.send_json(
pending_events[0], flags=zmq.DONTWAIT
)
except zmq.Again:
break
pending_events.popleft()
now = time.monotonic()
if now - last_reap_at >= _RESERVATION_REAP_INTERVAL_SECONDS:
self._reap()
last_reap_at = now
try:
request = socket.recv_json()
except zmq.Again:
continue
except Exception:
# The frame arrived but did not decode. REP still owes a
# reply, so answer before returning to the loop.
logger.exception(
"EC Mooncake control channel on port %d received an "
"undecodable request",
self.port,
)
socket.send_json(
{"ok": False, "error": "malformed control request"}
)
continue
try:
op = request.get("op")
result: Any = None
if op in ("reserve", "reserve_batch"):
items = (
request["items"] if op == "reserve_batch" else [request]
)
results = []
for item in items:
try:
reserved = self._reserve(item)
results.append({"ok": True, "result": reserved})
if reserved.get("ready"):
queue_ready(str(item["transfer_id"]))
except Exception as exc:
results.append({"ok": False, "error": str(exc)})
if op == "reserve_batch":
result = {"items": results}
elif not results[0]["ok"]:
raise RuntimeError(results[0]["error"])
else:
result = results[0]["result"]
elif op == "status":
result = self._status(str(request["transfer_id"]))
elif op == "event_port":
result = self.event_port
elif op == "peers":
result = {"ports": self.peer_ports}
elif op in ("complete", "complete_batch"):
items = (
request["items"]
if op == "complete_batch"
else [request]
)
completions = []
for item in items:
transfer_id = str(item["transfer_id"])
accepted, became_ready = self._complete(
transfer_id, str(item["reservation_id"])
)
for ready_id in self._drain_ready():
queue_ready(ready_id)
completions.append(
{
"completed": accepted,
"became_ready": became_ready,
}
)
if not became_ready:
continue
queue_ready(transfer_id)
result = (
{"items": completions}
if op == "complete_batch"
else completions[0]
)
elif op == "cancel":
result = {
"cancelled": self._cancel(
str(request["transfer_id"]),
str(request.get("reservation_id", "")),
bool(request.get("abandon", False)),
bool(request.get("refresh", False)),
)
}
else:
raise ValueError(f"unknown control op: {op!r}")
socket.send_json({"ok": True, "result": result})
except Exception as e:
socket.send_json({"ok": False, "error": str(e)})
except Exception:
# `finally` closes the sockets and the thread ends, so without
# this every later reserve against this shard would surface
# only as a control timeout with nothing to attribute it to.
logger.exception(
"EC Mooncake control channel on port %d stopped serving",
self.port,
)
raise
finally:
socket.close(linger=0)
event_socket.close(linger=0)
context.term()
self._thread = threading.Thread(
target=loop, name="ec-mooncake-control", daemon=True
)
self._thread.start()
if not self._started.wait(timeout=5):
raise RuntimeError("EC Mooncake control channel failed to start")
if self._startup_error is not None:
raise RuntimeError("EC Mooncake control channel failed to bind") from (
self._startup_error
)
logger.info(
"EC Mooncake control channel listening on tcp://%s:%d (events tcp://%s:%d)",
self.host,
self.port,
self.host,
self.event_port,
)
def close(self) -> None:
if self._thread is None:
return
self._stop.set()
self._thread.join()
self._thread = None
@@ -1,504 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Registered-memory allocation and residency for Mooncake transfers.
The Producer pool stages source tensors in a registered slab. The Consumer
pool owns destination allocations from reservation through publication,
resident reuse, CUDA-safe retirement, and pressure-driven reclamation.
"""
from __future__ import annotations
import bisect
import threading
from collections import OrderedDict
from collections.abc import Callable
from dataclasses import dataclass
from typing import Generic, TypeVar
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
MooncakeTransfer,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
_T = TypeVar("_T")
@dataclass
class MemoryAllocation:
"""Describe a tensor view carved from the Consumer receive slab."""
offset: int
size: int
tensor: torch.Tensor
@dataclass
class _ResidentEntry(Generic[_T]):
"""Store one resident value and its ownership accounting."""
value: _T
pinned: bool = True
leases: int = 0
@dataclass
class ResidentLease(Generic[_T]):
"""Represent a borrow of one resident entry."""
key: str
_entry: _ResidentEntry[_T]
_active: bool = True
@property
def value(self) -> _T:
return self._entry.value
class ContiguousAllocator:
"""Own a registered slab and its aligned regions; callers serialize access."""
def __init__(self, capacity: int, alignment: int = 256):
self.alignment = alignment
self._capacity = capacity
self._free = [(0, capacity)]
self.tensor: torch.Tensor | None = None
self._disabled = False
def prepare(self, device: torch.device, transfer: MooncakeTransfer) -> None:
if self.tensor is not None or self._disabled:
return
try:
tensor = torch.empty(self._capacity, dtype=torch.uint8, device=device)
ret = transfer.register_memory(tensor)
if ret != 0:
raise RuntimeError(f"Mooncake returned {ret}")
except (RuntimeError, torch.OutOfMemoryError) as error:
self._disabled = True
logger.warning("Could not initialize the EC registered buffer: %s", error)
return
self.tensor = tensor
self._free = [(0, tensor.nbytes)]
logger.info("Registered %d-byte buffer for Mooncake EC", tensor.nbytes)
def view(
self, offset: int, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
) -> torch.Tensor:
assert self.tensor is not None
return self.tensor.narrow(0, offset, nbytes).view(dtype).view(shape)
def close(self, transfer: MooncakeTransfer) -> bool:
if self.tensor is not None and not transfer.unregister_memory(self.tensor):
return False
self.tensor = None
self._free.clear()
return True
def allocate(self, nbytes: int) -> tuple[int, int] | None:
size = (nbytes + self.alignment - 1) // self.alignment * self.alignment
for index, (offset, available) in enumerate(self._free):
if size > available:
continue
if size == available:
self._free.pop(index)
else:
self._free[index] = (offset + size, available - size)
return offset, size
return None
def free(self, offset: int, size: int) -> None:
index = bisect.bisect_left(self._free, (offset, size))
self._free.insert(index, (offset, size))
if index + 1 < len(self._free):
next_offset, next_size = self._free[index + 1]
if offset + size == next_offset:
self._free[index] = (offset, size + next_size)
self._free.pop(index + 1)
if index > 0:
previous_offset, previous_size = self._free[index - 1]
current_offset, current_size = self._free[index]
if previous_offset + previous_size == current_offset:
self._free[index - 1] = (
previous_offset,
previous_size + current_size,
)
self._free.pop(index)
class ResidentPool(Generic[_T]):
"""Track resident values, active leases, and LRU eviction eligibility."""
def __init__(self):
self._entries: dict[str, _ResidentEntry[_T]] = {}
self._evictable: OrderedDict[str, None] = OrderedDict()
def referenced(self) -> list[str]:
return [key for key, entry in self._entries.items() if entry.pinned]
def get(self, key: str) -> _T | None:
entry = self._entries.get(key)
return entry.value if entry is not None else None
def insert(
self,
key: str,
value: _T,
) -> _T | None:
"""Pin an entry and return a displaced value that has no owners."""
previous = self._entries.get(key)
entry = _ResidentEntry(value)
self._entries[key] = entry
self._evictable.pop(key, None)
if previous is not None and previous.leases == 0:
return previous.value
return None
def pin(self, key: str) -> _T | None:
entry = self._entries.get(key)
if entry is None:
return None
self._evictable.pop(key, None)
entry.pinned = True
return entry.value
def acquire(self, key: str) -> ResidentLease[_T] | None:
entry = self._entries.get(key)
if entry is None:
return None
entry.leases += 1
self._evictable.pop(key, None)
return ResidentLease(key, entry)
def release(self, lease: ResidentLease[_T]) -> _T | None:
if not lease._active:
return None
lease._active = False
entry = lease._entry
entry.leases -= 1
current = self._entries.get(lease.key)
if current is not entry:
if entry.leases == 0:
return entry.value
return None
if not entry.pinned and entry.leases == 0:
self._evictable[lease.key] = None
return None
def consume(self, lease: ResidentLease[_T]) -> tuple[_T, _T | None]:
current = self._entries[lease.key]
current.pinned = True
self._evictable.pop(lease.key, None)
released = self.release(lease)
return current.value, released
def retire(self, key: str) -> None:
if key not in self._entries:
return
entry = self._entries[key]
entry.pinned = False
if entry.leases == 0:
self._evictable[key] = None
def evict_lru(self, evict: Callable[[str, _T], bool]) -> str | None:
for key in list(self._evictable):
entry = self._entries[key]
if not evict(key, entry.value):
continue
self._evictable.pop(key, None)
del self._entries[key]
return key
return None
def clear(self) -> None:
self._entries.clear()
self._evictable.clear()
@dataclass
class StagedSources:
"""Own Producer tensor views and their staging-slab regions."""
tensors: list[torch.Tensor]
regions: list[tuple[int, int]]
class ProducerMemoryPool:
"""Own the Producer staging slab and regions carved from it."""
def __init__(self, capacity: int, transfer: MooncakeTransfer) -> None:
self._transfer = transfer
self._allocator = ContiguousAllocator(capacity)
self._lock = threading.Lock()
self._local = threading.local()
@property
def tensor(self) -> torch.Tensor | None:
return self._allocator.tensor
def _free_regions(self, regions: list[tuple[int, int]]) -> None:
for offset, size in regions:
self._allocator.free(offset, size)
def stage(self, tensors: list[torch.Tensor]) -> StagedSources | None:
"""Copy tensors into one registered slab, or return None for fallback."""
if not tensors:
return StagedSources([], [])
allocator = self._allocator
staged: list[torch.Tensor] = []
regions: list[tuple[int, int]] = []
with self._lock:
allocator.prepare(tensors[0].device, self._transfer)
pool = allocator.tensor
if pool is None:
return None
for tensor in tensors:
region = allocator.allocate(tensor.nbytes)
if region is None:
self._free_regions(regions)
return None
regions.append(region)
staged.append(
allocator.view(
region[0], tensor.nbytes, tuple(tensor.shape), tensor.dtype
)
)
if pool.device.type == "cuda":
stream = getattr(self._local, "stream", None)
if stream is None:
stream = torch.cuda.Stream(device=pool.device)
self._local.stream = stream
with torch.cuda.stream(stream):
for destination, source in zip(staged, tensors):
destination.copy_(source, non_blocking=True)
stream.synchronize()
else:
for destination, source in zip(staged, tensors):
destination.copy_(source)
return StagedSources(staged, regions)
def release(self, staged: StagedSources) -> None:
if not staged.regions:
return
with self._lock:
self._free_regions(staged.regions)
def close(self) -> None:
with self._lock:
self._allocator.close(self._transfer)
class ConsumerMemoryPool:
"""Own the registered receive slab and resident allocation lifecycle."""
def __init__(
self,
capacity: int,
transfer: MooncakeTransfer,
) -> None:
self._transfer = transfer
self._allocator = ContiguousAllocator(capacity)
self._residents: ResidentPool[MemoryAllocation] = ResidentPool()
self._retire_events: dict[str, torch.Event] = {}
self._pending_frees: list[tuple[torch.Event, MemoryAllocation]] = []
self._reclaimed: set[str] = set()
self.lock = threading.RLock()
@property
def tensor(self) -> torch.Tensor | None:
return self._allocator.tensor
def prepare(
self,
device: torch.device,
) -> None:
with self.lock:
self._allocator.prepare(device, self._transfer)
def _free(self, allocation: MemoryAllocation) -> None:
self._allocator.free(allocation.offset, allocation.size)
def free(self, allocation: MemoryAllocation) -> None:
with self.lock:
self._free(allocation)
def _defer_or_free(
self, allocation: MemoryAllocation, event: torch.Event | None
) -> None:
if event is None or event.query():
self._free(allocation)
else:
self._pending_frees.append((event, allocation))
def _poll_frees_locked(self) -> None:
pending = []
for event, allocation in self._pending_frees:
if event.query():
self._free(allocation)
else:
pending.append((event, allocation))
self._pending_frees = pending
def _reclaim_locked(self, nbytes: int) -> tuple[int, int] | None:
def evict(mm_hash: str, allocation: MemoryAllocation) -> bool:
event = self._retire_events.pop(mm_hash, None)
self._defer_or_free(allocation, event)
self._reclaimed.add(mm_hash)
return True
while self._residents.evict_lru(evict) is not None:
# Eviction defers any free whose CUDA event is still pending, so
# those bytes reach the allocator only once the event is polled.
self._poll_frees_locked()
region = self._allocator.allocate(nbytes)
if region is not None:
return region
self._poll_frees_locked()
return self._allocator.allocate(nbytes)
def _make_allocation(
self,
region: tuple[int, int],
nbytes: int,
shape: tuple[int, ...],
dtype: torch.dtype,
) -> MemoryAllocation:
offset, size = region
tensor = self._allocator.view(offset, nbytes, shape, dtype)
return MemoryAllocation(offset, size, tensor)
def try_allocate(
self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
) -> MemoryAllocation | None:
with self.lock:
allocator = self._allocator
assert allocator.tensor is not None
self._poll_frees_locked()
region = allocator.allocate(nbytes)
if region is None:
return None
return self._make_allocation(region, nbytes, shape, dtype)
def reclaim_and_allocate(
self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
) -> MemoryAllocation | None:
with self.lock:
region = self._reclaim_locked(nbytes)
if region is None:
return None
return self._make_allocation(region, nbytes, shape, dtype)
def acquire_cached(
self, mm_hash: str, shape: tuple[int, ...], dtype: torch.dtype
) -> ResidentLease[MemoryAllocation] | None:
with self.lock:
allocation = self._residents.get(mm_hash)
if allocation is None:
return None
if (
tuple(allocation.tensor.shape) != shape
or allocation.tensor.dtype != dtype
):
raise ValueError("conflicting cached tensor for mm_hash")
return self._residents.acquire(mm_hash)
def take_resident(
self, mm_hash: str, shape: tuple[int, ...], dtype_name: str
) -> torch.Tensor | None:
with self.lock:
allocation = self._residents.get(mm_hash)
if allocation is None:
return None
tensor = allocation.tensor
if (
tuple(tensor.shape) != shape
or str(tensor.dtype).split(".")[-1] != dtype_name
):
return None
self._residents.pin(mm_hash)
self._retire_events.pop(mm_hash, None)
return tensor
def _record_release_event(self) -> torch.Event | None:
pool = self.tensor
if pool is None or pool.device.type != "cuda":
return None
event = torch.Event()
event.record(torch.accelerator.current_stream(pool.device))
return event
def release_cached(self, lease: ResidentLease[MemoryAllocation]) -> None:
with self.lock:
released = self._residents.release(lease)
if released is not None:
self._defer_or_free(released, self._record_release_event())
def publish(
self,
mm_hash: str,
allocation: MemoryAllocation,
lease: ResidentLease[MemoryAllocation] | None = None,
*,
pin: bool = True,
) -> MemoryAllocation:
with self.lock:
if lease is not None:
canonical, released = self._residents.consume(lease)
self._retire_events.pop(mm_hash, None)
if released is not None:
self._defer_or_free(released, self._record_release_event())
return canonical
previous = self._residents.get(mm_hash)
displaced = self._residents.insert(mm_hash, allocation)
if not pin:
self._residents.retire(mm_hash)
event = None
if previous is not None and previous is not allocation:
event = self._retire_events.pop(mm_hash, None)
if displaced is None or displaced is allocation:
return allocation
if event is None:
event = self._record_release_event()
self._defer_or_free(displaced, event)
return allocation
def retire_stale(
self,
encoder_cache: dict[str, torch.Tensor],
reserved_hashes: set[str] | None = None,
*,
freed: list[str] | None = None,
) -> None:
if self.tensor is None:
return
with self.lock:
for mm_hash in self._residents.referenced() if freed is None else freed:
allocation = self._residents.get(mm_hash)
if allocation is None:
continue
if encoder_cache.get(mm_hash) is allocation.tensor:
continue
if reserved_hashes and mm_hash in reserved_hashes:
continue
event = self._record_release_event()
if event is not None:
self._retire_events[mm_hash] = event
self._residents.retire(mm_hash)
self._poll_frees_locked()
def drain_reclaimed(self) -> set[str]:
with self.lock:
reclaimed = self._reclaimed
self._reclaimed = set()
return reclaimed
def close(self) -> None:
with self.lock:
if not self._allocator.close(self._transfer):
return
self._residents.clear()
self._retire_events.clear()
self._pending_frees.clear()
@@ -1,79 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Metadata exchanged by the Mooncake encoder-cache connector."""
from __future__ import annotations
from dataclasses import dataclass, field
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
ECConnectorWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
TensorSpec,
)
@dataclass
class ECMooncakeLoadSpec:
"""Describe a remote reservation or resident allocation to load."""
mm_hash: str
nbytes: int
shape: tuple[int, ...]
dtype: str
transfer_id: str
# The consumer pool still holds this item, so the load is a local handoff:
# no transfer, no producer.
local: bool = False
@dataclass
class ECMooncakePushSpec:
"""Describe a destination reservation prepared before a tensor is ready."""
mm_hash: str
nbytes: int
shape: tuple[int, ...]
dtype: str
consumer_zmq: str
transfer_id: str
request_id: str = ""
@dataclass
class ECMooncakeConnectorMetadata(ECConnectorMetadata):
"""Worker operations emitted for one Scheduler step."""
loads: list[ECMooncakeLoadSpec] = field(default_factory=list)
pushes: list[ECMooncakePushSpec] = field(default_factory=list)
freed: list[str] | None = None
store_candidates: dict[str, TensorSpec] = field(default_factory=dict)
@dataclass
class ECMooncakeWorkerMetadata(ECConnectorWorkerMetadata):
"""Completion state reported from Workers to the Scheduler."""
loaded: set[str] = field(default_factory=set)
failed_loads: set[str] = field(default_factory=set)
# Items the receive pool dropped under pressure. The scheduler assumes an
# evicted item stays resident until told otherwise.
reclaimed: set[str] = field(default_factory=set)
pending_saves: bool = False
failed_saves: set[str] = field(default_factory=set)
def aggregate(self, other: ECConnectorWorkerMetadata) -> ECMooncakeWorkerMetadata:
assert isinstance(other, ECMooncakeWorkerMetadata)
return ECMooncakeWorkerMetadata(
# Every tensor-parallel rank gathers the embedding from its own
# cache, so an item counts as loaded only where all of them have
# it; one rank falling short must fail the load rather than leave
# the scheduler believing it is ready.
loaded=self.loaded & other.loaded,
failed_loads=self.failed_loads | other.failed_loads,
reclaimed=self.reclaimed | other.reclaimed,
pending_saves=self.pending_saves or other.pending_saves,
failed_saves=self.failed_saves | other.failed_saves,
)
@@ -1,418 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Producer-side push lifecycle, futures, and source-tensor ownership.
This module records when a push may advance or release its source. Worker
orchestration performs the actual control exchanges and Mooncake writes.
"""
from __future__ import annotations
import threading
from collections import OrderedDict
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from contextlib import suppress
from dataclasses import dataclass, field
from enum import Enum, auto
from typing import Any
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakePushSpec,
)
def _same_destination(
existing: ECMooncakePushSpec, incoming: ECMooncakePushSpec
) -> bool:
"""Whether two specs describe one push, ignoring which request asked.
The same encoding requested twice shares a transfer id legitimately; only
a different payload or destination is a collision.
"""
return (
existing.mm_hash == incoming.mm_hash
and existing.nbytes == incoming.nbytes
and existing.shape == incoming.shape
and existing.dtype == incoming.dtype
and existing.consumer_zmq == incoming.consumer_zmq
)
class ProducerPushState(Enum):
"""Lifecycle of one Producer push from reservation to terminal state."""
# The reservation reply and source tensor may arrive in either order.
WAITING_INPUTS = auto()
WRITING = auto()
NOTIFYING = auto()
DONE = auto()
CANCEL_PENDING = auto()
CANCELLED = auto()
FAILED = auto()
@dataclass
class ProducerPushRecord:
"""Collect all asynchronous state for one Producer push."""
spec: ECMooncakePushSpec
state: ProducerPushState
reservation_future: Future[list[dict[str, Any]]]
reservations: list[dict[str, Any]] = field(default_factory=list)
shard_futures: list[Future[Any]] = field(default_factory=list)
source_tensor: torch.Tensor | None = None
source_event: torch.Event | None = None
batch_future: Future[None] | None = None
error: str | None = None
_ALLOWED_TRANSITIONS = {
ProducerPushState.WAITING_INPUTS: {
ProducerPushState.WRITING,
ProducerPushState.CANCEL_PENDING,
ProducerPushState.FAILED,
},
ProducerPushState.WRITING: {
ProducerPushState.NOTIFYING,
ProducerPushState.FAILED,
},
ProducerPushState.NOTIFYING: {
ProducerPushState.DONE,
ProducerPushState.FAILED,
},
ProducerPushState.CANCEL_PENDING: {
ProducerPushState.CANCELLED,
ProducerPushState.FAILED,
},
ProducerPushState.DONE: set(),
ProducerPushState.CANCELLED: set(),
ProducerPushState.FAILED: set(),
}
_TERMINAL_STATES = {
ProducerPushState.DONE,
ProducerPushState.CANCELLED,
ProducerPushState.FAILED,
}
_TERMINAL_LIMIT = 1 << 16
class ProducerPushManager:
"""Own Producer push records, transitions, and source tensor leases.
Terminal records are retained for duplicate-metadata idempotency. Separate
ordered indexes keep hot polling paths from scanning those tombstones.
"""
def __init__(self, wake: Callable[[], None] = lambda: None) -> None:
self._records: OrderedDict[str, ProducerPushRecord] = OrderedDict()
self._active_ids: OrderedDict[str, None] = OrderedDict()
self._reapable_terminal_ids: OrderedDict[str, None] = OrderedDict()
self._unreported_ids: OrderedDict[str, None] = OrderedDict()
self._batch_ids: OrderedDict[str, None] = OrderedDict()
self._source_waiters: dict[str, OrderedDict[str, None]] = {}
self._lock = threading.RLock()
self._wake = wake
def reserve(
self,
spec: ECMooncakePushSpec,
submit: Callable[[], Future[list[dict[str, Any]]]],
) -> tuple[ProducerPushRecord, bool]:
with self._lock:
existing = self._records.get(spec.transfer_id)
if existing is not None:
if not _same_destination(existing.spec, spec):
raise ValueError(
f"Conflicting EC destination for transfer_id={spec.transfer_id}"
)
return existing, False
record = ProducerPushRecord(
spec=spec,
state=ProducerPushState.WAITING_INPUTS,
reservation_future=submit(),
)
self._records[spec.transfer_id] = record
self._active_ids[spec.transfer_id] = None
self._source_waiters.setdefault(spec.mm_hash, OrderedDict())[
spec.transfer_id
] = None
record.reservation_future.add_done_callback(
lambda future: self._reservation_done(record, future)
)
return record, True
def bind_source(
self,
mm_hash: str,
tensor: torch.Tensor,
ready_event: torch.Event | None,
) -> None:
with self._lock:
waiters = self._source_waiters.pop(mm_hash, OrderedDict())
for transfer_id in waiters:
record = self._records[transfer_id]
if (
record.source_tensor is not None
or record.state is not ProducerPushState.WAITING_INPUTS
):
continue
record.source_tensor = tensor
record.source_event = ready_event
self._wake()
def submit_batches(
self,
executor: ThreadPoolExecutor,
run_batch: Callable[[list[ProducerPushRecord]], None],
*,
wait: bool = False,
) -> bool:
pending_event = False
with self._lock:
grouped: dict[str, list[ProducerPushRecord]] = {}
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if (
record.source_tensor is not None
and record.batch_future is None
and record.state is ProducerPushState.WAITING_INPUTS
):
if not record.reservation_future.done():
continue
if record.source_event is not None:
if wait:
record.source_event.synchronize()
elif not record.source_event.query():
pending_event = True
continue
grouped.setdefault(record.spec.consumer_zmq, []).append(record)
batches = list(grouped.values())
for records in batches:
future = executor.submit(run_batch, records)
for record in records:
record.batch_future = future
self._batch_ids[record.spec.transfer_id] = None
return pending_event
def resolve_reservations(self, record: ProducerPushRecord) -> list[dict[str, Any]]:
results = record.reservation_future.result()
with self._lock:
record.reservations = results
return list(results)
def finish_reservations(
self,
outcomes: list[tuple[ProducerPushRecord, list[dict[str, Any]] | Exception]],
) -> None:
with self._lock:
for record, result in outcomes:
if isinstance(result, Exception):
record.reservation_future.set_exception(result)
else:
record.reservation_future.set_result(result)
def _reservation_done(
self,
record: ProducerPushRecord,
future: Future[list[dict[str, Any]]],
) -> None:
try:
future.result()
except Exception as exc:
with self._lock:
self._set_error(record, exc)
if (
record.state is ProducerPushState.WAITING_INPUTS
and record.source_tensor is None
):
self._transition(record, ProducerPushState.FAILED)
finally:
self._wake()
def settle_all(self, records: list[ProducerPushRecord]) -> None:
for record in records:
with suppress(Exception):
record.reservation_future.result()
def replace_reservations(
self,
record: ProducerPushRecord,
reservations: list[dict[str, Any]],
) -> None:
with self._lock:
record.reservations = reservations
def track_shard_futures(
self,
records: list[ProducerPushRecord],
futures: list[Future[Any]],
) -> None:
with self._lock:
for record in records:
record.shard_futures.extend(futures)
def begin_writing(self, record: ProducerPushRecord) -> None:
with self._lock:
if record.source_tensor is None:
raise RuntimeError(
f"Producer push {record.spec.transfer_id!r} has no source tensor"
)
self._transition(record, ProducerPushState.WRITING)
def begin_notifying(self, records: list[ProducerPushRecord]) -> None:
with self._lock:
for record in records:
self._transition(record, ProducerPushState.NOTIFYING)
def complete(self, records: list[ProducerPushRecord]) -> None:
with self._lock:
self._check_sources_releasable(records)
for record in records:
self._transition(record, ProducerPushState.DONE)
self._release_source(record)
def fail(self, records: list[ProducerPushRecord], error: Exception) -> None:
with self._lock:
self._check_sources_releasable(records)
for record in records:
if record.state not in _TERMINAL_STATES:
self._set_error(record, error)
self._transition(record, ProducerPushState.FAILED)
self._release_source(record)
def cancel_requests(self, request_ids: set[str] | None) -> list[ProducerPushRecord]:
"""Cancel source-less waiters for requests, or all waiters at shutdown."""
cancelled = []
with self._lock:
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if (
(
request_ids is not None
and record.spec.request_id not in request_ids
)
or record.source_tensor is not None
or record.batch_future is not None
):
continue
if record.state is not ProducerPushState.WAITING_INPUTS:
continue
self._transition(record, ProducerPushState.CANCEL_PENDING)
cancelled.append(record)
return cancelled
def finish_cancel(self, record: ProducerPushRecord) -> None:
with self._lock:
if record.state is ProducerPushState.CANCEL_PENDING:
self._transition(record, ProducerPushState.CANCELLED)
def submit_cancel(
self,
record: ProducerPushRecord,
executor: ThreadPoolExecutor,
run_cancel: Callable[[ProducerPushRecord], None],
) -> None:
with self._lock:
future = executor.submit(run_cancel, record)
record.batch_future = future
transfer_id = record.spec.transfer_id
self._batch_ids[transfer_id] = None
if record.state in _TERMINAL_STATES:
self._reapable_terminal_ids.pop(transfer_id, None)
def poll(self) -> list[tuple[str, str]]:
failures = []
with self._lock:
for transfer_id in list(self._batch_ids):
record = self._records[transfer_id]
future = record.batch_future
assert future is not None
if not future.done():
continue
try:
future.result()
except Exception as exc:
self._set_error(record, exc)
self._batch_ids.pop(transfer_id)
self._mark_reapable(record)
for transfer_id in list(self._unreported_ids):
if transfer_id in self._batch_ids:
continue
record = self._records[transfer_id]
assert record.error is not None
failures.append((record.spec.mm_hash, record.error))
self._unreported_ids.pop(transfer_id)
self._mark_reapable(record)
self._reap_terminals()
return failures
@property
def pending(self) -> bool:
with self._lock:
return bool(self._active_ids or self._batch_ids)
def _release_source(self, record: ProducerPushRecord) -> None:
record.source_tensor = None
record.source_event = None
def _set_error(self, record: ProducerPushRecord, error: BaseException) -> None:
if record.error is None:
record.error = str(error)
if record.state in _TERMINAL_STATES:
transfer_id = record.spec.transfer_id
self._unreported_ids[transfer_id] = None
self._reapable_terminal_ids.pop(transfer_id, None)
def _reap_terminals(self) -> None:
while len(self._reapable_terminal_ids) > _TERMINAL_LIMIT:
transfer_id, _ = self._reapable_terminal_ids.popitem(last=False)
self._records.pop(transfer_id)
def _mark_reapable(self, record: ProducerPushRecord) -> None:
transfer_id = record.spec.transfer_id
if (
record.state in _TERMINAL_STATES
and transfer_id not in self._batch_ids
and transfer_id not in self._unreported_ids
):
self._reapable_terminal_ids[transfer_id] = None
def _drop_source_waiter(self, record: ProducerPushRecord) -> None:
waiters = self._source_waiters.get(record.spec.mm_hash)
if waiters is None:
return
waiters.pop(record.spec.transfer_id, None)
if not waiters:
self._source_waiters.pop(record.spec.mm_hash)
@staticmethod
def _check_sources_releasable(records: list[ProducerPushRecord]) -> None:
for record in records:
futures = [record.reservation_future, *record.shard_futures]
if record.source_tensor is not None and not all(
future.done() for future in futures
):
raise RuntimeError(
f"Producer push {record.spec.transfer_id!r} released its "
"source too early"
)
def _transition(self, record: ProducerPushRecord, state: ProducerPushState) -> None:
if state not in _ALLOWED_TRANSITIONS[record.state]:
raise RuntimeError(
f"Cannot transition producer push {record.spec.transfer_id!r} from "
f"{record.state.name} to {state.name}"
)
record.state = state
if state is not ProducerPushState.WAITING_INPUTS:
self._drop_source_waiter(record)
if state in _TERMINAL_STATES:
transfer_id = record.spec.transfer_id
self._active_ids.pop(transfer_id, None)
if record.error is not None:
self._unreported_ids[transfer_id] = None
self._mark_reapable(record)
@@ -1,487 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Consumer-side reservation lifecycle and destination-memory ownership.
Reservations make remote writes idempotent and ensure cancellation or expiry
cannot free a destination while Mooncake may still be writing into it.
"""
from __future__ import annotations
import threading
import time
import uuid
from collections import OrderedDict
from dataclasses import dataclass
from enum import Enum, auto
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import (
ConsumerMemoryPool,
MemoryAllocation,
ResidentLease,
)
class ConsumerReservationState(Enum):
"""Lifecycle of one Consumer destination reservation."""
WRITING = auto()
READY = auto()
CANCEL_PENDING = auto()
EXPIRE_PENDING = auto()
CANCELLED = auto()
EXPIRED = auto()
_ALLOWED_TRANSITIONS = {
ConsumerReservationState.WRITING: {
ConsumerReservationState.READY,
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
ConsumerReservationState.CANCELLED,
},
ConsumerReservationState.READY: {
ConsumerReservationState.CANCELLED,
ConsumerReservationState.EXPIRED,
},
ConsumerReservationState.CANCEL_PENDING: {ConsumerReservationState.CANCELLED},
ConsumerReservationState.EXPIRE_PENDING: {
ConsumerReservationState.CANCELLED,
ConsumerReservationState.EXPIRED,
},
ConsumerReservationState.CANCELLED: set(),
ConsumerReservationState.EXPIRED: set(),
}
_ACTIVE_STATES = {
ConsumerReservationState.WRITING,
ConsumerReservationState.READY,
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
}
_DEFERRED_STATES = {
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
}
_WRITER_OWNED_STATES = {
ConsumerReservationState.WRITING,
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.EXPIRE_PENDING,
}
@dataclass
class ConsumerReservation:
"""Own the identity and destination allocation of one remote push."""
transfer_id: str
mm_hash: str
reservation_id: str
state: ConsumerReservationState
shape: tuple[int, ...] = ()
dtype: str = ""
allocation: MemoryAllocation | None = None
lease: ResidentLease[MemoryAllocation] | None = None
expires_at: float = 0
writer_id: str = ""
class ConsumerReservationManager:
"""Own reservation transitions and destination allocation releases.
A remote writer owns a ``WRITING`` allocation. Cancellation and expiry are
therefore deferred until completion, unless refresh explicitly abandons
that writer before replacing its reservation.
"""
def __init__(
self,
memory: ConsumerMemoryPool,
lease_ttl: float,
tombstone_limit: int,
) -> None:
self._memory = memory
self._lease_ttl = lease_ttl
self._tombstone_limit = tombstone_limit
self._records: dict[str, ConsumerReservation] = {}
self._active_ids: dict[str, None] = {}
self._tombstones: OrderedDict[str, None] = OrderedDict()
self._writers: dict[str, ConsumerReservation] = {}
self._followers: dict[str, dict[str, None]] = {}
self._ready: list[str] = []
self._condition = threading.Condition(memory.lock)
self._shutting_down = False
def reserve(
self,
transfer_id: str,
mm_hash: str,
nbytes: int,
shape: tuple[int, ...],
dtype_name: str,
dtype: torch.dtype,
) -> tuple[ConsumerReservation | None, bool]:
with self._condition:
if self._shutting_down:
raise RuntimeError("Consumer reservation manager is shutting down")
existing = self._records.get(transfer_id)
if (
existing is not None
and existing.state is ConsumerReservationState.CANCELLED
):
return existing, False
if (
existing is not None
and existing.state is ConsumerReservationState.EXPIRED
):
self._remove(transfer_id)
existing = None
if (
existing is not None
and existing.state is ConsumerReservationState.EXPIRE_PENDING
):
raise RuntimeError(
f"Transfer {transfer_id!r} still has an active writer"
)
if existing is not None:
if (
existing.mm_hash != mm_hash
or existing.shape != shape
or existing.dtype != dtype_name
):
raise ValueError("conflicting reservation for transfer_id")
if existing.state not in _ACTIVE_STATES:
raise RuntimeError(
f"Cannot reserve transfer {transfer_id!r} in "
f"{existing.state.name}"
)
if existing.state is ConsumerReservationState.WRITING:
existing.expires_at = time.monotonic() + self._lease_ttl
return existing, False
lease = self._memory.acquire_cached(mm_hash, shape, dtype)
now = time.monotonic()
if lease is not None:
record = ConsumerReservation(
transfer_id,
mm_hash,
uuid.uuid4().hex,
ConsumerReservationState.READY,
shape,
dtype_name,
lease.value,
lease,
now + self._lease_ttl,
)
self._insert(record)
return record, False
writer = self._writers.get(mm_hash)
if writer is not None and writer.state is ConsumerReservationState.WRITING:
if writer.shape != shape or writer.dtype != dtype_name:
raise ValueError("conflicting in-flight tensor for mm_hash")
record = ConsumerReservation(
transfer_id,
mm_hash,
uuid.uuid4().hex,
ConsumerReservationState.WRITING,
shape,
dtype_name,
writer.allocation,
expires_at=now + self._lease_ttl,
writer_id=writer.transfer_id,
)
self._insert(record)
self._followers.setdefault(writer.transfer_id, {})[transfer_id] = None
return record, False
allocation = self._memory.try_allocate(nbytes, shape, dtype)
if allocation is None:
self._expire_locked(time.monotonic())
allocation = self._memory.try_allocate(nbytes, shape, dtype)
if allocation is None:
allocation = self._memory.reclaim_and_allocate(nbytes, shape, dtype)
if allocation is None:
return None, False
record = ConsumerReservation(
transfer_id,
mm_hash,
uuid.uuid4().hex,
ConsumerReservationState.WRITING,
shape,
dtype_name,
allocation,
None,
now + self._lease_ttl,
)
self._insert(record)
self._writers[mm_hash] = record
return record, True
def status(self, transfer_id: str) -> ConsumerReservation | None:
with self._memory.lock:
record = self._records.get(transfer_id)
if record is None or record.state not in _ACTIVE_STATES:
return None
return record
def complete(self, transfer_id: str, reservation_id: str) -> tuple[bool, bool]:
with self._condition:
try:
record = self._records.get(transfer_id)
if record is None or record.reservation_id != reservation_id:
return False, False
if record.state is ConsumerReservationState.READY:
return True, False
if record.writer_id:
return False, False
if record.state in _WRITER_OWNED_STATES:
self._publish_followers(record)
if record.state in _DEFERRED_STATES:
terminal = (
ConsumerReservationState.CANCELLED
if record.state is ConsumerReservationState.CANCEL_PENDING
else ConsumerReservationState.EXPIRED
)
self._terminate(record, terminal)
return True, False
if record.state is not ConsumerReservationState.WRITING:
return False, False
self._transition(record, ConsumerReservationState.READY)
record.expires_at = time.monotonic() + self._lease_ttl
return True, True
finally:
self._condition.notify_all()
def _publish_followers(self, writer: ConsumerReservation) -> None:
if self._writers.get(writer.mm_hash) is writer:
self._writers.pop(writer.mm_hash)
followers = self._followers.pop(writer.transfer_id, {})
if not followers:
return
assert writer.allocation is not None
allocation = self._memory.publish(writer.mm_hash, writer.allocation, pin=False)
for record in [writer, *(self._records[key] for key in followers)]:
record.allocation = allocation
record.lease = self._memory.acquire_cached(
record.mm_hash, record.shape, allocation.tensor.dtype
)
assert record.lease is not None
if record is not writer:
record.writer_id = ""
self._transition(record, ConsumerReservationState.READY)
record.expires_at = time.monotonic() + self._lease_ttl
self._ready.append(record.transfer_id)
def drain_ready(self) -> list[str]:
with self._memory.lock:
ready, self._ready = self._ready, []
return ready
def begin_shutdown(self) -> None:
"""Stop new reservations and cancel everything without a remote writer."""
with self._condition:
if self._shutting_down:
return
self._shutting_down = True
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if record.writer_id or record.state is ConsumerReservationState.READY:
self._terminate(record, ConsumerReservationState.CANCELLED)
elif record.state is ConsumerReservationState.WRITING:
self._defer(record, ConsumerReservationState.CANCEL_PENDING)
self._condition.notify_all()
def wait_for_writers(self, timeout: float) -> bool:
"""Wait until no reservation is still owned by a remote writer."""
def writers_finished() -> bool:
return not any(
self._records[transfer_id].state in _WRITER_OWNED_STATES
for transfer_id in self._active_ids
)
with self._condition:
return self._condition.wait_for(writers_finished, timeout=max(0, timeout))
def cancel(
self,
transfer_id: str,
reservation_id: str,
abandon: bool = False,
refresh: bool = False,
) -> bool:
with self._memory.lock:
record = self._records.get(transfer_id)
if (
record is not None
and reservation_id
and record.reservation_id != reservation_id
):
return False
if record is None:
record = ConsumerReservation(
transfer_id,
"",
"",
ConsumerReservationState.CANCELLED,
)
self._insert(record)
self._set_tombstone_deadline(record)
self._reap_tombstones(time.monotonic())
return True
if record.state is ConsumerReservationState.CANCELLED:
self._set_tombstone_deadline(record)
self._reap_tombstones(time.monotonic())
return True
if refresh:
if not abandon or record.state not in {
ConsumerReservationState.WRITING,
ConsumerReservationState.EXPIRE_PENDING,
}:
return False
if record.state is ConsumerReservationState.WRITING:
self._transition(record, ConsumerReservationState.EXPIRE_PENDING)
self._terminate(record, ConsumerReservationState.EXPIRED)
self._condition.notify_all()
self._reap_tombstones(time.monotonic())
return True
if record.writer_id:
self._terminate(record, ConsumerReservationState.CANCELLED)
return True
if record.state in _DEFERRED_STATES and not abandon:
return True
if record.state is ConsumerReservationState.WRITING and not abandon:
self._defer(record, ConsumerReservationState.CANCEL_PENDING)
return True
if record.state not in _ACTIVE_STATES:
return False
self._terminate(record, ConsumerReservationState.CANCELLED)
if self._shutting_down:
self._condition.notify_all()
self._reap_tombstones(time.monotonic())
return True
def take(self, transfer_id: str, mm_hash: str) -> MemoryAllocation:
with self._memory.lock:
record = self._records.get(transfer_id)
if (
record is None
or record.state is not ConsumerReservationState.READY
or record.mm_hash != mm_hash
or record.allocation is None
):
raise RuntimeError(
f"Pushed EC tensor is not ready for mm_hash={mm_hash}"
)
allocation = self._memory.publish(mm_hash, record.allocation, record.lease)
record.allocation = None
record.lease = None
self._remove(transfer_id)
return allocation
def expire(self) -> int:
with self._memory.lock:
return self._expire_locked(time.monotonic())
def retire_stale(
self, encoder_cache: dict[str, torch.Tensor], freed: list[str] | None = None
) -> None:
with self._memory.lock:
self._memory.retire_stale(encoder_cache, freed=freed)
def _expire_locked(self, now: float) -> int:
expired = 0
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if record.expires_at > now:
continue
if record.state is ConsumerReservationState.READY:
self._terminate(record, ConsumerReservationState.EXPIRED)
expired += 1
elif record.state is ConsumerReservationState.WRITING:
if record.writer_id:
self._terminate(record, ConsumerReservationState.CANCELLED)
else:
self._defer(record, ConsumerReservationState.EXPIRE_PENDING)
self._reap_tombstones(now)
return expired
def _defer(
self, record: ConsumerReservation, state: ConsumerReservationState
) -> None:
"""Keep the destination until the writer completes or abandons it."""
self._transition(record, state)
record.expires_at = float("inf")
def _terminate(
self, record: ConsumerReservation, state: ConsumerReservationState
) -> None:
self._transition(record, state)
self._release(record)
self._set_tombstone_deadline(record)
def _release(self, record: ConsumerReservation) -> None:
if record.writer_id:
followers = self._followers.get(record.writer_id, {})
followers.pop(record.transfer_id, None)
record.allocation = None
return
if self._writers.get(record.mm_hash) is record:
self._writers.pop(record.mm_hash)
for transfer_id in list(self._followers.pop(record.transfer_id, {})):
self._terminate(
self._records[transfer_id], ConsumerReservationState.CANCELLED
)
allocation = record.allocation
if allocation is None:
return
if record.lease is not None:
self._memory.release_cached(record.lease)
else:
self._memory.free(allocation)
record.allocation = None
record.lease = None
def _set_tombstone_deadline(self, record: ConsumerReservation) -> None:
record.expires_at = time.monotonic() + self._lease_ttl
self._tombstones[record.transfer_id] = None
self._tombstones.move_to_end(record.transfer_id)
def _reap_tombstones(self, now: float) -> int:
dropped = 0
while self._tombstones:
transfer_id = next(iter(self._tombstones))
record = self._records[transfer_id]
if (
record.expires_at > now
and len(self._tombstones) <= self._tombstone_limit
):
break
self._remove(transfer_id)
dropped += 1
return dropped
def _insert(self, record: ConsumerReservation) -> None:
self._records[record.transfer_id] = record
if record.state in _ACTIVE_STATES:
self._active_ids[record.transfer_id] = None
def _remove(self, transfer_id: str) -> None:
self._records.pop(transfer_id, None)
self._active_ids.pop(transfer_id, None)
self._tombstones.pop(transfer_id, None)
def _transition(
self, record: ConsumerReservation, state: ConsumerReservationState
) -> None:
if state not in _ALLOWED_TRANSITIONS[record.state]:
raise RuntimeError(
f"Cannot transition {record.transfer_id!r} from "
f"{record.state.name} to {state.name}"
)
record.state = state
if state in _ACTIVE_STATES:
self._active_ids[record.transfer_id] = None
else:
self._active_ids.pop(record.transfer_id, None)
@@ -1,561 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scheduler-side planning and observation for Mooncake cache transfers.
The Scheduler prepares Producer reservations, consumes Consumer readiness
events, emits per-step Worker metadata, and converts Worker results back into
request availability without owning tensor memory or running data transfers.
"""
from __future__ import annotations
import math
import time
from collections import OrderedDict
from concurrent.futures import Future, ThreadPoolExecutor
from typing import TYPE_CHECKING, Any
import torch
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
get_encoder_output_width,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
_RESERVATION_TTL_SECONDS,
MooncakeECConfig,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.control import (
ControlClient,
EventInbox,
make_cancel_request,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeConnectorMetadata,
ECMooncakeLoadSpec,
ECMooncakePushSpec,
ECMooncakeWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.state import (
SchedulerTransferState,
SchedulerTransferTable,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
ensure_mooncake_available,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
TensorSpec,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ECConnectorOutput
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
# Unmatched `ec_items` entries are a protocol desync, not a per-request
# event: warn on the first few, then only once in a while.
_MAX_UNRESOLVED_TRANSFER_ID_WARNINGS = 5
_DRAIN_MIN_INTERVAL = 0.005
_MAX_PENDING_EVENTS = 4096
_MAX_TERMINAL_TRANSFER_RECORDS = 1 << 16
_CANCEL_ATTEMPTS = 2
_CONTROL_WORKERS = 8
class ECMooncakeScheduler:
"""Coordinate Mooncake transfers from the vLLM Scheduler process."""
def __init__(self, vllm_config: VllmConfig) -> None:
ensure_mooncake_available()
config = MooncakeECConfig.from_vllm_config(vllm_config)
self._store_candidates: list[tuple[str, str, TensorSpec]] = []
self._cross_encoder_cache = config.cross_encoder_cache
self._is_producer = config.is_producer
self._is_consumer = config.is_consumer
self._control_addr = config.control_addr
self._push_wait_timeout = config.push_wait_timeout_s
self._vllm_config = vllm_config
self._model_config = vllm_config.model_config
self._control_client = ControlClient(config.control_timeout_ms)
self._control_executor = ThreadPoolExecutor(
max_workers=_CONTROL_WORKERS,
thread_name_prefix="ec-mooncake-control",
)
self._event_inbox = EventInbox(self._control_client, self._control_executor)
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
self._unresolved_transfer_ids = 0
self._drain_pending = True
self._drained_at = 0.0
self._pending_cancels: dict[str, Future[Any]] = {}
self._transfers = SchedulerTransferTable(
config.pool_size, _RESERVATION_TTL_SECONDS
)
self._scheduler_pending_work = False
self._pushes_to_prepare: dict[str, ECMooncakePushSpec] = {}
self._prepared_push_transfer_ids: set[str] = set()
self._event_ready_shards: OrderedDict[str, set[int]] = OrderedDict()
# Mirror of the engine's encoder cache, maintained from the alloc and
# free notifications the Scheduler already sends. Tracking it here
# keeps `ensure_cache_available` on the upstream two-argument shape.
self._local_cache: set[str] = set()
self._failed_saves: set[str] = set()
def _cancel_remote(self, consumer_zmq: str, transfer_id: str) -> bool:
pending = None
for _ in range(_CANCEL_ATTEMPTS):
pending = self._control_client.discover_shards(consumer_zmq)
if pending is not None:
break
if pending is None:
raise RuntimeError(
f"Could not discover every EC consumer shard at {consumer_zmq}"
)
cancelled = False
error: BaseException | None = None
for _ in range(_CANCEL_ATTEMPTS):
failed = []
for addr in pending:
try:
result = self._control_client.request(
addr,
make_cancel_request(transfer_id),
)
except Exception as exc:
if error is None:
error = exc
failed.append(addr)
continue
cancelled |= isinstance(result, dict) and bool(result.get("cancelled"))
if not failed:
return cancelled
pending = failed
if error is not None:
raise error
return cancelled
def _note_awaiting_push(
self,
mm_hash: str,
transfer_id: str,
request_id: str,
now: float,
) -> None:
record = self._transfers.wait_for_event(
transfer_id,
request_id,
mm_hash,
now + self._push_wait_timeout,
)
if record is None:
# The id names another encoding; the request is already failed.
return
if record.state is not SchedulerTransferState.WAITING_EVENT:
return
assert record.deadline is not None
if now < record.deadline:
return
elapsed = now - record.deadline + self._push_wait_timeout
self._transfers.mark_unavailable(transfer_id, now)
logger.warning(
"EC Mooncake waited %.1fs for a push of mm_hash=%s "
"(transfer_id=%s) that never arrived; "
"requests needing it fail with a retryable error.",
elapsed,
mm_hash,
transfer_id,
)
def take_unavailable_requests(self) -> set[str]:
failed, self._failed_saves = self._failed_saves, set()
return failed | self._transfers.take_unavailable_requests()
def _poll_pending_cancels(self) -> None:
pending = {}
for transfer_id, future in self._pending_cancels.items():
if not future.done():
pending[transfer_id] = future
continue
try:
future.result()
except Exception:
logger.warning(
"EC Mooncake reservation cancellation failed", exc_info=True
)
self._pending_cancels = pending
def _note_shard_ready(self, data: dict[str, Any]) -> bool:
shard_count = self._event_inbox.shard_count
if shard_count <= 1:
return True
transfer_id = str(data["transfer_id"])
record = self._transfers.get(transfer_id)
if (
record is not None
and record.state is not SchedulerTransferState.WAITING_EVENT
):
return False
shard = data.get("shard")
shards = self._event_ready_shards.setdefault(transfer_id, set())
self._event_ready_shards.move_to_end(transfer_id)
shards.add(int(shard) if shard is not None else len(shards))
if len(shards) < shard_count:
while len(self._event_ready_shards) > _MAX_PENDING_EVENTS:
self._event_ready_shards.popitem(last=False)
return False
self._event_ready_shards.pop(transfer_id, None)
return True
def _forget_shard_readiness(self, transfer_id: str) -> None:
self._event_ready_shards.pop(transfer_id, None)
def _store_pushed_spec(self, data: dict[str, Any]) -> bool:
transfer_id = str(data["transfer_id"])
identifier = str(data["mm_hash"])
_, accepted = self._transfers.observe_ready(
ECMooncakeLoadSpec(
mm_hash=identifier,
nbytes=int(data["nbytes"]),
shape=tuple(int(value) for value in data["shape"]),
dtype=str(data["dtype"]),
transfer_id=transfer_id,
),
time.monotonic() + _RESERVATION_TTL_SECONDS,
)
return accepted
def _queue_cancel(
self,
transfer_id: str,
mm_hash: str = "",
request_id: str = "",
) -> None:
if not self._transfers.cancel(
transfer_id,
time.monotonic(),
mm_hash=mm_hash,
request_id=request_id,
):
return
self._forget_shard_readiness(transfer_id)
self._pending_cancels[transfer_id] = self._control_executor.submit(
self._cancel_remote,
self._control_addr,
transfer_id,
)
def _expire_transfers(self) -> None:
now = time.monotonic()
expired = self._transfers.expire(now, _MAX_TERMINAL_TRANSFER_RECORDS)
for record in expired:
self._queue_cancel(record.transfer_id)
def _drain_push_notifications(self) -> None:
now = time.monotonic()
if not self._drain_pending and now - self._drained_at < _DRAIN_MIN_INTERVAL:
return
self._drain_pending = False
self._drained_at = now
self._poll_pending_cancels()
self._expire_transfers()
events = self._event_inbox.drain(self._control_addr)
for data in events:
try:
if not isinstance(data, dict):
raise TypeError("EC readiness event must be an object")
if not data.get("ready"):
continue
self._accept_ready_event(data)
except (KeyError, TypeError, ValueError):
# Readiness events cross a plain PULL socket, so a malformed
# one costs that event and not the engine.
logger.warning(
"Discarding a malformed EC readiness event.", exc_info=True
)
def _accept_ready_event(self, data: dict[str, Any]) -> None:
transfer_id = str(data["transfer_id"])
record = self._transfers.get(transfer_id)
if record is not None and record.state in {
SchedulerTransferState.CANCELLED,
SchedulerTransferState.UNAVAILABLE,
SchedulerTransferState.EXPIRED,
SchedulerTransferState.FAILED,
}:
return
if not self._note_shard_ready(data):
return
self._store_pushed_spec(data)
def has_cache_item(self, identifier: str) -> bool:
if not self._is_consumer:
return False
self._drain_push_notifications()
return self._transfers.has_state(
identifier,
(
SchedulerTransferState.READY,
SchedulerTransferState.RESIDENT,
SchedulerTransferState.AVAILABLE,
),
)
def _warn_unresolved_transfer_id(
self, request: Any, index: int, where: str
) -> None:
"""Report an `ec_items` entry that cannot be matched to a feature.
A caller that cannot resolve a transfer id has to fall back to a
locally invented one or skip its bookkeeping entirely, and either way
the two sides of a transfer stop agreeing on its name. That desyncs
silently -- reservations are never released and only surface minutes
later as a full buffer pool -- so say it out loud the first time.
`mm_hash` in `ec_items` must be a value some engine reported, never one
the caller derived itself: `mm_features[i].identifier` folds in the
engine's media_io_kwargs and mm_processor_kwargs, so a media uuid alone
no longer equals it. Omit the field to match by position instead.
"""
self._unresolved_transfer_ids += 1
count = self._unresolved_transfer_ids
if count > _MAX_UNRESOLVED_TRANSFER_ID_WARNINGS and count % 1000:
return
params = getattr(request, "ec_transfer_params", None)
items = (params or {}).get("ec_items") or []
logger.warning(
"EC Mooncake could not resolve a transfer id at %s for req=%s "
"index=%d: feature identifier=%s does not match any of the %d "
"ec_items %s (ec_transfer_params present=%s). Occurrence %d. "
"ec_items[].mm_hash must echo an engine-reported identifier, or "
"be omitted so the entry is matched by position.",
where,
request.request_id,
index,
request.mm_features[index].identifier[:16],
len(items),
[str(item.get("mm_hash"))[:16] for item in items[:4]],
params is not None,
count,
)
@staticmethod
def _request_transfer_id(request: Any, index: int) -> str | None:
params = getattr(request, "ec_transfer_params", None) or {}
items = params.get("ec_items") or []
mm_hash = request.mm_features[index].identifier
if index < len(items):
item = items[index]
if item.get("mm_hash") in (None, mm_hash) and item.get("transfer_id"):
return str(item["transfer_id"])
for item in items:
if item.get("mm_hash") == mm_hash and item.get("transfer_id"):
return str(item["transfer_id"])
return None
def ensure_cache_available(self, request: Any, num_computed_tokens: int) -> bool:
if self._is_producer:
for index, feature in enumerate(request.mm_features):
if (
feature.mm_position.offset + feature.mm_position.length
> num_computed_tokens
):
self._prepare_push_spec(request, index)
if not self._is_consumer:
return True
self._drain_push_notifications()
# One timestamp for the whole decision, so the deadlines it hands out
# cannot disagree between features.
now = time.monotonic()
all_ready = True
for index, feature in enumerate(request.mm_features):
if (
feature.mm_position.offset + feature.mm_position.length
<= num_computed_tokens
):
continue
mm_hash = feature.identifier
transfer_id = self._request_transfer_id(request, index)
if transfer_id is not None:
self._transfers.touch_available(
transfer_id, now + _RESERVATION_TTL_SECONDS
)
if mm_hash in self._local_cache:
continue
if self._transfers.has_state(mm_hash, (SchedulerTransferState.READY,)):
continue
if self._transfers.has_state(mm_hash, (SchedulerTransferState.LOADING,)):
all_ready = False
continue
record = self._transfers.begin_load(
mm_hash,
transfer_id,
request.request_id,
now + self._push_wait_timeout,
)
if record is not None:
self._scheduler_pending_work = True
all_ready = False
else:
waiting_id = transfer_id or f"{request.request_id}:{index}"
self._note_awaiting_push(mm_hash, waiting_id, request.request_id, now)
all_ready = False
return all_ready
def _prepare_push_spec(self, request: Any, index: int) -> None:
params = getattr(request, "ec_transfer_params", None) or {}
consumer_zmq = params.get("consumer_zmq")
mm_hash = request.mm_features[index].identifier
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
# Still push: after the proxy has rewritten the item to embeds the
# consumer has no media left to fall back on, so a nameless push
# beats none. But the consumer knows this transfer by the id it
# sent, not by the one invented here, so its cancel will never
# reach the reservation this push is about to take.
if consumer_zmq:
self._warn_unresolved_transfer_id(request, index, "push prepare")
transfer_id = f"{request.request_id}:{index}"
if not consumer_zmq or transfer_id in self._prepared_push_transfer_ids:
return
output = self._encoder_output_spec(request, index)
self._pushes_to_prepare[transfer_id] = ECMooncakePushSpec(
mm_hash=mm_hash,
nbytes=output.nbytes,
shape=output.shape,
dtype=output.dtype.removeprefix("torch."),
consumer_zmq=str(consumer_zmq),
transfer_id=transfer_id,
request_id=request.request_id,
)
self._prepared_push_transfer_ids.add(transfer_id)
def _encoder_output_spec(self, request: Any, index: int) -> TensorSpec:
dtype = self._model_config.dtype
assert isinstance(dtype, torch.dtype)
hidden_dim = get_encoder_output_width(
self._vllm_config, request.mm_features[index].modality
)
shape = (request.get_num_encoder_embeds(index), hidden_dim)
return TensorSpec(shape, str(dtype), math.prod(shape) * dtype.itemsize)
def update_state_after_alloc(self, request: Any, index: int) -> None:
self._local_cache.add(request.mm_features[index].identifier)
if self._is_producer:
self._prepare_push_spec(request, index)
if (
self._cross_encoder_cache
and request.mm_features[index].modality == "image"
):
self._store_candidates.append(
(
request.request_id,
request.mm_features[index].identifier,
self._encoder_output_spec(request, index),
)
)
def update_state_after_free(self, request: Any, index: int) -> None:
if not self._is_consumer:
return
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
self._warn_unresolved_transfer_id(request, index, "encoder-cache free")
return
self._queue_cancel(
transfer_id,
mm_hash=request.mm_features[index].identifier,
request_id=request.request_id,
)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> ECConnectorMetadata:
for mm_hash in scheduler_output.free_encoder_mm_hashes:
self._local_cache.discard(mm_hash)
self._transfers.release_ready(mm_hash, time.monotonic())
for transfer_id in self._transfers.drain_orphaned():
self._queue_cancel(transfer_id)
meta = ECMooncakeConnectorMetadata(
freed=scheduler_output.free_encoder_mm_hashes
)
preempted = scheduler_output.preempted_req_ids or set()
meta.store_candidates = {
identifier: output
for request_id, identifier, output in self._store_candidates
if request_id not in preempted
}
self._store_candidates.clear()
for push_spec in self._pushes_to_prepare.values():
meta.pushes.append(push_spec)
self._pushes_to_prepare.clear()
for record in self._transfers.take_loads_to_dispatch():
assert record.spec is not None
meta.loads.append(record.spec)
self._poll_pending_cancels()
self._drain_pending = True
return meta
def update_connector_output(self, connector_output: ECConnectorOutput) -> None:
meta = connector_output.ec_connector_worker_meta
if not isinstance(meta, ECMooncakeWorkerMetadata):
return
for mm_hash in meta.loaded:
self._transfers.complete_load(mm_hash)
for mm_hash in meta.failed_loads:
self._transfers.fail_load(mm_hash, time.monotonic())
for mm_hash in meta.reclaimed:
self._transfers.reclaim(mm_hash, time.monotonic())
self._scheduler_pending_work = meta.pending_saves
self._failed_saves.update(meta.failed_saves)
def has_pending_push_work(self) -> bool:
return self._scheduler_pending_work
def request_finished(self, request: Any) -> tuple[bool, dict[str, Any] | None]:
if self._is_consumer:
for index in range(len(request.mm_features)):
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
self._warn_unresolved_transfer_id(request, index, "request finish")
continue
self._queue_cancel(
transfer_id,
mm_hash=request.mm_features[index].identifier,
request_id=request.request_id,
)
if self._is_producer and self._prepared_push_transfer_ids:
for index in range(len(request.mm_features)):
transfer_id = self._request_transfer_id(request, index)
if transfer_id is None:
transfer_id = f"{request.request_id}:{index}"
self._prepared_push_transfer_ids.discard(transfer_id)
if not self._is_producer:
return False, None
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
for index, feature in enumerate(request.mm_features):
transfer_id = self._request_transfer_id(request, index)
if transfer_id is not None:
items[feature.identifier]["transfer_id"] = transfer_id
if not items:
return False, None
return False, items
def close(self) -> None:
self._control_executor.shutdown(wait=True, cancel_futures=True)
self._control_client.close()
self._event_inbox.close()
@@ -1,448 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Scheduler-owned lifecycle and indexes for Consumer-bound transfers."""
from __future__ import annotations
from collections import OrderedDict, deque
from collections.abc import Iterable
from dataclasses import dataclass, field, replace
from enum import Enum, auto
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeLoadSpec,
)
from vllm.logger import init_logger
logger = init_logger(__name__)
class SchedulerTransferState(Enum):
"""Lifecycle states visible to the Scheduler.
The states cover waiting for a Consumer event, dispatching a Worker load,
retaining a locally reusable tensor, and terminal failure conditions.
"""
WAITING_EVENT = auto()
AVAILABLE = auto()
LOADING = auto()
READY = auto()
RESIDENT = auto()
UNAVAILABLE = auto()
EXPIRED = auto()
FAILED = auto()
CANCELLED = auto()
@dataclass
class SchedulerTransfer:
"""Track one transfer as observed by the Scheduler."""
transfer_id: str
request_id: str
mm_hash: str
state: SchedulerTransferState
spec: ECMooncakeLoadSpec | None
deadline: float | None
notified_requests: set[str] = field(default_factory=set, repr=False)
_TERMINAL_STATES = {
SchedulerTransferState.UNAVAILABLE,
SchedulerTransferState.EXPIRED,
SchedulerTransferState.FAILED,
SchedulerTransferState.CANCELLED,
}
class SchedulerTransferTable:
"""Own Scheduler transfer state, lookup indexes, and dispatch queues.
The Scheduler is single-threaded, so direct transitions are sufficient;
the Producer and Consumer managers retain stricter transition matrices
because callbacks and control requests can race there.
"""
def __init__(self, resident_capacity: int, tombstone_ttl: float) -> None:
self._resident_capacity = resident_capacity
self._tombstone_ttl = tombstone_ttl
self._records: OrderedDict[str, SchedulerTransfer] = OrderedDict()
self._active_ids: dict[str, None] = {}
self._terminal_ids: OrderedDict[str, None] = OrderedDict()
self._resident_ids: OrderedDict[str, None] = OrderedDict()
self._resident_bytes = 0
self._hash_index: dict[str, deque[str]] = {}
self._loads_to_dispatch: OrderedDict[str, None] = OrderedDict()
self._unavailable_requests: set[str] = set()
# Records expired straight out of READY/RESIDENT: the consumer worker
# still holds their push reservation, and nothing else tells it to let
# go, so the destination buffer would sit pinned until the lease TTL.
self._orphaned: list[str] = []
def get(self, transfer_id: str) -> SchedulerTransfer | None:
return self._records.get(transfer_id)
def drain_orphaned(self) -> list[str]:
"""Transfer IDs expired out of READY/RESIDENT since the last drain.
The caller must release the consumer worker's reservation for each:
`cancel()` still accepts an EXPIRED record, so the usual cancel path
applies.
"""
orphaned = self._orphaned
self._orphaned = []
return orphaned
def records_for_hash(
self,
mm_hash: str,
states: Iterable[SchedulerTransferState],
) -> list[SchedulerTransfer]:
wanted = set(states)
return [
record
for transfer_id in self._hash_index.get(mm_hash, ())
if (record := self._records.get(transfer_id)) is not None
and record.state in wanted
]
def first_for_hash(
self,
mm_hash: str,
states: Iterable[SchedulerTransferState],
) -> SchedulerTransfer | None:
states = tuple(states)
if len(states) == 1:
wanted_state = states[0]
for transfer_id in self._hash_index.get(mm_hash, ()):
record = self._records.get(transfer_id)
if record is not None and record.state is wanted_state:
return record
return None
wanted_states = set(states)
for transfer_id in self._hash_index.get(mm_hash, ()):
record = self._records.get(transfer_id)
if record is not None and record.state in wanted_states:
return record
return None
def has_state(self, mm_hash: str, states: Iterable[SchedulerTransferState]) -> bool:
return self.first_for_hash(mm_hash, states) is not None
def wait_for_event(
self,
transfer_id: str,
request_id: str,
mm_hash: str,
deadline: float,
) -> SchedulerTransfer | None:
"""Track a request waiting for a push.
Returns None when `transfer_id` already names a different encoding.
The id comes from the request, so a collision has to fail that one
request; re-issuing it re-runs the encode under a fresh id.
"""
record = self._records.get(transfer_id)
if record is None:
record = SchedulerTransfer(
transfer_id=transfer_id,
request_id=request_id,
mm_hash=mm_hash,
state=SchedulerTransferState.WAITING_EVENT,
spec=None,
deadline=deadline,
)
self._insert(record)
elif not self._identity_matches(record, mm_hash):
self._refuse_colliding_id(record, mm_hash, request_id)
return None
else:
if not record.request_id:
record.request_id = request_id
if record.state in _TERMINAL_STATES and request_id:
self._notify_unavailable(record, request_id)
return record
def observe_ready(
self, spec: ECMooncakeLoadSpec, deadline: float
) -> tuple[SchedulerTransfer, bool]:
transfer_id = spec.transfer_id
record = self._records.get(transfer_id)
if record is None:
record = SchedulerTransfer(
transfer_id=transfer_id,
request_id="",
mm_hash=spec.mm_hash,
state=SchedulerTransferState.WAITING_EVENT,
spec=None,
deadline=None,
)
self._insert(record)
elif not self._identity_matches(record, spec.mm_hash):
self._refuse_colliding_id(record, spec.mm_hash, record.request_id)
return record, False
if record.state is not SchedulerTransferState.WAITING_EVENT:
return record, False
record.spec = spec
record.deadline = deadline
self._transition(record, SchedulerTransferState.AVAILABLE)
return record, True
def touch_available(self, transfer_id: str, deadline: float) -> None:
record = self._records.get(transfer_id)
if record is not None and record.state is SchedulerTransferState.AVAILABLE:
record.deadline = deadline
def begin_load(
self,
mm_hash: str,
transfer_id: str | None = None,
request_id: str = "",
deadline: float | None = None,
) -> SchedulerTransfer | None:
record = self._records.get(transfer_id) if transfer_id else None
if record is not None and (
record.mm_hash != mm_hash
or record.state is not SchedulerTransferState.AVAILABLE
):
record = None
if record is None:
record = self.first_for_hash(
mm_hash,
(
SchedulerTransferState.AVAILABLE,
SchedulerTransferState.RESIDENT,
),
)
if record is None or record.spec is None:
return None
if not record.request_id:
record.request_id = request_id
# A dispatched load that never reports back would otherwise hold this
# hash in LOADING for good, deferring every later request for it.
record.deadline = deadline
self._transition(record, SchedulerTransferState.LOADING)
self._loads_to_dispatch[record.transfer_id] = None
return record
def take_loads_to_dispatch(self) -> list[SchedulerTransfer]:
records = [
record
for transfer_id in self._loads_to_dispatch
if (record := self._records.get(transfer_id)) is not None
and record.state is SchedulerTransferState.LOADING
]
self._loads_to_dispatch.clear()
return records
def complete_load(self, mm_hash: str) -> bool:
record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,))
if record is None:
return self.has_state(mm_hash, (SchedulerTransferState.READY,))
self._transition(record, SchedulerTransferState.READY)
return True
def fail_load(self, mm_hash: str, now: float) -> bool:
record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,))
if record is None:
return self.has_state(mm_hash, (SchedulerTransferState.FAILED,))
self._transition(record, SchedulerTransferState.FAILED, now=now)
return True
def release_ready(self, mm_hash: str, now: float) -> None:
ready = self.records_for_hash(mm_hash, (SchedulerTransferState.READY,))
if ready:
canonical = ready[-1]
for record in self.records_for_hash(
mm_hash,
(SchedulerTransferState.READY, SchedulerTransferState.RESIDENT),
):
if record is not canonical:
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
if canonical.spec is None:
self._transition(canonical, SchedulerTransferState.EXPIRED, now=now)
else:
canonical.spec = replace(canonical.spec, local=True)
self._transition(canonical, SchedulerTransferState.RESIDENT)
self._evict_residents(now)
def reclaim(self, mm_hash: str, now: float) -> None:
for record in self.records_for_hash(
mm_hash,
(SchedulerTransferState.READY, SchedulerTransferState.RESIDENT),
):
if record.state is SchedulerTransferState.READY:
record.spec = None
else:
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
def mark_unavailable(self, transfer_id: str, now: float) -> None:
record = self._records[transfer_id]
self._transition(record, SchedulerTransferState.UNAVAILABLE, now=now)
if record.request_id:
self._notify_unavailable(record, record.request_id)
def cancel(
self,
transfer_id: str,
now: float,
mm_hash: str = "",
request_id: str = "",
) -> bool:
record = self._records.get(transfer_id)
if (
record is not None
and mm_hash
and not self._identity_matches(record, mm_hash)
):
return False
if record is None:
record = SchedulerTransfer(
transfer_id=transfer_id,
request_id=request_id,
mm_hash=mm_hash,
state=SchedulerTransferState.WAITING_EVENT,
spec=None,
deadline=None,
)
self._insert(record)
if record.state in {
SchedulerTransferState.CANCELLED,
SchedulerTransferState.READY,
SchedulerTransferState.RESIDENT,
SchedulerTransferState.FAILED,
}:
return False
self._transition(record, SchedulerTransferState.CANCELLED, now=now)
self._loads_to_dispatch.pop(transfer_id, None)
return True
def expire(self, now: float, terminal_limit: int) -> list[SchedulerTransfer]:
if terminal_limit < 0:
raise ValueError("terminal_limit must be non-negative")
expired = []
for transfer_id in list(self._active_ids):
record = self._records[transfer_id]
if record.deadline is None or record.deadline > now:
continue
if record.state is SchedulerTransferState.AVAILABLE:
self._transition(
record,
SchedulerTransferState.EXPIRED,
now=now,
)
expired.append(record)
elif record.state is SchedulerTransferState.LOADING:
self._transition(
record,
SchedulerTransferState.UNAVAILABLE,
now=now,
)
if record.request_id:
self._notify_unavailable(record, record.request_id)
expired.append(record)
while self._terminal_ids:
transfer_id = next(iter(self._terminal_ids))
record = self._records[transfer_id]
if (
record.deadline is not None
and record.deadline > now
and len(self._terminal_ids) <= terminal_limit
):
break
self._remove(transfer_id)
return expired
def take_unavailable_requests(self) -> set[str]:
unavailable = self._unavailable_requests
self._unavailable_requests = set()
return unavailable
def _notify_unavailable(self, record: SchedulerTransfer, request_id: str) -> None:
if request_id not in record.notified_requests:
record.notified_requests.add(request_id)
self._unavailable_requests.add(request_id)
def _insert(self, record: SchedulerTransfer) -> None:
self._records[record.transfer_id] = record
self._active_ids[record.transfer_id] = None
if record.mm_hash:
self._hash_index.setdefault(record.mm_hash, deque()).append(
record.transfer_id
)
@staticmethod
def _identity_matches(record: SchedulerTransfer, mm_hash: str) -> bool:
return not record.mm_hash or record.mm_hash == mm_hash
def _refuse_colliding_id(
self, record: SchedulerTransfer, mm_hash: str, request_id: str
) -> None:
"""Report a transfer id that already names another encoding.
Transfer ids arrive on the request, so a collision is reachable from
outside and must not reach the engine as an exception.
"""
logger.warning(
"EC Mooncake transfer_id=%s already names mm_hash=%s; refusing "
"mm_hash=%s for request %s",
record.transfer_id,
record.mm_hash[:16],
mm_hash[:16],
request_id or "<unknown>",
)
if request_id:
self._unavailable_requests.add(request_id)
def _transition(
self,
record: SchedulerTransfer,
state: SchedulerTransferState,
now: float | None = None,
) -> None:
if record.state is SchedulerTransferState.RESIDENT:
self._resident_ids.pop(record.transfer_id, None)
if record.spec is not None:
self._resident_bytes -= record.spec.nbytes
if state is SchedulerTransferState.RESIDENT:
self._resident_ids[record.transfer_id] = None
if record.spec is not None:
self._resident_bytes += record.spec.nbytes
if state in _TERMINAL_STATES:
if now is None:
raise ValueError("Terminal transition requires a timestamp")
# Keep a bounded tombstone so late events and repeat request IDs
# remain idempotent instead of reviving a finished transfer.
record.deadline = now + self._tombstone_ttl
self._active_ids.pop(record.transfer_id, None)
self._terminal_ids[record.transfer_id] = None
self._terminal_ids.move_to_end(record.transfer_id)
if state is SchedulerTransferState.EXPIRED and record.state in (
SchedulerTransferState.READY,
SchedulerTransferState.RESIDENT,
):
self._orphaned.append(record.transfer_id)
record.state = state
self._records.move_to_end(record.transfer_id)
def _evict_residents(self, now: float) -> None:
while self._resident_bytes > self._resident_capacity and self._resident_ids:
record = self._records[next(iter(self._resident_ids))]
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
def _remove(self, transfer_id: str) -> None:
record = self._records.pop(transfer_id, None)
self._active_ids.pop(transfer_id, None)
self._terminal_ids.pop(transfer_id, None)
if transfer_id in self._resident_ids:
self._resident_ids.pop(transfer_id)
if record is not None and record.spec is not None:
self._resident_bytes -= record.spec.nbytes
self._loads_to_dispatch.pop(transfer_id, None)
if record is None or not record.mm_hash:
return
transfer_ids = self._hash_index[record.mm_hash]
transfer_ids.remove(transfer_id)
if not transfer_ids:
self._hash_index.pop(record.mm_hash)
@@ -1,209 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake data-plane engine and memory-registration ownership.
The wrapper isolates optional-engine initialization, session addressing,
synchronous batched writes, and the lifetime of transient source registrations.
"""
from __future__ import annotations
import threading
from dataclasses import dataclass
import torch
from vllm.logger import init_logger
logger = init_logger(__name__)
_MOONCAKE_IMPORT_ERROR: ImportError | None = None
try:
from mooncake.engine import TransferEngine
except ImportError as error:
TransferEngine = None # type: ignore[misc, assignment]
_MOONCAKE_IMPORT_ERROR = error
def ensure_mooncake_available() -> None:
if _MOONCAKE_IMPORT_ERROR is not None:
raise ImportError(
"Install mooncake-transfer-engine to use ECMooncakeConnector."
) from _MOONCAKE_IMPORT_ERROR
@dataclass
class _SourceRegistration:
"""Retain one transient source range while batches reference it."""
tensor: torch.Tensor
nbytes: int
users: int = 1
class MooncakeTransfer:
"""Own a lazy Mooncake engine and transient memory registrations."""
def __init__(self, hostname: str, protocol: str) -> None:
self._hostname = hostname
self._protocol = protocol
self._engine: TransferEngine | None = None
self._engine_lock = threading.Lock()
self._source_registrations: dict[int, _SourceRegistration] = {}
self._pending_unregister: dict[int, torch.Tensor] = {}
self._registration_lock = threading.Lock()
self._closed = False
def _ensure_engine(self) -> TransferEngine:
if self._engine is not None:
return self._engine
with self._engine_lock:
if self._engine is not None:
return self._engine
engine = TransferEngine()
ret = engine.initialize(self._hostname, "P2PHANDSHAKE", self._protocol, "")
if ret != 0:
raise RuntimeError("Mooncake TransferEngine initialization failed.")
self._engine = engine
logger.info(
"ECMooncakeConnector TransferEngine ready at %s:%d",
self._hostname,
engine.get_rpc_port(),
)
return self._engine
def ensure_ready(self) -> None:
self._ensure_engine()
def local_session(self) -> str:
engine = self._ensure_engine()
return f"{self._hostname}:{engine.get_rpc_port()}"
def register_memory(self, tensor: torch.Tensor) -> int:
return self._ensure_engine().batch_register_memory(
[tensor.data_ptr()], [tensor.nbytes]
)
def unregister_memory(self, tensor: torch.Tensor) -> bool:
engine = self._ensure_engine()
address = tensor.data_ptr()
ret = engine.unregister_memory(address)
with self._registration_lock:
if ret != 0:
logger.error(
"Mooncake EC memory unregistration failed for address %d: %d",
address,
ret,
)
self._pending_unregister[address] = tensor
return False
self._pending_unregister.pop(address, None)
return True
@staticmethod
def _source_range(tensor: torch.Tensor) -> tuple[int, int]:
# Encoder batches commonly split one storage into sibling tensor views.
# Registering each view's exact bytes avoids overlapping memory regions.
return tensor.data_ptr(), tensor.nbytes
def acquire_sources(self, tensors: list[torch.Tensor]) -> list[int]:
ranges: dict[int, tuple[int, torch.Tensor]] = {}
for tensor in tensors:
address, nbytes = self._source_range(tensor)
ranges.setdefault(address, (nbytes, tensor))
engine = self._ensure_engine()
acquired: list[int] = []
new_addresses: list[int] = []
new_lengths: list[int] = []
with self._registration_lock:
for address, (nbytes, tensor) in ranges.items():
entry = self._source_registrations.get(address)
if entry is not None:
if entry.nbytes != nbytes:
raise RuntimeError(
"Mooncake EC source storage changed size while registered"
)
entry.users += 1
acquired.append(address)
continue
new_addresses.append(address)
new_lengths.append(nbytes)
self._source_registrations[address] = _SourceRegistration(
tensor=tensor,
nbytes=nbytes,
)
acquired.append(address)
if new_addresses:
ret = engine.batch_register_memory(new_addresses, new_lengths)
if ret != 0:
for address in acquired:
entry = self._source_registrations[address]
entry.users -= 1
if entry.users == 0:
del self._source_registrations[address]
raise RuntimeError("Mooncake EC source registration failed")
return acquired
def release_sources(self, addresses: list[int]) -> bool:
if not addresses:
return True
with self._registration_lock:
unused = []
for address in addresses:
entry = self._source_registrations.get(address)
if entry is None:
continue
entry.users -= 1
if entry.users == 0:
unused.append(address)
if not unused:
return True
ret = self._ensure_engine().batch_unregister_memory(unused)
if ret != 0:
logger.warning(
"Keeping %d EC source tensors registered after Mooncake "
"unregistration failure",
len(unused),
)
return False
for address in unused:
del self._source_registrations[address]
self._pending_unregister.pop(address, None)
return True
def write(
self,
session: str,
sources: list[int],
destinations: list[int],
lengths: list[int],
) -> None:
"""Write one synchronous batch, returning only at terminal status."""
ret = self._ensure_engine().batch_transfer_sync_write(
session, sources, destinations, lengths
)
if ret != 0:
raise RuntimeError(
f"Mooncake EC push to {session} failed with status {ret}"
)
def close(self) -> None:
if self._closed:
return
self._closed = True
engine = self._engine
if engine is None:
return
with self._registration_lock:
addresses = list(self._source_registrations)
addresses.extend(self._pending_unregister)
if not addresses:
return
ret = engine.batch_unregister_memory(list(dict.fromkeys(addresses)))
if ret != 0:
logger.error("Mooncake EC batch memory unregistration failed: %d", ret)
return
self._source_registrations.clear()
self._pending_unregister.clear()
@@ -1,988 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Worker-side orchestration of Mooncake control, memory, and data planes.
Consumer Workers expose rank-local reservations and publish received tensors.
Producer Workers reserve every destination shard, bind computed sources, run
batched Mooncake writes, and report asynchronous completion to the Scheduler.
"""
from __future__ import annotations
import math
import threading
import time
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
from functools import partial
from typing import TYPE_CHECKING, Any, TypeVar, cast
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
_RESERVATION_TTL_SECONDS,
MooncakeECConfig,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.control import (
ConsumerControlServer,
ControlClient,
make_cancel_request,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import (
ConsumerMemoryPool,
ProducerMemoryPool,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
ECMooncakeConnectorMetadata,
ECMooncakePushSpec,
ECMooncakeWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.producer import (
ProducerPushManager,
ProducerPushRecord,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.reservation import (
ConsumerReservationManager,
ConsumerReservationState,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
MooncakeTransfer,
ensure_mooncake_available,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
build_embedding_namespace,
)
from vllm.logger import init_logger
from vllm.utils.network_utils import get_ip
logger = init_logger(__name__)
_T = TypeVar("_T")
if TYPE_CHECKING:
from vllm.config import VllmConfig
from ..mooncake_store_embedding.backend import (
MooncakeEmbeddingStoreBackend,
)
_RESERVATION_REFRESH_SECONDS = _RESERVATION_TTL_SECONDS / 2
_MAX_CANCELLED_TRANSFER_IDS = 1 << 16
_CANCEL_ATTEMPTS = 2
_TRANSFER_WORKERS = 4
_CONTROL_WORKERS = 8
class _FanoutError(RuntimeError):
"""Retain shard outcomes after every started task settles."""
def __init__(self, error: BaseException, results: list[Any | None]) -> None:
self.results = results
super().__init__(str(error))
class ECMooncakeWorker:
"""Orchestrate consumer reservations and producer push batches.
Consumers may use TP, PP, and DP. Only the first PP stage receives encoder
outputs, and each TP rank exposes a consecutive control port and receives
the same source concurrently. Producers remain unsharded and unreplicated.
With DP, the caller must route both halves of a request to the same replica
and pass that replica's control address to the producer.
"""
def __init__(self, vllm_config: VllmConfig) -> None:
ensure_mooncake_available()
config = MooncakeECConfig.from_vllm_config(vllm_config)
self._store_config = (
(
build_embedding_namespace(vllm_config),
config.store_max_pending_items,
config.store_max_pending_bytes,
config.store_read_buffer_bytes,
)
if config.cross_encoder_cache
else None
)
self._output_store: MooncakeEmbeddingStoreBackend | None = None
self.is_producer = config.is_producer
self.is_consumer = config.is_consumer
self._buffer_device = config.buffer_device
self._control_host = config.control_host
self._control_port = config.control_port
self._transfer = MooncakeTransfer(get_ip(), config.protocol)
self._consumer_memory = ConsumerMemoryPool(
config.pool_size,
self._transfer,
)
self._reservations = ConsumerReservationManager(
self._consumer_memory,
_RESERVATION_TTL_SECONDS,
_MAX_CANCELLED_TRANSFER_IDS,
)
self._consumer_rank_resolved = False
self._is_receiving_rank = True
self._tp_rank = 0
self._tp_size = 1
self._control_server: ConsumerControlServer | None = None
# Worker producer
self._producer_memory = ProducerMemoryPool(
config.pool_size,
self._transfer,
)
self._control_client = ControlClient(config.control_timeout_ms)
self._shutdown_drain_timeout_s = config.control_timeout_ms / 1000
self._io_executor = ThreadPoolExecutor(
max_workers=_TRANSFER_WORKERS,
thread_name_prefix="ec-mooncake-transfer",
)
self._control_executor = ThreadPoolExecutor(
max_workers=_CONTROL_WORKERS,
thread_name_prefix="ec-mooncake-control",
)
self._shard_pool: ThreadPoolExecutor | None = None
self._shard_pool_lock = threading.Lock()
self._push_ready = threading.Event()
self._producer_pushes = ProducerPushManager(self._push_ready.set)
self._dispatch_stop = threading.Event()
self._dispatcher: threading.Thread | None = None
self._failed_saves: set[str] = set()
self._collecting_sources = False
self._completed_loads: set[str] = set()
self._failed_loads: set[str] = set()
self._shutdown = False
if self.is_producer:
self._dispatcher = threading.Thread(
target=self._dispatch_pushes, name="ec-mooncake-ready", daemon=True
)
self._dispatcher.start()
def _dispatch_pushes(self) -> None:
pending_event = False
while not self._dispatch_stop.is_set():
self._push_ready.wait(timeout=0.001 if pending_event else None)
self._push_ready.clear()
if self._dispatch_stop.is_set():
break
if self._collecting_sources:
pending_event = False
continue
pending_event = self._producer_pushes.submit_batches(
self._io_executor, self._push_batch
)
def _resolve_consumer_rank(self) -> None:
"""Place this worker in the consumer receive topology."""
if self._consumer_rank_resolved:
return
self._consumer_rank_resolved = True
try:
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
tp_group = get_tp_group()
self._tp_rank = tp_group.rank_in_group
self._tp_size = tp_group.world_size
self._is_receiving_rank = get_pp_group().is_first_rank
except AssertionError:
# Groups are only absent outside a distributed run, where this
# worker is the whole consumer.
self._tp_rank = 0
self._tp_size = 1
self._is_receiving_rank = True
def start_services(self) -> None:
if self._store_config is not None and self._output_store is None:
from ..mooncake_store_embedding.backend import (
MooncakeEmbeddingStoreBackend,
)
from ..mooncake_store_embedding.store_client import (
create_mooncake_embedding_store_client,
)
namespace, max_items, max_bytes, read_bytes = self._store_config
self._output_store = MooncakeEmbeddingStoreBackend(
create_mooncake_embedding_store_client(read_buffer_bytes=read_bytes),
namespace,
max_pending_items=max_items,
max_pending_bytes=max_bytes,
)
if not self.is_consumer or self._control_server is not None:
return
self._resolve_consumer_rank()
if not self._is_receiving_rank:
# Later pipeline stages hold no encoder outputs, so they need
# neither a receive pool nor a control channel.
return
self._consumer_memory.prepare(torch.device(self._buffer_device))
consumer_pool = self._consumer_memory.tensor
if consumer_pool is None:
raise RuntimeError(
"Mooncake push mode requires a registered consumer buffer pool."
)
base_port = self._control_port
self._control_server = ConsumerControlServer(
# The reservation channel hands out registered-memory addresses
# and takes cancellations, so it listens where the Consumer
# advertises itself (`ec_ip`), not on every interface.
self._control_host,
base_port + self._tp_rank,
self._reserve_push_destination,
self._push_status,
self._reservations.complete,
self._reservations.cancel,
self._reservations.expire,
peer_ports=[base_port + rank for rank in range(self._tp_size)],
device=consumer_pool.device,
drain_ready=self._reservations.drain_ready,
)
try:
self._control_server.start()
except Exception:
self._control_server.close()
self._control_server = None
raise
def _reserve_push_destination(self, payload: dict[str, Any]) -> dict[str, Any]:
transfer_id = str(payload["transfer_id"])
mm_hash = str(payload["mm_hash"])
nbytes = int(payload["nbytes"])
shape = tuple(int(value) for value in payload["shape"])
dtype_name = str(payload["dtype"])
dtype = getattr(torch, dtype_name, None)
if not isinstance(dtype, torch.dtype):
raise ValueError(f"Unsupported torch dtype string: {dtype_name!r}")
expected_nbytes = math.prod(shape) * dtype.itemsize
if expected_nbytes != nbytes:
raise ValueError("shape and dtype do not match nbytes")
self._reservations.expire()
reservation, should_write = self._reservations.reserve(
transfer_id, mm_hash, nbytes, shape, dtype_name, dtype
)
if reservation is None:
raise RuntimeError("EC consumer buffer pool is full")
if reservation.state in {
ConsumerReservationState.CANCEL_PENDING,
ConsumerReservationState.CANCELLED,
}:
return {
"reservation_id": "",
"dst_session": "",
"dst_ptr": 0,
"nbytes": nbytes,
"write": False,
"ready": False,
"cancelled": True,
}
assert reservation.allocation is not None
return {
"reservation_id": reservation.reservation_id,
"dst_session": self._transfer.local_session(),
"dst_ptr": reservation.allocation.tensor.data_ptr(),
"nbytes": reservation.allocation.tensor.nbytes,
"write": should_write,
"ready": reservation.state is ConsumerReservationState.READY,
"cached": reservation.lease is not None,
}
def _push_status(self, transfer_id: str) -> dict[str, Any] | None:
reservation = self._reservations.status(transfer_id)
if reservation is None:
return None
assert reservation.allocation is not None
return {
"mm_hash": reservation.mm_hash,
"ready": reservation.state is ConsumerReservationState.READY,
"reservation_id": reservation.reservation_id,
"nbytes": reservation.allocation.tensor.nbytes,
"shape": list(reservation.shape),
"dtype": reservation.dtype,
}
def _shard_executor(self) -> ThreadPoolExecutor:
"""Use a separate pool so nested shard fan-out cannot deadlock."""
with self._shard_pool_lock:
if self._shard_pool is None:
self._shard_pool = ThreadPoolExecutor(
max_workers=32, thread_name_prefix="ec-mooncake-shard"
)
return self._shard_pool
def _reserve_one(self, addr: str, spec: ECMooncakePushSpec) -> dict[str, Any]:
result = self._control_client.request(
addr,
{
"op": "reserve",
"transfer_id": spec.transfer_id,
"mm_hash": spec.mm_hash,
"nbytes": spec.nbytes,
"shape": list(spec.shape),
"dtype": spec.dtype,
},
)
if not isinstance(result, dict):
raise RuntimeError("Invalid EC reservation response")
result["_received_at"] = time.monotonic()
result["addr"] = addr
return result
def _run_fanout(
self,
tasks: list[Callable[[], _T]],
on_submit: Callable[[int, Future[_T]], None] | None = None,
) -> list[_T]:
"""Run every started shard task and retain partial results on failure.
Waiting for all submitted tasks is what makes source-memory release and
partial-reservation cleanup safe after one shard fails.
"""
if not tasks:
return []
futures: list[tuple[int, Future[_T]]] = []
results: list[_T | None] = [None] * len(tasks)
error: BaseException | None = None
for index, task in enumerate(tasks[1:], 1):
try:
future = self._shard_executor().submit(task)
except Exception as exc:
error = exc
break
futures.append((index, future))
if on_submit is not None:
on_submit(index, future)
if error is None:
try:
results[0] = tasks[0]()
except Exception as exc:
error = exc
for index, future in futures:
try:
results[index] = future.result()
except Exception as exc:
if error is None:
error = exc
if error is not None:
raise _FanoutError(error, results)
return cast(list[_T], results)
def _cancel_reservations(
self,
spec: ECMooncakePushSpec,
reservations: list[dict[str, Any]],
*,
refresh: bool = False,
record: ProducerPushRecord | None = None,
) -> None:
reservations = [
shard for shard in reservations if not shard.get("cancelled", False)
]
if not reservations:
return
def cancel(shard: dict[str, Any]) -> dict[str, Any]:
addr = shard.get("addr")
if not isinstance(addr, str) or not addr:
raise RuntimeError(
"EC reservation is missing a confirmed shard address"
)
result = self._control_client.request(
addr,
make_cancel_request(
spec.transfer_id,
str(shard.get("reservation_id", "")),
abandon=True,
refresh=refresh,
),
)
if not isinstance(result, dict) or not result.get("cancelled"):
raise RuntimeError(
f"Could not cancel EC reservation for mm_hash={spec.mm_hash}"
)
return shard
def track(_index: int, future: Future[dict[str, Any]]) -> None:
if record is not None:
self._producer_pushes.track_shard_futures([record], [future])
self._run_fanout([partial(cancel, shard) for shard in reservations], track)
def _retry_cancel_reservations(
self,
spec: ECMooncakePushSpec,
reservations: list[dict[str, Any]],
*,
record: ProducerPushRecord | None = None,
) -> None:
pending = [shard for shard in reservations if not shard.get("cancelled", False)]
error: _FanoutError | None = None
for _ in range(_CANCEL_ATTEMPTS):
try:
self._cancel_reservations(spec, pending, record=record)
except _FanoutError as exc:
error = exc
pending = [
shard
for index, shard in enumerate(pending)
if exc.results[index] is None
]
continue
return
assert error is not None
raise error
def _reserve_remote(self, spec: ECMooncakePushSpec) -> list[dict[str, Any]]:
"""Reserve a destination on every shard of the consumer."""
result = self._reserve_remote_many([spec])[0]
if isinstance(result, Exception):
raise result
return result
def _refresh_remote_reservations(
self,
spec: ECMooncakePushSpec,
reservations: list[dict[str, Any]],
record: ProducerPushRecord | None = None,
) -> list[dict[str, Any]]:
stale = [
shard
for shard in reservations
if not shard.get("ready", False)
and not shard.get("cached", False)
and not shard.get("cancelled", False)
]
try:
self._cancel_reservations(spec, stale, refresh=True, record=record)
except _FanoutError as exc:
pending = [
shard for index, shard in enumerate(stale) if exc.results[index] is None
]
try:
self._retry_cancel_reservations(spec, pending, record=record)
except _FanoutError as cleanup_error:
raise exc from cleanup_error
raise
return self._reserve_remote(spec)
@staticmethod
def _validate_push_source(push: ProducerPushRecord) -> None:
tensor = push.source_tensor
assert tensor is not None
spec = push.spec
if tuple(tensor.shape) != tuple(spec.shape):
raise ValueError(f"EC source shape mismatch for mm_hash={spec.mm_hash}")
if str(tensor.dtype).split(".")[-1] != spec.dtype:
raise ValueError(f"EC source dtype mismatch for mm_hash={spec.mm_hash}")
if not tensor.is_contiguous():
raise ValueError(f"EC source must be contiguous for mm_hash={spec.mm_hash}")
if tensor.nbytes != spec.nbytes:
raise ValueError(f"EC source size mismatch for mm_hash={spec.mm_hash}")
def start_save_caches(
self,
metadata: ECMooncakeConnectorMetadata,
encoder_cache: dict[str, torch.Tensor] | None = None,
**kwargs: Any,
) -> None:
self._collecting_sources = True
new: dict[str, list[ProducerPushRecord]] = {}
for spec in metadata.pushes:
try:
record, created = self._producer_pushes.reserve(spec, Future)
if created:
new.setdefault(spec.consumer_zmq, []).append(record)
except ValueError:
logger.warning("Rejected conflicting EC push", exc_info=True)
self._failed_saves.add(spec.request_id)
for records in new.values():
future = self._control_executor.submit(self._reserve_batch, records)
future.add_done_callback(partial(self._reservation_batch_done, records))
if not isinstance(encoder_cache, dict):
return
for mm_hash in dict.fromkeys(spec.mm_hash for spec in metadata.pushes):
tensor = encoder_cache.get(mm_hash)
if tensor is not None:
self._bind_push_source(tensor, mm_hash)
if self._output_store is not None:
loaded = self._output_store.resolve_inputs(
metadata.store_candidates, encoder_cache, self._buffer_device
)
# Loaded entries precede the runner's new-output snapshot, so they
# need explicit P2P source binding here.
for mm_hash in loaded:
self._bind_push_source(encoder_cache[mm_hash], mm_hash)
def _reserve_batch(self, records: list[ProducerPushRecord]) -> None:
"""Resolve independent item futures with one reserve RPC per shard."""
if len(records) == 1:
record = records[0]
try:
record.reservation_future.set_result(self._reserve_remote(record.spec))
except Exception as exc:
record.reservation_future.set_exception(exc)
return
outcomes = self._reserve_remote_many([record.spec for record in records])
self._producer_pushes.finish_reservations(list(zip(records, outcomes)))
def _reserve_remote_many(
self, specs: list[ECMooncakePushSpec]
) -> list[list[dict[str, Any]] | Exception]:
shards = None
for _ in range(_CANCEL_ATTEMPTS):
shards = self._control_client.discover_shards(specs[0].consumer_zmq)
if shards is not None:
break
if shards is None:
raise RuntimeError(
f"Could not discover every EC consumer shard at {specs[0].consumer_zmq}"
)
def reserve(addr: str) -> list[dict[str, Any]]:
if len(specs) == 1:
return [{"ok": True, "result": self._reserve_one(addr, specs[0])}]
response = self._control_client.request(
addr,
{
"op": "reserve_batch",
"items": [
{
"transfer_id": spec.transfer_id,
"mm_hash": spec.mm_hash,
"nbytes": spec.nbytes,
"shape": list(spec.shape),
"dtype": spec.dtype,
}
for spec in specs
],
},
)
items = response["items"]
if len(items) != len(specs):
raise RuntimeError("Malformed EC reservation batch response")
for item in items:
if item.get("ok"):
item["result"]["_received_at"] = time.monotonic()
return items
fanout_error = None
results: list[Any]
try:
results = self._run_fanout([partial(reserve, addr) for addr in shards])
except _FanoutError as exc:
results = exc.results
fanout_error = exc
outcomes: list[list[dict[str, Any]] | Exception] = []
for index, spec in enumerate(specs):
reservations: list[dict[str, Any]] = []
error = None
for addr, items in zip(shards, results):
item = items[index] if items is not None else {}
if item.get("ok"):
reservations.append({**item["result"], "addr": addr})
else:
error = fanout_error or RuntimeError(item["error"])
reservations.append({"addr": addr, "reservation_id": ""})
if error is None:
outcomes.append(reservations)
else:
failure = _FanoutError(error, list(reservations))
try:
self._retry_cancel_reservations(spec, reservations)
except Exception as cleanup_error:
failure.__cause__ = cleanup_error
logger.exception("Failed to release partial EC reservation batch")
outcomes.append(failure)
return outcomes
@staticmethod
def _reservation_batch_done(
records: list[ProducerPushRecord], future: Future[None]
) -> None:
error = future.exception()
if error is not None:
for record in records:
if not record.reservation_future.done():
record.reservation_future.set_exception(error)
def start_load_caches(
self,
metadata: ECMooncakeConnectorMetadata,
encoder_cache: dict[str, torch.Tensor],
**kwargs: Any,
) -> None:
self._resolve_consumer_rank()
if not self._is_receiving_rank:
# Later pipeline stages never gather multimodal embeddings.
return
self._transfer.ensure_ready()
if self._buffer_device == "cuda" and not torch.accelerator.is_available():
raise RuntimeError(
"ECMooncakeConnector requires CUDA for ec_buffer_device=cuda"
)
self._reservations.retire_stale(encoder_cache, metadata.freed)
for spec in metadata.loads:
if spec.mm_hash in encoder_cache:
if not spec.local:
# The spec's id is one shard's; cancel by transfer.
self._reservations.cancel(spec.transfer_id, "")
self._completed_loads.add(spec.mm_hash)
continue
if spec.local:
tensor = self._consumer_memory.take_resident(
spec.mm_hash, tuple(spec.shape), spec.dtype
)
else:
try:
allocation = self._reservations.take(spec.transfer_id, spec.mm_hash)
tensor = allocation.tensor
except RuntimeError as e:
logger.warning("EC Mooncake pushed load failed: %s", e)
tensor = None
if tensor is None:
self._failed_loads.add(spec.mm_hash)
else:
encoder_cache[spec.mm_hash] = tensor
self._completed_loads.add(spec.mm_hash)
def _push_batch(self, pushes: list[ProducerPushRecord]) -> None:
started_at = time.monotonic()
ready: list[tuple[ProducerPushRecord, dict[str, Any]]] = []
written_pushes: dict[str, ProducerPushRecord] = {}
failure: Exception | None = None
try:
for push in pushes:
self._validate_push_source(push)
reservations = self._producer_pushes.resolve_reservations(push)
stale = [
index
for index, shard in enumerate(reservations)
if not shard.get("ready", False)
and not shard.get("cancelled", False)
and time.monotonic() - float(shard.get("_received_at", started_at))
>= _RESERVATION_REFRESH_SECONDS
]
if stale:
reservations = self._refresh_remote_reservations(
push.spec, reservations, push
)
self._producer_pushes.replace_reservations(push, reservations)
self._producer_pushes.begin_writing(push)
writable = [
shard
for shard in reservations
if not shard.get("cached", False)
and not shard.get("cancelled", False)
and shard.get("write", True)
]
source = push.source_tensor
assert source is not None
for shard in writable:
if int(shard["nbytes"]) != source.nbytes:
raise RuntimeError(
"Reserved EC size does not match tensor for "
f"mm_hash={push.spec.mm_hash}"
)
ready.append((push, shard))
written_pushes.setdefault(push.spec.transfer_id, push)
if ready:
# Stage each source once, then write it to every destination.
source_index = {
push.spec.transfer_id: index
for index, push in enumerate(written_pushes.values())
}
tensors = [
cast(torch.Tensor, push.source_tensor)
for push in written_pushes.values()
if push.source_tensor is not None
]
lengths = [tensor.nbytes for tensor in tensors]
staged = self._producer_memory.stage(tensors)
registered_sources: list[int] = []
if staged is not None:
sources = staged.tensors
else:
sources = tensors
registered_sources = self._transfer.acquire_sources(tensors)
addresses = [tensor.data_ptr() for tensor in sources]
try:
by_session: dict[str, list[tuple[int, int]]] = {}
session_records: dict[str, dict[str, ProducerPushRecord]] = {}
for push, shard in ready:
session = str(shard["dst_session"])
by_session.setdefault(session, []).append(
(source_index[push.spec.transfer_id], int(shard["dst_ptr"]))
)
session_records.setdefault(session, {})[
push.spec.transfer_id
] = push
def write(session: str, items: list[tuple[int, int]]) -> None:
self._transfer.write(
session,
[addresses[index] for index, _ in items],
[dst for _, dst in items],
[lengths[index] for index, _ in items],
)
sessions = list(by_session.items())
# Write shards concurrently to avoid serial TP latency.
def track_write(index: int, future: Future[None]) -> None:
session = sessions[index][0]
self._producer_pushes.track_shard_futures(
list(session_records[session].values()), [future]
)
writes: list[Callable[[], None]] = [
partial(write, *session) for session in sessions
]
self._run_fanout(writes, track_write)
finally:
if staged is not None:
self._producer_memory.release(staged)
self._transfer.release_sources(registered_sources)
self._producer_pushes.begin_notifying(pushes)
self._notify_completions(ready)
self._producer_pushes.complete(pushes)
except Exception as exc:
# Report asynchronously; raising here would fail EngineCore.
failure = exc
logger.exception(
"EC Mooncake push batch failed for mm_hashes=%s",
[push.spec.mm_hash for push in pushes],
)
self._producer_pushes.settle_all(pushes)
self._abandon_pushes(pushes)
finally:
if failure is not None:
self._producer_pushes.fail(pushes, failure)
def _notify_completions(
self, notifications: list[tuple[ProducerPushRecord, dict[str, Any]]]
) -> None:
"""Tell the consumer, in one message per destination, what landed."""
if not notifications:
return
by_destination: dict[str, list[tuple[ProducerPushRecord, dict[str, Any]]]] = {}
for push, reservation in notifications:
by_destination.setdefault(
str(reservation.get("addr", push.spec.consumer_zmq)), []
).append((push, reservation))
destinations = list(by_destination.items())
def notify(
consumer_zmq: str,
items: list[tuple[ProducerPushRecord, dict[str, Any]]],
) -> None:
result = self._control_client.request(
consumer_zmq,
{
"op": "complete_batch",
"items": [
{
"transfer_id": push.spec.transfer_id,
"reservation_id": reservation["reservation_id"],
}
for push, reservation in items
],
},
)
completions = result.get("items", []) if isinstance(result, dict) else []
if len(completions) != len(items):
raise RuntimeError("Malformed EC completion response")
for (push, _), completion in zip(items, completions):
if not completion.get("completed"):
raise RuntimeError(
f"Unknown EC reservation for mm_hash={push.spec.mm_hash}"
)
def track(index: int, future: Future[None]) -> None:
records = {
push.spec.transfer_id: push for push, _ in destinations[index][1]
}
self._producer_pushes.track_shard_futures(list(records.values()), [future])
self._run_fanout(
[
partial(notify, destination, items)
for destination, items in destinations
],
track,
)
@staticmethod
def _known_reservations(record: ProducerPushRecord) -> list[dict[str, Any]]:
if record.reservations:
return list(record.reservations)
try:
return list(record.reservation_future.result())
except _FanoutError as exc:
return [result for result in exc.results if isinstance(result, dict)]
except Exception:
return []
def _abandon_pushes(self, pushes: list[ProducerPushRecord]) -> None:
"""Release the consumer-side reservations of a batch that failed."""
for push in pushes:
shards = self._known_reservations(push)
if not shards:
# No confirmed topology means there is no safe address to
# cancel. The original push failure remains observable via
# ProducerPushManager.fail below.
continue
try:
self._retry_cancel_reservations(push.spec, shards, record=push)
except _FanoutError:
logger.exception(
"Failed to abandon EC reservations for transfer_id=%s",
push.spec.transfer_id,
)
def _flush_pending_pushes(self) -> None:
self._producer_pushes.submit_batches(
self._io_executor,
self._push_batch,
)
def _bind_push_source(self, tensor: torch.Tensor, mm_hash: str) -> None:
ready_event = None
if tensor.device.type == "cuda":
ready_event = torch.Event()
ready_event.record(torch.accelerator.current_stream(tensor.device))
self._producer_pushes.bind_source(mm_hash, tensor, ready_event)
def _cancel_orphaned_reservation(self, record: ProducerPushRecord) -> None:
resolution_error: Exception | None = None
try:
reservations = self._producer_pushes.resolve_reservations(record)
except Exception as exc:
resolution_error = exc
reservations = self._known_reservations(record)
reservations = [
shard for shard in reservations if not shard.get("cancelled", False)
]
if not reservations:
if resolution_error is not None:
self._producer_pushes.fail([record], resolution_error)
else:
self._producer_pushes.finish_cancel(record)
return
try:
self._retry_cancel_reservations(record.spec, reservations, record=record)
except Exception as cleanup_error:
if resolution_error is not None:
combined_error = RuntimeError(
f"EC reservation resolution failed ({resolution_error}); "
f"cleanup also failed ({cleanup_error})"
)
combined_error.__cause__ = resolution_error
cleanup_error = combined_error
self._producer_pushes.fail([record], cleanup_error)
return
if resolution_error is not None:
self._producer_pushes.fail([record], resolution_error)
else:
self._producer_pushes.finish_cancel(record)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
if not self.is_producer:
return None, None
if self._output_store is not None:
self._output_store.reap()
for record in self._producer_pushes.cancel_requests(finished_req_ids):
self._producer_pushes.submit_cancel(
record,
self._io_executor,
self._cancel_orphaned_reservation,
)
return None, None
def save_caches(
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs: Any
) -> None:
if not self.is_producer:
return
tensor = encoder_cache[mm_hash]
self._bind_push_source(tensor, mm_hash)
if self._output_store is not None:
self._output_store.save_output(mm_hash, tensor)
def build_connector_worker_meta(self) -> ECMooncakeWorkerMetadata | None:
if self.is_consumer and not self._is_receiving_rank:
# `loaded` is intersected across reporting ranks, so a stage that
# never loads must not report at all rather than report nothing.
return None
self._collecting_sources = False
self._push_ready.set()
self._flush_pending_pushes()
failures = self._producer_pushes.poll()
for mm_hash, error in failures:
logger.error(
"EC Mooncake async save failed for mm_hash=%s: %s",
mm_hash,
error,
)
reclaimed = self._consumer_memory.drain_reclaimed()
meta = ECMooncakeWorkerMetadata(
loaded=self._completed_loads,
failed_loads=self._failed_loads,
reclaimed=reclaimed,
pending_saves=self._producer_pushes.pending,
failed_saves=self._failed_saves,
)
self._failed_saves = set()
self._completed_loads = set()
self._failed_loads = set()
return meta
def close(self) -> None:
if self._shutdown:
return
self._shutdown = True
dispatcher = getattr(self, "_dispatcher", None)
if dispatcher is not None:
self._dispatch_stop.set()
self._push_ready.set()
dispatcher.join()
if self._control_server is not None:
self._reservations.begin_shutdown()
self._flush_pending_pushes()
for record in self._producer_pushes.cancel_requests(None):
self._producer_pushes.submit_cancel(
record,
self._io_executor,
self._cancel_orphaned_reservation,
)
self._control_executor.shutdown(wait=True)
self._producer_pushes.submit_batches(
self._io_executor, self._push_batch, wait=True
)
self._io_executor.shutdown(wait=True)
if self._shard_pool is not None:
self._shard_pool.shutdown(wait=True)
# Every producer-side thread that could hold a control socket is stopped.
self._control_client.close()
drained = True
if self._control_server is not None:
drained = self._reservations.wait_for_writers(
self._shutdown_drain_timeout_s
)
self._control_server.close()
if drained:
self._consumer_memory.close()
else:
logger.error(
"Timed out waiting for Mooncake EC writers; keeping the consumer "
"receive pool registered"
)
self._producer_memory.close()
self._transfer.close()
if self._output_store is not None:
self._output_store.shutdown()
self._output_store = None
@@ -1,3 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake Store support for cross-Encoder output reuse."""
@@ -1,214 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Encoder-side Mooncake Store resolver and publisher."""
from __future__ import annotations
import threading
import traceback
from collections.abc import Mapping
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
import torch
from vllm.logger import init_logger
from .data import TensorSpec, make_embedding_key
from .store_client import (
EmbeddingStoreError,
EmbeddingStoreOperationError,
MooncakeEmbeddingStoreClient,
)
logger = init_logger(__name__)
@dataclass
class _StoreSave:
tensor: torch.Tensor | None
ready_event: torch.Event | None
storage_ptr: int
def release(self) -> None:
self.tensor = None
self.ready_event = None
class MooncakeEmbeddingStoreBackend:
"""Resolve immutable encoder outputs and publish misses asynchronously."""
def __init__(
self,
store_client: MooncakeEmbeddingStoreClient,
namespace: str,
*,
max_pending_items: int,
max_pending_bytes: int,
) -> None:
self.store_client = store_client
self.namespace = namespace
self.max_pending_items = max_pending_items
self.max_pending_bytes = max_pending_bytes
self._executor = ThreadPoolExecutor(
max_workers=1,
thread_name_prefix="ec-mooncake-store",
)
self._pending: dict[str, tuple[Future[None], _StoreSave]] = {}
self._pending_storages: dict[int, tuple[torch.UntypedStorage, int]] = {}
self._pending_lock = threading.Lock()
self._pending_bytes = 0
self._step_candidates: set[str] = set()
self._closed = False
self._fatal_error: BaseException | None = None
def save_output(self, identifier: str, tensor: torch.Tensor) -> None:
if identifier in self._step_candidates:
self.reap()
self._enqueue_save(identifier, tensor)
def resolve_inputs(
self,
expected_tensors: Mapping[str, TensorSpec],
encoder_cache: dict[str, torch.Tensor],
device: torch.device | str,
) -> set[str]:
"""Load outputs before encoding; unresolved items retain normal computation."""
self.reap()
self._step_candidates = set(expected_tensors)
candidates = [key for key in expected_tensors if key not in encoder_cache]
if not candidates:
return set()
pool_keys = [
make_embedding_key(self.namespace, identifier) for identifier in candidates
]
try:
exists = self.store_client.batch_exists(pool_keys)
except (EmbeddingStoreOperationError, OSError):
logger.warning(
"Mooncake embedding Store lookup failed; falling back to encoder",
exc_info=True,
)
return set()
hits = {
pool_key: expected_tensors[identifier]
for identifier, pool_key, hit in zip(
candidates, pool_keys, exists, strict=True
)
if hit
}
try:
tensors = self.store_client.load_tensors(hits, device)
except (EmbeddingStoreOperationError, OSError):
logger.warning(
"Mooncake embedding Store GET failed; falling back to encoder",
exc_info=True,
)
return set()
loaded: set[str] = set()
for identifier, pool_key in zip(candidates, pool_keys, strict=True):
if pool_key in tensors:
encoder_cache[identifier] = tensors[pool_key]
loaded.add(identifier)
return loaded
def _enqueue_save(self, identifier: str, tensor: torch.Tensor) -> bool:
"""Admit contiguous outputs, charging each retained storage once."""
if not tensor.is_contiguous():
return False
storage = tensor.untyped_storage()
storage_ptr = storage.data_ptr()
with self._pending_lock:
if self._fatal_error is not None:
raise EmbeddingStoreError(
"Store publisher has failed"
) from self._fatal_error
if self._closed or identifier in self._pending:
return False
_, ref_count = self._pending_storages.get(storage_ptr, (storage, 0))
budget_bytes = 0 if ref_count else storage.nbytes()
if (
len(self._pending) >= self.max_pending_items
or self._pending_bytes + budget_bytes > self.max_pending_bytes
):
return False
# Only admitted work creates an event. Runtime errors propagate.
save = _StoreSave(
tensor,
_record_tensor_ready_event(tensor),
storage_ptr,
)
future = self._executor.submit(
self._save, make_embedding_key(self.namespace, identifier), save
)
self._pending[identifier] = (future, save)
# Keep the storage alive until reaping so its address cannot be reused.
self._pending_storages[storage_ptr] = (storage, ref_count + 1)
self._pending_bytes += budget_bytes
return True
def _save(self, pool_key: str, save: _StoreSave) -> None:
try:
if self.store_client.exists(pool_key):
return
tensor = save.tensor
assert tensor is not None
if save.ready_event is not None:
save.ready_event.synchronize()
self.store_client.put_tensor(pool_key, tensor)
except BaseException as error:
# Returned client frames can otherwise keep tensors via a Future's
# traceback. Uncertain I/O owners remain retained by the client.
traceback.clear_frames(error.__traceback__)
raise
finally:
tensor = None
save.release()
def reap(self) -> None:
with self._pending_lock:
for identifier, (future, save) in list(self._pending.items()):
# Observe once; a racing completion is handled on the next poll.
if not future.done():
continue
try:
future.result() # Already done: never waits on Store I/O.
except (EmbeddingStoreOperationError, OSError) as error:
logger.warning("Store PUT failed for %s: %s", identifier, error)
# This failure is consumed; do not retain publisher/caller frames.
error.__traceback__ = None
except BaseException as error:
# The client may still own unsafe native-I/O buffers. Keep
# their charge and fatal result until worker termination.
if self._fatal_error is None:
self._fatal_error = error
continue
del self._pending[identifier]
storage, ref_count = self._pending_storages[save.storage_ptr]
if ref_count == 1:
del self._pending_storages[save.storage_ptr]
self._pending_bytes -= storage.nbytes()
else:
self._pending_storages[save.storage_ptr] = (storage, ref_count - 1)
fatal = self._fatal_error
if fatal is not None:
raise fatal
def shutdown(self) -> None:
with self._pending_lock:
if self._closed:
return
self._closed = True
self._executor.shutdown(wait=True, cancel_futures=False)
self.reap()
self.store_client.close()
def _record_tensor_ready_event(tensor: torch.Tensor) -> torch.Event | None:
if tensor.device.type != "cuda":
return None
event = torch.Event()
event.record(torch.accelerator.current_stream(tensor.device))
return event
@@ -1,51 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Content identities and output contracts for the shared Encoder Store."""
from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from vllm.config import VllmConfig
EMBEDDING_CACHE_KEY_VERSION = "v3"
def make_embedding_key(namespace: str, identifier: str) -> str:
return f"{namespace}@id:{identifier}"
@dataclass(frozen=True)
class TensorSpec:
"""Contiguous output contract used by the Encoder and Store codec."""
shape: tuple[int, ...]
dtype: str
nbytes: int
def build_embedding_namespace(vllm_config: VllmConfig) -> str:
model_config = vllm_config.model_config
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None
extra_config = ec_config.ec_connector_extra_config
multimodal_config = model_config.multimodal_config
encoder_hash = (
multimodal_config.compute_hash()
if multimodal_config is not None
else "encoder:default"
)
cache_prefix = str(extra_config.get("embedding_cache_prefix", ""))
model_name = str(extra_config.get("embedding_model_identity", model_config.model))
prefix = f"{cache_prefix}@" if cache_prefix else ""
return (
f"{prefix}embedding"
f"@model:{model_name}"
f"@revision:{model_config.revision or 'default'}"
f"@encoder:{encoder_hash}"
f"@dtype:{model_config.dtype}"
f"@protocol:{EMBEDDING_CACHE_KEY_VERSION}"
)
@@ -1,463 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Thin Mooncake Store client for embedding objects.
Originally adapted from vLLM PR #47302 (0d1f71f5d36c). The v3 key namespace
uses a compact header for complete 2D embeddings, not that PR's tensor format.
"""
from __future__ import annotations
import ctypes
import struct
import threading
from collections.abc import Callable, Iterator, Mapping
from contextlib import contextmanager
from enum import IntEnum
from typing import Any
import torch
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
TensorSpec,
)
from vllm.distributed.mooncake_store import MooncakeStoreConfig, setup_mooncake_store
from vllm.logger import init_logger
from vllm.utils.network_utils import get_ip
logger = init_logger(__name__)
# A failed transfer can still use registered memory after the binding returns.
# Poisoned clients retain their owners until process exit, including if normal
# worker teardown drops the connector. New operations are rejected.
_UNSAFE_CLIENTS: list[Any] = []
# Values from Mooncake v0.3.12 / v0.3.12.post1 types.h (ErrorCode).
# Python exposes integer returns, not this enum.
class _MooncakeErrorCode(IntEnum):
NO_AVAILABLE_HANDLE = -200
INVALID_PARAMS = -600
INVALID_REPLICA = -702
REPLICA_IS_NOT_READY = -703
OBJECT_NOT_FOUND = -704
OBJECT_ALREADY_EXISTS = -705
LEASE_EXPIRED = -707
RPC_FAIL = -900
RPC_TIMEOUT = -901
TENANT_QUOTA_EXCEEDED = -1700
# For RAM batch reads and single-replica writes, these rejections occur
# before submission or after completed I/O. Transfer timeouts can leave I/O
# in flight; all other failures retain registered buffers and poison the client.
_SAFE_IO_REJECTIONS = frozenset(
{
_MooncakeErrorCode.NO_AVAILABLE_HANDLE,
_MooncakeErrorCode.REPLICA_IS_NOT_READY,
_MooncakeErrorCode.OBJECT_NOT_FOUND,
_MooncakeErrorCode.OBJECT_ALREADY_EXISTS,
_MooncakeErrorCode.RPC_FAIL,
_MooncakeErrorCode.RPC_TIMEOUT,
}
)
# Batch GET validates capacity/replicas before I/O and checks leases afterwards.
# PUT rejects tenant quotas before allocating replicas or submitting transfers.
_SAFE_GET_REJECTIONS = _SAFE_IO_REJECTIONS | {
_MooncakeErrorCode.INVALID_PARAMS,
_MooncakeErrorCode.INVALID_REPLICA,
_MooncakeErrorCode.LEASE_EXPIRED,
}
_SAFE_PUT_REJECTIONS = _SAFE_IO_REJECTIONS | {_MooncakeErrorCode.TENANT_QUOTA_EXCEEDED}
_MOONCAKE_TENSOR_OBJECT_MAGIC = 0x4D4F4F4E
# Magic, dtype, token count, hidden dimension. Format versioning is in the key.
_MOONCAKE_TENSOR_HEADER = struct.Struct("<IIQQ")
_MOONCAKE_DTYPE_TO_TORCH_DTYPE = {
0: "torch.float32",
11: "torch.float16",
12: "torch.bfloat16",
}
_TORCH_DTYPE_TO_MOONCAKE_DTYPE = {
value: key for key, value in _MOONCAKE_DTYPE_TO_TORCH_DTYPE.items()
}
def create_mooncake_embedding_store_client(
read_buffer_bytes: int = 128 * 1024**2,
) -> MooncakeEmbeddingStoreClient:
try:
from mooncake.store import ( # type: ignore
MooncakeDistributedStore,
ObjectDataType,
ReplicateConfig,
)
except ImportError as e:
raise ImportError(
"Install mooncake with Store support to enable cross_encoder_cache."
) from e
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils
config = MooncakeStoreConfig.load_from_config()
if config.enable_offload:
raise ValueError(
"cross_encoder_cache supports a RAM Store; disable enable_offload"
)
store = MooncakeDistributedStore()
local_ip = get_ip()
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
setup_mooncake_store(store, config, local_hostname)
logger.info(
"Initialized embedding Mooncake store mode=%s global_segment_size=%d "
"local_buffer_size=%d",
config.mode,
config.global_segment_size,
config.local_buffer_size,
)
replicate_config = ReplicateConfig()
replicate_config.data_type = ObjectDataType.TENSOR
return MooncakeEmbeddingStoreClient(
store, replicate_config=replicate_config, read_buffer_bytes=read_buffer_bytes
)
class EmbeddingStoreError(RuntimeError):
"""A Store failure that must propagate to the worker."""
class EmbeddingStoreOperationError(EmbeddingStoreError):
"""A rejected or completed operation that permits normal Encoder fallback."""
def _batch_get_completed_safely(results: Any, capacities: list[int]) -> bool:
return (
isinstance(results, list)
and len(results) == len(capacities)
and all(
type(result) is int
and (0 <= result <= capacity or result in _SAFE_GET_REJECTIONS)
for result, capacity in zip(results, capacities, strict=True)
)
)
class MooncakeEmbeddingStoreClient:
"""Wraps Mooncake object and buffer APIs used by embedding transfer."""
def __init__(
self,
store: Any,
replicate_config: Any | None = None,
*,
read_buffer_bytes: int = 128 * 1024**2,
):
self._lifetime_lock = threading.Lock()
self._unsafe_owners: list[Any] = []
self._poisoned = False
self.store = store
self.replicate_config = replicate_config
self._read_buffer_bytes = read_buffer_bytes
self._read_buffer: torch.Tensor | None = None
self._put_header: ctypes.Array[ctypes.c_char] | None = None
def _check_healthy(self) -> None:
if self._poisoned:
raise EmbeddingStoreError(
"Mooncake I/O completion is unconfirmed; worker must exit"
)
def _poison(self, owners: list[Any]) -> None:
with self._lifetime_lock:
self._unsafe_owners.extend(owners)
if not self._poisoned:
self._poisoned = True
_UNSAFE_CLIENTS.append(self)
@contextmanager
def _registered_io(
self,
buffers: list[tuple[Any, int, int]],
) -> Iterator[Callable[[int], None]]:
self._check_healthy()
registered = []
submitted = False
confirmed = False
def confirm(result: int) -> None:
nonlocal confirmed
if type(result) is not int or (
result < 0 and result not in _SAFE_PUT_REJECTIONS
):
raise EmbeddingStoreError(
"Mooncake I/O completion is unconfirmed; retaining buffers"
)
confirmed = True
try:
for owner, addr, size in buffers:
ret = self.store.register_buffer(addr, size)
if ret != 0:
raise EmbeddingStoreOperationError(
f"Failed to register embedding buffer: {ret}"
)
registered.append(addr)
submitted = True
yield confirm
except BaseException as error:
if submitted and not confirmed:
self._poison([owner for owner, _, _ in buffers])
raise EmbeddingStoreError(
"Mooncake I/O completion is unconfirmed; retaining buffers"
) from error
raise
finally:
if not submitted or confirmed:
try:
for addr in reversed(registered):
self._unregister_buffer(addr)
except BaseException:
self._poison([owner for owner, _, _ in buffers])
raise
def close(self) -> None:
"""Close the supported Mooncake binding after all I/O is drained."""
self._check_healthy()
if self._read_buffer is not None:
try:
self._unregister_buffer(self._read_buffer.data_ptr())
except BaseException:
self._poison([self._read_buffer])
raise
self._read_buffer = None
if self._put_header is not None:
try:
self._unregister_buffer(ctypes.addressof(self._put_header))
except BaseException:
self._poison([self._put_header])
raise
self._put_header = None
ret = self.store.close()
if ret != 0:
raise EmbeddingStoreError(
f"failed to close embedding Mooncake Store: {ret}"
)
def exists(self, pool_key: str) -> bool:
return self.batch_exists([pool_key])[0]
def batch_exists(self, keys: list[str]) -> list[bool]:
self._check_healthy()
if not keys:
return []
states = self.store.batch_is_exist(keys)
if len(states) != len(keys):
raise RuntimeError(
"Mooncake Store returned an unexpected number of lookup results"
)
return [state == 1 for state in states]
def load_tensors(
self, expected: Mapping[str, TensorSpec], device: torch.device | str
) -> dict[str, torch.Tensor]:
"""Synchronously load runner inputs into independently owned tensors."""
self._check_healthy()
device = torch.device(device)
loaded: dict[str, torch.Tensor] = {}
chunk: dict[str, TensorSpec] = {}
chunk_bytes = 0
for key, spec in expected.items():
size = _MOONCAKE_TENSOR_HEADER.size + spec.nbytes
if size > self._read_buffer_bytes:
logger.warning(
"Skipping Store GET for %s: object bytes=%d exceed read buffer=%d",
key,
size,
self._read_buffer_bytes,
)
continue
if chunk_bytes + size > self._read_buffer_bytes:
loaded.update(self._load_chunk(chunk, device))
chunk = {}
chunk_bytes = 0
chunk[key] = spec
chunk_bytes += size
if chunk:
loaded.update(self._load_chunk(chunk, device))
return loaded
def _load_chunk(
self, expected: dict[str, TensorSpec], device: torch.device
) -> dict[str, torch.Tensor]:
self._check_healthy()
if self._read_buffer is None:
buffer = torch.empty(
self._read_buffer_bytes,
dtype=torch.uint8,
device="cpu",
pin_memory=device.type == "cuda",
)
ret = self.store.register_buffer(buffer.data_ptr(), buffer.nbytes)
if ret != 0:
raise EmbeddingStoreOperationError(
f"Failed to register embedding read buffer: {ret}"
)
self._read_buffer = buffer
buffer = self._read_buffer
base = buffer.data_ptr()
sizes = [
_MOONCAKE_TENSOR_HEADER.size + spec.nbytes for spec in expected.values()
]
offsets: list[int] = []
end = 0
for size in sizes:
offsets.append(end)
end += size
try:
results = self.store.batch_get_into(
list(expected), [base + offset for offset in offsets], sizes
)
if not _batch_get_completed_safely(results, sizes):
raise EmbeddingStoreError("Unsafe batch GET results")
except BaseException as error:
self._poison([buffer])
raise EmbeddingStoreError(
"Mooncake I/O completion is unconfirmed; retaining read buffer"
) from error
valid: list[tuple[str, TensorSpec, int]] = []
for (key, spec), offset, size, result in zip(
expected.items(), offsets, sizes, results, strict=True
):
if result != size:
logger.warning(
"Store GET skipped for %s: result=%d, expected bytes=%d",
key,
result,
size,
)
continue
try:
_validate_mooncake_tensor_metadata(
key,
ctypes.string_at(base + offset, _MOONCAKE_TENSOR_HEADER.size),
spec,
)
except EmbeddingStoreOperationError as error:
logger.warning("Store GET skipped for %s: %s", key, error)
continue
valid.append((key, spec, offset + _MOONCAKE_TENSOR_HEADER.size))
if not valid:
return {}
# Allocate every destination before submitting any asynchronous copy.
targets = {
key: torch.empty(
spec.shape,
dtype=getattr(torch, spec.dtype.removeprefix("torch.")),
device=device,
)
for key, spec, _ in valid
}
ready = torch.Event() if device.type == "cuda" else None
try:
for key, spec, offset in valid:
targets[key].view(torch.uint8).view(-1).copy_(
buffer[offset : offset + spec.nbytes],
non_blocking=ready is not None,
)
if ready is not None:
ready.record(torch.accelerator.current_stream(device))
ready.synchronize()
except BaseException as error:
if ready is not None:
self._poison([buffer, *targets.values(), ready])
raise EmbeddingStoreError(
"Embedding copy completion is unconfirmed; retaining buffers"
) from error
raise
return targets
def put_tensor(self, pool_key: str, tensor: torch.Tensor) -> None:
self._check_healthy()
if not tensor.is_contiguous():
raise EmbeddingStoreOperationError("embedding tensor must be contiguous")
metadata = _encode_mooncake_tensor_metadata(tensor)
# The single publisher reuses this header only after PUT completion.
if self._put_header is None:
header = ctypes.create_string_buffer(len(metadata))
ret = self.store.register_buffer(ctypes.addressof(header), len(metadata))
if ret != 0:
raise EmbeddingStoreOperationError(
f"Failed to register embedding header: {ret}"
)
self._put_header = header
header_ptr = ctypes.addressof(self._put_header)
ctypes.memmove(header_ptr, metadata, len(metadata))
data_size = tensor.numel() * tensor.element_size()
buffers = [
(tensor, tensor.data_ptr(), data_size),
]
with self._registered_io(buffers) as confirm:
[result] = self.store.batch_put_from_multi_buffers(
[pool_key],
[[header_ptr, tensor.data_ptr()]],
[[len(metadata), data_size]],
self.replicate_config,
)
confirm(result)
if result < 0:
raise EmbeddingStoreOperationError(
f"failed to put embedding tensor for {pool_key}"
)
def _unregister_buffer(self, addr: int) -> None:
try:
ret = self.store.unregister_buffer(addr)
except Exception as error:
raise EmbeddingStoreError(
f"could not confirm buffer unregistration addr={addr:#x}"
) from error
if ret != 0:
raise EmbeddingStoreError(
f"failed to unregister embedding buffer addr={addr:#x}: {ret}"
)
def _validate_mooncake_tensor_metadata(
pool_key: str,
metadata: bytes,
expected: TensorSpec,
) -> None:
magic, dtype, num_tokens, hidden_dim = _MOONCAKE_TENSOR_HEADER.unpack(metadata)
if magic != _MOONCAKE_TENSOR_OBJECT_MAGIC:
raise EmbeddingStoreOperationError(
f"invalid Mooncake tensor metadata header for {pool_key}"
)
if _MOONCAKE_DTYPE_TO_TORCH_DTYPE.get(dtype) != expected.dtype:
raise EmbeddingStoreOperationError(
f"unexpected Mooncake tensor dtype for {pool_key}: {dtype}"
)
if (num_tokens, hidden_dim) != expected.shape:
raise EmbeddingStoreOperationError(
f"embedding tensor shape mismatch for {pool_key}: expected={expected.shape}"
)
def _encode_mooncake_tensor_metadata(tensor: torch.Tensor) -> bytes:
dtype = str(tensor.dtype)
if dtype not in _TORCH_DTYPE_TO_MOONCAKE_DTYPE:
raise EmbeddingStoreOperationError(
f"unsupported embedding tensor dtype: {dtype}"
)
if tensor.ndim != 2:
raise EmbeddingStoreOperationError(
f"embedding tensor must be 2D: shape={tuple(tensor.shape)}"
)
return _MOONCAKE_TENSOR_HEADER.pack(
_MOONCAKE_TENSOR_OBJECT_MAGIC,
_TORCH_DTYPE_TO_MOONCAKE_DTYPE[dtype],
*tensor.shape,
)
@@ -150,30 +150,12 @@ class KVConnectorFactory:
# The registration should not be done in each individual file, as we want to
# only load the files corresponding to the current connector.
KVConnectorFactory.register_connector(
"ExampleConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.example_connector",
"ExampleConnector",
)
KVConnectorFactory.register_connector(
"ExampleHiddenStatesConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.example_hidden_states_connector",
"ExampleHiddenStatesConnector",
)
KVConnectorFactory.register_connector(
"LMCacheConnectorV1",
"vllm.distributed.kv_transfer.kv_connector.v1.lmcache_connector",
"LMCacheConnectorV1",
)
KVConnectorFactory.register_connector(
"LMCacheMPConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.lmcache_mp_connector",
"LMCacheMPConnector",
)
KVConnectorFactory.register_connector(
"NixlConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
@@ -222,21 +204,6 @@ KVConnectorFactory.register_connector(
"DecodeBenchConnector",
)
KVConnectorFactory.register_connector(
"MooncakeConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector",
"MooncakeConnector",
)
KVConnectorFactory.register_connector(
"MooncakeStoreConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.connector",
"MooncakeStoreConnector",
)
KVConnectorFactory.register_connector(
"FlexKVConnectorV1",
"vllm.distributed.kv_transfer.kv_connector.v1.flexkv_connector",
"FlexKVConnectorV1",
)
KVConnectorFactory.register_connector(
"SimpleCPUOffloadConnector",
"vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector",
@@ -1,270 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
from vllm.logger import init_logger
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
from vllm.distributed.kv_events import KVCacheEvent
from vllm.forward_context import ForwardContext
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
# FlexKV is a distributed KV Store and multi-level cache management system for
# ultra-large-scale LLM inference.
# GitHub: https://github.com/taco-project/FlexKV
# Install: git clone git@github.com:taco-project/FlexKV.git \
# && cd FlexKV && bash build.sh
class FlexKVConnectorV1(KVConnectorBase_V1):
"""KV Connector that offloads KV cache to FlexKV.
FlexKV is a distributed KV Store and multi-level cache management system
designed for ultra-large-scale LLM inference. It supports offloading KV
cache to CPU memory, SSD, and remote storage.
Installation:
See https://github.com/taco-project/FlexKV for installation instructions.
Quick start::
git clone git@github.com:taco-project/FlexKV.git
cd FlexKV && bash build.sh
Configuration:
Pass ``kv_connector="FlexKVConnectorV1"`` via ``--kv-transfer-config``::
--kv-transfer-config \
'{"kv_connector":"FlexKVConnectorV1","kv_role":"kv_both"}'
"""
def __init__(
self,
vllm_config: "VllmConfig",
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(
vllm_config=vllm_config, role=role, kv_cache_config=kv_cache_config
)
try:
from flexkv.integration.vllm.vllm_v1_adapter import FlexKVConnectorV1Impl
except ImportError as e:
raise ImportError(
"FlexKV is not installed. Please install it to use "
"FlexKVConnectorV1. See https://github.com/taco-project/FlexKV "
"for installation instructions."
) from e
self._flexkv_connector = FlexKVConnectorV1Impl(vllm_config, role)
def shutdown(self):
self._flexkv_connector.shutdown()
# ==============================
# Worker-side methods
# ==============================
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
"""No-op for FlexKV (currently).
FlexKV manages all KV transfers on the **scheduler side** via
``build_connector_meta`` (which calls ``launch_tasks``) and
``update_connector_output`` (which polls ``query_finished_task``).
KV blocks are transferred directly between the FlexKV server and
vLLM's GPU memory without worker-side intervention during the
forward pass — similar to how NIXL operates.
These worker-side hooks are kept (rather than omitted) to satisfy
the ``KVConnectorBase_V1`` interface contract and to serve as
extension points for a future worker-side layer-pipelining path.
Args:
forward_context (ForwardContext): the forward context.
**kwargs (Any): additional arguments (unused).
"""
self._flexkv_connector.start_load_kv(forward_context, **kwargs)
def wait_for_layer_load(self, layer_name: str) -> None:
"""No-op for FlexKV (currently).
FlexKV manages all KV transfers on the scheduler side.
This hook is retained for ``KVConnectorBase_V1`` API compatibility.
Args:
layer_name: the name of the layer (unused).
"""
self._flexkv_connector.wait_for_layer_load(layer_name)
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: "AttentionMetadata",
**kwargs,
) -> None:
"""No-op for FlexKV (currently).
FlexKV offloads KV cache asynchronously from the scheduler side
after a request finishes (see ``request_finished``). It does not
intercept individual layer tensors during the forward pass.
This hook is retained to satisfy ``KVConnectorBase_V1`` and as an
extension point for future per-layer async offload support.
Args:
layer_name (str): the name of the layer (unused).
kv_layer (torch.Tensor): the paged KV buffer (unused).
attn_metadata (AttentionMetadata): the attention metadata (unused).
**kwargs (Any): additional arguments (unused).
"""
self._flexkv_connector.save_kv_layer(
layer_name, kv_layer, attn_metadata, **kwargs
)
def wait_for_save(self):
"""No-op for FlexKV (currently).
KV offload tasks are tracked asynchronously by the scheduler
connector via ``request_finished`` / ``query_finished_task``.
There is no pending worker-side save to wait for at
forward-context exit.
Retained to satisfy ``KVConnectorBase_V1`` and as an extension
point for future worker-side save-completion signalling.
"""
self._flexkv_connector.wait_for_save()
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""Notify worker-side connector of requests that have finished
generating tokens.
Returns:
Tuple of (sending/saving ids, recving/loading ids) for requests
that have finished asynchronous transfer. The finished saves/sends
req ids must belong to a set provided in a call to this method
(this call or a prior one).
"""
return self._flexkv_connector.get_finished(finished_req_ids)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Initialize with the KV caches. Useful for pre-registering the
KV caches in the KVConnector (e.g. for NIXL).
Args:
kv_caches: dictionary of layer names to kv cache tensors.
"""
self._flexkv_connector.register_kv_caches(kv_caches)
# ==============================
# Scheduler-side methods
# ==============================
def get_num_new_matched_tokens(
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[int, bool]:
"""Get the number of new tokens that can be loaded from the
external KV cache beyond ``num_computed_tokens``.
Args:
request (Request): the request object.
num_computed_tokens (int): the number of locally computed
tokens for this request.
Returns:
Tuple of (num_external_tokens, is_ready) where
num_external_tokens is the number of additional tokens that
can be loaded from the external KV cache.
"""
return self._flexkv_connector.get_num_new_matched_tokens(
request, num_computed_tokens
)
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""Update KVConnector state after block allocation."""
self._flexkv_connector.update_state_after_alloc(
request, blocks, num_external_tokens
)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
return self._flexkv_connector.build_connector_meta(scheduler_output)
def update_connector_output(self, connector_output: KVConnectorOutput):
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
self._flexkv_connector.update_connector_output(connector_output)
def request_finished(
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""Called when a request has finished, before its blocks are freed.
Returns:
Tuple of (async_save, kv_transfer_params) where async_save is
True if the request is being saved/sent asynchronously and blocks
should not be freed until the request_id is returned from
:meth:`get_finished`. kv_transfer_params is an optional dict of
KVTransferParams to be included in the request outputs.
"""
return self._flexkv_connector.request_finished(request, block_ids)
def take_events(self) -> Iterable["KVCacheEvent"]:
"""Collect buffered KV cache events.
Returns:
New KV cache events since the last call.
"""
return self._flexkv_connector.take_events()
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""Get the KV connector stats collected during the last interval."""
return self._flexkv_connector.get_kv_connector_stats()
def get_block_ids_with_load_errors(self) -> set[int]:
"""Get the block ids that have failed to load."""
return self._flexkv_connector.get_block_ids_with_load_errors()
@@ -1,358 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_events import (
BlockStored,
KVCacheEvent,
KVConnectorKVEvents,
KVEventAggregator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
)
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
from vllm.forward_context import ForwardContext
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
logger = init_logger(__name__)
class LMCacheKVEvents(KVConnectorKVEvents):
"""Concrete implementation of KVConnectorKVEvents using KVEventAggregator."""
def __init__(self, num_workers: int) -> None:
self._aggregator = KVEventAggregator(num_workers)
def add_events(self, events: list[KVCacheEvent]) -> None:
self._aggregator.add_events(events)
def aggregate(self) -> "LMCacheKVEvents":
"""Aggregate KV events and retain only common events."""
common_events = self._aggregator.get_common_events()
self._aggregator.clear_events()
self._aggregator.add_events(common_events)
self._aggregator.reset_workers()
return self
def increment_workers(self, count: int = 1) -> None:
self._aggregator.increment_workers(count)
def get_all_events(self) -> list[KVCacheEvent]:
return self._aggregator.get_all_events()
def get_number_of_workers(self) -> int:
return self._aggregator.get_number_of_workers()
def clear_events(self) -> None:
self._aggregator.clear_events()
self._aggregator.reset_workers()
def __repr__(self) -> str:
return f"<LMCacheKVEvents events={self.get_all_events()}>"
class LMCacheConnectorV1(KVConnectorBase_V1):
@classmethod
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
"""LMCache requires PIECEWISE CUDA graph mode when layerwise
operations are enabled. The wait_for_layer_load and save_kv_layer
methods perform actual async synchronization that cannot be
captured in CUDA graphs.
"""
return extra_config.get("use_layerwise", False)
def __init__(
self,
vllm_config: "VllmConfig",
role: KVConnectorRole,
kv_cache_config: "KVCacheConfig",
):
super().__init__(
vllm_config=vllm_config, role=role, kv_cache_config=kv_cache_config
)
assert vllm_config.kv_transfer_config is not None
use_native = vllm_config.kv_transfer_config.get_from_extra_config(
"use_native", False
)
if use_native:
logger.info("Initializing native LMCache connector")
# lazy import
from vllm.distributed.kv_transfer.kv_connector.v1 import lmcache_integration
_adapter = lmcache_integration.vllm_v1_adapter
cls = _adapter.LMCacheConnectorV1Impl
else:
logger.info("Initializing latest dev LMCache connector")
# lazy import
from lmcache.integration.vllm.vllm_v1_adapter import (
LMCacheConnectorV1Impl as LMCacheConnectorLatestImpl,
)
cls = LMCacheConnectorLatestImpl
self._lmcache_engine = cls(vllm_config, role, self)
self._kv_cache_events: LMCacheKVEvents | None = None
# ==============================
# Worker-side methods
# ==============================
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Initialize with the KV caches. Useful for pre-registering the
KV Caches in the KVConnector (e.g. for NIXL).
Args:
kv_caches: dictionary of layer names, kv cache
"""
if hasattr(self._lmcache_engine, "register_kv_caches"):
self._lmcache_engine.register_kv_caches(kv_caches)
else:
logger.warning(
"LMCache engine does not support register_kv_caches, "
"please check and use the latest version"
)
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
super().bind_connector_metadata(connector_metadata)
engine = self._lmcache_engine
if hasattr(engine, "bind_connector_metadata"):
engine.bind_connector_metadata(connector_metadata)
else:
# Compat for engines predating the bind hook: start_load_kv may
# run after the forward (SchedulerOutput.has_sync_kv_loads), so
# the per-step layerwise state it used to reset must be reset
# here. Remove once LMCache ships bind_connector_metadata.
if hasattr(engine, "current_layer"):
engine.current_layer = 0
if hasattr(engine, "layerwise_retrievers"):
engine.layerwise_retrievers = []
engine.layerwise_storers = []
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""Start loading the KV cache from the connector to vLLM's paged
KV buffer. This is called from the forward context before the
forward pass to enable async loading during model execution.
Args:
forward_context (ForwardContext): the forward context.
**kwargs: additional arguments for the load operation
Note:
The number of elements in kv_caches and layer_names should be
the same.
"""
self._lmcache_engine.start_load_kv(forward_context, **kwargs)
def wait_for_layer_load(self, layer_name: str) -> None:
"""Block until the KV for a specific layer is loaded into vLLM's
paged buffer. This is called from within attention layer to ensure
async copying from start_load_kv is complete.
This interface will be useful for layer-by-layer pipelining.
Args:
layer_name: the name of that layer
"""
self._lmcache_engine.wait_for_layer_load(layer_name)
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
"""Start saving the a layer of KV cache from vLLM's paged buffer
to the connector. This is called from within attention layer to
enable async copying during execution.
Args:
layer_name (str): the name of the layer.
kv_layer (torch.Tensor): the paged KV buffer of the current
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
**kwargs: additional arguments for the save operation.
"""
self._lmcache_engine.save_kv_layer(
layer_name, kv_layer, attn_metadata, **kwargs
)
def wait_for_save(self):
"""Block until all the save operations is done. This is called
as the forward context exits to ensure that the async saving
from save_kv_layer is complete before finishing the forward.
This prevents overwrites of paged KV buffer before saving done.
"""
self._lmcache_engine.wait_for_save()
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""Notifies worker-side connector ids of requests that have
finished generating tokens.
Returns:
ids of requests that have finished asynchronous transfer
(requests that previously returned True from request_finished()),
tuple of (sending/saving ids, recving/loading ids).
The finished saves/sends req ids must belong to a set provided in a
call to this method (this call or a prior one).
"""
return self._lmcache_engine.get_finished(finished_req_ids)
def get_block_ids_with_load_errors(self) -> set[int]:
"""Get the set of block IDs that failed to load.
Returns:
Set of block IDs that encountered load errors.
Empty set if no load errors occurred.
"""
method = getattr(self._lmcache_engine, "get_block_ids_with_load_errors", None)
if callable(method):
return method()
# Fallback for older versions that don't support this method
return set()
def get_kv_connector_kv_cache_events(self) -> LMCacheKVEvents | None:
"""Get the KV connector kv cache events collected during the last interval."""
events = self._lmcache_engine.get_kv_events() # type: ignore [attr-defined]
if not events:
return None
blocks: list[BlockStored] = [
BlockStored(
block_hashes=e.block_hashes,
parent_block_hash=e.parent_block_hash,
token_ids=e.token_ids,
lora_id=e.lora_id,
block_size=e.block_size,
medium=e.medium,
lora_name=getattr(e, "lora_name", None),
)
for e in events
]
lmcache_kv_events = LMCacheKVEvents(num_workers=1)
lmcache_kv_events.add_events(blocks)
return lmcache_kv_events
# ==============================
# Scheduler-side methods
# ==============================
def get_num_new_matched_tokens(
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
request (Request): the request object.
num_computed_tokens (int): the number of locally
computed tokens for this request
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
return self._lmcache_engine.get_num_new_matched_tokens(
request, num_computed_tokens
), False
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""Update KVConnector state after block allocation."""
self._lmcache_engine.update_state_after_alloc(request, num_external_tokens)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
return self._lmcache_engine.build_connector_meta(scheduler_output)
def update_connector_output(self, connector_output: KVConnectorOutput):
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
# Get the KV events
kv_cache_events = connector_output.kv_cache_events
if not kv_cache_events or not isinstance(kv_cache_events, LMCacheKVEvents):
return
if self._kv_cache_events is None:
self._kv_cache_events = kv_cache_events
else:
self._kv_cache_events.add_events(kv_cache_events.get_all_events())
self._kv_cache_events.increment_workers(
kv_cache_events.get_number_of_workers()
)
return
def request_finished(
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""Called when a request has finished, before its blocks are freed.
Returns:
True if the request is being saved/sent asynchronously and blocks
should not be freed until the request_id is returned from
get_finished().
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
return self._lmcache_engine.request_finished(request, block_ids)
def take_events(self) -> Iterable["KVCacheEvent"]:
"""Take the KV cache events from the connector.
Yields:
New KV cache events since the last call.
"""
if self._kv_cache_events is not None:
self._kv_cache_events.aggregate()
kv_cache_events = self._kv_cache_events.get_all_events()
yield from kv_cache_events
self._kv_cache_events.clear_events()
self._kv_cache_events = None
@@ -1,20 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from . import multi_process_adapter, vllm_v1_adapter
from .multi_process_adapter import (
LMCacheMPSchedulerAdapter,
LMCacheMPWorkerAdapter,
LoadStoreOp,
ParallelStrategy,
)
__all__ = [
"vllm_v1_adapter",
"multi_process_adapter",
"LMCacheMPSchedulerAdapter",
"LMCacheMPWorkerAdapter",
"LoadStoreOp",
"ParallelStrategy",
]
@@ -1,735 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from collections.abc import Iterable
from dataclasses import dataclass
from itertools import islice
from typing import Any
import torch
import zmq
from lmcache.utils import _lmcache_nvtx_annotate, init_logger
from lmcache.v1.multiprocess.custom_types import (
CudaIPCWrapper,
IPCCacheEngineKey,
KVCache,
)
from lmcache.v1.multiprocess.mq import MessageQueueClient, MessagingFuture
from lmcache.v1.multiprocess.protocol import RequestType, get_response_class
logger = init_logger(__name__)
def wrap_kv_caches(kv_caches: dict[str, torch.Tensor]) -> KVCache:
logger.info("KV caches keys are %s", list(kv_caches.keys()))
return [CudaIPCWrapper(tensor) for tensor in kv_caches.values()]
def striding_block_hashes(
block_hashes: list[bytes], blocks_in_chunk: int
) -> Iterable[bytes]:
"""Extract chunk-level hashes from block hashes by striding.
In hash-based vLLM, each vLLM block has its own hash. LMCache chunks
span ``blocks_in_chunk`` consecutive blocks. The representative hash
for a chunk is the hash of the **last** block in that chunk (because
each block hash already encodes its prefix). So we start at index
``blocks_in_chunk - 1`` and stride by ``blocks_in_chunk``.
"""
return islice(block_hashes, blocks_in_chunk - 1, None, blocks_in_chunk)
def send_lmcache_request(
mq_client: MessageQueueClient,
request_type: RequestType,
payloads: list[Any],
) -> MessagingFuture[Any]:
"""Helper function to send the request to the LMCache multiprocess server.
Args:
mq_client: The LMCache multiprocess mode message queue client
request_type: The request type
payloads: The request payloads
Returns:
A messaging future for the request
"""
future = mq_client.submit_request(
request_type, payloads, get_response_class(request_type)
)
return future
def get_lmcache_chunk_size(
mq_client: MessageQueueClient,
) -> int:
"""Helper function to get the LMCache chunk size from the server.
Args:
mq_client: The LMCache multiprocess mode message queue client
Returns:
An integer representing the LMCache chunk size
"""
future = send_lmcache_request(mq_client, RequestType.GET_CHUNK_SIZE, [])
chunk_size = future.result()
return chunk_size
@dataclass
class ParallelStrategy:
use_mla: bool
"""Whether to use the MLA."""
kv_world_size: int
"""
The kv world size, kv_world_size may not be equal to the actual_world_size,
in the case of mla, it will 'exclude' the effect of TP, the value is
calculated by `extract_world_size_and_kv_rank` in `lmcache_mp_connector.py`.
"""
kv_worker_id: int
"""
The kv worker id of the sub-process, kv_worker_id may not be equal to the
actual_worker_id, in the case of mla, it will 'exclude' the effect of TP,
the value is calculated by `extract_world_size_and_kv_rank` in
`lmcache_mp_connector.py`.
"""
actual_world_size: int
"""The actual world size."""
actual_worker_id: int
"""The actual worker id of the sub-process."""
tp_size: int
"""The tensor parallel size."""
pp_size: int
"""The pipeline parallel size."""
@dataclass
class LoadStoreOp:
block_ids: list[int]
"""Block ids for the load/store operation"""
token_ids: list[int] | None = None
"""Token IDs for the load/store operation (token mode)"""
block_hashes: list[bytes] | None = None
"""Block hashes for the load/store operation (hash mode)"""
start: int = 0
"""Start token index (token mode only)"""
end: int = 0
"""End token index (token mode only)"""
def __len__(self) -> int:
return len(self.block_ids)
StoreResult = bool
RetrieveResult = list[bool]
LookupResult = int
class LMCacheMPSchedulerAdapter:
def __init__(
self,
server_url: str,
context: zmq.Context,
model_name: str,
vllm_block_size: int,
parallel_strategy: ParallelStrategy,
):
"""Args:
server_url: The server URL for the LMCache message queue
context: The ZMQ context
model_name: The model name used for LMCache keys
vllm_block_size: The block size used in vLLM
parallel_strategy:
The parallel strategy, which includes `use_mla`,
`world_size`, `worker_id` and so on
"""
self.mq_client = MessageQueueClient(server_url, context)
# Request futures
self.lookup_futures: dict[str, MessagingFuture[LookupResult]] = {}
self.model_name = model_name
self.parallel_strategy = parallel_strategy
# Read chunk size from lmcache
self.chunk_size = get_lmcache_chunk_size(self.mq_client)
assert self.chunk_size % vllm_block_size == 0, (
"LMCache chunk size should be a multiple of vLLM block size"
)
self.blocks_in_chunk = self.chunk_size // vllm_block_size
@property
def world_size(self) -> int:
"""The world size."""
return self.parallel_strategy.kv_world_size
@property
def worker_id(self) -> int:
"""The worker id."""
return self.parallel_strategy.kv_worker_id
@property
def tp_size(self) -> int:
"""The tensor parallel size."""
return self.parallel_strategy.tp_size
@_lmcache_nvtx_annotate
def maybe_submit_lookup_request(
self,
request_id: str,
block_hashes: list[bytes] | None = None,
token_ids: list[int] | None = None,
) -> None:
"""Submit a new lookup request to LMCache if there is no ongoing request.
Supports both token-based and hash-based vLLM:
- token_ids: token IDs (token-based vLLM) -> single token-mode key
- block_hashes: block hashes (hash-based vLLM) -> strided hash-mode keys
Exactly one of block_hashes or token_ids must be provided.
Args:
request_id: The ID of the lookup request. The same ID indicates it's
from the same request
block_hashes: Block hashes to lookup from LMCache (hash mode)
token_ids: Token IDs to lookup from LMCache (token mode)
Returns:
None
Notes:
This function will have a side-effect: submitting a look up request to
LMCache, which will essentially 'lock' the KV cache chunks in the LMCache
for later retrieve operations.
In the meantime, this function will record the lookup request, and the
status of the look up request can be checked by `check_lookup_result`.
"""
if request_id in self.lookup_futures:
# Skip if there is already a lookup request
return
assert (block_hashes is None) != (token_ids is None), (
"Exactly one of block_hashes or token_ids must be provided"
)
if block_hashes is not None:
# Hash mode: stride block hashes -> N hash-mode keys
chunk_hashes = list(
striding_block_hashes(block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
]
else:
# Token mode: truncate to chunk-aligned length
assert token_ids is not None
aligned_end = (len(token_ids) // self.chunk_size) * self.chunk_size
if aligned_end == 0:
return
keys = [
self._create_key(
token_ids,
start=0,
end=aligned_end,
request_id=request_id,
).no_worker_id_version()
]
future = send_lmcache_request(
self.mq_client,
RequestType.LOOKUP,
[keys],
)
self.lookup_futures[request_id] = future
@_lmcache_nvtx_annotate
def check_lookup_result(self, request_id: str) -> int | None:
"""Check the result of a previously submitted lookup request.
Args:
request_id: The ID of the lookup request submitted in
`maybe_submit_lookup_request`
Returns:
An integer representing the total number of tokens matched
in LMCache (prefix matching), or
None if the lookup request is not finished yet.
"""
assert request_id in self.lookup_futures, (
f"Lookup request for request_id={request_id} has not been submitted"
)
future = self.lookup_futures[request_id]
if not future.query():
return None
result = future.result()
num_chunks = result
return num_chunks * self.chunk_size
def num_blocks_per_chunk(self) -> int:
"""Returns:
The number of vllm blocks in a LMCache data chunk
"""
return self.blocks_in_chunk
def cleanup_lookup_result(self, request_id: str) -> None:
"""Clean up lookup future for a finished request to prevent memory leak.
Args:
request_id: The ID of the finished request.
"""
self.lookup_futures.pop(request_id, None)
def end_session(self, request_id: str) -> None:
"""Notify LMCache server to remove the session for a finished request.
Args:
request_id: The ID of the finished request.
"""
send_lmcache_request(
self.mq_client,
RequestType.END_SESSION,
[request_id],
)
# Helper functions
def _create_key(
self,
token_ids: list[int],
start: int = 0,
end: int = 0,
request_id: str | None = None,
) -> IPCCacheEngineKey:
"""Convert token IDs to an IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
worker_id=self.worker_id,
token_ids=tuple(token_ids),
start=start,
end=end,
request_id=request_id,
tp_size=self.tp_size,
)
def _create_hash_key(
self, chunk_hash: bytes, request_id: str | None = None
) -> IPCCacheEngineKey:
"""Create a hash-mode IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
worker_id=None,
chunk_hash=chunk_hash,
request_id=request_id,
tp_size=self.tp_size,
)
class LMCacheMPWorkerAdapter:
def __init__(
self,
server_url: str,
context: zmq.Context,
model_name: str,
vllm_block_size: int,
parallel_strategy: ParallelStrategy,
):
self.mq_client = MessageQueueClient(server_url, context)
# Instance id for GPU worker
self.instance_id = os.getpid()
# Registered kv caches from vLLM
self.kv_caches: dict[str, torch.Tensor] = {}
# Request futures
# request_id -> (future, other merged requests)
self.store_futures: dict[
str, tuple[MessagingFuture[StoreResult], list[str]]
] = {}
self.retrieve_futures: dict[
str, tuple[MessagingFuture[RetrieveResult], list[str]]
] = {}
# The store requests that have finished execution in LMCache
self.finished_stores: set[str] = set()
# The finished request ids that are passed via vLLM and also
# have corresponding store requests submitted to LMCache before
self.previously_finished: set[str] = set()
self.model_name = model_name
self.parallel_strategy = parallel_strategy
# Read chunk size from lmcache
chunk_size = get_lmcache_chunk_size(self.mq_client)
assert chunk_size % vllm_block_size == 0, (
"LMCache chunk size should be a multiple of vLLM block size"
)
self.blocks_in_chunk = chunk_size // vllm_block_size
@property
def world_size(self) -> int:
"""The world size."""
return self.parallel_strategy.kv_world_size
@property
def worker_id(self) -> int:
"""The worker id."""
return self.parallel_strategy.kv_worker_id
@property
def use_mla(self) -> bool:
"""Whether to use MLA."""
return self.parallel_strategy.use_mla
@property
def is_first_rank_of_pp_group(self) -> bool:
"""Is the first rank of the pipeline parallel group."""
return (
self.parallel_strategy.actual_worker_id % self.parallel_strategy.tp_size
== 0
)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Register the kv caches with LMCache server.
Args:
kv_caches: A dict of kv caches to register. The keys are the
layer names and the values are the corresponding tensors.
"""
# Register kv cache and send the request
self.kv_caches = kv_caches
logger.info("Registering kv caches")
future = send_lmcache_request(
self.mq_client,
RequestType.REGISTER_KV_CACHE,
[self.instance_id, wrap_kv_caches(kv_caches)],
)
future.result()
@_lmcache_nvtx_annotate
def submit_store_request(
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
):
"""Submit a KV cache store request to LMCache.
Args:
request_id: The ID of the request
op: The LoadStoreOp describing the store operation.
event: The CUDA event that is recorded after the current
model inference step
"""
if op.block_hashes is not None:
# Hash mode
chunk_hashes = list(
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
]
else:
# Token mode
assert op.token_ids is not None
keys = [
self._create_key(op.token_ids, op.start, op.end, request_id=request_id)
]
future = send_lmcache_request(
self.mq_client,
RequestType.STORE,
[keys, self.instance_id, op.block_ids, event.ipc_handle()],
).to_cuda_future()
self.store_futures[request_id] = (future, [])
@_lmcache_nvtx_annotate
def submit_retrieve_request(
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
):
"""Submit a KV cache retrieve request to LMCache.
Args:
request_id: The ID of the request
op: The LoadStoreOp describing the retrieve operation.
event: The CUDA event that is recorded after the current
model inference step
"""
if op.block_hashes is not None:
# Hash mode
chunk_hashes = list(
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
]
else:
# Token mode
assert op.token_ids is not None
keys = [
self._create_key(op.token_ids, op.start, op.end, request_id=request_id)
]
future = send_lmcache_request(
self.mq_client,
RequestType.RETRIEVE,
[keys, self.instance_id, op.block_ids, event.ipc_handle()],
).to_cuda_future()
self.retrieve_futures[request_id] = (future, [])
@_lmcache_nvtx_annotate
def batched_submit_store_requests(
self,
request_ids: list[str],
ops: list[LoadStoreOp],
event: torch.cuda.Event,
):
"""Submit a batched store request to LMCache.
Args:
request_ids: The IDs of the requests
ops: The LoadStoreOps describing the store operations. Should have
the same length as request_ids
event: The CUDA event that is recorded after the current
model inference step
"""
all_keys: list[IPCCacheEngineKey] = []
block_ids: list[int] = []
for request_id, op in zip(request_ids, ops, strict=False):
if op.block_hashes is not None:
chunk_hashes = list(
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id)
for ch in chunk_hashes
]
all_keys.extend(keys)
else:
assert op.token_ids is not None
all_keys.append(
self._create_key(
op.token_ids, op.start, op.end, request_id=request_id
)
)
block_ids.extend(op.block_ids)
future = send_lmcache_request(
self.mq_client,
RequestType.STORE,
[
all_keys,
self.instance_id,
block_ids,
event.ipc_handle(),
],
).to_cuda_future()
self.store_futures[request_ids[0]] = (future, list(request_ids[1:]))
@_lmcache_nvtx_annotate
def batched_submit_retrieve_requests(
self,
request_ids: list[str],
ops: list[LoadStoreOp],
event: torch.cuda.Event,
):
"""Submit a batched retrieve request to LMCache.
Args:
request_ids: The IDs of the requests
ops: The LoadStoreOps describing the retrieve operations. Should have
the same length as request_ids
event: The CUDA event that is recorded after the current
model inference step
"""
all_keys: list[IPCCacheEngineKey] = []
block_ids: list[int] = []
for request_id, op in zip(request_ids, ops, strict=False):
if op.block_hashes is not None:
chunk_hashes = list(
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
)
keys = [
self._create_hash_key(ch, request_id=request_id)
for ch in chunk_hashes
]
all_keys.extend(keys)
else:
assert op.token_ids is not None
all_keys.append(
self._create_key(
op.token_ids, op.start, op.end, request_id=request_id
)
)
block_ids.extend(op.block_ids)
future = send_lmcache_request(
self.mq_client,
RequestType.RETRIEVE,
[
all_keys,
self.instance_id,
block_ids,
event.ipc_handle(),
],
).to_cuda_future()
self.retrieve_futures[request_ids[0]] = (future, list(request_ids[1:]))
@_lmcache_nvtx_annotate
def get_finished(
self, finished_req_ids_from_engine: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""Check and get the finished store and retrieve requests.
Args:
finished_req_ids_from_engine: the set of request ids that are
reported as finished from the vLLM engine side.
Returns:
A tuple of two sets:
- The first set contains the finished store request ids. The returned
store request ids MUST be seen before in the
`finished_req_ids_from_engine`.
- The second set contains the finished retrieve request ids.
Notes:
When enabling async scheduling in vLLM, the same request ID may appear
multiple times in `finished_req_ids_from_engine`. The adapter should
take care of deduplicating the request IDs and only return the request
IDs that have not been returned before.
"""
finished_stores = set()
finished_retrieves = set()
for request_id, (s_future, other_reqs) in self.store_futures.items():
if not s_future.query():
continue
s_result = s_future.result()
finished_stores.add(request_id)
finished_stores.update(other_reqs)
if not s_result:
# TODO: add error handling here
logger.error(
"Something went wrong when processing the "
"store request for request_id=%s",
request_id,
)
for request_id, (r_future, other_reqs) in self.retrieve_futures.items():
if not r_future.query():
continue
r_result = r_future.result()
finished_retrieves.add(request_id)
finished_retrieves.update(other_reqs)
if not all(r_result):
# TODO: add error handing here
logger.error(
"Something went wrong when processing the "
"retrieve request for request_id=%s, result=%s",
request_id,
r_result,
)
# Remove the finished requests from the tracking dicts
for request_id in finished_stores:
self.store_futures.pop(request_id, None)
for request_id in finished_retrieves:
self.retrieve_futures.pop(request_id, None)
# Update the internal states
self.finished_stores.update(finished_stores)
ret_stores = set()
for req_id in finished_req_ids_from_engine:
if req_id in self.finished_stores or req_id in self.store_futures:
self.previously_finished.add(req_id)
else:
ret_stores.add(req_id)
# Calculate the final finished stores
ret_stores.update(self._update_and_get_finished_store())
return ret_stores, finished_retrieves
def num_blocks_per_chunk(self) -> int:
"""Returns:
The number of vllm blocks in a LMCache data chunk
"""
return self.blocks_in_chunk
def shutdown(self):
"""Shutdown the LMCache MP worker adapter."""
logger.info("Unregistering kv caches")
send_lmcache_request(
self.mq_client, RequestType.UNREGISTER_KV_CACHE, [self.instance_id]
).result()
self.mq_client.close()
# Helper functions
def _update_and_get_finished_store(
self,
) -> set[str]:
"""Converge the internal states about finished stores
and returns the 'safe finished store request ids' back
"""
safe_finished_s = self.finished_stores.intersection(self.previously_finished)
self.finished_stores.difference_update(self.previously_finished)
self.previously_finished.difference_update(safe_finished_s)
return safe_finished_s
def _create_key(
self,
token_ids: list[int],
start: int = 0,
end: int = 0,
request_id: str | None = None,
) -> IPCCacheEngineKey:
"""Convert token IDs to an IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
worker_id=self.worker_id,
token_ids=tuple(token_ids),
start=start,
end=end,
request_id=request_id,
)
def _create_hash_key(
self, chunk_hash: bytes, request_id: str | None = None
) -> IPCCacheEngineKey:
"""Create a hash-mode IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
worker_id=self.worker_id,
chunk_hash=chunk_hash,
request_id=request_id,
)
@@ -1,134 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Standard
import os
import threading
from typing import TYPE_CHECKING, Union
import torch
from lmcache.logging import init_logger
from lmcache.v1.config import LMCacheEngineConfig as V1Config
if TYPE_CHECKING:
from vllm.config import ModelConfig
from vllm.multimodal.inputs import PlaceholderRange
from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.request import Request
logger = init_logger(__name__)
ENGINE_NAME = "vllm-instance"
# Thread-safe singleton storage
_config_instance: V1Config | None = None
_config_lock = threading.Lock()
def lmcache_get_or_create_config() -> V1Config:
"""Get the LMCache configuration from the environment variable
`LMCACHE_CONFIG_FILE`. If the environment variable is not set, this
function will return the default configuration.
This function is thread-safe and implements singleton pattern,
ensuring the configuration is loaded only once.
"""
global _config_instance
# Double-checked locking for thread-safe singleton
if _config_instance is None:
with _config_lock:
if _config_instance is None: # Check again within lock
LMCacheEngineConfig = V1Config # type: ignore[assignment]
if "LMCACHE_CONFIG_FILE" not in os.environ:
logger.warning(
"No LMCache configuration file is set. Trying to read"
" configurations from the environment variables."
)
logger.warning(
"You can set the configuration file through "
"the environment variable: LMCACHE_CONFIG_FILE"
)
_config_instance = LMCacheEngineConfig.from_env()
else:
config_file = os.environ["LMCACHE_CONFIG_FILE"]
logger.info("Loading LMCache config file %s", config_file)
_config_instance = LMCacheEngineConfig.from_file(config_file)
# Update config from environment variables
_config_instance.update_config_from_env()
return _config_instance
def hex_hash_to_int16(s: str) -> int:
"""Convert a hex hash string to a 16-bit integer."""
return int(s, 16) & 0xFFFF
def apply_mm_hashes_to_token_ids(
token_ids: torch.Tensor,
mm_hashes: list[str],
mm_positions: list["PlaceholderRange"],
) -> torch.Tensor:
"""Overwrite token_ids in-place for multimodal placeholders using
efficient slice assignments.
"""
n = token_ids.size(0)
for hash_str, placeholder in zip(mm_hashes, mm_positions):
start, length = placeholder.offset, placeholder.length
if start >= n:
continue
end = min(start + length, n)
token_ids[start:end] = hex_hash_to_int16(hash_str)
return token_ids
def mla_enabled(model_config: "ModelConfig") -> bool:
return (
hasattr(model_config, "use_mla")
and isinstance(model_config.use_mla, bool)
and model_config.use_mla
)
def extract_mm_features(
request: Union["Request", "NewRequestData"], modify: bool = False
) -> tuple[list[str], list["PlaceholderRange"]]:
"""Normalize multimodal information from a Request into parallel lists.
This helper reads either:
1) `request.mm_features` (objects each exposing `.identifier` and
`.mm_position`), or
2) legacy fields `request.mm_hashes` and `request.mm_positions`.
It returns two equally sized lists: the multimodal hash identifiers and
their corresponding positions. If the request contains no multimodal info,
it returns `([], [])`.
Args:
request (Request): The source object.
modify (bool):
Controls copy semantics for the legacy-path return values.
- If True and legacy fields are used, shallow-copies are returned so
the caller can mutate the lists without affecting `request`.
- If False, the original legacy sequences are returned as-is
(zero-copy); treat them as read-only.
Returns:
tuple[list[str], list[PlaceholderRange]]: (`mm_hashes`, `mm_positions`).
May be `([], [])` when no multimodal data is present.
"""
if getattr(request, "mm_features", None):
mm_hashes, mm_positions = zip(
*((f.identifier, f.mm_position) for f in request.mm_features)
)
return (list(mm_hashes), list(mm_positions))
elif getattr(request, "mm_hashes", None):
if modify:
return (
request.mm_hashes.copy(), # type: ignore
request.mm_positions.copy(), # type: ignore
)
else:
return (request.mm_hashes, request.mm_positions) # type: ignore
else:
return ([], [])
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,178 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import socket
import threading
import time
from dataclasses import dataclass
import uvicorn
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from vllm.distributed.kv_transfer.kv_connector.utils import EngineId
from vllm.logger import init_logger
WorkerAddr = str
logger = init_logger(__name__)
class RegisterWorkerPayload(BaseModel):
engine_id: EngineId
dp_rank: int
tp_rank: int
pp_rank: int
addr: WorkerAddr
@dataclass
class EngineEntry:
engine_id: EngineId
# {tp_rank: {pp_rank: worker_addr}}
worker_addr: dict[int, dict[int, WorkerAddr]]
class MooncakeBootstrapServer:
"""A centralized registry for prefiller connection info (IP, port, ranks)."""
def __init__(self, host: str, port: int):
self.workers: dict[int, EngineEntry] = {}
self.host = host
self.port = port
self.app = FastAPI()
self._register_routes()
self.server_thread: threading.Thread | None = None
self.server: uvicorn.Server | None = None
self._socket: socket.socket | None = None
self._startup_error: BaseException | None = None
def __del__(self):
self.shutdown()
def _register_routes(self):
# All methods are async. No need to use lock to protect data.
self.app.post("/register")(self.register_worker)
self.app.get("/query", response_model=dict[int, EngineEntry])(self.query)
def start(self, *, timeout: float = 30.0):
if self.server_thread:
if (
self.server_thread.is_alive()
and self.server is not None
and self.server.started
and not self.server.should_exit
and self._socket is not None
and self._socket.fileno() >= 0
):
return
raise RuntimeError(
"Mooncake bootstrap server is still starting or stopping"
)
try:
# Bind in the caller so errors propagate, and retain the listener
# when handing it to Uvicorn, including for automatically chosen ports.
family = socket.AF_INET6 if ":" in self.host else socket.AF_INET
self._socket = socket.create_server((self.host, self.port), family=family)
self.port = self._socket.getsockname()[1]
config = uvicorn.Config(app=self.app, host=self.host, port=self.port)
server = self.server = uvicorn.Server(config=config)
listener = self._socket
self._startup_error = None
def run():
try:
server.run(sockets=[listener])
except BaseException as exc:
self._startup_error = exc
self.server_thread = threading.Thread(
target=run, name="mooncake_bootstrap_server", daemon=True
)
self.server_thread.start()
deadline = time.monotonic() + timeout
while not server.started:
if not self.server_thread.is_alive():
raise RuntimeError(
"Mooncake bootstrap server exited during startup"
) from self._startup_error
if time.monotonic() >= deadline:
raise TimeoutError(
"Mooncake bootstrap server did not start in time"
)
time.sleep(0.01)
except BaseException:
self.shutdown()
raise
logger.info("Mooncake Bootstrap Server started at %s:%d", self.host, self.port)
def shutdown(self):
was_started = self.server is not None and self.server.started
if self.server is not None:
self.server.should_exit = True
if self.server_thread is not None and self.server_thread.ident is not None:
self.server_thread.join(timeout=5)
if self._socket is not None:
self._socket.close()
self._socket = None
if self.server_thread is not None and self.server_thread.is_alive():
logger.warning("Mooncake bootstrap server did not stop in time")
else:
self.server_thread = None
self.server = None
if was_started:
logger.info("Mooncake Bootstrap Server stopped.")
async def register_worker(self, payload: RegisterWorkerPayload):
"""Handles registration of a prefiller worker."""
if payload.dp_rank not in self.workers:
self.workers[payload.dp_rank] = EngineEntry(
engine_id=payload.engine_id,
worker_addr={},
)
dp_entry = self.workers[payload.dp_rank]
if dp_entry.engine_id != payload.engine_id:
raise HTTPException(
status_code=400,
detail=(
f"Engine ID mismatch for dp_rank={payload.dp_rank}: "
f"expected {dp_entry.engine_id}, got {payload.engine_id}"
),
)
if payload.tp_rank not in dp_entry.worker_addr:
dp_entry.worker_addr[payload.tp_rank] = {}
tp_entry = dp_entry.worker_addr[payload.tp_rank]
existing = tp_entry.get(payload.pp_rank)
if existing is not None:
# A client timeout can fire after the server recorded the
# registration so an identical retry must not be an error.
if existing == payload.addr:
return {"status": "ok"}
raise HTTPException(
status_code=400,
detail=(
f"Worker with dp_rank={payload.dp_rank}, "
f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} "
f"is already registered at "
f"{existing}, "
f"but still want to register at {payload.addr}"
),
)
tp_entry[payload.pp_rank] = payload.addr
logger.debug(
"Registered worker: engine_id=%s, dp_rank=%d, tp_rank=%d, pp_rank=%d at %s",
payload.engine_id,
payload.dp_rank,
payload.tp_rank,
payload.pp_rank,
payload.addr,
)
return {"status": "ok"}
async def query(self) -> dict[int, EngineEntry]:
return self.workers
@@ -1,46 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Mooncake requester config helpers."""
from collections.abc import Mapping
from typing import Any
import vllm.envs as envs
from vllm.logger import init_logger
logger = init_logger(__name__)
def normalize_string_override(value: Any) -> str | None:
if not isinstance(value, str):
return None
normalized = value.strip()
return normalized or None
def get_requester_local_hostname(local_ip: str) -> str:
override = normalize_string_override(envs.MOONCAKE_REQUESTER_LOCAL_HOSTNAME)
if override is not None:
return override
return local_ip
def get_configured_preferred_segment(
extra_config: Mapping[str, Any],
) -> str | None:
preferred_segment = normalize_string_override(extra_config.get("preferred_segment"))
if preferred_segment is not None:
return preferred_segment
if extra_config.get("preferred_segment") is not None:
raise ValueError(
"Mooncake preferred_segment override must be a non-empty string"
)
env_value = normalize_string_override(envs.MOONCAKE_PREFERRED_SEGMENT)
if env_value is not None:
logger.info(
"Mooncake preferred_segment from MOONCAKE_PREFERRED_SEGMENT: %s",
env_value,
)
return env_value
return None
@@ -1,146 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Stats container for the Mooncake connector."""
import threading
from dataclasses import dataclass
from typing import Any
import numpy as np
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorStats,
)
# TODO(mooncake-stats): add MooncakePromMetrics (mirror NixlPromMetrics)
# and wire it via MooncakeConnector.build_prom_metrics in a follow-up PR.
@dataclass
class MooncakeKVConnectorStats(KVConnectorStats):
"""Container for Mooncake KV transfer performance metrics.
`_lock` serializes record_* against clone_and_reset so each row's
appends are atomic and column lengths stay aligned. Writers run on
the sender pool / receiver loop / sender loop; reader runs on the
main worker thread.
"""
def __post_init__(self):
self._lock = threading.Lock()
if not self.data:
self.reset()
# threading.Lock is not picklable; strip it from the wire form and
# rebuild a fresh per-process lock on the receiver side.
def __getstate__(self) -> dict[str, Any]:
state = self.__dict__.copy()
state.pop("_lock", None)
return state
def __setstate__(self, state: dict[str, Any]) -> None:
self.__dict__.update(state)
self._lock = threading.Lock()
def reset(self):
self.data: dict[str, list[float | int]] = {
"transfer_duration": [],
"bytes_transferred": [],
"num_descriptors": [],
"num_failed_transfers": [],
"num_failed_recvs": [],
"num_kv_expired_reqs": [],
}
def record_transfer(self, duration_s: float, total_bytes: int, num_descs: int):
with self._lock:
self.data["transfer_duration"].append(duration_s)
self.data["bytes_transferred"].append(total_bytes)
self.data["num_descriptors"].append(num_descs)
# Failure counters store a list of 1s so a future Prom counter can iterate
# with .inc(list_item), mirroring NIXL's NixlPromMetrics.observe.
def record_failed_transfer(self):
with self._lock:
self.data["num_failed_transfers"].append(1)
def record_failed_recv(self):
with self._lock:
self.data["num_failed_recvs"].append(1)
def record_kv_expired_req(self):
with self._lock:
self.data["num_kv_expired_reqs"].append(1)
def clone_and_reset(self) -> "MooncakeKVConnectorStats":
# Copy lists under the lock for length alignment; return a fresh
# instance so the snapshot has its own _lock.
with self._lock:
snapshot_data: dict[str, list[float | int]] = {
k: list(v) for k, v in self.data.items()
}
self.reset()
return MooncakeKVConnectorStats(data=snapshot_data)
def is_empty(self) -> bool:
return (
self.num_successful_transfers == 0
and len(self.data["num_failed_transfers"]) == 0
and len(self.data["num_failed_recvs"]) == 0
and len(self.data["num_kv_expired_reqs"]) == 0
)
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if not other.is_empty():
for k, v in other.data.items():
accumulator = self.data[k]
assert isinstance(accumulator, list)
accumulator.extend(v)
return self
def reduce(self) -> dict[str, int | float]:
num_failed_transfers = len(self.data["num_failed_transfers"])
num_failed_recvs = len(self.data["num_failed_recvs"])
num_kv_expired_reqs = len(self.data["num_kv_expired_reqs"])
if self.num_successful_transfers == 0:
return {
"Num successful transfers": 0,
"Avg xfer time (ms)": 0,
"P90 xfer time (ms)": 0,
"Avg MB per transfer": 0,
"Throughput (MB/s)": 0,
"Avg number of descriptors": 0,
"Num failed transfers": num_failed_transfers,
"Num failed recvs": num_failed_recvs,
"Num KV expired reqs": num_kv_expired_reqs,
}
xfer_time = np.asarray(self.data["transfer_duration"])
mb = np.asarray(self.data["bytes_transferred"]) / 2**20
descs = np.asarray(self.data["num_descriptors"], dtype=np.uint32)
n = len(descs)
assert n == self.num_successful_transfers
total_mb = mb.sum()
avg_mb = total_mb / n
total_time_seconds = xfer_time.sum()
throughput_mb_s = (
(total_mb / total_time_seconds).item() if total_time_seconds > 0 else 0.0
)
return {
"Num successful transfers": n,
"Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3),
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
"Avg MB per transfer": round(avg_mb.item(), 3),
"Throughput (MB/s)": round(throughput_mb_s, 3),
"Avg number of descriptors": round(descs.mean().item(), 1),
"Num failed transfers": num_failed_transfers,
"Num failed recvs": num_failed_recvs,
"Num KV expired reqs": num_kv_expired_reqs,
}
@property
def num_successful_transfers(self) -> int:
return len(self.data["transfer_duration"])
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,436 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Adapted from vllm-project/vllm-ascend
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
"""MooncakeStoreConnector - KV cache connector using MooncakeDistributedStore.
Unlike MooncakeConnector which does direct P2P transfer, this connector
uses MooncakeDistributedStore as a shared KV cache pool. Both producer
and consumer instances read/write KV to/from the store independently,
enabling prefix caching via hash-based deduplication.
"""
from collections import Counter
from collections.abc import Iterable, Sequence
from contextlib import AbstractContextManager
from typing import Any
import torch
from vllm.config import KVTransferConfig, VllmConfig
from vllm.distributed.kv_events import (
BlockStored,
KVCacheEvent,
KVConnectorKVEvents,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
KVConnectorWorkerMetadata,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
from .data import MooncakeStoreConnectorMetadata
from .metrics import MooncakeStoreConnectorStats, MooncakeStorePromMetrics
from .scheduler import MooncakeStoreScheduler
from .worker import MooncakeStoreWorker
logger = init_logger(__name__)
class MooncakeStoreKVEvents(KVConnectorKVEvents):
"""KV event aggregation for MooncakeStoreConnector."""
def __init__(
self,
num_workers: int,
group_tp_replication_factors: Sequence[int] = (1,),
) -> None:
if num_workers <= 0:
raise ValueError("num_workers must be greater than zero.")
if any(factor <= 0 for factor in group_tp_replication_factors):
raise ValueError("TP replication factors must be greater than zero.")
self._event_counter: Counter[KVCacheEvent] = Counter()
self._num_workers = num_workers
self._group_tp_replication_factors = tuple(group_tp_replication_factors)
def add_events(self, events: list[KVCacheEvent]) -> None:
if not isinstance(events, list):
raise TypeError("events must be a list of KVCacheEvent.")
self._event_counter.update(events)
def _replication_factor(self, event: KVCacheEvent) -> int:
if not isinstance(event, BlockStored) or event.group_idx is None:
return 1
return self._group_tp_replication_factors[event.group_idx]
def _is_common_event(self, event: KVCacheEvent, count: int) -> bool:
return count * self._replication_factor(event) >= self._num_workers
def aggregate(self) -> "MooncakeStoreKVEvents":
common_events = self.pop_common_events()
self._event_counter.clear()
self._event_counter.update(common_events)
self._num_workers = 1
return self
def pop_common_events(self) -> list[KVCacheEvent]:
common_events = [
event
for event, count in self._event_counter.items()
if self._is_common_event(event, count)
]
for event in common_events:
del self._event_counter[event]
return common_events
def has_events(self) -> bool:
return bool(self._event_counter)
def increment_workers(self, count: int = 1) -> None:
if count <= 0:
raise ValueError("count must be positive.")
self._num_workers += count
def get_all_events(self) -> list[KVCacheEvent]:
return list(self._event_counter.elements())
def get_number_of_workers(self) -> int:
return self._num_workers
def clear_events(self) -> None:
self._event_counter.clear()
self._num_workers = 1
def __repr__(self) -> str:
return f"<MooncakeStoreKVEvents events={self.get_all_events()}>"
class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
"""KV connector using MooncakeDistributedStore as shared KV pool."""
@classmethod
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
return False
@staticmethod
def _validate_kv_cache_config(
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
) -> None:
from vllm.v1.kv_cache_interface import CrossAttentionSpec, MambaSpec
unsupported: list[str] = []
store_group_ids = kv_cache_config.prefix_cacheable_group_ids
if not store_group_ids:
raise ValueError(
"MooncakeStore requires at least one prefix-cacheable KV cache group"
)
for group_id in store_group_ids:
spec = kv_cache_config.kv_cache_groups[group_id].kv_cache_spec
if isinstance(spec, CrossAttentionSpec):
unsupported.append(f"group {group_id}: CrossAttentionSpec")
if isinstance(spec, MambaSpec) and spec.mamba_cache_mode != "align":
unsupported.append(
f"group {group_id}: mamba_cache_mode="
f"{spec.mamba_cache_mode!r} != 'align'"
)
pcp = vllm_config.parallel_config.prefill_context_parallel_size
if len(store_group_ids) > 1 and pcp > 1:
unsupported.append(f"PCP > 1 (pcp={pcp}) with hybrid attention")
if unsupported:
raise ValueError(
"MooncakeStoreConnector does not support: " + "; ".join(unsupported)
)
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: KVCacheConfig | None = None,
):
super().__init__(
vllm_config=vllm_config,
role=role,
kv_cache_config=kv_cache_config, # type: ignore[arg-type]
)
assert vllm_config.kv_transfer_config is not None
assert kv_cache_config is not None, "kv_cache_config is required"
self.kv_role = vllm_config.kv_transfer_config.kv_role
extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
save_decode_cache = extra_config.get("save_decode_cache", False)
# Capacity-only: contributes its segment to the store pool but transfers
# no KV, so the KV-cache-shape invariants below cannot be reached.
self._capacity_only = (
self.kv_role == "kv_consumer"
and not extra_config.get("enable_lookup", True)
and not save_decode_cache
)
if not self._capacity_only:
self._validate_kv_cache_config(vllm_config, kv_cache_config)
self._kv_cache_config = kv_cache_config
self._kv_cache_events: MooncakeStoreKVEvents | None = None
self.connector_scheduler: MooncakeStoreScheduler | None = None
self.connector_worker: MooncakeStoreWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = MooncakeStoreScheduler(
vllm_config, kv_cache_config
)
else:
self.connector_worker = MooncakeStoreWorker(vllm_config, kv_cache_config)
def shutdown(self):
"""Release connector resources on teardown.
Closes the worker's MooncakeDistributedStore handle so its
TransferEngine and RDMA registrations are released. Invoked from the
engine's explicit shutdown path and as a backstop from ``__del__``;
a no-op on the scheduler role, which holds no store handle.
"""
worker = getattr(self, "connector_worker", None)
if worker is not None:
worker.close()
def __del__(self):
self.shutdown()
# ============================================================
# Scheduler-side methods
# ============================================================
def get_num_new_matched_tokens(
self,
request: Request,
num_computed_tokens: int,
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
)
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
):
assert self.connector_scheduler is not None
return self.connector_scheduler.update_state_after_alloc(
request, blocks, num_external_tokens
)
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
def has_pending_push_work(self) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.has_pending_push_work()
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
) -> KVConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
assert self.connector_worker is not None
return self.connector_worker.build_connector_worker_meta()
def request_finished(
self,
request: Request,
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
return self.request_finished_all_groups(request, (block_ids,))
def request_finished_all_groups(
self,
request: Request,
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
# An in-flight store job holds its own reference on the blocks it reads,
# so a finishing request never has to defer freeing them.
return False, None
def register_finished_partial_tail(
self,
request: Request,
block_ids: tuple[list[int], ...],
partial_tail_offloads: list[tuple[int, int, int]],
) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.register_finished_partial_tail(
request, block_ids, partial_tail_offloads
)
def reset_cache(self) -> bool | None:
"""Reset the external Mooncake store on prefix-cache reset.
Drains the worker send queue, then runs ``remove_all`` on the
Mooncake master. Caller must first pause generation (e.g.
``pause_generation``) so no new puts are enqueued during drain.
Returns True on ack, False on failure, None for the worker role.
"""
if self.role == KVConnectorRole.SCHEDULER:
assert self.connector_scheduler is not None
# Clear local references to keys we're about to wipe.
self.connector_scheduler.load_specs.clear()
self._kv_cache_events = None
return self.connector_scheduler.reset_store()
return None
def update_connector_output(self, connector_output: KVConnectorOutput):
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
kv_cache_events = connector_output.kv_cache_events
if not kv_cache_events or not isinstance(
kv_cache_events, MooncakeStoreKVEvents
):
return
if self._kv_cache_events is None:
self._kv_cache_events = kv_cache_events
else:
self._kv_cache_events.add_events(kv_cache_events.get_all_events())
def take_events(self) -> Iterable[KVCacheEvent]:
if self._kv_cache_events is not None:
events = self._kv_cache_events.pop_common_events()
if not self._kv_cache_events.has_events():
self._kv_cache_events = None
yield from events
# ============================================================
# Worker-side methods
# ============================================================
def get_mem_pool_context(self) -> AbstractContextManager | None:
"""Return a context manager for the custom MemPool, or None.
Called by the Worker before ``initialize_kv_cache`` so that KV
cache is allocated from the Mooncake-managed pool when
``custom_mem_pool`` is set in ``kv_connector_extra_config``.
"""
if self.connector_worker is None:
return None
return self.connector_worker.get_mem_pool_context()
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
self.connector_worker.start_load_kv(metadata)
def wait_for_layer_load(self, layer_name: str) -> None:
# No layerwise support - no-op
return
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
# No layerwise support - no-op
return
def wait_for_save(self):
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
self.connector_worker.wait_for_save(metadata)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
return self.connector_worker.get_finished(finished_req_ids, metadata)
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
return self.connector_worker.get_transfer_results(finished_req_ids, metadata)
def get_block_ids_with_load_errors(self) -> set[int]:
assert self.connector_worker is not None
return self.connector_worker.get_block_ids_with_load_errors()
def get_kv_connector_kv_cache_events(
self,
) -> MooncakeStoreKVEvents | None:
assert self.connector_worker is not None
if (
not self.connector_worker.enable_kv_events
or self.connector_worker.kv_send_thread is None
):
return None
events = self.connector_worker.get_kv_events()
# Empty containers still count this worker toward the poll's quorum.
kv_events = MooncakeStoreKVEvents(
num_workers=1,
group_tp_replication_factors=(
self.connector_worker.group_tp_replication_factors
),
)
kv_events.add_events(events)
return kv_events
def get_kv_connector_stats(self) -> KVConnectorStats | None:
if self.connector_worker is None:
return None
return self.connector_worker.get_kv_connector_stats()
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
MooncakeStoreConnectorStats(data=data)
if data is not None
else MooncakeStoreConnectorStats()
)
@classmethod
def build_prom_metrics(
cls,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> KVConnectorPromMetrics:
return MooncakeStorePromMetrics(
vllm_config, metric_types, labelnames, per_engine_labelvalues
)
@@ -1,515 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""External-store cache-hit coordinator for MooncakeStoreConnector."""
from collections.abc import Sequence
from dataclasses import replace
from typing import TYPE_CHECKING, NamedTuple, cast
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
chunk_hashes_for_block_size,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_coordinator import replay_boundaries
from vllm.v1.core.kv_cache_utils import (
BlockHash,
KVCacheBlock,
eagle_proof_margin,
resolve_dcp_kv_cache_spec,
)
from vllm.v1.core.single_type_kv_cache_manager import (
SingleTypeKVCacheManager,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
MambaSpec,
UniformTypeKVCacheSpecs,
)
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
if TYPE_CHECKING:
from vllm.config import VllmConfig
class StoreSpecGroup(NamedTuple):
spec: KVCacheSpec
group_ids: list[int]
manager_cls: type[SingleTypeKVCacheManager]
use_eagle: bool
class ExternalCachedBlockPool:
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
def __init__(
self,
hash_block_size: int,
exists: set[tuple[int, bytes]] | None = None,
) -> None:
# ``exists=None`` is used on the recv side where hit_length is already
# determined and we just want each spec's manager to apply its own mask.
self._exists = exists
self.hash_block_size = hash_block_size
self.null_block = KVCacheBlock(block_id=0)
# Dummy ID 1 for present block for duck-typing.
self._present_block = KVCacheBlock(block_id=1)
def get_cached_block(
self,
block_hash: BlockHash,
group_ids: list[int],
) -> list[KVCacheBlock] | None:
# Mirrors BlockPool.get_cached_block: hit only when every group_id
# (groups sharing a spec) has the hash cached.
if self._exists is None:
return [self._present_block] * len(group_ids)
h = bytes(block_hash)
if all((g, h) in self._exists for g in group_ids):
return [self._present_block] * len(group_ids)
return None
def contains(self, group_id: int, block_hash: BlockHash) -> bool:
"""Return whether a group has a loadable key for this hash."""
return (
self._exists is not None and (group_id, bytes(block_hash)) in self._exists
)
class MooncakeStoreCoordinator:
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
``ExternalCachedBlockPool``."""
def __init__(
self,
kv_cache_groups: list[KVCacheGroupSpec],
scheduler_block_size: int,
hash_block_size: int,
use_eagle: bool = False,
retention_interval: int | None = None,
enable_partial_hash_hits: bool = False,
) -> None:
# Mirrors core's resolve_kv_cache_block_sizes: the hash unit only has
# to divide groups that participate in prefix caching. Non-shareable
# scratch groups (e.g. GLM-5.3-Flash's kpool tail) are skipped by
# _verify_and_split_kv_cache_groups and never probed for hits.
assert all(
g.kv_cache_spec.block_size % hash_block_size == 0
for g in kv_cache_groups
if g.kv_cache_spec.prefix_cacheable
), "block_size must be divisible by hash_block_size"
assert scheduler_block_size % hash_block_size == 0, (
f"scheduler_block_size ({scheduler_block_size}) must be a multiple of "
f"hash_block_size ({hash_block_size})"
)
assert all(
scheduler_block_size % g.kv_cache_spec.block_size == 0
for g in kv_cache_groups
if g.kv_cache_spec.prefix_cacheable
), "scheduler_block_size must be a multiple of each group's block_size"
self.kv_cache_groups = kv_cache_groups
self.mamba_group_ids = {
group_id
for group_id, group in enumerate(kv_cache_groups)
if isinstance(_unwrap_spec(group.kv_cache_spec), MambaSpec)
}
self.hash_block_size = hash_block_size
self.lcm_block_size = scheduler_block_size
# Core's decision: whether hits land on the hash unit, below the block.
self.enable_partial_hash_hits = enable_partial_hash_hits
self.use_eagle = use_eagle
# Mirror vLLM core's KVCacheCoordinator.retention_interval.
self.retention_interval = retention_interval
self._verify_and_split_kv_cache_groups()
@classmethod
def from_kv_cache_config(
cls,
kv_cache_config: KVCacheConfig,
vllm_config: "VllmConfig",
scheduler_block_size: int,
hash_block_size: int,
) -> "MooncakeStoreCoordinator":
"""Build the coordinator the connector's scheduler and workers share.
Specs are DCP-resolved, so a block spans the tokens of one block-table
entry. Partial hits follow core's hit alignment, which checks every KV
cache group and its manager, not only the groups the store transfers;
unset (a config the engine core did not resolve) keeps hits on the
block.
"""
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
spec_config = vllm_config.speculative_config
alignment = kv_cache_config.cache_hit_alignment_tokens
return cls(
[
replace(
group,
kv_cache_spec=resolve_dcp_kv_cache_spec(
group.kv_cache_spec, dcp_world_size
),
)
for group in kv_cache_config.prefix_cacheable_groups
],
scheduler_block_size,
hash_block_size,
use_eagle=spec_config is not None and spec_config.use_eagle_block_drop(),
retention_interval=kv_cache_config.prefix_cache_retention_interval,
enable_partial_hash_hits=(
alignment is not None and alignment < scheduler_block_size
),
)
def align_lookup_length(self, length: int) -> int:
alignment = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
return length // alignment * alignment
def _verify_and_split_kv_cache_groups(self) -> None:
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
dispatches via spec_manager_map (we don't allocate managers).
"""
attention_groups: list[StoreSpecGroup] = []
for i, g in enumerate(self.kv_cache_groups):
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
# kpool tail): per-request scratch, never shareable, so they must
# not participate in hit lookup. Mirrors core's
# KVCacheCoordinator.verify_and_split_kv_cache_groups.
if not g.kv_cache_spec.prefix_cacheable:
continue
spec = _unwrap_spec(g.kv_cache_spec)
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
assert manager_cls is not None, (
f"No manager registered for KVCacheSpec {spec}"
)
for idx, group in enumerate(attention_groups):
if group.spec == spec:
assert manager_cls is group.manager_cls
group.group_ids.append(i)
if g.is_eagle_group and not group.use_eagle:
attention_groups[idx] = group._replace(use_eagle=True)
break
else:
attention_groups.append(
StoreSpecGroup(spec, [i], manager_cls, g.is_eagle_group)
)
# Full attention first (matches upstream convergence ordering).
attention_groups.sort(key=lambda g: not isinstance(g.spec, FullAttentionSpec))
# Conservatively flag all groups when use_eagle is set but none is flagged.
if self.use_eagle and not any(g.use_eagle for g in attention_groups):
attention_groups = [g._replace(use_eagle=True) for g in attention_groups]
self.attention_groups = attention_groups
# Per-group eagle bits. SpecGroup carries use_eagle for the whole
# merged spec group, so the per-group store/lookup masks agree with
# the merged-group hit check, which applies the eagle drop to every
# group sharing the spec.
self.eagle_group_ids = {
gid for g in attention_groups if g.use_eagle for gid in g.group_ids
}
self.eagle_proof_margin_by_group = {
gid: eagle_proof_margin(
group.spec.block_size,
self.hash_block_size,
self.enable_partial_hash_hits
and group.manager_cls.supports_fine_grained_hash_lookup,
)
for group in attention_groups
if group.use_eagle and not isinstance(group.spec, MambaSpec)
for gid in group.group_ids
}
def find_longest_cache_hit(
self,
block_hashes: Sequence[BlockHash],
max_length: int,
cached_block_pool: ExternalCachedBlockPool,
*,
apply_eagle: bool = True,
) -> tuple[tuple[list[bool], ...], int]:
"""Returns ``(load_mask_per_group, hit_length)``. ``mask[g][i]`` is True iff
group ``g`` populates chunk ``i`` locally (e.g. SWA and Mamba tail-only);
recv-side callers skip False slots.
``apply_eagle`` controls whether the per-spec ``use_eagle`` last-block
pop is applied. Lookup callers want it (the drafter requires recomputing
the last block); per-chunk mask callers must not, because ``token_len``
already reflects the eagle-pruned hit length and a second pop would
leave the trailing block unloaded.
"""
blocks_per_group, hit_length = self._find_hit_blocks(
block_hashes, max_length, cached_block_pool, apply_eagle=apply_eagle
)
masks = tuple(
[blk is not cached_block_pool.null_block for blk in blocks]
for blocks in blocks_per_group
)
return masks, hit_length
def load_mask(
self,
block_hashes: Sequence[BlockHash],
token_len: int,
) -> tuple[list[bool], ...]:
"""Per-group load masks: ``mask[g][i]`` is True iff group ``g``'s
spec would populate chunk ``i`` locally at length ``token_len``
(e.g. SWA / Mamba tail-only).
"""
# ``apply_eagle=False`` because ``token_len`` is already the
# eagle-pruned hit length returned by ``client.lookup``. Re-applying
# the pop here would shorten the mask by one extra block; the recv
# thread would then silently skip the trailing chunk yielded by
# ``db.process_tokens`` and leave that block uninitialized in the
# local KV pool.
masks, _ = self.find_longest_cache_hit(
block_hashes,
token_len,
ExternalCachedBlockPool(self.hash_block_size),
apply_eagle=False,
)
return masks
def store_mask(
self,
aligned_token_len: int,
start_token: int = 0,
num_prompt_tokens: int | None = None,
) -> tuple[list[bool] | None, ...]:
"""Per-group store masks for the suffix starting at ``start_token``.
``mask[g][i]`` is True iff the i-th chunk of group ``g`` *after*
``start_token`` should be written to the store so a future cache hit
can consume it. ``None`` is the all-True sentinel for the suffix.
Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask``
so the store retains exactly the blocks the local prefix cache would.
Mamba groups are always all-False: the normal save resolves blocks
positionally from the connector's append-only block-ID snapshot, but
an align-mode mamba block table is not append-only (interior state
blocks are nulled/freed, and speculative decoding relocates the spec
blocks in place), so a positional read may hit a null, freed, or live
speculative-state block and persist wrong bytes under a valid prefix
hash. Mamba state is persisted only through the connector-pinned exact
block hand-off path
(``SchedulerOutput.kv_connector_block_state.boundary_state_offloads``).
"""
return self._reachable_masks(
aligned_token_len,
start_token,
retention_interval=self.retention_interval,
num_prompt_tokens=num_prompt_tokens,
exclude_mamba=True,
)
def lookup_mask(
self,
aligned_token_len: int,
) -> tuple[list[bool] | None, ...]:
"""Per-group lookup masks.
``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be
looked up as an aligned hit boundary. ``None`` is the all-True
sentinel.
"""
return self._reachable_masks(
aligned_token_len,
0,
retention_interval=None,
num_prompt_tokens=None,
)
def _reachable_masks(
self,
aligned_token_len: int,
start_token: int,
*,
retention_interval: int | None,
num_prompt_tokens: int | None,
exclude_mamba: bool = False,
) -> tuple[list[bool] | None, ...]:
mask_alignment = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
assert aligned_token_len % mask_alignment == 0, (
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
f"{mask_alignment}"
)
reachable_boundaries = (
()
if num_prompt_tokens is None
else replay_boundaries(
num_prompt_tokens,
self.lcm_block_size,
use_eagle=bool(self.eagle_group_ids),
)
)
masks: list[list[bool] | None] = []
for g_idx, g in enumerate(self.kv_cache_groups):
spec = _unwrap_spec(g.kv_cache_spec)
end_chunk = aligned_token_len // spec.block_size
start_chunk = min(end_chunk, max(0, cdiv(start_token, spec.block_size)))
if exclude_mamba and isinstance(spec, MambaSpec):
masks.append([False] * (end_chunk - start_chunk))
continue
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
assert manager_cls is not None
use_eagle = g_idx in self.eagle_group_ids
mask = manager_cls.reachable_block_mask(
start_block=start_chunk,
end_block=end_chunk,
alignment_tokens=self.lcm_block_size,
kv_cache_spec=spec,
use_eagle=use_eagle,
retention_interval=retention_interval,
reachable_boundaries=reachable_boundaries,
# ``spec`` is already DCP-resolved (worker.py applies
# resolve_dcp_kv_cache_spec) and ``end_chunk`` is indexed in
# that scaled block size, so the mask must not scale again.
dcp_world_size=1,
)
if mask is not None:
assert len(mask) == end_chunk - start_chunk
masks.append(mask)
return tuple(masks)
def block_hashes_for_spec(
self, block_hashes: Sequence[BlockHash], spec: KVCacheSpec
) -> Sequence[BlockHash]:
return chunk_hashes_for_block_size(
block_hashes, self.hash_block_size, spec.block_size
)
def _find_hit_blocks(
self,
block_hashes: Sequence[BlockHash],
max_length: int,
cached_block_pool: ExternalCachedBlockPool,
*,
apply_eagle: bool = True,
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
"""Mirrors HybridKVCacheCoordinator.find_longest_cache_hit but
dispatches via spec_manager_map (we don't allocate managers).
When ``apply_eagle`` is False, ignore each group's ``use_eagle`` —
used by ``load_mask`` to avoid popping a second block on top of the
one already removed by the lookup.
"""
alignment_tokens = (
self.hash_block_size
if self.enable_partial_hash_hits
else self.lcm_block_size
)
if len(self.attention_groups) == 1:
spec, group_ids, manager_cls, group_eagle = self.attention_groups[0]
hit_blocks, hit_length = manager_cls.find_longest_cache_hit(
block_hashes=block_hashes, # type: ignore[arg-type]
max_length=max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
drop_eagle_block=apply_eagle and group_eagle,
alignment_tokens=alignment_tokens,
)
num_groups = len(self.kv_cache_groups)
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
for gid, blks in zip(group_ids, hit_blocks, strict=True):
blocks_by_group[gid] = blks
return tuple(blocks_by_group), hit_length
num_groups = len(self.kv_cache_groups)
hit_length = max_length
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
hit_length_by_group: list[int] = [0] * num_groups
is_simple_hybrid = len(self.attention_groups) == 2 and isinstance(
self.attention_groups[0].spec, FullAttentionSpec
)
eagle_verified: set[int] = set()
while True:
curr_hit_length = hit_length
for idx, (spec, group_ids, manager_cls, group_eagle) in enumerate(
self.attention_groups
):
first_group_id = group_ids[0]
cached = hit_blocks_by_group[first_group_id]
if isinstance(spec, FullAttentionSpec) and cached is not None:
curr_hit_length = min(
curr_hit_length, hit_length_by_group[first_group_id]
)
continue
drop_eagle_block = (
apply_eagle and group_eagle and idx not in eagle_verified
)
_max_length = curr_hit_length
# No eagle peek margin for a recurrent (Mamba) group: its finder
# never drops a block, so a widened bound would match past the
# attention-verified hit and resume from speculative state (#43559).
if drop_eagle_block and not isinstance(spec, MambaSpec):
eagle_margin = eagle_proof_margin(
spec.block_size,
self.hash_block_size,
self.enable_partial_hash_hits
and manager_cls.supports_fine_grained_hash_lookup,
)
_max_length = min(
curr_hit_length + eagle_margin,
len(block_hashes) * self.hash_block_size,
)
hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit(
block_hashes=block_hashes, # type: ignore[arg-type]
max_length=_max_length,
kv_cache_group_ids=group_ids,
block_pool=cast(BlockPool, cached_block_pool),
kv_cache_spec=spec,
drop_eagle_block=drop_eagle_block,
alignment_tokens=alignment_tokens,
)
if drop_eagle_block:
eagle_verified.add(idx)
elif _new_hit_length < curr_hit_length:
eagle_verified.clear()
curr_hit_length = _new_hit_length
for gid, blocks in zip(group_ids, hit_blocks, strict=True):
hit_blocks_by_group[gid] = blocks
hit_length_by_group[gid] = _new_hit_length
if curr_hit_length >= hit_length:
break
hit_length = curr_hit_length
if is_simple_hybrid:
break
# Truncate full-attention hit_blocks to final converged length;
# other specs already trim themselves inside their hit logic. cdiv keeps
# the partial tail block when hit_length is not block-aligned.
for group in self.attention_groups:
if not isinstance(group.spec, FullAttentionSpec):
continue
num_blocks = cdiv(hit_length, group.spec.block_size)
for group_id in group.group_ids:
full_blks = hit_blocks_by_group[group_id]
assert full_blks is not None
del full_blks[num_blocks:]
hit_length_by_group[group_id] = hit_length
return (
tuple(blks if blks is not None else [] for blks in hit_blocks_by_group),
hit_length,
)
def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
if isinstance(spec, UniformTypeKVCacheSpecs):
return next(iter(spec.kv_cache_specs.values()))
return spec
@@ -1,898 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Adapted from vllm-project/vllm-ascend
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
"""Data classes for MooncakeStoreConnector."""
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from typing import NamedTuple, cast
import numpy as np
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
KVConnectorWorkerMetadata,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import is_non_overlapping_and_dense
from vllm.v1.core.kv_cache_utils import (
BlockHash,
BlockHashListWithBlockSize,
)
logger = init_logger(__name__)
class BlobBlockHashes(Sequence[BlockHash]):
"""Lazy view over a flat buffer of fixed-size block hashes to avoid the overhead
of materializing all hashes upfront.
"""
def __init__(self, blob: memoryview, hash_len: int):
self._blob = blob
self._hash_len = hash_len
self._n = len(blob) // hash_len if hash_len else 0
def __len__(self) -> int:
return self._n
def __getitem__(self, idx):
if isinstance(idx, slice):
return [self[i] for i in range(*idx.indices(self._n))]
if idx < 0:
idx += self._n
if not 0 <= idx < self._n:
raise IndexError(idx)
off = idx * self._hash_len
return BlockHash(self._blob[off : off + self._hash_len])
class _CompactChunkHashList(BlockHashListWithBlockSize):
"""View that keys each ``block_size`` chunk by the last constituent
``hash_block_size`` hash instead of concatenating all of them.
The engine chains block hashes (each hash folds in the previous one), so the
final sub-block hash of a chunk already uniquely identifies the whole chunk
and its prefix. Using it keeps a Mooncake key at a single hash digest
regardless of the ``block_size`` / ``hash_block_size`` ratio, instead of
growing the key linearly with it (e.g. 64x for ``block_size=256``,
``hash_block_size=4``).
"""
def __init__(
self,
block_hashes: Sequence[BlockHash],
hash_block_size: int,
target_block_size: int,
):
# Accept any indexable sequence (e.g. the lazy ``BlobBlockHashes``), not
# just ``list``; the base only indexes/sizes it.
assert target_block_size % hash_block_size == 0
self.block_hashes = block_hashes # type: ignore[assignment]
self.scale_factor = target_block_size // hash_block_size
def _get_value_at(self, idx: int) -> BlockHash:
return self.block_hashes[idx * self.scale_factor + self.scale_factor - 1]
def chunk_hashes_for_block_size(
block_hashes: Sequence[BlockHash],
hash_block_size: int,
block_size: int,
) -> Sequence[BlockHash]:
"""Map ``hash_block_size``-granular block hashes to one compact hash per
``block_size`` chunk (the chunk's last sub-hash). Returns ``block_hashes``
unchanged when the two sizes are equal.
"""
if block_size == hash_block_size:
return block_hashes
# Structurally a Sequence[BlockHash] (indexable + sized); the base class
# just isn't declared as one.
return cast(
"Sequence[BlockHash]",
_CompactChunkHashList(block_hashes, hash_block_size, block_size),
)
@dataclass
class KeyMetadata:
"""Metadata for constructing pool keys."""
model_name: str
tp_rank: int
pcp_rank: int
dcp_rank: int
pp_rank: int
group_id: int = 0
# Optional namespace prepended to every key. Lets separate deployments
# share one Mooncake master without colliding on identical block hashes.
# Empty (the default) keeps keys byte-identical to the unprefixed format.
cache_prefix: str = ""
# Complete namespace for an opt-in Store payload format. Empty keeps
# historical keys byte-identical.
store_namespace: str = ""
@dataclass(order=True)
class PoolKey:
"""Key for addressing KV cache blocks in the distributed store."""
key_metadata: KeyMetadata
chunk_hash: str
def __hash__(self):
return hash(
(
self.key_metadata.cache_prefix,
self.key_metadata.model_name,
self.key_metadata.store_namespace,
self.key_metadata.tp_rank,
self.key_metadata.pcp_rank,
self.key_metadata.dcp_rank,
self.key_metadata.pp_rank,
self.key_metadata.group_id,
self.chunk_hash,
)
)
@staticmethod
def build_prefix(
key_metadata: KeyMetadata,
*,
tp_rank: int | None = None,
pcp_rank: int | None = None,
dcp_rank: int | None = None,
pp_rank: int | None = None,
) -> str:
"""Return the stable prefix for a Mooncake pool key."""
prefix = f"{key_metadata.cache_prefix}@" if key_metadata.cache_prefix else ""
return (
f"{prefix}"
f"{key_metadata.model_name}"
f"{key_metadata.store_namespace}"
f"@tp_rank:{key_metadata.tp_rank if tp_rank is None else tp_rank}"
f"@pcp{key_metadata.pcp_rank if pcp_rank is None else pcp_rank}"
f"@dcp{key_metadata.dcp_rank if dcp_rank is None else dcp_rank}"
f"@pp_rank:{key_metadata.pp_rank if pp_rank is None else pp_rank}"
f"@group:{key_metadata.group_id}"
)
@staticmethod
def build_key_string(key_prefix: str, chunk_hash: str) -> str:
return f"{key_prefix}@{chunk_hash}"
def to_string(self) -> str:
return self.build_key_string(
self.build_prefix(self.key_metadata), self.chunk_hash
)
StoreShardId = int
# Physical ranks encoded in a rank-local key: (TP, PCP, DCP, PP).
RankNamespace = tuple[int, int, int, int]
class StoreLayout:
"""Store payload layout for one KV-cache group on one local TP rank."""
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int,
) -> None:
self.metadata = metadata
self.block_size = block_size
self.hash_block_size = hash_block_size
@property
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
"""Store objects contributed or consumed by this local rank."""
raise NotImplementedError
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
raise NotImplementedError
def lookup_key_prefixes(
self,
rank_namespaces: Sequence[RankNamespace],
) -> tuple[str, ...]:
"""Prefixes that must exist for a logical block to be reusable."""
raise NotImplementedError
def register_kv_caches(
self,
kv_caches: Sequence[torch.Tensor],
num_blocks: int,
) -> None:
"""Build descriptors from the local KV cache tensors."""
raise NotImplementedError
def prepare_values(
self,
chunks: Sequence[tuple[int, int]],
block_ids: list[int],
shard_ids: Sequence[StoreShardId],
) -> tuple[list[list[int]], list[list[int]], list[int]]:
"""Build Store multi-buffer descriptors for logical chunks."""
raise NotImplementedError
class RankLocalStoreLayout(StoreLayout):
"""Historical rank-local Store payload layout."""
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int,
) -> None:
super().__init__(metadata, block_size, hash_block_size)
self._key_prefix = PoolKey.build_prefix(metadata)
self.kv_caches_base_addr: list[int] = []
self.block_len: list[int] = []
@property
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
return (0,)
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
assert shard_id == 0
return PoolKey.build_key_string(self._key_prefix, chunk_hash.hex())
def lookup_key_prefixes(
self,
rank_namespaces: Sequence[RankNamespace],
) -> tuple[str, ...]:
return tuple(
PoolKey.build_prefix(
self.metadata,
tp_rank=tp_rank,
pcp_rank=pcp_rank,
dcp_rank=dcp_rank,
pp_rank=pp_rank,
)
for tp_rank, pcp_rank, dcp_rank, pp_rank in rank_namespaces
)
def set_kv_caches_base_addr(self, base_addrs: list[int]) -> None:
self.kv_caches_base_addr = base_addrs
def set_block_len(self, block_lens: list[int]) -> None:
self.block_len = block_lens
def register_kv_caches(
self,
kv_caches: Sequence[torch.Tensor],
num_blocks: int,
) -> None:
seen_region_ptrs: set[int] = set()
base_addrs: list[int] = []
block_lens: list[int] = []
for cache in kv_caches:
storage = cache.untyped_storage()
storage_addr = storage.data_ptr()
storage_len = storage.nbytes()
if not is_non_overlapping_and_dense(cache[0]):
for head_idx in range(cache.shape[1]):
head_cache = cache[:, head_idx]
assert is_non_overlapping_and_dense(head_cache[0])
region_addr = head_cache.data_ptr()
if region_addr in seen_region_ptrs:
continue
seen_region_ptrs.add(region_addr)
base_addrs.append(region_addr)
block_lens.append(head_cache.stride(0) * head_cache.element_size())
elif cache.stride(0) * cache.element_size() * num_blocks == storage_len:
if storage_addr in seen_region_ptrs:
continue
seen_region_ptrs.add(storage_addr)
base_addrs.append(storage_addr)
block_lens.append(storage_len // num_blocks)
else:
region_addr = cache.data_ptr()
if region_addr in seen_region_ptrs:
continue
seen_region_ptrs.add(region_addr)
base_addrs.append(region_addr)
block_lens.append(cache.stride(0) * cache.element_size())
self.kv_caches_base_addr = base_addrs
self.block_len = block_lens
def prepare_values(
self,
chunks: Sequence[tuple[int, int]],
block_ids: list[int],
shard_ids: Sequence[StoreShardId],
) -> tuple[list[list[int]], list[list[int]], list[int]]:
if not chunks:
return [], [], []
assert len(chunks) == len(shard_ids) and all(
shard_id == 0 for shard_id in shard_ids
)
base = np.asarray(self.kv_caches_base_addr, dtype=np.int64)
length = len(self.block_len)
blen = np.asarray(
[self.block_len[i % length] for i in range(base.shape[0])],
dtype=np.int64,
)
n = len(chunks)
starts = np.fromiter((c[0] for c in chunks), dtype=np.int64, count=n)
spans = np.fromiter((c[1] for c in chunks), dtype=np.int64, count=n) - starts
assert np.all(spans % self.hash_block_size == 0)
bids = np.fromiter(
(block_ids[i] for i in (starts // self.block_size).tolist()),
dtype=np.int64,
count=n,
)
addrs = base[None, :] + bids[:, None] * blen[None, :]
block_counts = (spans + self.block_size - 1) // self.block_size
sizes = blen[None, :] * block_counts[:, None]
return addrs.tolist(), sizes.tolist(), bids.tolist()
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
length = len(self.block_len)
return (
[
base_addr + block_id * self.block_len[index % length]
for index, base_addr in enumerate(self.kv_caches_base_addr)
],
[
self.block_len[index % length]
for index in range(len(self.kv_caches_base_addr))
],
)
class TPShardedStoreLayout(StoreLayout):
"""Store layout shared by divisible TP sizes."""
store_format: str
@classmethod
def shared_namespace(cls, store_tp_size: int, pp_size: int) -> str:
return (
f"@store_tp:{store_tp_size}@store_pp:{pp_size}"
f"@store_format:{cls.store_format}"
)
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int,
local_tp_size: int,
store_tp_size: int,
tp_rank: int,
num_kv_heads: int,
) -> None:
super().__init__(metadata, block_size, hash_block_size)
self.shards_per_rank = store_tp_size // local_tp_size
first_shard = tp_rank * self.shards_per_rank
self.store_shard_ids = tuple(
range(first_shard, first_shard + self.shards_per_rank)
)
self.store_tp_size = store_tp_size
self.heads_per_store_shard = num_kv_heads // store_tp_size
self.local_num_kv_heads = num_kv_heads // local_tp_size
self._shard_key_prefixes = {
shard_id: PoolKey.build_prefix(metadata, tp_rank=shard_id)
for shard_id in self.store_shard_ids
}
self._shard_addr_bases = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
self._shard_block_strides = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
self._shard_sizes = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
@property
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
return self.store_shard_ids
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
return PoolKey.build_key_string(
self._shard_key_prefixes[shard_id], chunk_hash.hex()
)
def lookup_key_prefixes(
self,
rank_namespaces: Sequence[RankNamespace],
) -> tuple[str, ...]:
pp_ranks = sorted({namespace[3] for namespace in rank_namespaces})
return tuple(
PoolKey.build_prefix(self.metadata, tp_rank=shard_id, pp_rank=pp_rank)
for shard_id in range(self.store_tp_size)
for pp_rank in pp_ranks
)
def _set_segment_templates(
self, templates: Sequence[tuple[list[int], list[int], list[int]]]
) -> None:
self._shard_addr_bases = np.asarray(
[template[0] for template in templates], dtype=np.uint64
)
self._shard_block_strides = np.asarray(
[template[1] for template in templates], dtype=np.uint64
)
self._shard_sizes = np.asarray(
[template[2] for template in templates], dtype=np.uint64
)
def _has_packed_strides(
self, head_stride: int, token_stride: int, content_bytes: int
) -> bool:
raise NotImplementedError
def _shard_segments(
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
) -> Iterable[tuple[int, int]]:
raise NotImplementedError
def register_kv_caches(
self,
kv_caches: Sequence[torch.Tensor],
num_blocks: int,
) -> None:
templates: list[tuple[list[int], list[int], list[int]]] = [
([], [], []) for _ in range(self.shards_per_rank)
]
for cache in kv_caches:
if cache.ndim != 4 or tuple(cache.shape[:3]) != (
num_blocks,
self.local_num_kv_heads,
self.block_size,
):
raise ValueError(
"TP-shared Mooncake store requires packed KV caches with "
"logical shape (num_blocks, local_kv_heads, block_size, content)"
)
element_size = cache.element_size()
block_stride, head_stride, token_stride, content_stride = (
stride * element_size for stride in cache.stride()
)
content_bytes = cache.shape[3] * element_size
if content_stride != element_size or not self._has_packed_strides(
head_stride, token_stride, content_bytes
):
raise ValueError(
"TP-shared Mooncake store requires packed "
f"{self.store_format} KV layout"
)
for local_shard, (addr_bases, block_strides, sizes) in enumerate(templates):
head_start = local_shard * self.heads_per_store_shard
for offset, size in self._shard_segments(
head_start, head_stride, token_stride, content_bytes
):
addr_bases.append(cache.data_ptr() + offset)
block_strides.append(block_stride)
sizes.append(size)
self._set_segment_templates(templates)
def prepare_values(
self,
chunks: Sequence[tuple[int, int]],
block_ids: list[int],
shard_ids: Sequence[StoreShardId],
) -> tuple[list[list[int]], list[list[int]], list[int]]:
if not chunks:
return [], [], []
if len(chunks) != len(shard_ids):
raise ValueError("Each Store chunk must have one shard ID")
count = len(chunks)
starts = np.fromiter(
(chunk[0] for chunk in chunks), dtype=np.int64, count=count
)
ends = np.fromiter((chunk[1] for chunk in chunks), dtype=np.int64, count=count)
if np.any(ends - starts != self.block_size):
raise ValueError("TP-shared Mooncake store requires full KV blocks")
first_shard = self.store_shard_ids[0]
local_shards = np.fromiter(
(shard_id - first_shard for shard_id in shard_ids),
dtype=np.int64,
count=count,
)
if np.any(local_shards < 0) or np.any(local_shards >= self.shards_per_rank):
raise ValueError("Store shard is not owned by this TP rank")
chunk_block_ids = np.fromiter(
(block_ids[index] for index in (starts // self.block_size).tolist()),
dtype=np.uint64,
count=count,
)
addr_bases = self._shard_addr_bases[local_shards]
block_strides = self._shard_block_strides[local_shards]
addrs = addr_bases + chunk_block_ids[:, None] * block_strides
sizes = self._shard_sizes[local_shards]
return addrs.tolist(), sizes.tolist(), chunk_block_ids.tolist()
class LBHNCStoreLayout(TPShardedStoreLayout):
"""Native head-major layout shared by divisible TP sizes."""
store_format = "tp_shared_lbhnc"
def _has_packed_strides(
self, head_stride: int, token_stride: int, content_bytes: int
) -> bool:
return (
token_stride == content_bytes
and head_stride == self.block_size * content_bytes
)
def _shard_segments(
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
) -> Iterable[tuple[int, int]]:
yield head_start * head_stride, self.heads_per_store_shard * head_stride
class LBNHCStoreLayout(TPShardedStoreLayout):
"""Native token-major layout shared by divisible TP sizes."""
store_format = "tp_shared_lbnhc"
def _has_packed_strides(
self, head_stride: int, token_stride: int, content_bytes: int
) -> bool:
return (
head_stride == content_bytes
and token_stride == self.local_num_kv_heads * content_bytes
)
def _shard_segments(
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
) -> Iterable[tuple[int, int]]:
for token_idx in range(self.block_size):
yield (
token_idx * token_stride + head_start * head_stride,
self.heads_per_store_shard * content_bytes,
)
class ChunkedTokenDatabase:
"""Enumerates logical token chunks and their hashes."""
def __init__(
self,
metadata: KeyMetadata,
block_size: int,
hash_block_size: int | None = None,
store_layout: StoreLayout | None = None,
):
self.metadata = metadata
self.block_size = block_size
self.hash_block_size = hash_block_size or block_size
if self.block_size % self.hash_block_size != 0:
raise ValueError(
f"block_size ({self.block_size}) must be a multiple of "
f"hash_block_size ({self.hash_block_size})"
)
self.store_layout = store_layout or RankLocalStoreLayout(
metadata, block_size, self.hash_block_size
)
@property
def kv_caches_base_addr(self) -> list[int]:
return self._rank_local_layout().kv_caches_base_addr
@property
def block_len(self) -> list[int]:
return self._rank_local_layout().block_len
def _rank_local_layout(self) -> RankLocalStoreLayout:
if not isinstance(self.store_layout, RankLocalStoreLayout):
raise RuntimeError("This operation requires a rank-local Store layout")
return self.store_layout
def key_for(self, chunk_hash: BlockHash) -> str:
return self._rank_local_layout().key_for(0, chunk_hash)
def set_kv_caches_base_addr(self, kv_caches_base_addr: list[int]):
self._rank_local_layout().set_kv_caches_base_addr(kv_caches_base_addr)
def set_block_len(self, block_len: list[int]):
self._rank_local_layout().set_block_len(block_len)
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
"""Return addresses and sizes for one physical block slot."""
return self._rank_local_layout().prepare_value_for_block(block_id)
def process_tokens(
self,
token_len: int,
block_hashes: list[BlockHash],
mask_num: int = 0,
*,
chunk_mask: list[bool] | None = None,
put_step: int = 1,
put_step_rank: int = 0,
) -> Iterable[tuple[int, int, BlockHash]]:
"""Process tokens and yield (start_idx, end_idx, block_hash) tuples.
When there are fewer KV heads than TP ranks, chunks are distributed
across TP ranks to avoid duplicate load/store. The assignment keys off
the absolute ``chunk_id`` so a given chunk always lands on the same
rank regardless of where the processed suffix begins.
Args:
token_len: Total number of tokens. Must be hash-block aligned and
covered by ``block_hashes`` when hashes are present.
block_hashes: Block hashes computed at ``hash_block_size`` granularity.
When ``block_size > hash_block_size`` each group's ``block_size`` chunk
is keyed by its last sub-hash via ``chunk_hashes_for_block_size``.
mask_num: Number of tokens to skip from the beginning.
chunk_mask: Optional mask relative to the first chunk after
``mask_num``. False entries are skipped before hash access.
put_step: Stride for distributing chunks across ranks.
put_step_rank: ``chunk_id % put_step`` value this rank stores.
"""
assert put_step > 0
if not block_hashes:
return
assert token_len % self.hash_block_size == 0
assert token_len // self.hash_block_size <= len(block_hashes)
start_chunk = max(0, cdiv(mask_num, self.block_size))
max_chunks = cdiv(token_len, self.block_size)
if chunk_mask is not None:
max_chunks = min(max_chunks, start_chunk + len(chunk_mask))
for chunk_id in range(start_chunk, max_chunks):
if chunk_mask is not None and not chunk_mask[chunk_id - start_chunk]:
continue
if chunk_id % put_step != put_step_rank:
continue
start_idx = chunk_id * self.block_size
end_idx = min(start_idx + self.block_size, token_len)
h = block_hashes[end_idx // self.hash_block_size - 1]
yield start_idx, end_idx, h
@dataclass(frozen=True)
class TailKeyBoundary:
"""Hash boundary used to key a group's tail block in the store.
Attributes:
group_id: KV-cache group containing the tail block.
num_tokens: Token boundary whose prefix hash identifies the matched
stored block. The loader uses
``block_hashes[num_tokens // hash_block_size - 1]`` instead of the
hash implied by ``MooncakeLookupResult.hit_length``. This changes
only the load key, not the reusable prefix.
"""
group_id: int
num_tokens: int
@dataclass
class MooncakeLookupResult:
"""Lookup result used to build the subsequent load request.
Attributes:
hit_length: Longest prefix that every KV-cache group can reuse after
their individual cache hits converge.
tail_key_boundaries: Hash boundary used to store each cache group's
tail block when ``hit_length`` does not identify its store key.
There is one entry per group for every nonzero hit.
"""
hit_length: int
tail_key_boundaries: tuple[TailKeyBoundary, ...] = ()
@dataclass
class LoadSpec:
"""Specification for loading KV cache from external store."""
vllm_cached_tokens: int
kvpool_cached_tokens: int
can_load: bool
token_len: int = 0
tail_key_boundaries: tuple[TailKeyBoundary, ...] = ()
@dataclass
class RequestTracker:
"""Tracks per-request state across scheduler ticks."""
req_id: str
token_len: int
allocated_block_ids: tuple[list[int], ...]
num_saved_tokens: int = 0
token_ids: list[int] | None = None
has_pending_offload: bool = False
# Snapshot of the prefill range length at tracker creation time.
# For a fresh request this is len(prompt). For a resumed-from-preemption
# request it includes previously-generated tokens, which are re-prefilled.
prefill_end_tokens: int = 0
partial_tail_sent: bool = False
def reset(self) -> None:
self.token_len = 0
self.allocated_block_ids = ()
self.num_saved_tokens = 0
self.token_ids = None
self.has_pending_offload = False
self.prefill_end_tokens = 0
self.partial_tail_sent = False
def update(
self,
new_block_ids: tuple[list[int], ...] | list[int],
) -> None:
# Backward-compat: accept a single list (broadcast to single group).
if isinstance(new_block_ids, list):
new_block_ids = (new_block_ids,)
if len(new_block_ids) != len(self.allocated_block_ids):
raise ValueError(
f"Group count mismatch: tracker has "
f"{len(self.allocated_block_ids)} groups, update has "
f"{len(new_block_ids)}"
)
for existing, new in zip(self.allocated_block_ids, new_block_ids, strict=True):
if new:
existing.extend(new)
class BoundaryPut(NamedTuple):
"""A block stored under the hash of the prefix ending at ``num_tokens``."""
group_id: int
block_id: int
num_tokens: int
@dataclass
class ReqMeta:
"""Per-request metadata for store put/get operations."""
req_id: str
token_len_chunk: int
block_ids: tuple[list[int], ...]
block_hashes: list[BlockHash]
can_save: bool | None = None
load_spec: LoadSpec | None = None
current_event: torch.cuda.Event | None = None
token_ids: list[int] | None = None
# Absolute request offset represented by token_ids[0].
token_ids_start: int = 0
num_prompt_tokens: int | None = None
# Total prefill tokens, including generated tokens replayed after preemption.
prefill_end_tokens: int | None = None
# Identifies this store job for the engine's lifetime. A request id cannot
# serve that purpose: it is reused once a preempted request resumes, so it
# would release the wrong job's blocks.
store_job_id: int | None = None
# Blocks stored under an explicit prefix hash rather than by position:
# core-provided mamba "align" states (a block-aligned entry is a committed
# snapshot, a non-aligned one the sub-block CoW prompt tail) and the
# scheduler-resolved non-Mamba blocks of the prompt's partial tail. The
# store-job reference keeps each block alive until every worker rank
# finishes the job.
boundary_puts: list[BoundaryPut] | None = None
# Total computed prefix length at the end of this step.
completed_token_len: int | None = None
# Set on the save that first covers the whole prompt, so the scheduler
# publishes the prompt's partial tail once rather than on every save.
publish_partial_tail: bool = False
@staticmethod
def from_request_tracker(
tracker: RequestTracker,
block_size: int,
load_spec: LoadSpec | None = None,
skip_save: bool | None = False,
block_hashes: list[BlockHash] | None = None,
save_partial_tail: bool = False,
num_prompt_tokens: int | None = None,
) -> "ReqMeta | None":
"""Create ReqMeta from a RequestTracker."""
if save_partial_tail:
assert num_prompt_tokens is not None
if block_hashes is None:
block_hashes = []
input_token_len = tracker.token_len
token_ids_start = tracker.num_saved_tokens
chunk_boundary = cdiv(token_ids_start + 1, block_size) * block_size
num_tokens_to_save = input_token_len // block_size * block_size
publish_tail = (
save_partial_tail
and not tracker.partial_tail_sent
and num_prompt_tokens is not None
and 0 < num_prompt_tokens <= input_token_len
)
skip_save = skip_save or (
num_tokens_to_save < chunk_boundary and not publish_tail
)
# A ReqMeta must never carry both a save AND a load.
# The save would also be wasted work — the bytes are being looked up
# in the store right now. Later cached_reqs steps save new tokens
# normally.
if load_spec is not None and load_spec.can_load:
skip_save = True
if skip_save and load_spec is None:
return None
if not skip_save:
tracker.num_saved_tokens = num_tokens_to_save
tracker.partial_tail_sent |= publish_tail
token_ids = None
if tracker.token_ids and not skip_save:
# Scheduler tracking continues while this job is handled by an
# asynchronous worker, so metadata must own a stable snapshot.
token_ids = tracker.token_ids[token_ids_start:num_tokens_to_save]
if load_spec is not None and load_spec.can_load:
logger.debug(
"Scheduled to load %d tokens for request %s",
load_spec.kvpool_cached_tokens,
tracker.req_id,
)
else:
load_spec = None
logger.debug(
"request:%s, meta save spec:%s, meta load spec:%s",
tracker.req_id,
not skip_save,
load_spec,
)
return ReqMeta(
req_id=tracker.req_id,
token_len_chunk=num_tokens_to_save,
block_ids=tracker.allocated_block_ids,
can_save=not skip_save,
load_spec=load_spec,
block_hashes=block_hashes,
token_ids=token_ids,
token_ids_start=token_ids_start,
num_prompt_tokens=num_prompt_tokens,
prefill_end_tokens=tracker.prefill_end_tokens,
completed_token_len=input_token_len,
publish_partial_tail=publish_tail and not skip_save,
)
@dataclass
class MooncakeStoreWorkerMetadata(KVConnectorWorkerMetadata):
"""Maps ``ReqMeta.store_job_id`` to the number of ranks done with that job."""
completed_saves: dict[int, int] = field(default_factory=dict)
def aggregate(
self, other: "KVConnectorWorkerMetadata"
) -> "MooncakeStoreWorkerMetadata":
assert isinstance(other, MooncakeStoreWorkerMetadata)
for store_job_id, count in other.completed_saves.items():
self.completed_saves[store_job_id] = (
self.completed_saves.get(store_job_id, 0) + count
)
return self
class MooncakeStoreConnectorMetadata(KVConnectorMetadata):
"""Metadata passed from scheduler to worker."""
def __init__(
self,
unfinished_request_ids: set[str],
preempted_req_ids: set[str],
):
self.requests: list[ReqMeta] = []
self.unfinished_request_ids = unfinished_request_ids
self.preempted_req_ids = preempted_req_ids
def add_request(self, req_meta: ReqMeta) -> None:
self.requests.append(req_meta)
@@ -1,189 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Per-operation telemetry for MooncakeStoreConnector.
Records one row per Mooncake RPC (``save_exists``, ``save_put``, ``load_get``,
``lookup_exists``) with duration, key/byte counts, status, and failed-key
count. Exposed to the logger via ``KVConnectorLogging`` and to Prometheus
via ``MooncakeStorePromMetrics``.
"""
from dataclasses import dataclass
from statistics import fmean
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
KVConnectorPromMetrics,
KVConnectorStats,
PromMetric,
PromMetricT,
)
def _nearest_rank_percentile(values: list[float], percentile: float) -> float:
if not values:
return 0.0
sorted_values = sorted(values)
rank = max(
0, min(len(sorted_values) - 1, int(percentile * len(sorted_values) - 1e-12))
)
return sorted_values[rank]
@dataclass
class MooncakeStoreConnectorStats(KVConnectorStats):
"""Serializable Mooncake store communication telemetry."""
def __post_init__(self):
if not self.data:
self.reset()
def reset(self):
self.data: dict[str, list[dict[str, int | float | str]]] = {}
def is_empty(self) -> bool:
return not self.data
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
if other.is_empty():
return self
for operation, records in other.data.items():
self.data.setdefault(operation, []).extend(records)
return self
def reduce(self) -> dict[str, int | float]:
reduced: dict[str, int | float] = {}
for operation, records in sorted(self.data.items()):
if not records:
continue
durations = [float(record["duration_seconds"]) for record in records]
reduced[f"{operation}_count"] = len(records)
reduced[f"{operation}_avg_ms"] = round(fmean(durations) * 1e3, 3)
reduced[f"{operation}_p90_ms"] = round(
_nearest_rank_percentile(durations, 0.9) * 1e3, 3
)
reduced[f"{operation}_total_keys"] = sum(
int(record["num_keys"]) for record in records
)
reduced[f"{operation}_total_bytes"] = sum(
int(record["num_bytes"]) for record in records
)
reduced[f"{operation}_failed_keys"] = sum(
int(record["num_failed_keys"]) for record in records
)
reduced[f"{operation}_error_count"] = sum(
1 for record in records if record["status"] == "error"
)
return reduced
def record_operation(
self,
operation: str,
duration_seconds: float,
num_keys: int,
*,
num_bytes: int = 0,
status: str = "ok",
num_failed_keys: int = 0,
) -> None:
self.data.setdefault(operation, []).append(
{
"duration_seconds": duration_seconds,
"num_keys": num_keys,
"num_bytes": num_bytes,
"status": status,
"num_failed_keys": num_failed_keys,
}
)
class MooncakeStorePromMetrics(KVConnectorPromMetrics):
"""Prometheus metrics for Mooncake store communication."""
def __init__(
self,
vllm_config: VllmConfig,
metric_types: dict[type[PromMetric], type[PromMetricT]],
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
):
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
metric_labelnames = labelnames + ["operation", "status"]
self._metric_cache: dict[tuple[int, str, str], dict[str, PromMetric]] = {}
self._histogram_operation_time = self._histogram_cls(
name="vllm:mooncake_store_operation_time_seconds",
documentation="Histogram of Mooncake store communication time.",
buckets=[
1e-3,
5e-3,
1e-2,
5e-2,
1e-1,
2e-1,
3e-1,
4e-1,
5e-1,
7.5e-1,
1.0,
1.5,
2.0,
3.0,
4.0,
],
labelnames=metric_labelnames,
)
self._counter_operation_calls = self._counter_cls(
name="vllm:mooncake_store_operation_total",
documentation="Number of Mooncake store communication operations.",
labelnames=metric_labelnames,
)
self._counter_operation_keys = self._counter_cls(
name="vllm:mooncake_store_operation_keys_total",
documentation="Number of Mooncake store keys touched by operations.",
labelnames=metric_labelnames,
)
self._counter_operation_bytes = self._counter_cls(
name="vllm:mooncake_store_operation_bytes_total",
documentation="Number of bytes transferred by Mooncake store operations.",
labelnames=metric_labelnames,
)
self._counter_failed_keys = self._counter_cls(
name="vllm:mooncake_store_operation_failed_keys_total",
documentation="Number of Mooncake store keys that failed in operations.",
labelnames=metric_labelnames,
)
def _get_metrics(
self,
engine_idx: int,
operation: str,
status: str,
) -> dict[str, PromMetric]:
cache_key = (engine_idx, operation, status)
if cache_key not in self._metric_cache:
label_values = self.per_engine_labelvalues[engine_idx] + [operation, status]
self._metric_cache[cache_key] = {
"time": self._histogram_operation_time.labels(*label_values),
"calls": self._counter_operation_calls.labels(*label_values),
"keys": self._counter_operation_keys.labels(*label_values),
"bytes": self._counter_operation_bytes.labels(*label_values),
"failed_keys": self._counter_failed_keys.labels(*label_values),
}
return self._metric_cache[cache_key]
def observe(self, transfer_stats_data: dict[str, Any] | None, engine_idx: int = 0):
if not transfer_stats_data:
return
for operation, records in transfer_stats_data.items():
assert isinstance(records, list)
for record in records:
assert isinstance(record, dict)
status = str(record["status"])
metrics = self._get_metrics(engine_idx, operation, status)
metrics["time"].observe(float(record["duration_seconds"]))
metrics["calls"].inc()
metrics["keys"].inc(int(record["num_keys"]))
metrics["bytes"].inc(int(record["num_bytes"]))
metrics["failed_keys"].inc(int(record["num_failed_keys"]))
@@ -1,81 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Wire-format constants for the LookupKey ZMQ admin channel.
This is the single source of truth shared by ``LookupKeyClient`` and
``LookupKeyServer`` on the scheduler<->worker rank-0 admin channel.
Wire format (REQ/REP over IPC):
Request: [msg_type: bytes] [payload_frames...]
msg_type == LOOKUP_MSG:
frame 1: num_tokens (u32 big-endian, 4 bytes); the worker derives
the aligned lookup length
frame 2: hash_len (u16 big-endian, 2 bytes) — byte length of each
fixed-size block hash (0 when there are no hashes)
frame 3: raw block hashes concatenated back-to-back (each hash_len
bytes); the server splits on hash_len
Response: hit_length (u32 big-endian, first 4 bytes), followed by zero
or more 8-byte tail-key boundaries. Each entry is group_id
(u32), then boundary_tokens (u32).
msg_type == RESET_MSG:
(no payload frames)
Response: [RESP_OK] or [RESP_ERR]
The first frame of every request is a named bytes tag (not a numeric
sentinel that aliases the data field) so the protocol stays
self-describing and extensible: adding new admin commands requires
only a new tag and a new dispatch branch.
Mirrors the named-tag convention used by the NIXL connector (see
``vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py``).
"""
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
MooncakeLookupResult,
TailKeyBoundary,
)
# Request message-type tags. Frame 0 of every request.
LOOKUP_MSG: bytes = b"lookup"
RESET_MSG: bytes = b"reset"
# Single-byte response status codes for admin commands.
RESP_OK: bytes = b"\x01"
RESP_ERR: bytes = b"\x00"
# group_id (u32), boundary_tokens (u32).
TAIL_KEY_BOUNDARY_ENTRY_SIZE: int = 8
def encode_lookup_response(result: MooncakeLookupResult) -> bytes:
hit_length = result.hit_length.to_bytes(4, "big")
if not result.tail_key_boundaries:
return hit_length
payload = bytearray(hit_length)
for boundary in result.tail_key_boundaries:
payload.extend(boundary.group_id.to_bytes(4, "big"))
payload.extend(boundary.num_tokens.to_bytes(4, "big"))
return bytes(payload)
def decode_lookup_response(payload: bytes) -> MooncakeLookupResult:
if len(payload) < 4 or (len(payload) - 4) % TAIL_KEY_BOUNDARY_ENTRY_SIZE:
raise ValueError("Invalid Mooncake lookup response")
hit_length = int.from_bytes(payload[:4], "big")
if len(payload) == 4:
return MooncakeLookupResult(hit_length)
boundaries = []
for offset in range(4, len(payload), TAIL_KEY_BOUNDARY_ENTRY_SIZE):
boundaries.append(
TailKeyBoundary(
group_id=int.from_bytes(payload[offset : offset + 4], "big"),
num_tokens=int.from_bytes(payload[offset + 4 : offset + 8], "big"),
)
)
return MooncakeLookupResult(hit_length, tuple(boundaries))
@@ -1,788 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Adapted from vllm-project/vllm-ascend
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
"""Scheduler-side logic for MooncakeStoreConnector."""
from collections.abc import Sequence
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import (
MooncakeStoreCoordinator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
BoundaryPut,
LoadSpec,
MooncakeStoreConnectorMetadata,
MooncakeStoreWorkerMetadata,
ReqMeta,
RequestTracker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( # noqa: E501
LookupKeyClient,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import (
resolve_kv_cache_block_sizes,
)
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
MambaSpec,
get_mamba_prefill_checkpoint_position,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
def _new_req_prefill_tokens(request: NewRequestData) -> list[int]:
"""Tokens this prefill will compute KV for.
Under the v2 model runner, resumed-from-preemption requests appear in
``scheduled_new_reqs`` with ``prefill_token_ids`` set to the request's full
token list (prompt + previously-generated). For all other cases this falls
back to the original prompt.
"""
if request.prefill_token_ids is not None:
return request.prefill_token_ids
assert request.prompt_token_ids is not None
return request.prompt_token_ids
def _partial_tail_non_mamba_puts(
coord: MooncakeStoreCoordinator,
req_meta: ReqMeta,
block_sizes: Sequence[int],
) -> list[BoundaryPut]:
"""Locate the non-Mamba blocks a partial-tail save publishes.
A later request resumes at the prompt's Mamba checkpoint (``boundary``) only
if both pieces of KV are stored:
- Mamba groups: the state at ``boundary``, handed off by the core as a
boundary put of its own, so Mamba groups are not listed here.
- Other groups: the KV from the last LCM-aligned normal save up to
``boundary`` plus the group's EAGLE proof margin (``proof_end``). Each
block is keyed by the hash at its end, clipped to ``proof_end``.
Groups whose proof is not computed yet are omitted. For example, with LCM
16, blocks of 4 tokens, and a checkpoint at 44, the full-attention blocks
are 8-10, keyed at 36, 40 and 44.
A Mamba state handed off at any other position, i.e. a shared-prefix
junction (``--enable-mamba-shared-prefix-checkpoint``), adds no puts here.
``block_sizes`` are the per-group block sizes, indexed like
``req_meta.block_ids``.
"""
mamba_tails = [
put.num_tokens
for put in req_meta.boundary_puts or []
if put.group_id in coord.mamba_group_ids
and put.num_tokens % block_sizes[put.group_id]
]
# The tail is due on the save that first covers the prompt, or when Mamba
# hands off its checkpoint state.
if not mamba_tails and not req_meta.publish_partial_tail:
return []
prompt_tokens = req_meta.num_prompt_tokens
completed = req_meta.completed_token_len
assert prompt_tokens is not None and completed is not None
hash_block_size = coord.hash_block_size
# Core drops the Mamba checkpoint whenever EAGLE block drop is on, whichever
# groups carry the eagle flag.
boundary = get_mamba_prefill_checkpoint_position(
prompt_tokens, hash_block_size, coord.use_eagle
)
# Any other sub-block Mamba hand-off is a shared-prefix junction. Its
# attention KV, EAGLE proof included, ends on an attention block end no later
# than this checkpoint's proof, so the normal and prompt-tail saves store it.
if not req_meta.publish_partial_tail and boundary not in mamba_tails:
return []
num_hashes = len(req_meta.block_hashes)
if boundary == 0 or boundary // hash_block_size > num_hashes:
return []
start = boundary // coord.lcm_block_size * coord.lcm_block_size
puts: list[BoundaryPut] = []
for group_id, block_size in enumerate(block_sizes):
if group_id in coord.mamba_group_ids:
continue
proof_end = boundary + coord.eagle_proof_margin_by_group.get(group_id, 0)
if proof_end > completed or proof_end // hash_block_size > num_hashes:
continue
group_blocks = req_meta.block_ids[group_id]
for block_idx in range(start // block_size, cdiv(proof_end, block_size)):
if block_idx >= len(group_blocks):
break
block_id = group_blocks[block_idx]
if block_id == NULL_BLOCK_ID:
continue
num_tokens = min((block_idx + 1) * block_size, proof_end)
puts.append(BoundaryPut(group_id, block_id, num_tokens))
return puts
class MooncakeStoreScheduler:
"""Scheduler-side component for MooncakeStoreConnector."""
def __init__(
self,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
):
assert vllm_config.kv_transfer_config is not None
self.kv_role = vllm_config.kv_transfer_config.kv_role
kvc_extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
self.load_async = kvc_extra_config.get("load_async", True)
self.lookup_async = kvc_extra_config.get("lookup_async", False)
# Skips lookup CPU cost on instances that never load KV from the store.
self.enable_lookup = kvc_extra_config.get("enable_lookup", True)
self.save_decode_cache = kvc_extra_config.get("save_decode_cache", False)
kv_event_config = vllm_config.kv_events_config
self.enable_kv_events = bool(
kv_event_config and kv_event_config.enable_kv_cache_events
)
self.client = LookupKeyClient(vllm_config)
self.kv_cache_config = kv_cache_config
self._store_group_ids = kv_cache_config.prefix_cacheable_group_ids
# Map scheduler group IDs to store group indices. Groups outside the
# store projection, such as the QSA ring, map to None and are skipped.
self._store_group_id_by_kv_cache_group_id = {
group_id: store_group_id
for store_group_id, group_id in enumerate(self._store_group_ids)
}
store_groups = kv_cache_config.prefix_cacheable_groups
# Align with the engine's own scheduler_block_size and hash_block_size.
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
mamba_groups = {
group_id: group.kv_cache_spec
for group_id, group in enumerate(store_groups)
if isinstance(group.kv_cache_spec, MambaSpec)
}
self._boundary_state_group_ids = frozenset(mamba_groups)
# Mirrors MooncakeStoreConnector._capacity_only.
self._capacity_only = (
self.kv_role == "kv_consumer"
and not self.enable_lookup
and not self.save_decode_cache
)
self._store_coord: MooncakeStoreCoordinator | None = None
self.enable_partial_hash_hits = False
if not self._capacity_only:
assert all(
spec.mamba_cache_mode == "align" for spec in mamba_groups.values()
), "MooncakeStoreScheduler requires mamba_cache_mode='align'"
self._store_coord = MooncakeStoreCoordinator.from_kv_cache_config(
kv_cache_config, vllm_config, self._block_size, self._hash_block_size
)
self.enable_partial_hash_hits = self._store_coord.enable_partial_hash_hits
self._gpu_block_pool: BlockPool | None = None
self._num_workers = vllm_config.parallel_config.world_size
self._next_store_job_id = 0
# store_job_id -> (referenced block ids, ranks yet to report completion)
self._pinned_saves: dict[int, tuple[list[int], int]] = {}
# Per-request state
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
self._unfinished_request_ids: set[str] = set()
self._finished_partial_tail_metas: dict[str, ReqMeta] = {}
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
self._gpu_block_pool = gpu_block_pool
def get_num_new_matched_tokens(
self,
request: Request,
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""Check for external KV cache hit.
Returns ``(None, False)`` when an async lookup is still in flight,
signaling the scheduler to retry this request on a later step.
"""
if not self.enable_lookup:
return 0, False
# Fine-grained hits may land on a hash boundary inside a block; without
# partial hits, prefixes shorter than one physical block are skipped.
align = (
self._hash_block_size if self.enable_partial_hash_hits else self._block_size
)
if request.num_tokens < align:
return 0, False
lookup_result = self.client.lookup(
request.request_id,
request.num_tokens,
request.block_hashes,
non_block=self.lookup_async,
)
if lookup_result is None:
# Lookup not ready yet; scheduler will retry on a later step.
return None, False
num_external_hit_tokens = lookup_result.hit_length
if num_external_hit_tokens < num_computed_tokens:
need_to_allocate = 0
else:
need_to_allocate = num_external_hit_tokens - num_computed_tokens
logger.debug(
"Reqid: %s, Total tokens %d, kvpool hit tokens: %d, need to load: %d",
request.request_id,
request.num_tokens,
num_external_hit_tokens,
need_to_allocate,
)
if need_to_allocate <= 0:
return 0, False
self.load_specs[request.request_id] = LoadSpec(
vllm_cached_tokens=num_computed_tokens,
kvpool_cached_tokens=num_external_hit_tokens,
can_load=False,
tail_key_boundaries=lookup_result.tail_key_boundaries,
)
return need_to_allocate, self.load_async
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
):
"""Update state after block allocation."""
local_block_ids: tuple[list[int], ...] = ()
if num_external_tokens > 0:
local_block_ids = blocks.get_block_ids(
group_ids=self._store_group_ids,
)
self._unfinished_requests[request.request_id] = (request, local_block_ids)
self._unfinished_request_ids.add(request.request_id)
if request.request_id not in self.load_specs:
return
if num_external_tokens == 0:
self.load_specs[request.request_id].can_load = False
return
assert (
num_external_tokens > 0
and num_external_tokens
== self.load_specs[request.request_id].kvpool_cached_tokens
- self.load_specs[request.request_id].vllm_cached_tokens
), (
f"Mismatch in number of tokens: {num_external_tokens} vs "
f"{self.load_specs[request.request_id].kvpool_cached_tokens} - "
f"{self.load_specs[request.request_id].vllm_cached_tokens}"
f" for request {request.request_id}"
)
self.load_specs[request.request_id].can_load = True
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""Build connector metadata for this scheduler step."""
is_consumer = self.kv_role == "kv_consumer"
can_process_cached = not is_consumer or self.save_decode_cache
for finished_req_id in scheduler_output.finished_req_ids:
self.client.discard(finished_req_id)
self.load_specs.pop(finished_req_id, None)
self._request_trackers.pop(finished_req_id, None)
self._unfinished_requests.pop(finished_req_id, None)
self._unfinished_request_ids.discard(finished_req_id)
preempted_ids = scheduler_output.preempted_req_ids or set()
for req_id in preempted_ids:
self.load_specs.pop(req_id, None)
if request_tracker := self._request_trackers.get(req_id):
request_tracker.reset()
self._unfinished_requests.pop(req_id, None)
meta = MooncakeStoreConnectorMetadata(
self._unfinished_request_ids,
preempted_ids,
)
# Handle new requests
for request in scheduler_output.scheduled_new_reqs:
load_spec = self.load_specs.pop(request.req_id, None)
num_tokens_to_compute = (
request.num_computed_tokens
+ scheduler_output.num_scheduled_tokens[request.req_id]
)
assert request.req_id in self._unfinished_requests
request_tuple = self._unfinished_requests.get(request.req_id)
request_real = request_tuple[0] # type: ignore[index]
unfolded_block_ids = tuple(
request.block_ids[group_id].copy() for group_id in self._store_group_ids
)
prefill_tokens = _new_req_prefill_tokens(request)
request_tracker = RequestTracker(
req_id=request.req_id,
token_len=num_tokens_to_compute,
allocated_block_ids=unfolded_block_ids,
num_saved_tokens=0,
token_ids=(
prefill_tokens[:num_tokens_to_compute]
if self.enable_kv_events
else None
),
prefill_end_tokens=len(prefill_tokens),
)
self._request_trackers[request.req_id] = request_tracker
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=load_spec,
# A consumer may write decode KV without becoming a prefill
# producer. Loads are still carried by the same metadata.
skip_save=is_consumer,
block_hashes=request_real.block_hashes,
num_prompt_tokens=request_real.num_prompt_tokens,
save_partial_tail=self.enable_partial_hash_hits,
)
if req_meta is not None:
meta.add_request(req_meta)
# Handle cached (running, or MRV1 resumed-from-preemption) requests
cached_reqs = scheduler_output.scheduled_cached_reqs
if can_process_cached:
for i, req_id in enumerate(cached_reqs.req_ids):
new_block_ids = cached_reqs.new_block_ids[i]
if new_block_ids:
new_block_ids = tuple(
new_block_ids[group_id] for group_id in self._store_group_ids
)
req_meta = None
if req_id in cached_reqs.resumed_req_ids:
# Resumed after preemption
if not new_block_ids:
continue
new_block_ids = tuple(b.copy() for b in new_block_ids)
load_spec = self.load_specs.pop(req_id, None)
request_tuple = self._unfinished_requests.get(req_id)
request_real = request_tuple[0] # type: ignore[index]
num_tokens_to_compute = (
request_real.num_computed_tokens
+ scheduler_output.num_scheduled_tokens[req_id]
)
# On resume, the request re-prefills prompt + previously
# generated tokens (all_token_ids).
prefill_tokens = list(request_real.all_token_ids)
request_tracker = RequestTracker(
req_id=req_id,
token_len=num_tokens_to_compute,
allocated_block_ids=new_block_ids,
num_saved_tokens=0,
token_ids=(
prefill_tokens[:num_tokens_to_compute]
if self.enable_kv_events
else None
),
prefill_end_tokens=len(prefill_tokens),
)
self._request_trackers[req_id] = request_tracker
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=load_spec,
skip_save=is_consumer,
block_hashes=request_real.block_hashes,
num_prompt_tokens=request_real.num_prompt_tokens,
save_partial_tail=self.enable_partial_hash_hits,
)
else:
# Decode/chunked request
request_tracker = self._request_trackers[req_id]
num_computed_token = cached_reqs.num_computed_tokens[i]
# Use the tracker's snapshot of the prefill range so resumed
# requests keep saving past the original prompt boundary.
prefill_end = request_tracker.prefill_end_tokens
is_decode = num_computed_token >= prefill_end
if is_decode and not self.save_decode_cache:
continue
num_new_tokens = scheduler_output.num_scheduled_tokens[req_id]
req_tuple = self._unfinished_requests.get(req_id)
if req_tuple:
unfinished_req = req_tuple[0]
num_current_tokens = request_tracker.token_len
new_token_ids = unfinished_req.all_token_ids[
num_current_tokens : num_current_tokens + num_new_tokens
]
request_tracker.token_len += len(new_token_ids)
if request_tracker.token_ids is not None:
request_tracker.token_ids.extend(new_token_ids)
else:
raise ValueError(
f"Request {req_id} is not in _unfinished_requests"
)
# A block is usually allocated before the step that fills
# it, so reaching a save boundary does not imply that this
# step has new block ids.
if new_block_ids:
request_tracker.update(new_block_ids)
if is_consumer and not is_decode:
continue
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=None,
skip_save=False,
block_hashes=unfinished_req.block_hashes,
num_prompt_tokens=unfinished_req.num_prompt_tokens,
save_partial_tail=self.enable_partial_hash_hits,
)
if req_meta is not None:
meta.add_request(req_meta)
# Handle requests with pending load specs not yet scheduled
request_ids = [req.req_id for req in scheduler_output.scheduled_new_reqs]
for request_id, (
unfinished_req,
block_ids,
) in self._unfinished_requests.items():
if request_id not in request_ids and request_id not in cached_reqs.req_ids:
load_spec = self.load_specs.pop(request_id, None)
# A load spec may have been proposed by this connector's
# lookup but rejected by MultiConnector in favor of another
# connector. Only the chosen connector may issue the pending
# load; the normal store path gets its blocks later from
# SchedulerOutput once the request is actually scheduled.
if load_spec is None or not load_spec.can_load:
continue
num_tokens_to_compute = load_spec.kvpool_cached_tokens
request_tracker = RequestTracker(
req_id=request_id,
token_len=num_tokens_to_compute,
allocated_block_ids=block_ids,
num_saved_tokens=0,
)
self._request_trackers[request_id] = request_tracker
req_meta = ReqMeta.from_request_tracker(
request_tracker,
self._block_size,
load_spec=load_spec,
skip_save=None,
block_hashes=unfinished_req.block_hashes,
num_prompt_tokens=unfinished_req.num_prompt_tokens,
)
if req_meta is not None:
meta.add_request(req_meta)
block_state = getattr(scheduler_output, "kv_connector_block_state", None)
if (
block_state is not None
and block_state.boundary_state_offloads
and not is_consumer
):
self._handle_boundary_state_offloads(
block_state.boundary_state_offloads, meta
)
self._apply_current_save_block_ids(meta, scheduler_output)
# Finish-time handoffs arrive after the producing step's metadata was
# built. Their exact blocks were pinned when they were registered, so
# they remain valid after request cleanup and need no current snapshot.
for req_meta in self._finished_partial_tail_metas.values():
meta.add_request(req_meta)
self._finished_partial_tail_metas.clear()
if not self._capacity_only:
self._reference_save_blocks(meta)
return meta
def _apply_current_save_block_ids(
self,
meta: MooncakeStoreConnectorMetadata,
scheduler_output: SchedulerOutput,
) -> None:
"""Replace append-only mirrors with the core's current block tables."""
save_metas = [req_meta for req_meta in meta.requests if req_meta.can_save]
if not save_metas:
return
block_state = scheduler_output.kv_connector_block_state
assert block_state is not None, (
"Current block tables are required for Mooncake store jobs"
)
for req_meta in save_metas:
block_ids = block_state.get_block_ids(req_meta.req_id)
assert block_ids is not None, (
f"Missing current block table for store request {req_meta.req_id}"
)
req_meta.block_ids = tuple(
block_ids[group_id] for group_id in self._store_group_ids
)
def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None:
"""Take a GPU block reference for every store job this step emits.
The worker DMAs out of these blocks after the step that scheduled them,
so a reference keeps them out of the free queue even once the request
itself is freed, until every rank reports the job done.
"""
pool = self._gpu_block_pool
coord = self._store_coord
assert coord is not None
block_sizes = [g.kv_cache_spec.block_size for g in coord.kv_cache_groups]
for req_meta in meta.requests:
if not req_meta.can_save:
continue
assert pool is not None, (
"GPU block pool must be bound before any store job is emitted"
)
if req_meta.store_job_id is not None:
assert req_meta.store_job_id in self._pinned_saves
continue
if tail := _partial_tail_non_mamba_puts(coord, req_meta, block_sizes):
req_meta.boundary_puts = [*tail, *(req_meta.boundary_puts or [])]
req_meta.store_job_id = store_job_id = self._next_store_job_id
self._next_store_job_id += 1
block_ids = [put.block_id for put in req_meta.boundary_puts or []]
assert NULL_BLOCK_ID not in block_ids, (
"A null block cannot back a boundary put"
)
if req_meta.token_len_chunk:
# Normal prefix save: pin all non-Mamba sources for retries.
# Every allocated block is referenced, not just the ones covering
# this job's token range: a rank resumes from its own last
# successful offset, which lags the scheduler's whenever a save was
# skipped or failed, so it may read anywhere below the range.
block_ids.extend(
block_id
for group_id, group in enumerate(req_meta.block_ids)
if group_id not in self._boundary_state_group_ids
for block_id in group
if block_id != NULL_BLOCK_ID
)
# An aligned boundary block may also be present in the request's
# block table. Take and release exactly one reference per block.
block_ids = list(dict.fromkeys(block_ids))
assert NULL_BLOCK_ID not in block_ids
if not block_ids:
continue
self._pinned_saves[store_job_id] = (block_ids, self._num_workers)
pool.touch([pool.blocks[block_id] for block_id in block_ids])
def register_finished_partial_tail(
self,
request: Request,
block_ids: tuple[list[int], ...],
partial_tail_offloads: list[tuple[int, int, int]],
) -> bool:
"""Queue and pin a finish-time tail for the next connector step."""
if self.kv_role == "kv_consumer" or not partial_tail_offloads:
return False
tracker = self._request_trackers.get(request.request_id)
if tracker is None or not any(block_ids):
return False
boundaries = {boundary for _, _, boundary in partial_tail_offloads}
if len(boundaries) != 1:
raise ValueError(
"Partial-tail offloads for one request must share a boundary"
)
boundary_tokens = next(iter(boundaries))
if boundary_tokens > tracker.prefill_end_tokens:
return False
mamba_puts: list[BoundaryPut] = []
for group_id, block_id, boundary in partial_tail_offloads:
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
if store_group_id not in self._boundary_state_group_ids:
return False
if block_id == NULL_BLOCK_ID:
return False
mamba_puts.append(BoundaryPut(store_group_id, block_id, boundary))
req_meta = ReqMeta(
req_id=request.request_id,
token_len_chunk=0,
block_ids=tuple(
block_ids[group_id].copy() for group_id in self._store_group_ids
),
block_hashes=list(request.block_hashes),
can_save=True,
num_prompt_tokens=request.num_prompt_tokens,
prefill_end_tokens=tracker.prefill_end_tokens,
boundary_puts=mamba_puts,
completed_token_len=request.num_computed_tokens,
)
coord = self._store_coord
assert coord is not None
req_meta.boundary_puts = [
*_partial_tail_non_mamba_puts(
coord,
req_meta,
[g.kv_cache_spec.block_size for g in coord.kv_cache_groups],
),
*mamba_puts,
]
pinned_block_ids = list(
dict.fromkeys(put.block_id for put in req_meta.boundary_puts)
)
pool = self._gpu_block_pool
assert pool is not None, (
"GPU block pool must be bound before a finish-time handoff"
)
assert request.request_id not in self._finished_partial_tail_metas
store_job_id = self._next_store_job_id
self._next_store_job_id += 1
self._pinned_saves[store_job_id] = (pinned_block_ids, self._num_workers)
pool.touch([pool.blocks[block_id] for block_id in pinned_block_ids])
req_meta.store_job_id = store_job_id
self._finished_partial_tail_metas[request.request_id] = req_meta
tracker.has_pending_offload = True
# The store job owns exact block refs, so request cleanup need not wait.
return False
def _handle_boundary_state_offloads(
self,
offloads: dict[str, list[tuple[int, int, int]]],
meta: MooncakeStoreConnectorMetadata,
) -> None:
"""Attach exact boundary-state blocks to store jobs for this step.
Flushed in the step they arrive: the CoW copy is enqueued before the
connector event records, so this step's event fences the exact block.
Entries ride the request's save meta when present, else an
offload-only ReqMeta (``token_len_chunk=0`` skips the normal save,
``can_save=True`` takes the normal enqueue and store-job pinning path).
"""
save_metas = {m.req_id: m for m in meta.requests if m.can_save}
for req_id, entries in offloads.items():
tracker = self._request_trackers.get(req_id)
req_tuple = self._unfinished_requests.get(req_id)
if tracker is None or req_tuple is None:
# Request finished/preempted within this step; its blocks are
# going away, so the offload is conservatively dropped.
logger.debug("Dropping boundary-state offload for request %s", req_id)
continue
accepted: list[BoundaryPut] = []
for group_id, block_id, boundary_tokens in entries:
# Every other group stops saving at the end of this prefill, so
# a mamba-only key past it can never complete a joint hybrid
# hit. `prefill_end_tokens` — not the original prompt length —
# is the boundary: a resumed request re-prefills and re-saves
# its previously generated tokens for every group.
if boundary_tokens > tracker.prefill_end_tokens:
continue
if block_id == NULL_BLOCK_ID:
continue
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
if store_group_id not in self._boundary_state_group_ids:
continue
accepted.append(BoundaryPut(store_group_id, block_id, boundary_tokens))
if not accepted:
continue
tracker.has_pending_offload = True
if (req_meta := save_metas.get(req_id)) is not None:
req_meta.boundary_puts = accepted
continue
meta.add_request(
ReqMeta(
req_id=req_id,
token_len_chunk=0,
block_ids=tracker.allocated_block_ids,
block_hashes=req_tuple[0].block_hashes,
can_save=True,
num_prompt_tokens=req_tuple[0].num_prompt_tokens,
prefill_end_tokens=tracker.prefill_end_tokens,
boundary_puts=accepted,
completed_token_len=tracker.token_len,
)
)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Drop the block references of store jobs every rank has finished."""
meta = connector_output.kv_connector_worker_meta
if not isinstance(meta, MooncakeStoreWorkerMetadata):
return
pool = self._gpu_block_pool
assert pool is not None
for store_job_id, count in meta.completed_saves.items():
pinned = self._pinned_saves.get(store_job_id)
if pinned is None:
# The job referenced no blocks, so nothing was recorded for it.
continue
block_ids, remaining = pinned
remaining -= count
if remaining > 0:
self._pinned_saves[store_job_id] = (block_ids, remaining)
continue
assert remaining == 0, (
f"store job {store_job_id} reported by too many ranks"
)
del self._pinned_saves[store_job_id]
# Tail-first, as elsewhere, so the shared prefix is evicted last.
pool.free_blocks(pool.blocks[bid] for bid in reversed(block_ids))
def has_pending_push_work(self) -> bool:
"""Keep the engine stepping while any store job still holds block refs.
Completions only reach the scheduler as worker metadata on a step, so an
engine that quiesced with jobs in flight would leave those references
held indefinitely. Nothing else keeps it alive now that a finishing
request no longer defers its own free.
"""
return bool(self._pinned_saves)
def reset_store(self) -> bool:
"""Trigger a global ``remove_all(force=True)`` on the Mooncake master.
Routes through the existing LookupKey ZMQ admin channel to worker
rank 0, which owns the ``MooncakeDistributedStore`` handle.
Ordering assumption: caller (typically
``Scheduler.reset_connector_cache``, invoked via
``reset_prefix_cache(reset_connector=True)``) MUST ensure no
in-flight Mooncake lookups or transfers. For RL workflows this is
satisfied at the step boundary after weight updates and rollout
drain. Violating this can allow stale KV to be served on the next
request, defeating the hard-reset guarantee.
Returns True on ACK from worker, False on NACK or RPC error.
"""
try:
ok = self.client.reset()
if ok:
logger.info("Mooncake store reset via remove_all succeeded.")
else:
logger.warning("Mooncake store reset returned NACK from worker.")
return ok
except Exception as e:
logger.error("Mooncake reset_store RPC failed: %s", e)
return False
File diff suppressed because it is too large Load Diff
@@ -1,41 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .connector import UMBPStoreConnector
from .data import (
BlockIdentityCodec,
BlockTransferPlan,
KVLayoutDescriptor,
KVLayoutPlanner,
KVRange,
KVRegion,
LoadSpec,
RankTopology,
RequestTracker,
TransferJobState,
TransferJobStatus,
UMBPConnectorMetadata,
UMBPConnectorWorkerMetadata,
UMBPNamespace,
)
from .runtime import UMBPRuntimeConfig, UMBPRuntimeFactory
__all__ = [
"UMBPStoreConnector",
"BlockIdentityCodec",
"BlockTransferPlan",
"KVLayoutPlanner",
"KVLayoutDescriptor",
"KVRange",
"KVRegion",
"LoadSpec",
"RequestTracker",
"RankTopology",
"TransferJobState",
"TransferJobStatus",
"UMBPConnectorMetadata",
"UMBPConnectorWorkerMetadata",
"UMBPNamespace",
"UMBPRuntimeConfig",
"UMBPRuntimeFactory",
]
@@ -1,220 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM-facing shared UMBP connector."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorTransferResults,
)
from vllm.forward_context import ForwardContext
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from .data import (
BlockIdentityCodec,
KVLayoutPlanner,
RankTopology,
UMBPConnectorMetadata,
UMBPConnectorWorkerMetadata,
UMBPNamespace,
)
from .runtime import UMBPRuntimeConfig, UMBPRuntimeFactory
from .scheduler import UMBPStoreConnectorScheduler
from .worker import UMBPStoreConnectorWorker
if TYPE_CHECKING:
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.request import Request
class UMBPStoreConnector(KVConnectorBase_V1):
"""vLLM connector lifecycle shared by every UMBP runtime mode."""
def __init__(
self,
vllm_config: VllmConfig,
role: KVConnectorRole,
kv_cache_config: KVCacheConfig,
) -> None:
super().__init__(vllm_config, role, kv_cache_config)
topology = RankTopology.from_vllm_config(vllm_config)
runtime_config = UMBPRuntimeConfig.from_vllm(
vllm_config
).resolve_for_rank_count(topology.rank_count)
if topology.pp_size > 1:
raise NotImplementedError(
f"{runtime_config.mode} UMBP does not support pipeline parallelism"
)
runtime = UMBPRuntimeFactory.build(runtime_config)
namespace = UMBPNamespace.from_vllm_config(vllm_config, kv_cache_config)
codec = BlockIdentityCodec(
namespace=namespace,
tp_rank=topology.tp_rank,
pp_rank=topology.pp_rank,
pcp_rank=topology.pcp_rank,
dcp_rank=topology.dcp_rank,
)
layout = KVLayoutPlanner.from_kv_cache_config(kv_cache_config)
layout_descriptor = layout.describe(topology)
self._runtime = runtime
self._store_submitted = False
self.connector_scheduler: UMBPStoreConnectorScheduler | None = None
self.connector_worker: UMBPStoreConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = UMBPStoreConnectorScheduler(
vllm_config,
kv_cache_config,
runtime.create_scheduler_handle(
namespace.value, topology, layout_descriptor
),
codec,
topology,
)
else:
self.connector_worker = UMBPStoreConnectorWorker(
runtime.create_worker_handle(
namespace.value, topology, layout_descriptor
),
layout,
codec=codec,
)
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
)
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.update_state_after_alloc(
request, blocks, num_external_tokens
)
def bind_gpu_block_pool(self, gpu_block_pool: Any) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def request_finished(
self, request: Request, block_ids: list[int]
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, (block_ids,))
def update_connector_output(self, connector_output: Any) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
def has_pending_push_work(self) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.has_pending_push_work()
def reset_cache(self) -> bool | None:
if self.connector_scheduler is None:
return None
return self.connector_scheduler.reset_store()
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
super().bind_connector_metadata(connector_metadata)
self._store_submitted = False
def clear_connector_metadata(self) -> None:
# No-forward steps skip the model runner's wait_for_save hook.
if self.connector_worker is not None and self.has_connector_metadata():
self.wait_for_save()
super().clear_connector_metadata()
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata) -> None:
assert self.connector_worker is not None
assert isinstance(kv_connector_metadata, UMBPConnectorMetadata)
self.connector_worker.handle_preemptions(kv_connector_metadata)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
del kwargs
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, UMBPConnectorMetadata)
self.connector_worker.start_load_kv(forward_context, metadata)
def wait_for_layer_load(self, layer_name: str) -> None:
assert self.connector_worker is not None
self.connector_worker.wait_for_layer_load(layer_name)
def save_kv_layer(
self,
layer_name: str,
kv_layer: torch.Tensor,
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
# Stores are submitted once per step, from wait_for_save.
return
def wait_for_save(self) -> None:
if self._store_submitted:
return
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, UMBPConnectorMetadata)
self.connector_worker.enqueue_stores(metadata)
self.connector_worker.wait_for_save()
self._store_submitted = True
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
assert self.connector_worker is not None
return self.connector_worker.get_finished(finished_req_ids)
def get_transfer_results(
self, finished_req_ids: set[str]
) -> KVConnectorTransferResults:
assert self.connector_worker is not None
finished_sending, finished_recving = self.connector_worker.get_finished(
finished_req_ids
)
return KVConnectorTransferResults(
finished_sending=set(finished_sending or ()),
finished_recving=set(finished_recving or ()),
)
def get_block_ids_with_load_errors(self) -> set[int]:
assert self.connector_worker is not None
return self.connector_worker.get_block_ids_with_load_errors()
def build_connector_worker_meta(self) -> UMBPConnectorWorkerMetadata:
assert self.connector_worker is not None
return self.connector_worker.build_connector_worker_meta()
def shutdown(self) -> None:
if self.connector_scheduler is not None:
self.connector_scheduler.close()
if self.connector_worker is not None:
self.connector_worker.close()
@@ -1,620 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared data types for the UMBP KV connector.
The types in this module deliberately do not depend on a UMBP client. They
describe vLLM's logical KV objects and the physical ranges a runtime adapter
will transfer.
"""
from __future__ import annotations
import hashlib
import json
from collections.abc import Sequence
from dataclasses import dataclass, field, replace
from enum import Enum
from typing import Any, NamedTuple
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
KVConnectorWorkerMetadata,
)
from vllm.v1.kv_cache_interface import KVCacheConfig
@dataclass(frozen=True)
class RankTopology:
"""Model-parallel ranks that participate in one KV object."""
tp_rank: int = 0
tp_size: int = 1
pp_rank: int = 0
pp_size: int = 1
pcp_rank: int = 0
pcp_size: int = 1
dcp_rank: int = 0
dcp_size: int = 1
def __post_init__(self) -> None:
for name in ("tp_size", "pp_size", "pcp_size", "dcp_size"):
if getattr(self, name) <= 0:
raise ValueError(f"{name} must be positive")
for rank, size, name in (
(self.tp_rank, self.tp_size, "tp_rank"),
(self.pp_rank, self.pp_size, "pp_rank"),
(self.pcp_rank, self.pcp_size, "pcp_rank"),
(self.dcp_rank, self.dcp_size, "dcp_rank"),
):
if rank < 0 or rank >= size:
raise ValueError(f"{name}={rank} is outside [0, {size})")
@classmethod
def from_vllm_config(cls, vllm_config: Any) -> RankTopology:
parallel = vllm_config.parallel_config
tp_size = getattr(parallel, "tensor_parallel_size", 1)
pp_size = getattr(parallel, "pipeline_parallel_size", 1)
pcp_size = getattr(parallel, "prefill_context_parallel_size", 1)
# An engine's workers are laid out as [PP, PCP, TP], TP innermost
# (initialize_model_parallel).
rank = getattr(parallel, "rank", 0)
tp_rank = rank % tp_size
dcp_size = getattr(parallel, "decode_context_parallel_size", 1)
return cls(
tp_rank=tp_rank,
tp_size=tp_size,
pp_rank=rank // (tp_size * pcp_size) % pp_size,
pp_size=pp_size,
pcp_rank=rank // tp_size % pcp_size,
pcp_size=pcp_size,
# DCP partitions the TP workers; it does not create a second
# Cartesian worker dimension. ParallelConfig is constructed before
# process groups, so derive this stable identity from TP rank.
dcp_rank=tp_rank % dcp_size,
dcp_size=dcp_size,
)
@property
def local_namespace(self) -> tuple[int, int, int, int]:
return (self.tp_rank, self.pcp_rank, self.dcp_rank, self.pp_rank)
@property
def rank_count(self) -> int:
return len(self.all_namespaces())
def all_namespaces(self) -> tuple[tuple[int, int, int, int], ...]:
if self.dcp_size > 1:
if self.tp_size % self.dcp_size:
raise ValueError("dcp_size must divide tp_size")
return tuple(
(tp, pcp, tp % self.dcp_size, pp)
for pp in range(self.pp_size)
for pcp in range(self.pcp_size)
for tp in range(self.tp_size)
)
return tuple(
(tp, pcp, dcp, pp)
for pp in range(self.pp_size)
for dcp in range(self.dcp_size)
for pcp in range(self.pcp_size)
for tp in range(self.tp_size)
)
@dataclass(frozen=True)
class UMBPNamespace:
"""Compatibility namespace for objects stored outside the engine."""
value: str
@classmethod
def from_vllm_config(
cls,
vllm_config: Any,
kv_cache_config: KVCacheConfig,
) -> UMBPNamespace:
extra = vllm_config.kv_transfer_config.kv_connector_extra_config
configured = extra.get("key_namespace", "auto")
if configured != "auto":
if not isinstance(configured, str) or not configured:
raise ValueError("key_namespace must be a non-empty string or 'auto'")
return cls(configured)
model_config = vllm_config.model_config
parallel_config = vllm_config.parallel_config
# Draft-model KV (e.g. DSpark/EAGLE layers) is stored alongside the
# target's, so a different drafter must not reuse it.
speculative_config = getattr(vllm_config, "speculative_config", None)
draft_model_config = getattr(speculative_config, "draft_model_config", None)
speculative = (
None
if speculative_config is None
else {
"method": getattr(speculative_config, "method", None),
"model": getattr(draft_model_config, "model", None),
"revision": getattr(draft_model_config, "revision", None),
}
)
fields = {
"model": getattr(model_config, "model", ""),
"revision": getattr(model_config, "revision", None),
"block_size": vllm_config.cache_config.block_size,
"cache_layout": getattr(kv_cache_config, "kv_cache_layout", None),
"groups": [
{
"layers": group.layer_names,
"block_size": group.kv_cache_spec.block_size,
"kind": type(group.kv_cache_spec).__name__,
"dtype": str(getattr(group.kv_cache_spec, "dtype", None)),
}
for group in kv_cache_config.kv_cache_groups
],
"tp": parallel_config.tensor_parallel_size,
"pp": parallel_config.pipeline_parallel_size,
"pcp": getattr(parallel_config, "prefill_context_parallel_size", 1),
"dcp": getattr(parallel_config, "decode_context_parallel_size", 1),
"layout_version": 1,
}
if speculative is not None:
# Added only when set, so namespaces without a drafter are unchanged.
fields["speculative"] = speculative
payload = json.dumps(fields, sort_keys=True, separators=(",", ":")).encode()
return cls(hashlib.sha256(payload).hexdigest()[:32])
@dataclass(frozen=True)
class BlockIdentityCodec:
"""Build stable rank-local keys from vLLM block hashes."""
namespace: UMBPNamespace
tp_rank: int = 0
pp_rank: int = 0
pcp_rank: int = 0
dcp_rank: int = 0
def key(self, block_hash: bytes, group_id: int = 0) -> str:
return self.key_for_namespace(
block_hash,
group_id,
(self.tp_rank, self.pcp_rank, self.dcp_rank, self.pp_rank),
)
def key_for_namespace(
self,
block_hash: bytes,
group_id: int,
namespace: tuple[int, int, int, int],
) -> str:
if not isinstance(block_hash, bytes) or not block_hash:
raise ValueError("block_hash must be non-empty bytes")
if group_id < 0:
raise ValueError("group_id must be non-negative")
tp_rank, pcp_rank, dcp_rank, pp_rank = namespace
return (
f"umbp:vllm:v1:{self.namespace.value}:"
f"tp{tp_rank}:pcp{pcp_rank}:dcp{dcp_rank}:"
f"pp{pp_rank}:g{group_id}:"
f"{block_hash.hex()}"
)
def keys_for_topology(
self,
block_hash: bytes,
topology: RankTopology,
group_ids: Sequence[int],
) -> tuple[str, ...]:
return tuple(
self.key_for_namespace(block_hash, group_id, namespace)
for namespace in topology.all_namespaces()
for group_id in group_ids
)
@dataclass(frozen=True)
class KVRegion:
"""One deterministic region in a rank-local KV object."""
layer_name: str
group_id: int
block_stride: int
block_bytes: int
object_offset: int
block_size: int = 1
def __post_init__(self) -> None:
if self.block_stride <= 0 or self.block_bytes <= 0:
raise ValueError("KV region sizes must be positive")
if self.block_size <= 0:
raise ValueError("KV region block_size must be positive")
if self.block_bytes > self.block_stride:
raise ValueError(
f"KV region bytes ({self.block_bytes}) exceed its stride "
f"({self.block_stride}) for {self.layer_name!r}"
)
@dataclass(frozen=True)
class KVLayoutDescriptor:
"""Backend-neutral description of one rank-local KV object layout."""
regions: tuple[KVRegion, ...]
topology: RankTopology
layout_format: str = "rank_local"
def __post_init__(self) -> None:
if not self.layout_format:
raise ValueError("layout_format must not be empty")
names = [region.layer_name for region in self.regions]
if len(names) != len(set(names)):
raise ValueError("KV layout contains duplicate layer regions")
@property
def object_size(self) -> int:
return sum(region.block_bytes for region in self.regions)
class KVLayoutPlanner:
"""Create a deterministic all-layer object layout from KVCacheConfig."""
def __init__(self, regions: Sequence[KVRegion]) -> None:
self.regions = tuple(regions)
self._regions_by_group = {
group_id: tuple(r for r in self.regions if r.group_id == group_id)
for group_id in {r.group_id for r in self.regions}
}
self._base_addresses: dict[str, int] = {}
self._physical_strides: dict[str, int] = {}
@classmethod
def from_kv_cache_config(cls, config: KVCacheConfig) -> KVLayoutPlanner:
group_by_layer = {
layer: group_id
for group_id, group in enumerate(config.kv_cache_groups)
for layer in group.layer_names
if group.enable_kv_transfer
}
regions: list[KVRegion] = []
offset = 0
for tensor in config.kv_cache_tensors:
for layer_name in tensor.layers:
group_id = group_by_layer.get(layer_name)
if group_id is None:
continue
spec = config.kv_cache_groups[group_id].kv_cache_spec
block_bytes = spec.page_size_bytes
regions.append(
KVRegion(
layer_name=layer_name,
group_id=group_id,
block_stride=tensor.block_stride,
block_bytes=block_bytes,
object_offset=offset,
block_size=spec.block_size,
)
)
offset += block_bytes
return cls(regions)
@property
def object_size(self) -> int:
return sum(region.block_bytes for region in self.regions)
def describe(
self,
topology: RankTopology,
layout_format: str = "rank_local",
) -> KVLayoutDescriptor:
return KVLayoutDescriptor(
regions=self.regions,
topology=topology,
layout_format=layout_format,
)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
"""Capture stable base addresses for the planned KV regions."""
required = {region.layer_name for region in self.regions}
missing = required - kv_caches.keys()
if missing:
raise ValueError(f"missing KV cache layers: {sorted(missing)}")
addresses: dict[str, int] = {}
for layer_name in required:
cache = kv_caches[layer_name]
if cache.ndim == 0:
raise ValueError(f"KV cache {layer_name!r} must be a tensor")
block_stride = cache.stride(0) * cache.element_size()
addresses[layer_name] = cache.data_ptr()
for region in self.regions:
if region.layer_name != layer_name:
continue
if (
block_stride > region.block_stride
or region.block_stride % block_stride != 0
or (
region.block_bytes > block_stride
and region.block_bytes % block_stride != 0
)
):
raise ValueError(
f"KV block stride mismatch for {layer_name!r}: "
f"config={region.block_stride}, tensor={block_stride}"
)
self._physical_strides[layer_name] = block_stride
self._base_addresses = addresses
def plan_registered_block(
self,
key: str,
block_id: int,
*,
group_id: int | None = None,
request_id: str | None = None,
) -> BlockTransferPlan:
if not self._base_addresses:
raise RuntimeError("KV caches must be registered before planning")
return self.plan_for_block(
key,
block_id,
self._base_addresses,
group_id=group_id,
request_id=request_id,
)
def plan_for_block(
self,
key: str,
block_id: int,
base_addresses: dict[str, int],
*,
group_id: int | None = None,
request_id: str | None = None,
) -> BlockTransferPlan:
"""Lower one GPU block into a deterministic scatter/gather plan."""
return BlockTransferPlan(
key=key,
block_id=block_id,
ranges=self._ranges_for_block(block_id, base_addresses, group_id),
request_id=request_id,
)
def materialize(self, plan: BlockTransferPlan) -> BlockTransferPlan:
if not self._base_addresses:
raise RuntimeError("KV caches must be registered before planning")
return replace(
plan,
ranges=self._ranges_for_block(
plan.block_id, self._base_addresses, plan.group_id
),
)
def _ranges_for_block(
self,
block_id: int,
base_addresses: dict[str, int],
group_id: int | None,
) -> tuple[KVRange, ...]:
if block_id < 0:
raise ValueError("block_id must be non-negative")
regions = (
self.regions
if group_id is None
else self._regions_by_group.get(group_id, ())
)
if group_id is not None and not regions:
raise ValueError(f"KV layout does not contain cache group {group_id}")
ranges: list[KVRange] = []
object_offset = 0
for region in regions:
try:
base_address = base_addresses[region.layer_name]
except KeyError as exc:
raise ValueError(
f"missing base address for layer {region.layer_name!r}"
) from exc
physical_stride = self._physical_strides.get(
region.layer_name, region.block_stride
)
if region.block_bytes <= physical_stride:
ranges.append(
KVRange(
layer_name=region.layer_name,
group_id=region.group_id,
block_id=block_id,
base_address=base_address + block_id * physical_stride,
stride=physical_stride,
length=region.block_bytes,
object_offset=(
object_offset
if group_id is not None
else region.object_offset
),
)
)
object_offset += region.block_bytes
continue
physical_parts = max(1, region.block_bytes // physical_stride)
for part in range(physical_parts):
part_start = part * physical_stride
part_end = min(part_start + physical_stride, region.block_bytes)
physical_block_id = block_id * physical_parts + part
ranges.append(
KVRange(
layer_name=region.layer_name,
group_id=region.group_id,
block_id=block_id,
base_address=base_address + physical_block_id * physical_stride,
stride=physical_stride,
length=part_end - part_start,
object_offset=(
object_offset + part_start
if group_id is not None
else region.object_offset + part * physical_stride
),
)
)
object_offset += region.block_bytes
return tuple(ranges)
class KVRange(NamedTuple):
"""A source/destination range in a runtime transfer."""
layer_name: str
group_id: int
block_id: int
base_address: int
stride: int
length: int
object_offset: int
@dataclass(frozen=True)
class BlockTransferPlan:
"""One complete logical block transfer."""
key: str
block_id: int
ranges: tuple[KVRange, ...] = ()
request_id: str | None = None
group_id: int | None = None
@dataclass
class LoadSpec:
"""Scheduler decision for one externally loaded prefix."""
local_tokens: int
external_tokens: int
@property
def num_tokens_to_load(self) -> int:
return max(self.external_tokens - self.local_tokens, 0)
@dataclass
class RequestTracker:
"""Core request state shared by every runtime mode."""
saved_tokens: int = 0
block_ids: tuple[list[int], ...] = ()
load_spec: LoadSpec | None = None
def reset(self) -> None:
self.saved_tokens = 0
self.block_ids = ()
self.load_spec = None
def update_blocks(self, block_ids: tuple[list[int], ...]) -> None:
if not self.block_ids:
self.block_ids = tuple(list(group) for group in block_ids)
return
if len(self.block_ids) != len(block_ids):
raise ValueError("KV cache group count changed for a request")
for current, new in zip(self.block_ids, block_ids, strict=True):
if new[: len(current)] == current:
current.extend(new[len(current) :])
elif not new or current[-len(new) :] == new:
continue
else:
current.extend(new)
def replace_blocks(self, block_ids: tuple[list[int], ...]) -> None:
"""Replace stale pre-preemption tables with resumed full tables."""
self.block_ids = tuple(list(group) for group in block_ids)
self.saved_tokens = 0
def mark_saved(self, token_count: int, block_size: int) -> int:
"""Advance the save watermark only to a complete block boundary."""
complete_tokens = token_count // block_size * block_size
if complete_tokens < self.saved_tokens:
return self.saved_tokens
self.saved_tokens = complete_tokens
return complete_tokens
class TransferJobStatus(str, Enum):
PENDING = "pending"
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
@dataclass
class TransferJobState:
"""Per-key completion state; one failed key does not fail its siblings."""
plans: tuple[BlockTransferPlan, ...]
status: TransferJobStatus = TransferJobStatus.PENDING
completed_keys: set[str] = field(default_factory=set)
failed_keys: set[str] = field(default_factory=set)
error: str | None = None
@property
def keys(self) -> Sequence[str]:
return tuple(plan.key for plan in self.plans)
def start(self) -> None:
if self.status != TransferJobStatus.PENDING:
raise RuntimeError(f"cannot start job in state {self.status}")
self.status = TransferJobStatus.RUNNING
def complete(self, keys: Sequence[str] = ()) -> None:
self.completed_keys.update(keys or self.keys)
self._finish_if_done()
def fail(self, keys: Sequence[str], error: str | None = None) -> None:
self.failed_keys.update(keys)
self.error = error
self._finish_if_done()
@property
def failed_block_ids(self) -> set[int]:
return {plan.block_id for plan in self.plans if plan.key in self.failed_keys}
def _finish_if_done(self) -> None:
keys = set(self.keys)
if self.completed_keys | self.failed_keys >= keys:
self.status = (
TransferJobStatus.FAILED
if self.failed_keys
else TransferJobStatus.COMPLETED
)
@dataclass
class UMBPConnectorMetadata(KVConnectorMetadata):
"""Scheduler-to-worker metadata for one engine step."""
async_load: bool = False
store_plans: list[BlockTransferPlan] = field(default_factory=list)
load_requests: dict[str, list[BlockTransferPlan]] = field(default_factory=dict)
store_requests: dict[str, list[BlockTransferPlan]] = field(default_factory=dict)
preempted_request_ids: set[str] = field(default_factory=set)
store_event: int = -1
@dataclass
class StoreEventResult:
"""Number of workers that finished one store event."""
completed_workers: int = 0
def merge(self, other: StoreEventResult) -> None:
self.completed_workers += other.completed_workers
@dataclass
class UMBPConnectorWorkerMetadata(KVConnectorWorkerMetadata):
"""Worker completion metadata aggregated across ranks."""
store_events: dict[int, StoreEventResult] = field(default_factory=dict)
def aggregate(
self, other: KVConnectorWorkerMetadata
) -> UMBPConnectorWorkerMetadata:
if not isinstance(other, UMBPConnectorWorkerMetadata):
raise TypeError("cannot aggregate incompatible UMBP worker metadata")
for event, result in other.store_events.items():
self.store_events.setdefault(event, StoreEventResult()).merge(result)
return self
@@ -1,13 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .base import IUMBPRuntime, UMBPSchedulerHandle, UMBPWorkerHandle
from .factory import UMBPRuntimeConfig, UMBPRuntimeFactory
__all__ = [
"IUMBPRuntime",
"UMBPSchedulerHandle",
"UMBPWorkerHandle",
"UMBPRuntimeConfig",
"UMBPRuntimeFactory",
]
@@ -1,69 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Backend-neutral runtime contract for UMBP."""
from __future__ import annotations
from collections.abc import Sequence
from typing import Protocol
import torch
from ..data import (
BlockTransferPlan,
KVLayoutDescriptor,
RankTopology,
TransferJobState,
)
class UMBPSchedulerHandle(Protocol):
def lookup(self, keys: Sequence[str]) -> Sequence[bool]:
"""Return authoritative existence for each key."""
def clear(self) -> bool:
"""Clear all objects visible to this scheduler handle."""
def close(self) -> None:
"""Release scheduler-side resources."""
class UMBPWorkerHandle(Protocol):
def register_buffers(self, kv_caches: dict[str, torch.Tensor]) -> None:
"""Register each unique KV allocation with the runtime."""
def load(self, plans: Sequence[BlockTransferPlan]) -> TransferJobState:
"""Start loading the requested plans."""
def store(self, plans: Sequence[BlockTransferPlan]) -> TransferJobState:
"""Start storing the requested plans."""
def wait(self, job: TransferJobState) -> TransferJobState:
"""Block until the job is final and its buffers are safe to reuse."""
def poll(self, job: TransferJobState) -> TransferJobState | None:
"""Return a finished job without blocking, or None if still pending."""
def publish(self, job: TransferJobState) -> None:
"""Accept a completed store; a runtime may defer visibility until then."""
def close(self) -> None:
"""Release worker-side resources."""
class IUMBPRuntime(Protocol):
def create_scheduler_handle(
self,
namespace: str,
topology: RankTopology,
layout: KVLayoutDescriptor,
) -> UMBPSchedulerHandle:
"""Create a scheduler-side handle."""
def create_worker_handle(
self,
namespace: str,
topology: RankTopology,
layout: KVLayoutDescriptor,
) -> UMBPWorkerHandle:
"""Create a worker-side handle."""
@@ -1,76 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Runtime configuration and adapter factory for UMBP."""
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass, replace
from typing import Any
from vllm.config import VllmConfig
from .base import IUMBPRuntime
@dataclass(frozen=True)
class UMBPRuntimeConfig:
"""Shared connector options passed to the selected runtime adapter."""
mode: str
options: dict[str, Any]
rank_count: int = 1
dp_index: int = 0
@classmethod
def from_vllm(cls, vllm_config: VllmConfig) -> UMBPRuntimeConfig:
transfer_config = vllm_config.kv_transfer_config
if transfer_config is None:
raise ValueError("UMBP requires kv_transfer_config")
options = dict(transfer_config.kv_connector_extra_config)
mode = options.get("mode", "embedded")
if mode not in {"embedded"}:
raise ValueError(f"unknown UMBP mode: {mode!r}")
namespace = options.get("key_namespace", "auto")
if namespace != "auto" and (not isinstance(namespace, str) or not namespace):
raise ValueError("key_namespace must be a non-empty string or 'auto'")
return cls(
mode=mode,
options=options,
dp_index=int(
getattr(vllm_config.parallel_config, "data_parallel_index", 0) or 0
),
)
def resolve_for_rank_count(self, rank_count: int) -> UMBPRuntimeConfig:
"""Pass topology to the adapter without interpreting its capacity options."""
if rank_count <= 0:
raise ValueError("rank_count must be positive")
return replace(self, rank_count=rank_count)
RuntimeBuilder = Callable[[UMBPRuntimeConfig], IUMBPRuntime]
class UMBPRuntimeFactory:
"""Registry used by UMBP adapters without importing MORI in vLLM."""
_builders: dict[str, RuntimeBuilder] = {}
@classmethod
def register(cls, mode: str, builder: RuntimeBuilder) -> None:
if mode not in {"embedded"}:
raise ValueError(f"unknown UMBP mode: {mode!r}")
if mode in cls._builders:
raise ValueError(f"UMBP runtime mode {mode!r} is already registered")
cls._builders[mode] = builder
@classmethod
def build(cls, config: UMBPRuntimeConfig) -> IUMBPRuntime:
try:
builder = cls._builders[config.mode]
except KeyError as exc:
raise RuntimeError(
f"no UMBP runtime adapter is registered for mode {config.mode!r}"
) from exc
return builder(config)
@@ -1,518 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared scheduler-side UMBP connector behavior."""
from __future__ import annotations
import logging
from collections.abc import Sequence
from dataclasses import dataclass, field
from functools import lru_cache
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.logger import init_logger
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import (
resolve_dcp_kv_block_size,
resolve_kv_cache_block_sizes,
)
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
)
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
from .data import (
BlockIdentityCodec,
BlockTransferPlan,
LoadSpec,
RankTopology,
RequestTracker,
StoreEventResult,
UMBPConnectorMetadata,
UMBPConnectorWorkerMetadata,
)
from .runtime import UMBPSchedulerHandle
logger = init_logger(__name__)
@lru_cache(maxsize=8192)
def _rank_keys(
codec: BlockIdentityCodec,
topology: RankTopology,
block_hash: bytes,
group_id: int,
) -> tuple[str, ...]:
"""Reuse immutable object identities, not runtime residency."""
return codec.keys_for_topology(block_hash, topology, (group_id,))
@dataclass
class _StoreEvent:
plans: tuple[BlockTransferPlan, ...]
result: StoreEventResult = field(default_factory=StoreEventResult)
class UMBPStoreConnectorScheduler:
"""Own vLLM prefix matching while the runtime owns lookup transport."""
def __init__(
self,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
runtime: UMBPSchedulerHandle,
codec: BlockIdentityCodec,
topology: RankTopology | None = None,
) -> None:
self.block_size, resolved_hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
self.kv_cache_config = kv_cache_config
dcp_size = vllm_config.parallel_config.decode_context_parallel_size
self.group_block_sizes = {
group_id: resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_size)
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
}
self.runtime = runtime
self.codec = codec
self.topology = topology or RankTopology()
transfer_config = vllm_config.kv_transfer_config
assert transfer_config is not None
extra = transfer_config.kv_connector_extra_config
self.load_async = bool(extra.get("load_async", True))
self.enable_lookup = bool(extra.get("enable_lookup", True))
self.save_decode_cache = bool(extra.get("save_decode_cache", False))
# A kv_consumer only restores from the pool, as with other store
# connectors; it never offloads its own KV.
self.store_enabled = getattr(transfer_config, "kv_role", None) != "kv_consumer"
# Request.block_hashes are computed at this size; keys index into them.
self.hash_block_size = resolved_hash_block_size
speculative_config = getattr(vllm_config, "speculative_config", None)
self.use_eagle = bool(
speculative_config is not None and speculative_config.use_eagle_block_drop()
)
self._pending_loads: dict[str, list[BlockTransferPlan]] = {}
self._load_specs: dict[str, LoadSpec] = {}
self._lookup_units = {
group_id: self.group_block_sizes[group_id]
for group_id in kv_cache_config.prefix_cacheable_group_ids
}
self._requests: dict[str, Request] = {}
self._request_trackers: dict[str, RequestTracker] = {}
self._gpu_block_pool: BlockPool | None = None
self._store_events: dict[int, _StoreEvent] = {}
self._num_workers = getattr(vllm_config.parallel_config, "world_size", 1)
self._store_event_counter = 0
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
self._gpu_block_pool = gpu_block_pool
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
) -> tuple[int | None, bool]:
if not self.enable_lookup:
return 0, False
hashes = request.block_hashes
if not hashes or request.num_tokens < self.block_size:
return 0, False
if num_computed_tokens % self.block_size != 0:
return 0, False
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
# vLLM must execute at least the final prompt token. Returning a hit
# for the entire prompt leaves the scheduler with no new token to run.
max_external_token = request.num_tokens - 1
if max_external_token <= num_computed_tokens:
self._load_specs.pop(request.request_id, None)
return 0, False
logical_objects_by_group, keys = self._lookup_objects(
request, num_computed_tokens, hashes
)
try:
hits = list(self.runtime.lookup(keys))
except Exception as exc:
logger.debug("UMBP lookup failed request=%s: %s", request.request_id, exc)
return 0, False
if len(hits) != len(keys):
logger.debug(
"UMBP lookup returned an invalid result length request=%s",
request.request_id,
)
return 0, False
per_rank = self.topology.rank_count
offset = 0
logical_hits_by_group: dict[int, list[bool]] = {}
for group_id in group_ids:
group_size = len(logical_objects_by_group[group_id]) * per_rank
group_hits = hits[offset : offset + group_size]
offset += group_size
logical_hits_by_group[group_id] = (
group_hits
if per_rank == 1
else [
all(group_hits[index : index + per_rank])
for index in range(0, len(group_hits), per_rank)
]
)
group_id = group_ids[0]
matched_blocks = 0
for hit in logical_hits_by_group[group_id]:
if not hit:
break
matched_blocks += 1
if self.use_eagle:
# As core's coordinator: the drafter recomputes the last block.
matched_blocks = max(0, matched_blocks - 1)
need_to_load = matched_blocks * self._lookup_units[group_id]
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"UMBP lookup result request=%s keys=%s group_hits=%s load_tokens=%d",
request.request_id,
keys,
logical_hits_by_group,
need_to_load,
)
matched_tokens = num_computed_tokens + need_to_load
if need_to_load <= 0:
return 0, False
self._load_specs[request.request_id] = LoadSpec(
local_tokens=num_computed_tokens,
external_tokens=matched_tokens,
)
return need_to_load, self.load_async
def _lookup_objects(
self, request: Request, local_tokens: int, hashes: Sequence[bytes]
) -> tuple[dict[int, list[tuple[bytes, tuple[str, ...]]]], list[str]]:
objects: dict[int, list[tuple[bytes, tuple[str, ...]]]] = {}
keys: list[str] = []
for group_id, unit in self._lookup_units.items():
group_objects = []
for token_end in range(local_tokens + unit, request.num_tokens, unit):
block_hash = self._object_hash_at_token_end(hashes, token_end)
rank_keys = _rank_keys(self.codec, self.topology, block_hash, group_id)
group_objects.append((block_hash, rank_keys))
keys.extend(rank_keys)
objects[group_id] = group_objects
return objects, keys
def update_state_after_alloc(
self,
request: Request,
blocks: KVCacheBlocks,
num_external_tokens: int,
) -> None:
self._requests[request.request_id] = request
block_groups = blocks.get_block_ids(
group_ids=self.kv_cache_config.prefix_cacheable_group_ids
)
if not block_groups:
return
tracker = self._tracker_for_request(request.request_id)
# Core passes the request's whole table on every admission. A request
# whose load failed is recomputed into newly allocated blocks.
tracker.replace_blocks(tuple(list(group) for group in block_groups))
if num_external_tokens <= 0:
self._pending_loads.pop(request.request_id, None)
self._load_specs.pop(request.request_id, None)
return
spec = self._load_specs.get(request.request_id)
if spec is not None:
tracker.load_spec = spec
# A request admitted without its load would wait for it forever.
assert num_external_tokens == spec.num_tokens_to_load, (
f"UMBP matched {spec.num_tokens_to_load} external tokens for "
f"request {request.request_id} but {num_external_tokens} were "
"allocated"
)
plans = self._load_plans_for_external_tokens(
request,
tracker,
block_groups,
num_external_tokens,
)
self._pending_loads[request.request_id] = plans
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
meta = UMBPConnectorMetadata(async_load=self.load_async)
for request_id in scheduler_output.finished_req_ids:
self._pending_loads.pop(request_id, None)
self._load_specs.pop(request_id, None)
self._request_trackers.pop(request_id, None)
self._requests.pop(request_id, None)
for request in scheduler_output.scheduled_new_reqs:
load_plans = self._pending_loads.pop(request.req_id, [])
self._load_specs.pop(request.req_id, None)
if load_plans:
meta.load_requests[request.req_id] = load_plans
else:
tracker = self._tracker_for_request(request.req_id)
total_tokens = (
request.num_computed_tokens
+ scheduler_output.num_scheduled_tokens[request.req_id]
)
tracked_request = self._requests.get(request.req_id)
selected_block_ids = tuple(
request.block_ids[group_id]
for group_id in (self.kv_cache_config.prefix_cacheable_group_ids)
)
store_plans = (
[]
if not self.store_enabled or tracked_request is None
else self._store_plans(
tracked_request,
tracker,
total_tokens,
block_ids_override=selected_block_ids,
)
)
logger.debug(
"UMBP store planning request=%s tokens=%d plans=%d keys=%s",
request.req_id,
total_tokens,
len(store_plans),
tuple(plan.key for plan in store_plans),
)
meta.store_plans.extend(store_plans)
if store_plans:
meta.store_requests[request.req_id] = store_plans
block_state = getattr(scheduler_output, "kv_connector_block_state", None)
cached_reqs = scheduler_output.scheduled_cached_reqs
resumed_ids: set[str] = set(getattr(cached_reqs, "resumed_req_ids", ()))
for request_id in cached_reqs.req_ids:
load_plans = self._pending_loads.pop(request_id, [])
self._load_specs.pop(request_id, None)
if load_plans:
meta.load_requests[request_id] = load_plans
elif self.store_enabled:
cached_request = self._requests.get(request_id)
cached_tracker = self._request_trackers.get(request_id)
if cached_request is not None and cached_tracker is not None:
request_index = cached_reqs.req_ids.index(request_id)
num_computed_tokens = cached_reqs.num_computed_tokens[request_index]
# num_tokens counts sampled tokens, so a decode step is still
# short of it; only the prompt bounds the prefill.
is_prefill = num_computed_tokens < cached_request.num_prompt_tokens
if not is_prefill and not self.save_decode_cache:
continue
new_block_ids = cached_reqs.new_block_ids[request_index]
if new_block_ids:
selected = tuple(
new_block_ids[group_id]
for group_id in (
self.kv_cache_config.prefix_cacheable_group_ids
)
)
if request_id in resumed_ids:
cached_tracker.replace_blocks(selected)
else:
cached_tracker.update_blocks(selected)
# The appended table keeps blocks core has since freed, such
# as sliding-window blocks out of the window, which a store
# would read; core's current table does not.
current = (
block_state.get_block_ids(request_id)
if block_state is not None
else None
)
if current is not None:
cached_tracker.block_ids = tuple(
list(current[group_id])
for group_id in (
self.kv_cache_config.prefix_cacheable_group_ids
)
)
total_tokens = (
num_computed_tokens
+ scheduler_output.num_scheduled_tokens[request_id]
)
if not self.save_decode_cache:
total_tokens = min(
total_tokens, cached_request.num_prompt_tokens
)
store_plans = self._store_plans(
cached_request,
cached_tracker,
total_tokens,
block_ids_override=cached_tracker.block_ids,
)
meta.store_plans.extend(store_plans)
if store_plans:
meta.store_requests[request_id] = store_plans
# Async loads are admitted without scheduling model tokens, so those
# requests are absent from both scheduled request collections above.
# Forward their plans explicitly so workers can start the transfer and
# eventually move the request out of WAITING_FOR_REMOTE_KVS.
for request_id, load_plans in list(self._pending_loads.items()):
self._pending_loads.pop(request_id, None)
self._load_specs.pop(request_id, None)
if load_plans:
meta.load_requests[request_id] = load_plans
for request_id in scheduler_output.preempted_req_ids or set():
meta.preempted_request_ids.add(request_id)
self._load_specs.pop(request_id, None)
if preempted_tracker := self._request_trackers.get(request_id):
preempted_tracker.reset()
event_plans = tuple(meta.store_plans)
if event_plans:
event = self._store_event_counter
self._store_event_counter += 1
meta.store_event = event
self._store_events[event] = _StoreEvent(event_plans)
if self._gpu_block_pool is not None:
self._gpu_block_pool.touch(
[
self._gpu_block_pool.blocks[block_id]
for block_id in dict.fromkeys(
plan.block_id for plan in event_plans
)
]
)
return meta
def _tracker_for_request(self, request_id: str) -> RequestTracker:
return self._request_trackers.setdefault(request_id, RequestTracker())
def _load_plans_for_external_tokens(
self,
request: Any,
tracker: RequestTracker,
block_groups: tuple[list[int], ...],
num_external_tokens: int,
) -> list[BlockTransferPlan]:
"""Plan the logical objects selected by the external hit window."""
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
spec = tracker.load_spec
local_tokens = spec.local_tokens if spec is not None else 0
end_tokens = local_tokens + num_external_tokens
plans: list[BlockTransferPlan] = []
for group_index, group_id in enumerate(group_ids):
group_block_size = self.group_block_sizes[group_id]
start_block = local_tokens // group_block_size
num_full_blocks = end_tokens // group_block_size
for block_index in range(start_block, num_full_blocks):
token_end = (block_index + 1) * group_block_size
block_hash = self._object_hash_at_token_end(
request.block_hashes, token_end
)
plans.append(
BlockTransferPlan(
key=self.codec.key(block_hash, group_id=group_id),
block_id=block_groups[group_index][block_index],
request_id=request.request_id,
group_id=group_id,
)
)
return plans
def _object_hash_at_token_end(
self, hashes: Sequence[bytes], token_end: int
) -> bytes:
hash_index = token_end // self.hash_block_size - 1
if hash_index < 0 or hash_index >= len(hashes):
raise ValueError(
f"request has {len(hashes)} hashes, needs index {hash_index}"
)
return hashes[hash_index]
def _store_plans(
self,
request: Any,
tracker: RequestTracker,
token_count: int,
block_ids_override: tuple[list[int], ...] | None = None,
) -> list[BlockTransferPlan]:
"""Describe full blocks produced by a scheduled prefill."""
request_id = getattr(request, "request_id", None) or request.req_id
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
# Scheduled speculative tokens are not yet verified and have no block
# hash; a block is storable only once its hash exists.
token_count = min(token_count, len(request.block_hashes) * self.hash_block_size)
previous_saved_tokens = tracker.saved_tokens
# Hash-aligned rather than block-aligned: a group with smaller blocks
# than the largest must not wait for that group's block to fill.
save_to = tracker.mark_saved(token_count, self.hash_block_size)
plans: list[BlockTransferPlan] = []
block_groups = (
block_ids_override
if block_ids_override is not None
else tuple(request.block_ids[group_id] for group_id in group_ids)
)
block_ranges = {
group_id: range(
previous_saved_tokens // self.group_block_sizes[group_id],
min(save_to // self.group_block_sizes[group_id], len(block_ids)),
)
for group_id, block_ids in zip(group_ids, block_groups)
}
for group_id, block_ids in zip(group_ids, block_groups):
if group_id not in block_ranges:
continue
group_block_size = self.group_block_sizes[group_id]
for index in block_ranges[group_id]:
if block_ids[index] == NULL_BLOCK_ID:
continue
token_end = (index + 1) * group_block_size
block_hash = self._object_hash_at_token_end(
request.block_hashes, token_end
)
plans.append(
BlockTransferPlan(
key=self.codec.key(block_hash, group_id=group_id),
block_id=block_ids[index],
request_id=request_id,
group_id=group_id,
)
)
return plans
def request_finished(
self, request: Request, block_ids: tuple[list[int], ...]
) -> tuple[bool, dict[str, Any] | None]:
self._pending_loads.pop(request.request_id, None)
self._load_specs.pop(request.request_id, None)
self._requests.pop(request.request_id, None)
return False, None
def update_connector_output(self, output: KVConnectorOutput) -> None:
metadata = output.kv_connector_worker_meta
if not isinstance(metadata, UMBPConnectorWorkerMetadata):
return
pool = self._gpu_block_pool
for event_id, result in metadata.store_events.items():
event = self._store_events.get(event_id)
if event is None:
continue
event.result.merge(result)
if event.result.completed_workers < self._num_workers:
continue
del self._store_events[event_id]
if pool is not None:
block_ids = dict.fromkeys(plan.block_id for plan in event.plans)
pool.free_blocks(
pool.blocks[block_id] for block_id in reversed(block_ids)
)
def has_pending_push_work(self) -> bool:
return bool(self._store_events)
def reset_store(self) -> bool:
if self._store_events:
return False
self._load_specs.clear()
self._pending_loads.clear()
return self.runtime.clear()
def close(self) -> None:
self.runtime.close()
@@ -1,240 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared worker-side UMBP connector behavior."""
from __future__ import annotations
from dataclasses import dataclass, replace
import torch
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from .data import (
BlockIdentityCodec,
BlockTransferPlan,
KVLayoutPlanner,
StoreEventResult,
TransferJobState,
TransferJobStatus,
UMBPConnectorMetadata,
UMBPConnectorWorkerMetadata,
)
from .runtime import UMBPWorkerHandle
logger = init_logger(__name__)
@dataclass
class _StoreBatch:
event_id: int
jobs: dict[str, TransferJobState]
class UMBPStoreConnectorWorker:
"""Translate shared metadata into runtime load/store jobs."""
def __init__(
self,
runtime: UMBPWorkerHandle,
layout: KVLayoutPlanner | None = None,
*,
codec: BlockIdentityCodec | None = None,
) -> None:
self.runtime = runtime
self.layout = layout
self.codec = codec
self._load_jobs: dict[str, dict[str | None, TransferJobState]] = {}
self._store_jobs: dict[str, TransferJobState] = {}
self._pending_stores: list[_StoreBatch] = []
self._worker_meta = UMBPConnectorWorkerMetadata()
self._finished_recving: set[str] = set()
self._load_error_block_ids: set[int] = set()
self._report_load_completions = False
self._active_store_event = -1
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
if self.layout is not None and self.layout.regions:
self.layout.register_kv_caches(kv_caches)
self.runtime.register_buffers(kv_caches)
def _materialize_plans(
self, plans: list[BlockTransferPlan]
) -> list[BlockTransferPlan]:
plans = [self._localize_plan(plan) for plan in plans]
if self.layout is None or not self.layout.regions:
return plans
materialized: list[BlockTransferPlan] = []
for plan in plans:
if plan.ranges:
materialized.append(plan)
continue
materialized.append(self.layout.materialize(plan))
return materialized
def _localize_plan(self, plan: BlockTransferPlan) -> BlockTransferPlan:
if self.codec is None or plan.group_id is None:
return plan
key = self._localize_key(plan.key, plan.group_id)
if key == plan.key:
return plan
return replace(plan, key=key)
def _localize_key(self, key: str, group_id: int) -> str:
if self.codec is None:
return key
try:
block_hash = bytes.fromhex(key.rsplit(":", 1)[1])
except (IndexError, ValueError):
return key
return self.codec.key(block_hash, group_id)
def start_load_kv(
self, forward_context: ForwardContext, metadata: UMBPConnectorMetadata
) -> None:
del forward_context
self._report_load_completions = metadata.async_load
for request_id, plans in metadata.load_requests.items():
if plans:
self._submit_loads(request_id, plans)
def _submit_loads(self, request_id: str, plans: list[BlockTransferPlan]) -> None:
materialized = self._materialize_plans(plans)
logger.debug(
"UMBP load submitted request=%s plans=%d ranges=%d bytes=%d",
request_id,
len(materialized),
sum(len(plan.ranges) for plan in materialized),
sum(item.length for plan in materialized for item in plan.ranges),
)
self._load_jobs[request_id] = {None: self.runtime.load(materialized)}
def _finish_job(
self, request_id: str, result: TransferJobState, *, is_load: bool
) -> bool:
operation = "load" if is_load else "store"
succeeded = result.status == TransferJobStatus.COMPLETED
if not succeeded:
logger.debug(
"UMBP %s failed request=%s status=%s error=%s",
operation,
request_id,
result.status.value,
result.error,
)
if is_load:
self._load_error_block_ids.update(result.failed_block_ids)
elif succeeded:
self.runtime.publish(result)
return succeeded
def wait_for_layer_load(self, layer_name: str) -> None:
if self._report_load_completions:
# Asynchronous loads fill blocks of requests that do not run until
# get_finished reports them, so no forward pass reads those blocks.
return
self._drain_load_jobs(wait=True)
def wait_for_save(self) -> None:
if self._store_jobs or self._active_store_event >= 0:
self._pending_stores.append(
_StoreBatch(self._active_store_event, self._store_jobs)
)
self._store_jobs = {}
self._active_store_event = -1
self._drain_store_jobs(wait=False)
def _drain_store_jobs(self, *, wait: bool) -> None:
for batch in list(self._pending_stores):
for request_id, job in list(batch.jobs.items()):
result = self.runtime.wait(job) if wait else self.runtime.poll(job)
if result is None:
continue
# A partly failed job is not published, including its stored keys.
self._finish_job(request_id, result, is_load=False)
del batch.jobs[request_id]
if not batch.jobs:
if batch.event_id >= 0:
self._worker_meta.store_events[batch.event_id] = StoreEventResult(
completed_workers=1
)
self._pending_stores.remove(batch)
def _submit_store_plans(
self, plans: list[BlockTransferPlan], request_id: str
) -> None:
if not plans:
return
materialized = self._materialize_plans(plans)
self._store_jobs[request_id] = self.runtime.store(materialized)
def enqueue_stores(self, metadata: UMBPConnectorMetadata) -> None:
self._active_store_event = metadata.store_event
store_requests = {
request_id: list(plans)
for request_id, plans in metadata.store_requests.items()
}
grouped = {plan for plans in store_requests.values() for plan in plans}
for plan in metadata.store_plans:
if plan not in grouped:
store_requests.setdefault(
plan.request_id or "__umbp_batch__", []
).append(plan)
grouped.add(plan)
for request_id, plans in store_requests.items():
self._submit_store_plans(plans, request_id)
def handle_preemptions(self, metadata: UMBPConnectorMetadata) -> None:
"""Finish request-local jobs before vLLM reuses their GPU blocks."""
preempted = metadata.preempted_request_ids
if not preempted:
return
for request_id in preempted:
for job in self._load_jobs.pop(request_id, {}).values():
self.runtime.wait(job)
for jobs in [self._store_jobs, *(batch.jobs for batch in self._pending_stores)]:
for request_id, job in jobs.items():
if request_id in preempted or any(
plan.request_id in preempted for plan in job.plans
):
jobs[request_id] = self.runtime.wait(job)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
del finished_req_ids
self._drain_load_jobs(wait=False)
self._drain_store_jobs(wait=False)
recving = set(self._finished_recving)
self._finished_recving.difference_update(recving)
return None, recving or None
def _drain_load_jobs(self, *, wait: bool) -> None:
for request_id, jobs in list(self._load_jobs.items()):
for layer, job in list(jobs.items()):
result = self.runtime.wait(job) if wait else self.runtime.poll(job)
if result is None:
continue
del jobs[layer]
self._finish_job(request_id, result, is_load=True)
if not jobs:
del self._load_jobs[request_id]
if self._report_load_completions:
self._finished_recving.add(request_id)
def build_connector_worker_meta(self) -> UMBPConnectorWorkerMetadata:
result = self._worker_meta
self._worker_meta = UMBPConnectorWorkerMetadata()
return result
def get_block_ids_with_load_errors(self) -> set[int]:
failed = self._load_error_block_ids
self._load_error_block_ids = set()
return failed
def close(self) -> None:
self.wait_for_save()
self._drain_store_jobs(wait=True)
self._drain_load_jobs(wait=True)
self.runtime.close()
-144
View File
@@ -1,144 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared configuration and setup for EC and KV Mooncake Store clients."""
import json
import os
from dataclasses import dataclass
from typing import Any, Literal
import regex as re
DEFAULT_GLOBAL_SEGMENT_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
DEFAULT_LOCAL_BUFFER_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
DEFAULT_TENANT_ID = "default"
MooncakeMode = Literal["embedded", "standalone-store"]
@dataclass
class MooncakeStoreConfig:
"""Configuration for MooncakeDistributedStore.
``mode`` selects the topology: ``embedded`` (each rank contributes
``global_segment_size`` in-process) or ``standalone-store`` (rank
contributes 0; an external ``mooncake_client`` process owns the pool
and the SSD tier).
"""
metadata_server: str
master_server_address: str
protocol: str
device_name: str
mode: MooncakeMode = "embedded"
global_segment_size: int = DEFAULT_GLOBAL_SEGMENT_SIZE
local_buffer_size: int = DEFAULT_LOCAL_BUFFER_SIZE
enable_offload: bool = False
tenant_id: str = DEFAULT_TENANT_ID
def __post_init__(self) -> None:
if self.mode not in ("embedded", "standalone-store"):
raise ValueError(f"unknown Mooncake mode: {self.mode!r}")
if self.local_buffer_size <= 0:
raise ValueError("local_buffer_size must be > 0")
if self.mode == "embedded" and self.global_segment_size == 0:
raise ValueError("embedded mode requires global_segment_size > 0")
if self.mode == "standalone-store" and self.global_segment_size != 0:
raise ValueError("standalone-store mode requires global_segment_size == 0")
@staticmethod
def from_file(file_path: str) -> "MooncakeStoreConfig":
with open(file_path) as file:
config = json.load(file)
return MooncakeStoreConfig(
metadata_server=config.get("metadata_server", ""),
master_server_address=config.get("master_server_address", ""),
protocol=config.get("protocol", "rdma"),
device_name=config.get("device_name", ""),
mode=config.get("mode", "embedded"),
global_segment_size=_parse_size(
config.get("global_segment_size", DEFAULT_GLOBAL_SEGMENT_SIZE)
),
local_buffer_size=_parse_size(
config.get("local_buffer_size", DEFAULT_LOCAL_BUFFER_SIZE)
),
enable_offload=bool(config.get("enable_offload", False)),
tenant_id=_normalize_tenant_id(config.get("tenant_id", DEFAULT_TENANT_ID)),
)
@staticmethod
def load_from_config() -> "MooncakeStoreConfig":
config_path = os.getenv("MOONCAKE_CONFIG_PATH")
if not config_path:
raise ValueError(
"The environment variable 'MOONCAKE_CONFIG_PATH' is not set."
)
return MooncakeStoreConfig.from_file(config_path)
def _normalize_tenant_id(value: Any) -> str:
if value is None:
return DEFAULT_TENANT_ID
if not isinstance(value, str):
raise TypeError(
f"tenant_id must be a string or null, got {type(value).__name__}: {value!r}"
)
tenant_id = value.strip()
return tenant_id if tenant_id else DEFAULT_TENANT_ID
def _parse_size(value: Any) -> int:
"""Parse storage size strings with units: GB, MB, KB, B."""
if isinstance(value, int):
return value
if not isinstance(value, str):
try:
return int(value)
except (TypeError, ValueError) as e:
raise TypeError(f"Unsupported type for size: {type(value)}") from e
cleaned = value.strip().lower()
if not cleaned:
raise ValueError("Size cannot be empty.")
unit_multipliers = {
"gb": 1024**3,
"mb": 1024**2,
"kb": 1024,
"b": 1,
}
match = re.match(r"^\s*([\d.]+)\s*(gb|mb|kb|b)?\s*$", cleaned)
if not match:
raise ValueError(f"Invalid format: '{value}'")
number_str = match.group(1)
unit = match.group(2) or "b"
multiplier = unit_multipliers[unit]
try:
numeric_value = float(number_str)
except ValueError as exc:
raise ValueError(f"Invalid numeric value '{number_str}' in: '{value}'") from exc
return int(numeric_value * multiplier)
def setup_mooncake_store(
store: Any, config: MooncakeStoreConfig, local_hostname: str
) -> None:
"""Initialize a native Store, preserving explicit tenant configuration."""
setup_kwargs: dict[str, str] = {}
if config.tenant_id != DEFAULT_TENANT_ID:
setup_kwargs["tenant_id"] = config.tenant_id
ret = store.setup(
local_hostname,
config.metadata_server,
config.global_segment_size,
config.local_buffer_size,
config.protocol,
config.device_name,
config.master_server_address,
**setup_kwargs,
)
if ret != 0:
raise RuntimeError("Initialize MooncakeDistributedStore failed.")