[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,296 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .base_device_communicator import DeviceCommunicatorBase
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class XpuCommunicator(DeviceCommunicatorBase):
|
||||
def __init__(
|
||||
self,
|
||||
cpu_group: ProcessGroup,
|
||||
device: torch.device | None = None,
|
||||
device_group: ProcessGroup | None = None,
|
||||
unique_name: str = "",
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
|
||||
)
|
||||
self.ca_comm: None = None
|
||||
if self.use_all2all:
|
||||
if self.all2all_backend in ("naive", "allgather_reducescatter"):
|
||||
from .all2all import AgRsAll2AllManager
|
||||
|
||||
self.all2all_manager = AgRsAll2AllManager(self.cpu_group)
|
||||
logger.info("Using AgRs manager on XPU device.")
|
||||
|
||||
else: # type: ignore[has-type]
|
||||
logger.warning(
|
||||
"`%s` all2all manager is not supported on XPU. "
|
||||
"Falling back to AgRs manager for XPU, "
|
||||
"which is the Default backend",
|
||||
self.all2all_backend, # type: ignore[has-type]
|
||||
)
|
||||
from .all2all import AgRsAll2AllManager
|
||||
|
||||
self.all2all_manager = AgRsAll2AllManager(self.cpu_group)
|
||||
logger.info("Using AgRs manager on XPU device.")
|
||||
|
||||
def _fixed_rank_sum(self, input_: torch.Tensor) -> torch.Tensor:
|
||||
flat_input = input_.reshape(-1)
|
||||
gathered = torch.empty(
|
||||
(self.world_size, flat_input.numel()),
|
||||
dtype=input_.dtype,
|
||||
device=input_.device,
|
||||
)
|
||||
dist.all_gather_single(gathered.view(-1), flat_input, group=self.device_group)
|
||||
output = gathered[0].clone()
|
||||
for rank in range(1, self.world_size):
|
||||
output.add_(gathered[rank])
|
||||
return output.view(input_.shape)
|
||||
|
||||
def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
# XCCL changes its reduction order with message size. Gather first
|
||||
# and accumulate in rank order, independent of the token batch.
|
||||
return self._fixed_rank_sum(input_)
|
||||
|
||||
output = input_.clone()
|
||||
dist.all_reduce(output, group=self.device_group)
|
||||
return output
|
||||
|
||||
def reduce_scatter(self, input_: torch.Tensor, dim: int = -1):
|
||||
world_size = self.world_size
|
||||
|
||||
if dim < 0:
|
||||
# Convert negative dim to positive.
|
||||
dim += input_.dim()
|
||||
|
||||
# Note: This will produce an incorrect answer if we don't make
|
||||
# the input_tensor contiguous. Possible bug in reduce_scatter_tensor?
|
||||
input_tensor = input_.movedim(0, dim).contiguous()
|
||||
|
||||
assert input_tensor.shape[0] % world_size == 0
|
||||
chunk_size = input_tensor.shape[0] // world_size
|
||||
output_shape = (chunk_size,) + input_tensor.shape[1:]
|
||||
|
||||
output = torch.empty(
|
||||
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
|
||||
)
|
||||
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
reduced = self._fixed_rank_sum(input_tensor)
|
||||
start = self.rank_in_group * chunk_size
|
||||
output.copy_(reduced.narrow(0, start, chunk_size))
|
||||
else:
|
||||
dist.reduce_scatter_tensor(output, input_tensor, group=self.device_group)
|
||||
|
||||
# Reshape before returning
|
||||
return output.movedim(0, dim).contiguous()
|
||||
|
||||
def reduce_scatterv(
|
||||
self, input_: torch.Tensor, dim: int = -1, sizes: list[int] | None = None
|
||||
):
|
||||
world_size = self.world_size
|
||||
|
||||
if dim < 0:
|
||||
# Convert negative dim to positive.
|
||||
dim += input_.dim()
|
||||
|
||||
# Note: This will produce an incorrect answer if we don't make
|
||||
# the input_tensor contiguous. Possible bug in reduce_scatter_tensor?
|
||||
input_tensor = input_.movedim(0, dim).contiguous()
|
||||
|
||||
if sizes is not None:
|
||||
assert len(sizes) == world_size
|
||||
assert input_tensor.shape[0] == sum(sizes)
|
||||
chunk_size = sizes[self.rank_in_group]
|
||||
else:
|
||||
assert input_tensor.shape[0] % world_size == 0
|
||||
chunk_size = input_tensor.shape[0] // world_size
|
||||
output_shape = (chunk_size,) + input_tensor.shape[1:]
|
||||
|
||||
output = torch.empty(
|
||||
output_shape, dtype=input_tensor.dtype, device=input_tensor.device
|
||||
)
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
reduced = self._fixed_rank_sum(input_tensor)
|
||||
start = (
|
||||
self.rank_in_group * chunk_size
|
||||
if sizes is None
|
||||
else sum(sizes[: self.rank_in_group])
|
||||
)
|
||||
output.copy_(reduced.narrow(0, start, chunk_size))
|
||||
elif sizes is None or sizes.count(sizes[0]) == len(sizes):
|
||||
dist.reduce_scatter_tensor(output, input_tensor, group=self.device_group)
|
||||
elif torch.xpu.device_count() < self.world_size:
|
||||
# uneven reduce_scatter is not supported when the
|
||||
# process cannot see every device in the group (
|
||||
# e.g. external-LBDP).
|
||||
max_size = max(sizes)
|
||||
padded = input_tensor.new_zeros(
|
||||
(world_size * max_size,) + input_tensor.shape[1:]
|
||||
)
|
||||
for rank, split in enumerate(input_tensor.split(sizes, dim=0)):
|
||||
padded[rank * max_size : rank * max_size + sizes[rank]] = split
|
||||
padded_output = input_tensor.new_empty((max_size,) + input_tensor.shape[1:])
|
||||
dist.reduce_scatter_tensor(padded_output, padded, group=self.device_group)
|
||||
output = padded_output[:chunk_size]
|
||||
else:
|
||||
input_splits = list(input_tensor.split(sizes, dim=0))
|
||||
dist.reduce_scatter(output, input_splits, group=self.device_group)
|
||||
# Reshape before returning
|
||||
return output.movedim(0, dim).contiguous()
|
||||
|
||||
def all_gatherv(
|
||||
self,
|
||||
input_: torch.Tensor | list[torch.Tensor],
|
||||
dim: int = 0,
|
||||
sizes: list[int] | None = None,
|
||||
):
|
||||
if dim != 0:
|
||||
raise NotImplementedError("only dim 0 all-gatherv is supported")
|
||||
world_size = self.world_size
|
||||
|
||||
# 'sizes' is not needed if all inputs in the same group have the same
|
||||
# shape
|
||||
if sizes is not None and all(s == sizes[0] for s in sizes):
|
||||
sizes = None
|
||||
|
||||
def _all_gather_single(input_: torch.Tensor, sizes: list[int] | None = None):
|
||||
input_size = input_.size()
|
||||
if sizes is not None:
|
||||
assert len(sizes) == world_size
|
||||
assert input_.shape[dim] == sizes[self.rank_in_group], (
|
||||
f"{input_.shape[dim]} != {sizes[self.rank_in_group]}"
|
||||
)
|
||||
output_size = (sum(sizes),) + input_size[1:]
|
||||
else:
|
||||
output_size = (input_size[0] * world_size,) + input_size[1:]
|
||||
# Allocate output tensor.
|
||||
output_tensor = torch.empty(
|
||||
output_size, dtype=input_.dtype, device=input_.device
|
||||
)
|
||||
|
||||
if sizes is not None:
|
||||
all_gather_list = []
|
||||
for size in sizes:
|
||||
all_gather_list.append(
|
||||
torch.empty(
|
||||
(size,) + input_.shape[1:],
|
||||
dtype=input_.dtype,
|
||||
device=input_.device,
|
||||
)
|
||||
)
|
||||
dist.all_gather(all_gather_list, input_, group=self.device_group)
|
||||
output_tensor = torch.cat(all_gather_list, dim=0)
|
||||
else:
|
||||
dist.all_gather([output_tensor], input_, group=self.device_group)
|
||||
return output_tensor
|
||||
|
||||
if isinstance(input_, torch.Tensor):
|
||||
return _all_gather_single(input_, sizes)
|
||||
|
||||
output_list = []
|
||||
for inp in input_:
|
||||
output_list.append(_all_gather_single(inp, sizes=sizes))
|
||||
return output_list
|
||||
|
||||
def gather(
|
||||
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
|
||||
) -> torch.Tensor | None:
|
||||
assert -input_.dim() <= dim < input_.dim(), (
|
||||
f"Invalid dim ({dim}) for input tensor with shape {input_.size()}"
|
||||
)
|
||||
if dim < 0:
|
||||
# Convert negative dim to positive.
|
||||
dim += input_.dim()
|
||||
# For xpu path, gather doesn't work properly together with ray
|
||||
# cluster so we use all_gather instead for now.
|
||||
input_size = input_.size()
|
||||
# Allocate output tensor.
|
||||
output_tensor = torch.empty(
|
||||
(self.world_size,) + input_size, dtype=input_.dtype, device=input_.device
|
||||
)
|
||||
# All-gather.
|
||||
dist.all_gather_single(output_tensor, input_, group=self.device_group)
|
||||
if self.rank_in_group == dst:
|
||||
# Reshape
|
||||
output_tensor = output_tensor.movedim(0, dim)
|
||||
output_tensor = output_tensor.reshape(
|
||||
input_size[:dim]
|
||||
+ (self.world_size * input_size[dim],)
|
||||
+ input_size[dim + 1 :]
|
||||
)
|
||||
else:
|
||||
output_tensor = None
|
||||
return output_tensor
|
||||
|
||||
def broadcast(self, input_: torch.Tensor, src: int = 0) -> None:
|
||||
dist.broadcast(input_, src=src, group=self.device_group)
|
||||
|
||||
def dispatch_router_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
is_sequence_parallel: bool = False,
|
||||
extra_tensors: list[torch.Tensor] | None = None,
|
||||
) -> (
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.dispatch_router_logits(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
is_sequence_parallel,
|
||||
extra_tensors,
|
||||
)
|
||||
|
||||
def dispatch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
is_sequence_parallel: bool = False,
|
||||
extra_tensors: list[torch.Tensor] | None = None,
|
||||
) -> (
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.dispatch(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
is_sequence_parallel,
|
||||
extra_tensors=extra_tensors,
|
||||
)
|
||||
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.combine(
|
||||
hidden_states,
|
||||
is_sequence_parallel,
|
||||
)
|
||||
@@ -1,9 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Internal building blocks for the Mooncake encoder-cache connector.
|
||||
|
||||
The public connector remains in :mod:`mooncake_ec_connector`. This package
|
||||
separates configuration, control-plane messaging, transfer state, registered
|
||||
memory, and worker/scheduler orchestration so that each component has one
|
||||
owner and can be tested independently.
|
||||
"""
|
||||
@@ -1,155 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.utils.network_utils import make_zmq_path
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
# Scheduler and Worker must agree on when a remote reservation becomes stale.
|
||||
_RESERVATION_TTL_SECONDS = 300
|
||||
|
||||
|
||||
def _positive_int(name: str, value: object) -> int:
|
||||
message = f"ECMooncakeConnector requires {name} to be a positive integer."
|
||||
if isinstance(value, bool):
|
||||
raise ValueError(message)
|
||||
if isinstance(value, int):
|
||||
result = value
|
||||
elif isinstance(value, float) and math.isfinite(value) and value.is_integer():
|
||||
result = int(value)
|
||||
elif isinstance(value, str):
|
||||
try:
|
||||
result = int(value)
|
||||
except ValueError as error:
|
||||
raise ValueError(message) from error
|
||||
else:
|
||||
raise ValueError(message)
|
||||
if result <= 0:
|
||||
raise ValueError(f"ECMooncakeConnector requires {name} > 0.")
|
||||
return result
|
||||
|
||||
|
||||
def _positive_float(name: str, value: object) -> float:
|
||||
message = f"ECMooncakeConnector requires {name} > 0."
|
||||
if isinstance(value, bool):
|
||||
raise ValueError(message)
|
||||
try:
|
||||
result = float(value) # type: ignore[arg-type]
|
||||
except (TypeError, ValueError) as error:
|
||||
raise ValueError(message) from error
|
||||
if not math.isfinite(result) or result <= 0:
|
||||
raise ValueError(message)
|
||||
return result
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MooncakeECConfig:
|
||||
"""Validated settings shared by the Scheduler and Worker roles.
|
||||
|
||||
``control_port`` is the first TP-shard port after the DP offset;
|
||||
``control_addr`` targets that shard, which advertises the full topology.
|
||||
``control_host`` is the address this instance both advertises and binds,
|
||||
so reaching a Consumer means being routed to it rather than finding it on
|
||||
every interface.
|
||||
"""
|
||||
|
||||
is_producer: bool
|
||||
is_consumer: bool
|
||||
protocol: str
|
||||
buffer_device: str
|
||||
control_host: str
|
||||
control_port: int
|
||||
control_addr: str
|
||||
control_timeout_ms: int
|
||||
push_wait_timeout_s: float
|
||||
pool_size: int
|
||||
cross_encoder_cache: bool = False
|
||||
store_max_pending_items: int = 32
|
||||
store_max_pending_bytes: int = 2 * 1024**3
|
||||
store_read_buffer_bytes: int = 128 * 1024**2
|
||||
|
||||
@classmethod
|
||||
def from_vllm_config(cls, vllm_config: VllmConfig) -> MooncakeECConfig:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
ec_config = vllm_config.ec_transfer_config
|
||||
assert ec_config is not None
|
||||
|
||||
if ec_config.is_ec_producer:
|
||||
if parallel_config.tensor_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"ECMooncakeConnector producers require tensor_parallel_size=1."
|
||||
)
|
||||
if parallel_config.pipeline_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"ECMooncakeConnector producers do not support pipeline parallelism."
|
||||
)
|
||||
if parallel_config.data_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"ECMooncakeConnector producers require data_parallel_size=1."
|
||||
)
|
||||
|
||||
registered_buffer_size = _positive_int(
|
||||
"ec_buffer_size", ec_config.ec_buffer_size
|
||||
)
|
||||
get = ec_config.get_from_extra_config
|
||||
shared_reuse = get("cross_encoder_cache", False)
|
||||
if not isinstance(shared_reuse, bool):
|
||||
raise ValueError("cross_encoder_cache must be a boolean")
|
||||
if shared_reuse:
|
||||
if not ec_config.is_encode_only:
|
||||
raise ValueError(
|
||||
"cross_encoder_cache requires an Encoder-only producer"
|
||||
)
|
||||
if not vllm_config.use_v2_model_runner:
|
||||
raise ValueError("cross_encoder_cache requires Model Runner V2")
|
||||
if vllm_config.lora_config is not None:
|
||||
raise ValueError("cross_encoder_cache does not support LoRA")
|
||||
mm_config = vllm_config.model_config.multimodal_config
|
||||
if mm_config is not None and mm_config.mm_processor_cache_gb == 0:
|
||||
raise ValueError(
|
||||
"cross_encoder_cache requires mm_processor_cache_gb > 0 "
|
||||
"to preserve content identifiers across Encoders and restarts"
|
||||
)
|
||||
control_port = int(ec_config.ec_port) + (
|
||||
parallel_config.data_parallel_index * parallel_config.tensor_parallel_size
|
||||
)
|
||||
highest_port = control_port + parallel_config.tensor_parallel_size - 1
|
||||
if not 1 <= control_port <= highest_port <= 65535:
|
||||
raise ValueError("ECMooncakeConnector ec_port must be in 1..65535.")
|
||||
|
||||
return cls(
|
||||
is_producer=ec_config.is_ec_producer,
|
||||
is_consumer=ec_config.is_ec_consumer,
|
||||
protocol=str(get("mooncake_protocol", "rdma")),
|
||||
buffer_device=str(ec_config.ec_buffer_device or "cuda").lower(),
|
||||
control_host=str(ec_config.ec_ip),
|
||||
control_port=control_port,
|
||||
control_addr=make_zmq_path("tcp", ec_config.ec_ip, control_port),
|
||||
control_timeout_ms=max(
|
||||
1,
|
||||
math.ceil(
|
||||
_positive_float("control_timeout_s", get("control_timeout_s", 30))
|
||||
* 1000
|
||||
),
|
||||
),
|
||||
push_wait_timeout_s=_positive_float(
|
||||
"push_wait_timeout_s", get("push_wait_timeout_s", 60)
|
||||
),
|
||||
pool_size=registered_buffer_size,
|
||||
cross_encoder_cache=shared_reuse,
|
||||
store_max_pending_items=_positive_int(
|
||||
"store_max_pending_items", get("store_max_pending_items", 32)
|
||||
),
|
||||
store_max_pending_bytes=_positive_int(
|
||||
"store_max_pending_bytes", get("store_max_pending_bytes", 2 * 1024**3)
|
||||
),
|
||||
store_read_buffer_bytes=_positive_int(
|
||||
"store_read_buffer_bytes", get("store_read_buffer_bytes", 128 * 1024**2)
|
||||
),
|
||||
)
|
||||
@@ -1,460 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ZMQ control plane for Mooncake encoder-cache reservations and events.
|
||||
|
||||
Tensor bytes never travel through this module. It coordinates destination
|
||||
reservations, completion/cancellation, TP-shard discovery, and readiness
|
||||
notifications while :mod:`transfer` owns the Mooncake data plane.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Executor, Future
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import zmq
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import make_zmq_path
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_MAX_PENDING_EVENTS = 4096
|
||||
_RESERVATION_REAP_INTERVAL_SECONDS = 1
|
||||
|
||||
|
||||
ControlRequest = dict[str, Any]
|
||||
ControlResponse = dict[str, Any]
|
||||
|
||||
|
||||
class ControlClient:
|
||||
"""Send control requests through reusable, thread-local REQ sockets."""
|
||||
|
||||
def __init__(self, timeout_ms: int) -> None:
|
||||
self._context = zmq.Context()
|
||||
self._timeout_ms = timeout_ms
|
||||
self._local = threading.local()
|
||||
self._topologies: dict[str, list[str]] = {}
|
||||
self._closed = False
|
||||
|
||||
def _sockets(self) -> dict[str, zmq.Socket]:
|
||||
sockets = getattr(self._local, "sockets", None)
|
||||
if sockets is None:
|
||||
sockets = {}
|
||||
self._local.sockets = sockets
|
||||
return sockets
|
||||
|
||||
def _discard(self, addr: str) -> None:
|
||||
socket = self._sockets().pop(addr, None)
|
||||
if socket is not None:
|
||||
socket.close(linger=0)
|
||||
|
||||
def _exchange(self, addr: str, payload: ControlRequest) -> ControlResponse:
|
||||
sockets = self._sockets()
|
||||
socket = sockets.get(addr)
|
||||
if socket is None:
|
||||
socket = self._context.socket(zmq.REQ)
|
||||
socket.setsockopt(zmq.IPV6, 1)
|
||||
socket.setsockopt(zmq.RCVTIMEO, self._timeout_ms)
|
||||
socket.setsockopt(zmq.SNDTIMEO, self._timeout_ms)
|
||||
socket.setsockopt(zmq.LINGER, 0)
|
||||
socket.connect(addr)
|
||||
sockets[addr] = socket
|
||||
try:
|
||||
socket.send_json(payload)
|
||||
response = socket.recv_json()
|
||||
except Exception:
|
||||
self._discard(addr)
|
||||
raise
|
||||
assert isinstance(response, dict)
|
||||
return response
|
||||
|
||||
def request(self, addr: str, payload: ControlRequest) -> Any:
|
||||
response = self._exchange(addr, payload)
|
||||
if not response.get("ok"):
|
||||
raise RuntimeError(response.get("error", "EC control request failed"))
|
||||
return response.get("result")
|
||||
|
||||
def discover_shards(self, base_addr: str) -> list[str] | None:
|
||||
if base_addr in self._topologies:
|
||||
return self._topologies[base_addr]
|
||||
try:
|
||||
reply = self.request(base_addr, {"op": "peers"})
|
||||
ports = reply.get("ports") if isinstance(reply, dict) else None
|
||||
if not isinstance(ports, list) or not ports:
|
||||
raise ValueError("invalid or empty peer list")
|
||||
prefix = base_addr.rsplit(":", 1)[0]
|
||||
shards = [f"{prefix}:{int(port)}" for port in ports]
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"EC Mooncake consumer at %s did not report its shards",
|
||||
base_addr,
|
||||
exc_info=True,
|
||||
)
|
||||
return None
|
||||
self._topologies[base_addr] = shards
|
||||
return shards
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
self._context.destroy(linger=0)
|
||||
|
||||
|
||||
def make_cancel_request(
|
||||
transfer_id: str,
|
||||
reservation_id: str = "",
|
||||
*,
|
||||
abandon: bool = False,
|
||||
refresh: bool = False,
|
||||
) -> ControlRequest:
|
||||
request: ControlRequest = {
|
||||
"op": "cancel",
|
||||
"transfer_id": transfer_id,
|
||||
"reservation_id": reservation_id,
|
||||
}
|
||||
if abandon:
|
||||
request["abandon"] = True
|
||||
if refresh:
|
||||
request["refresh"] = True
|
||||
return request
|
||||
|
||||
|
||||
class EventInbox:
|
||||
"""Receive Consumer readiness events without blocking the Scheduler.
|
||||
|
||||
Subscribing needs one blocking control request per shard, so it runs on
|
||||
`executor` when one is given: an unreachable Consumer must cost the
|
||||
Scheduler nothing, not one control timeout per drain.
|
||||
"""
|
||||
|
||||
def __init__(self, client: ControlClient, executor: Executor | None = None) -> None:
|
||||
self._client = client
|
||||
self._executor = executor
|
||||
self._discovery: Future[list[str] | None] | None = None
|
||||
self._context: zmq.Context | None = None
|
||||
self._socket: zmq.Socket | None = None
|
||||
self._closed = False
|
||||
self.shard_count = 1
|
||||
|
||||
def _endpoints(self, base_addr: str) -> list[str] | None:
|
||||
"""Ask every shard where it publishes readiness events.
|
||||
|
||||
Blocking: called on `self._executor` unless there is none.
|
||||
"""
|
||||
shards = self._client.discover_shards(base_addr)
|
||||
if shards is None:
|
||||
return None
|
||||
endpoints = []
|
||||
for addr in shards:
|
||||
try:
|
||||
event_port = self._client.request(addr, {"op": "event_port"})
|
||||
address, _ = addr.rsplit(":", 1)
|
||||
endpoints.append(f"{address}:{int(event_port)}")
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"EC Mooncake could not subscribe to the event channel of "
|
||||
"consumer shard %s; retrying the complete topology later.",
|
||||
addr,
|
||||
)
|
||||
return None
|
||||
return endpoints
|
||||
|
||||
def _connect(self, base_addr: str) -> None:
|
||||
if self._socket is not None or self._closed:
|
||||
return
|
||||
if self._executor is None:
|
||||
self._install(self._endpoints(base_addr))
|
||||
return
|
||||
if self._discovery is None:
|
||||
self._discovery = self._executor.submit(self._endpoints, base_addr)
|
||||
return
|
||||
if not self._discovery.done():
|
||||
return
|
||||
discovery, self._discovery = self._discovery, None
|
||||
try:
|
||||
self._install(discovery.result())
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"EC Mooncake event-channel discovery for %s failed; retrying.",
|
||||
base_addr,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
def _install(self, endpoints: list[str] | None) -> None:
|
||||
"""Adopt a discovered topology.
|
||||
|
||||
Runs on the caller's thread so the socket is only ever touched there.
|
||||
"""
|
||||
if not endpoints or self._closed:
|
||||
return
|
||||
context = zmq.Context()
|
||||
socket = context.socket(zmq.PULL)
|
||||
socket.setsockopt(zmq.IPV6, 1)
|
||||
for endpoint in endpoints:
|
||||
socket.connect(endpoint)
|
||||
self._context = context
|
||||
self._socket = socket
|
||||
self.shard_count = len(endpoints)
|
||||
|
||||
def drain(self, base_addr: str) -> list[dict[str, Any]]:
|
||||
self._connect(base_addr)
|
||||
if self._socket is None:
|
||||
return []
|
||||
events = []
|
||||
while True:
|
||||
try:
|
||||
events.append(self._socket.recv_json(flags=zmq.DONTWAIT))
|
||||
except zmq.Again:
|
||||
return events
|
||||
except zmq.ZMQError:
|
||||
logger.warning("EC Mooncake event channel failed", exc_info=True)
|
||||
return events
|
||||
except (ValueError, UnicodeError):
|
||||
# The event channel is a plain PULL socket: an undecodable
|
||||
# frame must cost one frame, not the engine.
|
||||
logger.warning(
|
||||
"Discarding an undecodable EC Mooncake event.", exc_info=True
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
if self._discovery is not None:
|
||||
self._discovery.cancel()
|
||||
self._discovery = None
|
||||
if self._socket is not None:
|
||||
self._socket.close(linger=0)
|
||||
self._socket = None
|
||||
if self._context is not None:
|
||||
self._context.term()
|
||||
|
||||
|
||||
class ConsumerControlServer:
|
||||
"""Expose Consumer reservations and readiness events over ZMQ.
|
||||
|
||||
One server runs on every receiving TP rank. The REP channel handles
|
||||
reservation operations, while a PUSH channel publishes newly ready items
|
||||
to the Scheduler.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int,
|
||||
reserve: Callable[[dict[str, Any]], dict[str, Any]],
|
||||
status: Callable[[str], dict[str, Any] | None],
|
||||
complete: Callable[[str, str], tuple[bool, bool]],
|
||||
cancel: Callable[[str, str, bool, bool], bool],
|
||||
reap: Callable[[], int],
|
||||
peer_ports: list[int] | None = None,
|
||||
device: torch.device | None = None,
|
||||
drain_ready: Callable[[], list[str]] = lambda: [],
|
||||
) -> None:
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.peer_ports = peer_ports or [port]
|
||||
self._device = device
|
||||
self.event_port: int | None = None
|
||||
self._reserve = reserve
|
||||
self._status = status
|
||||
self._complete = complete
|
||||
self._cancel = cancel
|
||||
self._reap = reap
|
||||
self._drain_ready = drain_ready
|
||||
self._stop = threading.Event()
|
||||
self._started = threading.Event()
|
||||
self._thread: threading.Thread | None = None
|
||||
self._startup_error: Exception | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
def loop() -> None:
|
||||
if self._device is not None and self._device.type == "cuda":
|
||||
torch.accelerator.set_device_index(self._device.index or 0)
|
||||
context = zmq.Context()
|
||||
socket = context.socket(zmq.REP)
|
||||
event_socket = context.socket(zmq.PUSH)
|
||||
socket.setsockopt(zmq.IPV6, 1)
|
||||
event_socket.setsockopt(zmq.IPV6, 1)
|
||||
pending_events: deque[dict[str, Any]] = deque()
|
||||
|
||||
def queue_event(event: dict[str, Any]) -> None:
|
||||
event["shard"] = self.port
|
||||
if len(pending_events) >= _MAX_PENDING_EVENTS:
|
||||
dropped = pending_events.popleft()
|
||||
logger.warning(
|
||||
"EC Mooncake event backlog full on port %d; dropping "
|
||||
"readiness for transfer_id=%s",
|
||||
self.port,
|
||||
dropped.get("transfer_id"),
|
||||
)
|
||||
pending_events.append(event)
|
||||
|
||||
def queue_ready(transfer_id: str) -> None:
|
||||
status = self._status(transfer_id)
|
||||
if status is not None:
|
||||
queue_event({"transfer_id": transfer_id, **status})
|
||||
|
||||
last_reap_at = time.monotonic()
|
||||
socket.setsockopt(zmq.RCVTIMEO, 100)
|
||||
try:
|
||||
socket.bind(make_zmq_path("tcp", self.host, self.port))
|
||||
event_socket.bind(make_zmq_path("tcp", self.host, 0))
|
||||
self.event_port = int(
|
||||
event_socket.getsockopt(zmq.LAST_ENDPOINT)
|
||||
.decode()
|
||||
.rsplit(":", 1)[1]
|
||||
)
|
||||
except Exception as e:
|
||||
self._startup_error = e
|
||||
self._started.set()
|
||||
socket.close(linger=0)
|
||||
event_socket.close(linger=0)
|
||||
context.term()
|
||||
return
|
||||
self._started.set()
|
||||
try:
|
||||
while not self._stop.is_set():
|
||||
while pending_events:
|
||||
try:
|
||||
event_socket.send_json(
|
||||
pending_events[0], flags=zmq.DONTWAIT
|
||||
)
|
||||
except zmq.Again:
|
||||
break
|
||||
pending_events.popleft()
|
||||
now = time.monotonic()
|
||||
if now - last_reap_at >= _RESERVATION_REAP_INTERVAL_SECONDS:
|
||||
self._reap()
|
||||
last_reap_at = now
|
||||
try:
|
||||
request = socket.recv_json()
|
||||
except zmq.Again:
|
||||
continue
|
||||
except Exception:
|
||||
# The frame arrived but did not decode. REP still owes a
|
||||
# reply, so answer before returning to the loop.
|
||||
logger.exception(
|
||||
"EC Mooncake control channel on port %d received an "
|
||||
"undecodable request",
|
||||
self.port,
|
||||
)
|
||||
socket.send_json(
|
||||
{"ok": False, "error": "malformed control request"}
|
||||
)
|
||||
continue
|
||||
try:
|
||||
op = request.get("op")
|
||||
result: Any = None
|
||||
if op in ("reserve", "reserve_batch"):
|
||||
items = (
|
||||
request["items"] if op == "reserve_batch" else [request]
|
||||
)
|
||||
results = []
|
||||
for item in items:
|
||||
try:
|
||||
reserved = self._reserve(item)
|
||||
results.append({"ok": True, "result": reserved})
|
||||
if reserved.get("ready"):
|
||||
queue_ready(str(item["transfer_id"]))
|
||||
except Exception as exc:
|
||||
results.append({"ok": False, "error": str(exc)})
|
||||
if op == "reserve_batch":
|
||||
result = {"items": results}
|
||||
elif not results[0]["ok"]:
|
||||
raise RuntimeError(results[0]["error"])
|
||||
else:
|
||||
result = results[0]["result"]
|
||||
elif op == "status":
|
||||
result = self._status(str(request["transfer_id"]))
|
||||
elif op == "event_port":
|
||||
result = self.event_port
|
||||
elif op == "peers":
|
||||
result = {"ports": self.peer_ports}
|
||||
elif op in ("complete", "complete_batch"):
|
||||
items = (
|
||||
request["items"]
|
||||
if op == "complete_batch"
|
||||
else [request]
|
||||
)
|
||||
completions = []
|
||||
for item in items:
|
||||
transfer_id = str(item["transfer_id"])
|
||||
accepted, became_ready = self._complete(
|
||||
transfer_id, str(item["reservation_id"])
|
||||
)
|
||||
for ready_id in self._drain_ready():
|
||||
queue_ready(ready_id)
|
||||
completions.append(
|
||||
{
|
||||
"completed": accepted,
|
||||
"became_ready": became_ready,
|
||||
}
|
||||
)
|
||||
if not became_ready:
|
||||
continue
|
||||
queue_ready(transfer_id)
|
||||
result = (
|
||||
{"items": completions}
|
||||
if op == "complete_batch"
|
||||
else completions[0]
|
||||
)
|
||||
elif op == "cancel":
|
||||
result = {
|
||||
"cancelled": self._cancel(
|
||||
str(request["transfer_id"]),
|
||||
str(request.get("reservation_id", "")),
|
||||
bool(request.get("abandon", False)),
|
||||
bool(request.get("refresh", False)),
|
||||
)
|
||||
}
|
||||
else:
|
||||
raise ValueError(f"unknown control op: {op!r}")
|
||||
socket.send_json({"ok": True, "result": result})
|
||||
except Exception as e:
|
||||
socket.send_json({"ok": False, "error": str(e)})
|
||||
except Exception:
|
||||
# `finally` closes the sockets and the thread ends, so without
|
||||
# this every later reserve against this shard would surface
|
||||
# only as a control timeout with nothing to attribute it to.
|
||||
logger.exception(
|
||||
"EC Mooncake control channel on port %d stopped serving",
|
||||
self.port,
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
socket.close(linger=0)
|
||||
event_socket.close(linger=0)
|
||||
context.term()
|
||||
|
||||
self._thread = threading.Thread(
|
||||
target=loop, name="ec-mooncake-control", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
if not self._started.wait(timeout=5):
|
||||
raise RuntimeError("EC Mooncake control channel failed to start")
|
||||
if self._startup_error is not None:
|
||||
raise RuntimeError("EC Mooncake control channel failed to bind") from (
|
||||
self._startup_error
|
||||
)
|
||||
logger.info(
|
||||
"EC Mooncake control channel listening on tcp://%s:%d (events tcp://%s:%d)",
|
||||
self.host,
|
||||
self.port,
|
||||
self.host,
|
||||
self.event_port,
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
if self._thread is None:
|
||||
return
|
||||
self._stop.set()
|
||||
self._thread.join()
|
||||
self._thread = None
|
||||
@@ -1,504 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Registered-memory allocation and residency for Mooncake transfers.
|
||||
|
||||
The Producer pool stages source tensors in a registered slab. The Consumer
|
||||
pool owns destination allocations from reservation through publication,
|
||||
resident reuse, CUDA-safe retirement, and pressure-driven reclamation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import bisect
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Generic, TypeVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
|
||||
MooncakeTransfer,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
@dataclass
|
||||
class MemoryAllocation:
|
||||
"""Describe a tensor view carved from the Consumer receive slab."""
|
||||
|
||||
offset: int
|
||||
size: int
|
||||
tensor: torch.Tensor
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ResidentEntry(Generic[_T]):
|
||||
"""Store one resident value and its ownership accounting."""
|
||||
|
||||
value: _T
|
||||
pinned: bool = True
|
||||
leases: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResidentLease(Generic[_T]):
|
||||
"""Represent a borrow of one resident entry."""
|
||||
|
||||
key: str
|
||||
_entry: _ResidentEntry[_T]
|
||||
_active: bool = True
|
||||
|
||||
@property
|
||||
def value(self) -> _T:
|
||||
return self._entry.value
|
||||
|
||||
|
||||
class ContiguousAllocator:
|
||||
"""Own a registered slab and its aligned regions; callers serialize access."""
|
||||
|
||||
def __init__(self, capacity: int, alignment: int = 256):
|
||||
self.alignment = alignment
|
||||
self._capacity = capacity
|
||||
self._free = [(0, capacity)]
|
||||
self.tensor: torch.Tensor | None = None
|
||||
self._disabled = False
|
||||
|
||||
def prepare(self, device: torch.device, transfer: MooncakeTransfer) -> None:
|
||||
if self.tensor is not None or self._disabled:
|
||||
return
|
||||
try:
|
||||
tensor = torch.empty(self._capacity, dtype=torch.uint8, device=device)
|
||||
ret = transfer.register_memory(tensor)
|
||||
if ret != 0:
|
||||
raise RuntimeError(f"Mooncake returned {ret}")
|
||||
except (RuntimeError, torch.OutOfMemoryError) as error:
|
||||
self._disabled = True
|
||||
logger.warning("Could not initialize the EC registered buffer: %s", error)
|
||||
return
|
||||
self.tensor = tensor
|
||||
self._free = [(0, tensor.nbytes)]
|
||||
logger.info("Registered %d-byte buffer for Mooncake EC", tensor.nbytes)
|
||||
|
||||
def view(
|
||||
self, offset: int, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
assert self.tensor is not None
|
||||
return self.tensor.narrow(0, offset, nbytes).view(dtype).view(shape)
|
||||
|
||||
def close(self, transfer: MooncakeTransfer) -> bool:
|
||||
if self.tensor is not None and not transfer.unregister_memory(self.tensor):
|
||||
return False
|
||||
self.tensor = None
|
||||
self._free.clear()
|
||||
return True
|
||||
|
||||
def allocate(self, nbytes: int) -> tuple[int, int] | None:
|
||||
size = (nbytes + self.alignment - 1) // self.alignment * self.alignment
|
||||
for index, (offset, available) in enumerate(self._free):
|
||||
if size > available:
|
||||
continue
|
||||
if size == available:
|
||||
self._free.pop(index)
|
||||
else:
|
||||
self._free[index] = (offset + size, available - size)
|
||||
return offset, size
|
||||
return None
|
||||
|
||||
def free(self, offset: int, size: int) -> None:
|
||||
index = bisect.bisect_left(self._free, (offset, size))
|
||||
self._free.insert(index, (offset, size))
|
||||
if index + 1 < len(self._free):
|
||||
next_offset, next_size = self._free[index + 1]
|
||||
if offset + size == next_offset:
|
||||
self._free[index] = (offset, size + next_size)
|
||||
self._free.pop(index + 1)
|
||||
if index > 0:
|
||||
previous_offset, previous_size = self._free[index - 1]
|
||||
current_offset, current_size = self._free[index]
|
||||
if previous_offset + previous_size == current_offset:
|
||||
self._free[index - 1] = (
|
||||
previous_offset,
|
||||
previous_size + current_size,
|
||||
)
|
||||
self._free.pop(index)
|
||||
|
||||
|
||||
class ResidentPool(Generic[_T]):
|
||||
"""Track resident values, active leases, and LRU eviction eligibility."""
|
||||
|
||||
def __init__(self):
|
||||
self._entries: dict[str, _ResidentEntry[_T]] = {}
|
||||
self._evictable: OrderedDict[str, None] = OrderedDict()
|
||||
|
||||
def referenced(self) -> list[str]:
|
||||
return [key for key, entry in self._entries.items() if entry.pinned]
|
||||
|
||||
def get(self, key: str) -> _T | None:
|
||||
entry = self._entries.get(key)
|
||||
return entry.value if entry is not None else None
|
||||
|
||||
def insert(
|
||||
self,
|
||||
key: str,
|
||||
value: _T,
|
||||
) -> _T | None:
|
||||
"""Pin an entry and return a displaced value that has no owners."""
|
||||
previous = self._entries.get(key)
|
||||
entry = _ResidentEntry(value)
|
||||
self._entries[key] = entry
|
||||
self._evictable.pop(key, None)
|
||||
if previous is not None and previous.leases == 0:
|
||||
return previous.value
|
||||
return None
|
||||
|
||||
def pin(self, key: str) -> _T | None:
|
||||
entry = self._entries.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
self._evictable.pop(key, None)
|
||||
entry.pinned = True
|
||||
return entry.value
|
||||
|
||||
def acquire(self, key: str) -> ResidentLease[_T] | None:
|
||||
entry = self._entries.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
entry.leases += 1
|
||||
self._evictable.pop(key, None)
|
||||
return ResidentLease(key, entry)
|
||||
|
||||
def release(self, lease: ResidentLease[_T]) -> _T | None:
|
||||
if not lease._active:
|
||||
return None
|
||||
lease._active = False
|
||||
entry = lease._entry
|
||||
entry.leases -= 1
|
||||
current = self._entries.get(lease.key)
|
||||
if current is not entry:
|
||||
if entry.leases == 0:
|
||||
return entry.value
|
||||
return None
|
||||
if not entry.pinned and entry.leases == 0:
|
||||
self._evictable[lease.key] = None
|
||||
return None
|
||||
|
||||
def consume(self, lease: ResidentLease[_T]) -> tuple[_T, _T | None]:
|
||||
current = self._entries[lease.key]
|
||||
current.pinned = True
|
||||
self._evictable.pop(lease.key, None)
|
||||
released = self.release(lease)
|
||||
return current.value, released
|
||||
|
||||
def retire(self, key: str) -> None:
|
||||
if key not in self._entries:
|
||||
return
|
||||
entry = self._entries[key]
|
||||
entry.pinned = False
|
||||
if entry.leases == 0:
|
||||
self._evictable[key] = None
|
||||
|
||||
def evict_lru(self, evict: Callable[[str, _T], bool]) -> str | None:
|
||||
for key in list(self._evictable):
|
||||
entry = self._entries[key]
|
||||
if not evict(key, entry.value):
|
||||
continue
|
||||
self._evictable.pop(key, None)
|
||||
del self._entries[key]
|
||||
return key
|
||||
return None
|
||||
|
||||
def clear(self) -> None:
|
||||
self._entries.clear()
|
||||
self._evictable.clear()
|
||||
|
||||
|
||||
@dataclass
|
||||
class StagedSources:
|
||||
"""Own Producer tensor views and their staging-slab regions."""
|
||||
|
||||
tensors: list[torch.Tensor]
|
||||
regions: list[tuple[int, int]]
|
||||
|
||||
|
||||
class ProducerMemoryPool:
|
||||
"""Own the Producer staging slab and regions carved from it."""
|
||||
|
||||
def __init__(self, capacity: int, transfer: MooncakeTransfer) -> None:
|
||||
self._transfer = transfer
|
||||
self._allocator = ContiguousAllocator(capacity)
|
||||
self._lock = threading.Lock()
|
||||
self._local = threading.local()
|
||||
|
||||
@property
|
||||
def tensor(self) -> torch.Tensor | None:
|
||||
return self._allocator.tensor
|
||||
|
||||
def _free_regions(self, regions: list[tuple[int, int]]) -> None:
|
||||
for offset, size in regions:
|
||||
self._allocator.free(offset, size)
|
||||
|
||||
def stage(self, tensors: list[torch.Tensor]) -> StagedSources | None:
|
||||
"""Copy tensors into one registered slab, or return None for fallback."""
|
||||
if not tensors:
|
||||
return StagedSources([], [])
|
||||
allocator = self._allocator
|
||||
staged: list[torch.Tensor] = []
|
||||
regions: list[tuple[int, int]] = []
|
||||
with self._lock:
|
||||
allocator.prepare(tensors[0].device, self._transfer)
|
||||
pool = allocator.tensor
|
||||
if pool is None:
|
||||
return None
|
||||
for tensor in tensors:
|
||||
region = allocator.allocate(tensor.nbytes)
|
||||
if region is None:
|
||||
self._free_regions(regions)
|
||||
return None
|
||||
regions.append(region)
|
||||
staged.append(
|
||||
allocator.view(
|
||||
region[0], tensor.nbytes, tuple(tensor.shape), tensor.dtype
|
||||
)
|
||||
)
|
||||
if pool.device.type == "cuda":
|
||||
stream = getattr(self._local, "stream", None)
|
||||
if stream is None:
|
||||
stream = torch.cuda.Stream(device=pool.device)
|
||||
self._local.stream = stream
|
||||
with torch.cuda.stream(stream):
|
||||
for destination, source in zip(staged, tensors):
|
||||
destination.copy_(source, non_blocking=True)
|
||||
stream.synchronize()
|
||||
else:
|
||||
for destination, source in zip(staged, tensors):
|
||||
destination.copy_(source)
|
||||
return StagedSources(staged, regions)
|
||||
|
||||
def release(self, staged: StagedSources) -> None:
|
||||
if not staged.regions:
|
||||
return
|
||||
with self._lock:
|
||||
self._free_regions(staged.regions)
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
self._allocator.close(self._transfer)
|
||||
|
||||
|
||||
class ConsumerMemoryPool:
|
||||
"""Own the registered receive slab and resident allocation lifecycle."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
capacity: int,
|
||||
transfer: MooncakeTransfer,
|
||||
) -> None:
|
||||
self._transfer = transfer
|
||||
self._allocator = ContiguousAllocator(capacity)
|
||||
self._residents: ResidentPool[MemoryAllocation] = ResidentPool()
|
||||
self._retire_events: dict[str, torch.Event] = {}
|
||||
self._pending_frees: list[tuple[torch.Event, MemoryAllocation]] = []
|
||||
self._reclaimed: set[str] = set()
|
||||
self.lock = threading.RLock()
|
||||
|
||||
@property
|
||||
def tensor(self) -> torch.Tensor | None:
|
||||
return self._allocator.tensor
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
with self.lock:
|
||||
self._allocator.prepare(device, self._transfer)
|
||||
|
||||
def _free(self, allocation: MemoryAllocation) -> None:
|
||||
self._allocator.free(allocation.offset, allocation.size)
|
||||
|
||||
def free(self, allocation: MemoryAllocation) -> None:
|
||||
with self.lock:
|
||||
self._free(allocation)
|
||||
|
||||
def _defer_or_free(
|
||||
self, allocation: MemoryAllocation, event: torch.Event | None
|
||||
) -> None:
|
||||
if event is None or event.query():
|
||||
self._free(allocation)
|
||||
else:
|
||||
self._pending_frees.append((event, allocation))
|
||||
|
||||
def _poll_frees_locked(self) -> None:
|
||||
pending = []
|
||||
for event, allocation in self._pending_frees:
|
||||
if event.query():
|
||||
self._free(allocation)
|
||||
else:
|
||||
pending.append((event, allocation))
|
||||
self._pending_frees = pending
|
||||
|
||||
def _reclaim_locked(self, nbytes: int) -> tuple[int, int] | None:
|
||||
def evict(mm_hash: str, allocation: MemoryAllocation) -> bool:
|
||||
event = self._retire_events.pop(mm_hash, None)
|
||||
self._defer_or_free(allocation, event)
|
||||
self._reclaimed.add(mm_hash)
|
||||
return True
|
||||
|
||||
while self._residents.evict_lru(evict) is not None:
|
||||
# Eviction defers any free whose CUDA event is still pending, so
|
||||
# those bytes reach the allocator only once the event is polled.
|
||||
self._poll_frees_locked()
|
||||
region = self._allocator.allocate(nbytes)
|
||||
if region is not None:
|
||||
return region
|
||||
self._poll_frees_locked()
|
||||
return self._allocator.allocate(nbytes)
|
||||
|
||||
def _make_allocation(
|
||||
self,
|
||||
region: tuple[int, int],
|
||||
nbytes: int,
|
||||
shape: tuple[int, ...],
|
||||
dtype: torch.dtype,
|
||||
) -> MemoryAllocation:
|
||||
offset, size = region
|
||||
tensor = self._allocator.view(offset, nbytes, shape, dtype)
|
||||
return MemoryAllocation(offset, size, tensor)
|
||||
|
||||
def try_allocate(
|
||||
self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
|
||||
) -> MemoryAllocation | None:
|
||||
with self.lock:
|
||||
allocator = self._allocator
|
||||
assert allocator.tensor is not None
|
||||
self._poll_frees_locked()
|
||||
region = allocator.allocate(nbytes)
|
||||
if region is None:
|
||||
return None
|
||||
return self._make_allocation(region, nbytes, shape, dtype)
|
||||
|
||||
def reclaim_and_allocate(
|
||||
self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype
|
||||
) -> MemoryAllocation | None:
|
||||
with self.lock:
|
||||
region = self._reclaim_locked(nbytes)
|
||||
if region is None:
|
||||
return None
|
||||
return self._make_allocation(region, nbytes, shape, dtype)
|
||||
|
||||
def acquire_cached(
|
||||
self, mm_hash: str, shape: tuple[int, ...], dtype: torch.dtype
|
||||
) -> ResidentLease[MemoryAllocation] | None:
|
||||
with self.lock:
|
||||
allocation = self._residents.get(mm_hash)
|
||||
if allocation is None:
|
||||
return None
|
||||
if (
|
||||
tuple(allocation.tensor.shape) != shape
|
||||
or allocation.tensor.dtype != dtype
|
||||
):
|
||||
raise ValueError("conflicting cached tensor for mm_hash")
|
||||
return self._residents.acquire(mm_hash)
|
||||
|
||||
def take_resident(
|
||||
self, mm_hash: str, shape: tuple[int, ...], dtype_name: str
|
||||
) -> torch.Tensor | None:
|
||||
with self.lock:
|
||||
allocation = self._residents.get(mm_hash)
|
||||
if allocation is None:
|
||||
return None
|
||||
tensor = allocation.tensor
|
||||
if (
|
||||
tuple(tensor.shape) != shape
|
||||
or str(tensor.dtype).split(".")[-1] != dtype_name
|
||||
):
|
||||
return None
|
||||
self._residents.pin(mm_hash)
|
||||
self._retire_events.pop(mm_hash, None)
|
||||
return tensor
|
||||
|
||||
def _record_release_event(self) -> torch.Event | None:
|
||||
pool = self.tensor
|
||||
if pool is None or pool.device.type != "cuda":
|
||||
return None
|
||||
event = torch.Event()
|
||||
event.record(torch.accelerator.current_stream(pool.device))
|
||||
return event
|
||||
|
||||
def release_cached(self, lease: ResidentLease[MemoryAllocation]) -> None:
|
||||
with self.lock:
|
||||
released = self._residents.release(lease)
|
||||
if released is not None:
|
||||
self._defer_or_free(released, self._record_release_event())
|
||||
|
||||
def publish(
|
||||
self,
|
||||
mm_hash: str,
|
||||
allocation: MemoryAllocation,
|
||||
lease: ResidentLease[MemoryAllocation] | None = None,
|
||||
*,
|
||||
pin: bool = True,
|
||||
) -> MemoryAllocation:
|
||||
with self.lock:
|
||||
if lease is not None:
|
||||
canonical, released = self._residents.consume(lease)
|
||||
self._retire_events.pop(mm_hash, None)
|
||||
if released is not None:
|
||||
self._defer_or_free(released, self._record_release_event())
|
||||
return canonical
|
||||
previous = self._residents.get(mm_hash)
|
||||
displaced = self._residents.insert(mm_hash, allocation)
|
||||
if not pin:
|
||||
self._residents.retire(mm_hash)
|
||||
event = None
|
||||
if previous is not None and previous is not allocation:
|
||||
event = self._retire_events.pop(mm_hash, None)
|
||||
if displaced is None or displaced is allocation:
|
||||
return allocation
|
||||
if event is None:
|
||||
event = self._record_release_event()
|
||||
self._defer_or_free(displaced, event)
|
||||
return allocation
|
||||
|
||||
def retire_stale(
|
||||
self,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
reserved_hashes: set[str] | None = None,
|
||||
*,
|
||||
freed: list[str] | None = None,
|
||||
) -> None:
|
||||
if self.tensor is None:
|
||||
return
|
||||
with self.lock:
|
||||
for mm_hash in self._residents.referenced() if freed is None else freed:
|
||||
allocation = self._residents.get(mm_hash)
|
||||
if allocation is None:
|
||||
continue
|
||||
if encoder_cache.get(mm_hash) is allocation.tensor:
|
||||
continue
|
||||
if reserved_hashes and mm_hash in reserved_hashes:
|
||||
continue
|
||||
event = self._record_release_event()
|
||||
if event is not None:
|
||||
self._retire_events[mm_hash] = event
|
||||
self._residents.retire(mm_hash)
|
||||
self._poll_frees_locked()
|
||||
|
||||
def drain_reclaimed(self) -> set[str]:
|
||||
with self.lock:
|
||||
reclaimed = self._reclaimed
|
||||
self._reclaimed = set()
|
||||
return reclaimed
|
||||
|
||||
def close(self) -> None:
|
||||
with self.lock:
|
||||
if not self._allocator.close(self._transfer):
|
||||
return
|
||||
self._residents.clear()
|
||||
self._retire_events.clear()
|
||||
self._pending_frees.clear()
|
||||
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Metadata exchanged by the Mooncake encoder-cache connector."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorMetadata,
|
||||
ECConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
|
||||
TensorSpec,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECMooncakeLoadSpec:
|
||||
"""Describe a remote reservation or resident allocation to load."""
|
||||
|
||||
mm_hash: str
|
||||
nbytes: int
|
||||
shape: tuple[int, ...]
|
||||
dtype: str
|
||||
transfer_id: str
|
||||
# The consumer pool still holds this item, so the load is a local handoff:
|
||||
# no transfer, no producer.
|
||||
local: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECMooncakePushSpec:
|
||||
"""Describe a destination reservation prepared before a tensor is ready."""
|
||||
|
||||
mm_hash: str
|
||||
nbytes: int
|
||||
shape: tuple[int, ...]
|
||||
dtype: str
|
||||
consumer_zmq: str
|
||||
transfer_id: str
|
||||
request_id: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECMooncakeConnectorMetadata(ECConnectorMetadata):
|
||||
"""Worker operations emitted for one Scheduler step."""
|
||||
|
||||
loads: list[ECMooncakeLoadSpec] = field(default_factory=list)
|
||||
pushes: list[ECMooncakePushSpec] = field(default_factory=list)
|
||||
freed: list[str] | None = None
|
||||
store_candidates: dict[str, TensorSpec] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECMooncakeWorkerMetadata(ECConnectorWorkerMetadata):
|
||||
"""Completion state reported from Workers to the Scheduler."""
|
||||
|
||||
loaded: set[str] = field(default_factory=set)
|
||||
failed_loads: set[str] = field(default_factory=set)
|
||||
# Items the receive pool dropped under pressure. The scheduler assumes an
|
||||
# evicted item stays resident until told otherwise.
|
||||
reclaimed: set[str] = field(default_factory=set)
|
||||
pending_saves: bool = False
|
||||
failed_saves: set[str] = field(default_factory=set)
|
||||
|
||||
def aggregate(self, other: ECConnectorWorkerMetadata) -> ECMooncakeWorkerMetadata:
|
||||
assert isinstance(other, ECMooncakeWorkerMetadata)
|
||||
return ECMooncakeWorkerMetadata(
|
||||
# Every tensor-parallel rank gathers the embedding from its own
|
||||
# cache, so an item counts as loaded only where all of them have
|
||||
# it; one rank falling short must fail the load rather than leave
|
||||
# the scheduler believing it is ready.
|
||||
loaded=self.loaded & other.loaded,
|
||||
failed_loads=self.failed_loads | other.failed_loads,
|
||||
reclaimed=self.reclaimed | other.reclaimed,
|
||||
pending_saves=self.pending_saves or other.pending_saves,
|
||||
failed_saves=self.failed_saves | other.failed_saves,
|
||||
)
|
||||
@@ -1,418 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Producer-side push lifecycle, futures, and source-tensor ownership.
|
||||
|
||||
This module records when a push may advance or release its source. Worker
|
||||
orchestration performs the actual control exchanges and Mooncake writes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum, auto
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
|
||||
ECMooncakePushSpec,
|
||||
)
|
||||
|
||||
|
||||
def _same_destination(
|
||||
existing: ECMooncakePushSpec, incoming: ECMooncakePushSpec
|
||||
) -> bool:
|
||||
"""Whether two specs describe one push, ignoring which request asked.
|
||||
|
||||
The same encoding requested twice shares a transfer id legitimately; only
|
||||
a different payload or destination is a collision.
|
||||
"""
|
||||
return (
|
||||
existing.mm_hash == incoming.mm_hash
|
||||
and existing.nbytes == incoming.nbytes
|
||||
and existing.shape == incoming.shape
|
||||
and existing.dtype == incoming.dtype
|
||||
and existing.consumer_zmq == incoming.consumer_zmq
|
||||
)
|
||||
|
||||
|
||||
class ProducerPushState(Enum):
|
||||
"""Lifecycle of one Producer push from reservation to terminal state."""
|
||||
|
||||
# The reservation reply and source tensor may arrive in either order.
|
||||
WAITING_INPUTS = auto()
|
||||
WRITING = auto()
|
||||
NOTIFYING = auto()
|
||||
DONE = auto()
|
||||
CANCEL_PENDING = auto()
|
||||
CANCELLED = auto()
|
||||
FAILED = auto()
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProducerPushRecord:
|
||||
"""Collect all asynchronous state for one Producer push."""
|
||||
|
||||
spec: ECMooncakePushSpec
|
||||
state: ProducerPushState
|
||||
reservation_future: Future[list[dict[str, Any]]]
|
||||
reservations: list[dict[str, Any]] = field(default_factory=list)
|
||||
shard_futures: list[Future[Any]] = field(default_factory=list)
|
||||
source_tensor: torch.Tensor | None = None
|
||||
source_event: torch.Event | None = None
|
||||
batch_future: Future[None] | None = None
|
||||
error: str | None = None
|
||||
|
||||
|
||||
_ALLOWED_TRANSITIONS = {
|
||||
ProducerPushState.WAITING_INPUTS: {
|
||||
ProducerPushState.WRITING,
|
||||
ProducerPushState.CANCEL_PENDING,
|
||||
ProducerPushState.FAILED,
|
||||
},
|
||||
ProducerPushState.WRITING: {
|
||||
ProducerPushState.NOTIFYING,
|
||||
ProducerPushState.FAILED,
|
||||
},
|
||||
ProducerPushState.NOTIFYING: {
|
||||
ProducerPushState.DONE,
|
||||
ProducerPushState.FAILED,
|
||||
},
|
||||
ProducerPushState.CANCEL_PENDING: {
|
||||
ProducerPushState.CANCELLED,
|
||||
ProducerPushState.FAILED,
|
||||
},
|
||||
ProducerPushState.DONE: set(),
|
||||
ProducerPushState.CANCELLED: set(),
|
||||
ProducerPushState.FAILED: set(),
|
||||
}
|
||||
|
||||
_TERMINAL_STATES = {
|
||||
ProducerPushState.DONE,
|
||||
ProducerPushState.CANCELLED,
|
||||
ProducerPushState.FAILED,
|
||||
}
|
||||
_TERMINAL_LIMIT = 1 << 16
|
||||
|
||||
|
||||
class ProducerPushManager:
|
||||
"""Own Producer push records, transitions, and source tensor leases.
|
||||
|
||||
Terminal records are retained for duplicate-metadata idempotency. Separate
|
||||
ordered indexes keep hot polling paths from scanning those tombstones.
|
||||
"""
|
||||
|
||||
def __init__(self, wake: Callable[[], None] = lambda: None) -> None:
|
||||
self._records: OrderedDict[str, ProducerPushRecord] = OrderedDict()
|
||||
self._active_ids: OrderedDict[str, None] = OrderedDict()
|
||||
self._reapable_terminal_ids: OrderedDict[str, None] = OrderedDict()
|
||||
self._unreported_ids: OrderedDict[str, None] = OrderedDict()
|
||||
self._batch_ids: OrderedDict[str, None] = OrderedDict()
|
||||
self._source_waiters: dict[str, OrderedDict[str, None]] = {}
|
||||
self._lock = threading.RLock()
|
||||
self._wake = wake
|
||||
|
||||
def reserve(
|
||||
self,
|
||||
spec: ECMooncakePushSpec,
|
||||
submit: Callable[[], Future[list[dict[str, Any]]]],
|
||||
) -> tuple[ProducerPushRecord, bool]:
|
||||
with self._lock:
|
||||
existing = self._records.get(spec.transfer_id)
|
||||
if existing is not None:
|
||||
if not _same_destination(existing.spec, spec):
|
||||
raise ValueError(
|
||||
f"Conflicting EC destination for transfer_id={spec.transfer_id}"
|
||||
)
|
||||
return existing, False
|
||||
record = ProducerPushRecord(
|
||||
spec=spec,
|
||||
state=ProducerPushState.WAITING_INPUTS,
|
||||
reservation_future=submit(),
|
||||
)
|
||||
self._records[spec.transfer_id] = record
|
||||
self._active_ids[spec.transfer_id] = None
|
||||
self._source_waiters.setdefault(spec.mm_hash, OrderedDict())[
|
||||
spec.transfer_id
|
||||
] = None
|
||||
record.reservation_future.add_done_callback(
|
||||
lambda future: self._reservation_done(record, future)
|
||||
)
|
||||
return record, True
|
||||
|
||||
def bind_source(
|
||||
self,
|
||||
mm_hash: str,
|
||||
tensor: torch.Tensor,
|
||||
ready_event: torch.Event | None,
|
||||
) -> None:
|
||||
with self._lock:
|
||||
waiters = self._source_waiters.pop(mm_hash, OrderedDict())
|
||||
for transfer_id in waiters:
|
||||
record = self._records[transfer_id]
|
||||
if (
|
||||
record.source_tensor is not None
|
||||
or record.state is not ProducerPushState.WAITING_INPUTS
|
||||
):
|
||||
continue
|
||||
record.source_tensor = tensor
|
||||
record.source_event = ready_event
|
||||
self._wake()
|
||||
|
||||
def submit_batches(
|
||||
self,
|
||||
executor: ThreadPoolExecutor,
|
||||
run_batch: Callable[[list[ProducerPushRecord]], None],
|
||||
*,
|
||||
wait: bool = False,
|
||||
) -> bool:
|
||||
pending_event = False
|
||||
with self._lock:
|
||||
grouped: dict[str, list[ProducerPushRecord]] = {}
|
||||
for transfer_id in list(self._active_ids):
|
||||
record = self._records[transfer_id]
|
||||
if (
|
||||
record.source_tensor is not None
|
||||
and record.batch_future is None
|
||||
and record.state is ProducerPushState.WAITING_INPUTS
|
||||
):
|
||||
if not record.reservation_future.done():
|
||||
continue
|
||||
if record.source_event is not None:
|
||||
if wait:
|
||||
record.source_event.synchronize()
|
||||
elif not record.source_event.query():
|
||||
pending_event = True
|
||||
continue
|
||||
grouped.setdefault(record.spec.consumer_zmq, []).append(record)
|
||||
batches = list(grouped.values())
|
||||
for records in batches:
|
||||
future = executor.submit(run_batch, records)
|
||||
for record in records:
|
||||
record.batch_future = future
|
||||
self._batch_ids[record.spec.transfer_id] = None
|
||||
return pending_event
|
||||
|
||||
def resolve_reservations(self, record: ProducerPushRecord) -> list[dict[str, Any]]:
|
||||
results = record.reservation_future.result()
|
||||
with self._lock:
|
||||
record.reservations = results
|
||||
return list(results)
|
||||
|
||||
def finish_reservations(
|
||||
self,
|
||||
outcomes: list[tuple[ProducerPushRecord, list[dict[str, Any]] | Exception]],
|
||||
) -> None:
|
||||
with self._lock:
|
||||
for record, result in outcomes:
|
||||
if isinstance(result, Exception):
|
||||
record.reservation_future.set_exception(result)
|
||||
else:
|
||||
record.reservation_future.set_result(result)
|
||||
|
||||
def _reservation_done(
|
||||
self,
|
||||
record: ProducerPushRecord,
|
||||
future: Future[list[dict[str, Any]]],
|
||||
) -> None:
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc:
|
||||
with self._lock:
|
||||
self._set_error(record, exc)
|
||||
if (
|
||||
record.state is ProducerPushState.WAITING_INPUTS
|
||||
and record.source_tensor is None
|
||||
):
|
||||
self._transition(record, ProducerPushState.FAILED)
|
||||
finally:
|
||||
self._wake()
|
||||
|
||||
def settle_all(self, records: list[ProducerPushRecord]) -> None:
|
||||
for record in records:
|
||||
with suppress(Exception):
|
||||
record.reservation_future.result()
|
||||
|
||||
def replace_reservations(
|
||||
self,
|
||||
record: ProducerPushRecord,
|
||||
reservations: list[dict[str, Any]],
|
||||
) -> None:
|
||||
with self._lock:
|
||||
record.reservations = reservations
|
||||
|
||||
def track_shard_futures(
|
||||
self,
|
||||
records: list[ProducerPushRecord],
|
||||
futures: list[Future[Any]],
|
||||
) -> None:
|
||||
with self._lock:
|
||||
for record in records:
|
||||
record.shard_futures.extend(futures)
|
||||
|
||||
def begin_writing(self, record: ProducerPushRecord) -> None:
|
||||
with self._lock:
|
||||
if record.source_tensor is None:
|
||||
raise RuntimeError(
|
||||
f"Producer push {record.spec.transfer_id!r} has no source tensor"
|
||||
)
|
||||
self._transition(record, ProducerPushState.WRITING)
|
||||
|
||||
def begin_notifying(self, records: list[ProducerPushRecord]) -> None:
|
||||
with self._lock:
|
||||
for record in records:
|
||||
self._transition(record, ProducerPushState.NOTIFYING)
|
||||
|
||||
def complete(self, records: list[ProducerPushRecord]) -> None:
|
||||
with self._lock:
|
||||
self._check_sources_releasable(records)
|
||||
for record in records:
|
||||
self._transition(record, ProducerPushState.DONE)
|
||||
self._release_source(record)
|
||||
|
||||
def fail(self, records: list[ProducerPushRecord], error: Exception) -> None:
|
||||
with self._lock:
|
||||
self._check_sources_releasable(records)
|
||||
for record in records:
|
||||
if record.state not in _TERMINAL_STATES:
|
||||
self._set_error(record, error)
|
||||
self._transition(record, ProducerPushState.FAILED)
|
||||
self._release_source(record)
|
||||
|
||||
def cancel_requests(self, request_ids: set[str] | None) -> list[ProducerPushRecord]:
|
||||
"""Cancel source-less waiters for requests, or all waiters at shutdown."""
|
||||
cancelled = []
|
||||
with self._lock:
|
||||
for transfer_id in list(self._active_ids):
|
||||
record = self._records[transfer_id]
|
||||
if (
|
||||
(
|
||||
request_ids is not None
|
||||
and record.spec.request_id not in request_ids
|
||||
)
|
||||
or record.source_tensor is not None
|
||||
or record.batch_future is not None
|
||||
):
|
||||
continue
|
||||
if record.state is not ProducerPushState.WAITING_INPUTS:
|
||||
continue
|
||||
self._transition(record, ProducerPushState.CANCEL_PENDING)
|
||||
cancelled.append(record)
|
||||
return cancelled
|
||||
|
||||
def finish_cancel(self, record: ProducerPushRecord) -> None:
|
||||
with self._lock:
|
||||
if record.state is ProducerPushState.CANCEL_PENDING:
|
||||
self._transition(record, ProducerPushState.CANCELLED)
|
||||
|
||||
def submit_cancel(
|
||||
self,
|
||||
record: ProducerPushRecord,
|
||||
executor: ThreadPoolExecutor,
|
||||
run_cancel: Callable[[ProducerPushRecord], None],
|
||||
) -> None:
|
||||
with self._lock:
|
||||
future = executor.submit(run_cancel, record)
|
||||
record.batch_future = future
|
||||
transfer_id = record.spec.transfer_id
|
||||
self._batch_ids[transfer_id] = None
|
||||
if record.state in _TERMINAL_STATES:
|
||||
self._reapable_terminal_ids.pop(transfer_id, None)
|
||||
|
||||
def poll(self) -> list[tuple[str, str]]:
|
||||
failures = []
|
||||
with self._lock:
|
||||
for transfer_id in list(self._batch_ids):
|
||||
record = self._records[transfer_id]
|
||||
future = record.batch_future
|
||||
assert future is not None
|
||||
if not future.done():
|
||||
continue
|
||||
try:
|
||||
future.result()
|
||||
except Exception as exc:
|
||||
self._set_error(record, exc)
|
||||
self._batch_ids.pop(transfer_id)
|
||||
self._mark_reapable(record)
|
||||
for transfer_id in list(self._unreported_ids):
|
||||
if transfer_id in self._batch_ids:
|
||||
continue
|
||||
record = self._records[transfer_id]
|
||||
assert record.error is not None
|
||||
failures.append((record.spec.mm_hash, record.error))
|
||||
self._unreported_ids.pop(transfer_id)
|
||||
self._mark_reapable(record)
|
||||
self._reap_terminals()
|
||||
return failures
|
||||
|
||||
@property
|
||||
def pending(self) -> bool:
|
||||
with self._lock:
|
||||
return bool(self._active_ids or self._batch_ids)
|
||||
|
||||
def _release_source(self, record: ProducerPushRecord) -> None:
|
||||
record.source_tensor = None
|
||||
record.source_event = None
|
||||
|
||||
def _set_error(self, record: ProducerPushRecord, error: BaseException) -> None:
|
||||
if record.error is None:
|
||||
record.error = str(error)
|
||||
if record.state in _TERMINAL_STATES:
|
||||
transfer_id = record.spec.transfer_id
|
||||
self._unreported_ids[transfer_id] = None
|
||||
self._reapable_terminal_ids.pop(transfer_id, None)
|
||||
|
||||
def _reap_terminals(self) -> None:
|
||||
while len(self._reapable_terminal_ids) > _TERMINAL_LIMIT:
|
||||
transfer_id, _ = self._reapable_terminal_ids.popitem(last=False)
|
||||
self._records.pop(transfer_id)
|
||||
|
||||
def _mark_reapable(self, record: ProducerPushRecord) -> None:
|
||||
transfer_id = record.spec.transfer_id
|
||||
if (
|
||||
record.state in _TERMINAL_STATES
|
||||
and transfer_id not in self._batch_ids
|
||||
and transfer_id not in self._unreported_ids
|
||||
):
|
||||
self._reapable_terminal_ids[transfer_id] = None
|
||||
|
||||
def _drop_source_waiter(self, record: ProducerPushRecord) -> None:
|
||||
waiters = self._source_waiters.get(record.spec.mm_hash)
|
||||
if waiters is None:
|
||||
return
|
||||
waiters.pop(record.spec.transfer_id, None)
|
||||
if not waiters:
|
||||
self._source_waiters.pop(record.spec.mm_hash)
|
||||
|
||||
@staticmethod
|
||||
def _check_sources_releasable(records: list[ProducerPushRecord]) -> None:
|
||||
for record in records:
|
||||
futures = [record.reservation_future, *record.shard_futures]
|
||||
if record.source_tensor is not None and not all(
|
||||
future.done() for future in futures
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"Producer push {record.spec.transfer_id!r} released its "
|
||||
"source too early"
|
||||
)
|
||||
|
||||
def _transition(self, record: ProducerPushRecord, state: ProducerPushState) -> None:
|
||||
if state not in _ALLOWED_TRANSITIONS[record.state]:
|
||||
raise RuntimeError(
|
||||
f"Cannot transition producer push {record.spec.transfer_id!r} from "
|
||||
f"{record.state.name} to {state.name}"
|
||||
)
|
||||
record.state = state
|
||||
if state is not ProducerPushState.WAITING_INPUTS:
|
||||
self._drop_source_waiter(record)
|
||||
if state in _TERMINAL_STATES:
|
||||
transfer_id = record.spec.transfer_id
|
||||
self._active_ids.pop(transfer_id, None)
|
||||
if record.error is not None:
|
||||
self._unreported_ids[transfer_id] = None
|
||||
self._mark_reapable(record)
|
||||
@@ -1,487 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Consumer-side reservation lifecycle and destination-memory ownership.
|
||||
|
||||
Reservations make remote writes idempotent and ensure cancellation or expiry
|
||||
cannot free a destination while Mooncake may still be writing into it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import (
|
||||
ConsumerMemoryPool,
|
||||
MemoryAllocation,
|
||||
ResidentLease,
|
||||
)
|
||||
|
||||
|
||||
class ConsumerReservationState(Enum):
|
||||
"""Lifecycle of one Consumer destination reservation."""
|
||||
|
||||
WRITING = auto()
|
||||
READY = auto()
|
||||
CANCEL_PENDING = auto()
|
||||
EXPIRE_PENDING = auto()
|
||||
CANCELLED = auto()
|
||||
EXPIRED = auto()
|
||||
|
||||
|
||||
_ALLOWED_TRANSITIONS = {
|
||||
ConsumerReservationState.WRITING: {
|
||||
ConsumerReservationState.READY,
|
||||
ConsumerReservationState.CANCEL_PENDING,
|
||||
ConsumerReservationState.EXPIRE_PENDING,
|
||||
ConsumerReservationState.CANCELLED,
|
||||
},
|
||||
ConsumerReservationState.READY: {
|
||||
ConsumerReservationState.CANCELLED,
|
||||
ConsumerReservationState.EXPIRED,
|
||||
},
|
||||
ConsumerReservationState.CANCEL_PENDING: {ConsumerReservationState.CANCELLED},
|
||||
ConsumerReservationState.EXPIRE_PENDING: {
|
||||
ConsumerReservationState.CANCELLED,
|
||||
ConsumerReservationState.EXPIRED,
|
||||
},
|
||||
ConsumerReservationState.CANCELLED: set(),
|
||||
ConsumerReservationState.EXPIRED: set(),
|
||||
}
|
||||
|
||||
_ACTIVE_STATES = {
|
||||
ConsumerReservationState.WRITING,
|
||||
ConsumerReservationState.READY,
|
||||
ConsumerReservationState.CANCEL_PENDING,
|
||||
ConsumerReservationState.EXPIRE_PENDING,
|
||||
}
|
||||
_DEFERRED_STATES = {
|
||||
ConsumerReservationState.CANCEL_PENDING,
|
||||
ConsumerReservationState.EXPIRE_PENDING,
|
||||
}
|
||||
_WRITER_OWNED_STATES = {
|
||||
ConsumerReservationState.WRITING,
|
||||
ConsumerReservationState.CANCEL_PENDING,
|
||||
ConsumerReservationState.EXPIRE_PENDING,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConsumerReservation:
|
||||
"""Own the identity and destination allocation of one remote push."""
|
||||
|
||||
transfer_id: str
|
||||
mm_hash: str
|
||||
reservation_id: str
|
||||
state: ConsumerReservationState
|
||||
shape: tuple[int, ...] = ()
|
||||
dtype: str = ""
|
||||
allocation: MemoryAllocation | None = None
|
||||
lease: ResidentLease[MemoryAllocation] | None = None
|
||||
expires_at: float = 0
|
||||
writer_id: str = ""
|
||||
|
||||
|
||||
class ConsumerReservationManager:
|
||||
"""Own reservation transitions and destination allocation releases.
|
||||
|
||||
A remote writer owns a ``WRITING`` allocation. Cancellation and expiry are
|
||||
therefore deferred until completion, unless refresh explicitly abandons
|
||||
that writer before replacing its reservation.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
memory: ConsumerMemoryPool,
|
||||
lease_ttl: float,
|
||||
tombstone_limit: int,
|
||||
) -> None:
|
||||
self._memory = memory
|
||||
self._lease_ttl = lease_ttl
|
||||
self._tombstone_limit = tombstone_limit
|
||||
self._records: dict[str, ConsumerReservation] = {}
|
||||
self._active_ids: dict[str, None] = {}
|
||||
self._tombstones: OrderedDict[str, None] = OrderedDict()
|
||||
self._writers: dict[str, ConsumerReservation] = {}
|
||||
self._followers: dict[str, dict[str, None]] = {}
|
||||
self._ready: list[str] = []
|
||||
self._condition = threading.Condition(memory.lock)
|
||||
self._shutting_down = False
|
||||
|
||||
def reserve(
|
||||
self,
|
||||
transfer_id: str,
|
||||
mm_hash: str,
|
||||
nbytes: int,
|
||||
shape: tuple[int, ...],
|
||||
dtype_name: str,
|
||||
dtype: torch.dtype,
|
||||
) -> tuple[ConsumerReservation | None, bool]:
|
||||
with self._condition:
|
||||
if self._shutting_down:
|
||||
raise RuntimeError("Consumer reservation manager is shutting down")
|
||||
existing = self._records.get(transfer_id)
|
||||
if (
|
||||
existing is not None
|
||||
and existing.state is ConsumerReservationState.CANCELLED
|
||||
):
|
||||
return existing, False
|
||||
if (
|
||||
existing is not None
|
||||
and existing.state is ConsumerReservationState.EXPIRED
|
||||
):
|
||||
self._remove(transfer_id)
|
||||
existing = None
|
||||
if (
|
||||
existing is not None
|
||||
and existing.state is ConsumerReservationState.EXPIRE_PENDING
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"Transfer {transfer_id!r} still has an active writer"
|
||||
)
|
||||
if existing is not None:
|
||||
if (
|
||||
existing.mm_hash != mm_hash
|
||||
or existing.shape != shape
|
||||
or existing.dtype != dtype_name
|
||||
):
|
||||
raise ValueError("conflicting reservation for transfer_id")
|
||||
if existing.state not in _ACTIVE_STATES:
|
||||
raise RuntimeError(
|
||||
f"Cannot reserve transfer {transfer_id!r} in "
|
||||
f"{existing.state.name}"
|
||||
)
|
||||
if existing.state is ConsumerReservationState.WRITING:
|
||||
existing.expires_at = time.monotonic() + self._lease_ttl
|
||||
return existing, False
|
||||
|
||||
lease = self._memory.acquire_cached(mm_hash, shape, dtype)
|
||||
now = time.monotonic()
|
||||
if lease is not None:
|
||||
record = ConsumerReservation(
|
||||
transfer_id,
|
||||
mm_hash,
|
||||
uuid.uuid4().hex,
|
||||
ConsumerReservationState.READY,
|
||||
shape,
|
||||
dtype_name,
|
||||
lease.value,
|
||||
lease,
|
||||
now + self._lease_ttl,
|
||||
)
|
||||
self._insert(record)
|
||||
return record, False
|
||||
writer = self._writers.get(mm_hash)
|
||||
if writer is not None and writer.state is ConsumerReservationState.WRITING:
|
||||
if writer.shape != shape or writer.dtype != dtype_name:
|
||||
raise ValueError("conflicting in-flight tensor for mm_hash")
|
||||
record = ConsumerReservation(
|
||||
transfer_id,
|
||||
mm_hash,
|
||||
uuid.uuid4().hex,
|
||||
ConsumerReservationState.WRITING,
|
||||
shape,
|
||||
dtype_name,
|
||||
writer.allocation,
|
||||
expires_at=now + self._lease_ttl,
|
||||
writer_id=writer.transfer_id,
|
||||
)
|
||||
self._insert(record)
|
||||
self._followers.setdefault(writer.transfer_id, {})[transfer_id] = None
|
||||
return record, False
|
||||
allocation = self._memory.try_allocate(nbytes, shape, dtype)
|
||||
if allocation is None:
|
||||
self._expire_locked(time.monotonic())
|
||||
allocation = self._memory.try_allocate(nbytes, shape, dtype)
|
||||
if allocation is None:
|
||||
allocation = self._memory.reclaim_and_allocate(nbytes, shape, dtype)
|
||||
if allocation is None:
|
||||
return None, False
|
||||
record = ConsumerReservation(
|
||||
transfer_id,
|
||||
mm_hash,
|
||||
uuid.uuid4().hex,
|
||||
ConsumerReservationState.WRITING,
|
||||
shape,
|
||||
dtype_name,
|
||||
allocation,
|
||||
None,
|
||||
now + self._lease_ttl,
|
||||
)
|
||||
self._insert(record)
|
||||
self._writers[mm_hash] = record
|
||||
return record, True
|
||||
|
||||
def status(self, transfer_id: str) -> ConsumerReservation | None:
|
||||
with self._memory.lock:
|
||||
record = self._records.get(transfer_id)
|
||||
if record is None or record.state not in _ACTIVE_STATES:
|
||||
return None
|
||||
return record
|
||||
|
||||
def complete(self, transfer_id: str, reservation_id: str) -> tuple[bool, bool]:
|
||||
with self._condition:
|
||||
try:
|
||||
record = self._records.get(transfer_id)
|
||||
if record is None or record.reservation_id != reservation_id:
|
||||
return False, False
|
||||
if record.state is ConsumerReservationState.READY:
|
||||
return True, False
|
||||
if record.writer_id:
|
||||
return False, False
|
||||
if record.state in _WRITER_OWNED_STATES:
|
||||
self._publish_followers(record)
|
||||
if record.state in _DEFERRED_STATES:
|
||||
terminal = (
|
||||
ConsumerReservationState.CANCELLED
|
||||
if record.state is ConsumerReservationState.CANCEL_PENDING
|
||||
else ConsumerReservationState.EXPIRED
|
||||
)
|
||||
self._terminate(record, terminal)
|
||||
return True, False
|
||||
if record.state is not ConsumerReservationState.WRITING:
|
||||
return False, False
|
||||
self._transition(record, ConsumerReservationState.READY)
|
||||
record.expires_at = time.monotonic() + self._lease_ttl
|
||||
return True, True
|
||||
finally:
|
||||
self._condition.notify_all()
|
||||
|
||||
def _publish_followers(self, writer: ConsumerReservation) -> None:
|
||||
if self._writers.get(writer.mm_hash) is writer:
|
||||
self._writers.pop(writer.mm_hash)
|
||||
followers = self._followers.pop(writer.transfer_id, {})
|
||||
if not followers:
|
||||
return
|
||||
assert writer.allocation is not None
|
||||
allocation = self._memory.publish(writer.mm_hash, writer.allocation, pin=False)
|
||||
for record in [writer, *(self._records[key] for key in followers)]:
|
||||
record.allocation = allocation
|
||||
record.lease = self._memory.acquire_cached(
|
||||
record.mm_hash, record.shape, allocation.tensor.dtype
|
||||
)
|
||||
assert record.lease is not None
|
||||
if record is not writer:
|
||||
record.writer_id = ""
|
||||
self._transition(record, ConsumerReservationState.READY)
|
||||
record.expires_at = time.monotonic() + self._lease_ttl
|
||||
self._ready.append(record.transfer_id)
|
||||
|
||||
def drain_ready(self) -> list[str]:
|
||||
with self._memory.lock:
|
||||
ready, self._ready = self._ready, []
|
||||
return ready
|
||||
|
||||
def begin_shutdown(self) -> None:
|
||||
"""Stop new reservations and cancel everything without a remote writer."""
|
||||
with self._condition:
|
||||
if self._shutting_down:
|
||||
return
|
||||
self._shutting_down = True
|
||||
for transfer_id in list(self._active_ids):
|
||||
record = self._records[transfer_id]
|
||||
if record.writer_id or record.state is ConsumerReservationState.READY:
|
||||
self._terminate(record, ConsumerReservationState.CANCELLED)
|
||||
elif record.state is ConsumerReservationState.WRITING:
|
||||
self._defer(record, ConsumerReservationState.CANCEL_PENDING)
|
||||
self._condition.notify_all()
|
||||
|
||||
def wait_for_writers(self, timeout: float) -> bool:
|
||||
"""Wait until no reservation is still owned by a remote writer."""
|
||||
|
||||
def writers_finished() -> bool:
|
||||
return not any(
|
||||
self._records[transfer_id].state in _WRITER_OWNED_STATES
|
||||
for transfer_id in self._active_ids
|
||||
)
|
||||
|
||||
with self._condition:
|
||||
return self._condition.wait_for(writers_finished, timeout=max(0, timeout))
|
||||
|
||||
def cancel(
|
||||
self,
|
||||
transfer_id: str,
|
||||
reservation_id: str,
|
||||
abandon: bool = False,
|
||||
refresh: bool = False,
|
||||
) -> bool:
|
||||
with self._memory.lock:
|
||||
record = self._records.get(transfer_id)
|
||||
if (
|
||||
record is not None
|
||||
and reservation_id
|
||||
and record.reservation_id != reservation_id
|
||||
):
|
||||
return False
|
||||
if record is None:
|
||||
record = ConsumerReservation(
|
||||
transfer_id,
|
||||
"",
|
||||
"",
|
||||
ConsumerReservationState.CANCELLED,
|
||||
)
|
||||
self._insert(record)
|
||||
self._set_tombstone_deadline(record)
|
||||
self._reap_tombstones(time.monotonic())
|
||||
return True
|
||||
if record.state is ConsumerReservationState.CANCELLED:
|
||||
self._set_tombstone_deadline(record)
|
||||
self._reap_tombstones(time.monotonic())
|
||||
return True
|
||||
if refresh:
|
||||
if not abandon or record.state not in {
|
||||
ConsumerReservationState.WRITING,
|
||||
ConsumerReservationState.EXPIRE_PENDING,
|
||||
}:
|
||||
return False
|
||||
if record.state is ConsumerReservationState.WRITING:
|
||||
self._transition(record, ConsumerReservationState.EXPIRE_PENDING)
|
||||
self._terminate(record, ConsumerReservationState.EXPIRED)
|
||||
self._condition.notify_all()
|
||||
self._reap_tombstones(time.monotonic())
|
||||
return True
|
||||
if record.writer_id:
|
||||
self._terminate(record, ConsumerReservationState.CANCELLED)
|
||||
return True
|
||||
if record.state in _DEFERRED_STATES and not abandon:
|
||||
return True
|
||||
if record.state is ConsumerReservationState.WRITING and not abandon:
|
||||
self._defer(record, ConsumerReservationState.CANCEL_PENDING)
|
||||
return True
|
||||
if record.state not in _ACTIVE_STATES:
|
||||
return False
|
||||
self._terminate(record, ConsumerReservationState.CANCELLED)
|
||||
if self._shutting_down:
|
||||
self._condition.notify_all()
|
||||
self._reap_tombstones(time.monotonic())
|
||||
return True
|
||||
|
||||
def take(self, transfer_id: str, mm_hash: str) -> MemoryAllocation:
|
||||
with self._memory.lock:
|
||||
record = self._records.get(transfer_id)
|
||||
if (
|
||||
record is None
|
||||
or record.state is not ConsumerReservationState.READY
|
||||
or record.mm_hash != mm_hash
|
||||
or record.allocation is None
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"Pushed EC tensor is not ready for mm_hash={mm_hash}"
|
||||
)
|
||||
allocation = self._memory.publish(mm_hash, record.allocation, record.lease)
|
||||
record.allocation = None
|
||||
record.lease = None
|
||||
self._remove(transfer_id)
|
||||
return allocation
|
||||
|
||||
def expire(self) -> int:
|
||||
with self._memory.lock:
|
||||
return self._expire_locked(time.monotonic())
|
||||
|
||||
def retire_stale(
|
||||
self, encoder_cache: dict[str, torch.Tensor], freed: list[str] | None = None
|
||||
) -> None:
|
||||
with self._memory.lock:
|
||||
self._memory.retire_stale(encoder_cache, freed=freed)
|
||||
|
||||
def _expire_locked(self, now: float) -> int:
|
||||
expired = 0
|
||||
for transfer_id in list(self._active_ids):
|
||||
record = self._records[transfer_id]
|
||||
if record.expires_at > now:
|
||||
continue
|
||||
if record.state is ConsumerReservationState.READY:
|
||||
self._terminate(record, ConsumerReservationState.EXPIRED)
|
||||
expired += 1
|
||||
elif record.state is ConsumerReservationState.WRITING:
|
||||
if record.writer_id:
|
||||
self._terminate(record, ConsumerReservationState.CANCELLED)
|
||||
else:
|
||||
self._defer(record, ConsumerReservationState.EXPIRE_PENDING)
|
||||
self._reap_tombstones(now)
|
||||
return expired
|
||||
|
||||
def _defer(
|
||||
self, record: ConsumerReservation, state: ConsumerReservationState
|
||||
) -> None:
|
||||
"""Keep the destination until the writer completes or abandons it."""
|
||||
self._transition(record, state)
|
||||
record.expires_at = float("inf")
|
||||
|
||||
def _terminate(
|
||||
self, record: ConsumerReservation, state: ConsumerReservationState
|
||||
) -> None:
|
||||
self._transition(record, state)
|
||||
self._release(record)
|
||||
self._set_tombstone_deadline(record)
|
||||
|
||||
def _release(self, record: ConsumerReservation) -> None:
|
||||
if record.writer_id:
|
||||
followers = self._followers.get(record.writer_id, {})
|
||||
followers.pop(record.transfer_id, None)
|
||||
record.allocation = None
|
||||
return
|
||||
if self._writers.get(record.mm_hash) is record:
|
||||
self._writers.pop(record.mm_hash)
|
||||
for transfer_id in list(self._followers.pop(record.transfer_id, {})):
|
||||
self._terminate(
|
||||
self._records[transfer_id], ConsumerReservationState.CANCELLED
|
||||
)
|
||||
allocation = record.allocation
|
||||
if allocation is None:
|
||||
return
|
||||
if record.lease is not None:
|
||||
self._memory.release_cached(record.lease)
|
||||
else:
|
||||
self._memory.free(allocation)
|
||||
record.allocation = None
|
||||
record.lease = None
|
||||
|
||||
def _set_tombstone_deadline(self, record: ConsumerReservation) -> None:
|
||||
record.expires_at = time.monotonic() + self._lease_ttl
|
||||
self._tombstones[record.transfer_id] = None
|
||||
self._tombstones.move_to_end(record.transfer_id)
|
||||
|
||||
def _reap_tombstones(self, now: float) -> int:
|
||||
dropped = 0
|
||||
while self._tombstones:
|
||||
transfer_id = next(iter(self._tombstones))
|
||||
record = self._records[transfer_id]
|
||||
if (
|
||||
record.expires_at > now
|
||||
and len(self._tombstones) <= self._tombstone_limit
|
||||
):
|
||||
break
|
||||
self._remove(transfer_id)
|
||||
dropped += 1
|
||||
return dropped
|
||||
|
||||
def _insert(self, record: ConsumerReservation) -> None:
|
||||
self._records[record.transfer_id] = record
|
||||
if record.state in _ACTIVE_STATES:
|
||||
self._active_ids[record.transfer_id] = None
|
||||
|
||||
def _remove(self, transfer_id: str) -> None:
|
||||
self._records.pop(transfer_id, None)
|
||||
self._active_ids.pop(transfer_id, None)
|
||||
self._tombstones.pop(transfer_id, None)
|
||||
|
||||
def _transition(
|
||||
self, record: ConsumerReservation, state: ConsumerReservationState
|
||||
) -> None:
|
||||
if state not in _ALLOWED_TRANSITIONS[record.state]:
|
||||
raise RuntimeError(
|
||||
f"Cannot transition {record.transfer_id!r} from "
|
||||
f"{record.state.name} to {state.name}"
|
||||
)
|
||||
record.state = state
|
||||
if state in _ACTIVE_STATES:
|
||||
self._active_ids[record.transfer_id] = None
|
||||
else:
|
||||
self._active_ids.pop(record.transfer_id, None)
|
||||
@@ -1,561 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Scheduler-side planning and observation for Mooncake cache transfers.
|
||||
|
||||
The Scheduler prepares Producer reservations, consumes Consumer readiness
|
||||
events, emits per-step Worker metadata, and converts Worker results back into
|
||||
request availability without owning tensor memory or running data transfers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
|
||||
get_encoder_output_width,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
|
||||
_RESERVATION_TTL_SECONDS,
|
||||
MooncakeECConfig,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.control import (
|
||||
ControlClient,
|
||||
EventInbox,
|
||||
make_cancel_request,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
|
||||
ECMooncakeConnectorMetadata,
|
||||
ECMooncakeLoadSpec,
|
||||
ECMooncakePushSpec,
|
||||
ECMooncakeWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.state import (
|
||||
SchedulerTransferState,
|
||||
SchedulerTransferTable,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
|
||||
ensure_mooncake_available,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
|
||||
TensorSpec,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
PlaceholderMetadataResolver,
|
||||
collect_ec_item_metadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ECConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Unmatched `ec_items` entries are a protocol desync, not a per-request
|
||||
# event: warn on the first few, then only once in a while.
|
||||
_MAX_UNRESOLVED_TRANSFER_ID_WARNINGS = 5
|
||||
|
||||
_DRAIN_MIN_INTERVAL = 0.005
|
||||
_MAX_PENDING_EVENTS = 4096
|
||||
_MAX_TERMINAL_TRANSFER_RECORDS = 1 << 16
|
||||
_CANCEL_ATTEMPTS = 2
|
||||
_CONTROL_WORKERS = 8
|
||||
|
||||
|
||||
class ECMooncakeScheduler:
|
||||
"""Coordinate Mooncake transfers from the vLLM Scheduler process."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig) -> None:
|
||||
ensure_mooncake_available()
|
||||
config = MooncakeECConfig.from_vllm_config(vllm_config)
|
||||
|
||||
self._store_candidates: list[tuple[str, str, TensorSpec]] = []
|
||||
self._cross_encoder_cache = config.cross_encoder_cache
|
||||
self._is_producer = config.is_producer
|
||||
self._is_consumer = config.is_consumer
|
||||
self._control_addr = config.control_addr
|
||||
self._push_wait_timeout = config.push_wait_timeout_s
|
||||
self._vllm_config = vllm_config
|
||||
self._model_config = vllm_config.model_config
|
||||
self._control_client = ControlClient(config.control_timeout_ms)
|
||||
self._control_executor = ThreadPoolExecutor(
|
||||
max_workers=_CONTROL_WORKERS,
|
||||
thread_name_prefix="ec-mooncake-control",
|
||||
)
|
||||
self._event_inbox = EventInbox(self._control_client, self._control_executor)
|
||||
|
||||
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
|
||||
self._unresolved_transfer_ids = 0
|
||||
self._drain_pending = True
|
||||
self._drained_at = 0.0
|
||||
self._pending_cancels: dict[str, Future[Any]] = {}
|
||||
self._transfers = SchedulerTransferTable(
|
||||
config.pool_size, _RESERVATION_TTL_SECONDS
|
||||
)
|
||||
self._scheduler_pending_work = False
|
||||
self._pushes_to_prepare: dict[str, ECMooncakePushSpec] = {}
|
||||
self._prepared_push_transfer_ids: set[str] = set()
|
||||
self._event_ready_shards: OrderedDict[str, set[int]] = OrderedDict()
|
||||
# Mirror of the engine's encoder cache, maintained from the alloc and
|
||||
# free notifications the Scheduler already sends. Tracking it here
|
||||
# keeps `ensure_cache_available` on the upstream two-argument shape.
|
||||
self._local_cache: set[str] = set()
|
||||
self._failed_saves: set[str] = set()
|
||||
|
||||
def _cancel_remote(self, consumer_zmq: str, transfer_id: str) -> bool:
|
||||
pending = None
|
||||
for _ in range(_CANCEL_ATTEMPTS):
|
||||
pending = self._control_client.discover_shards(consumer_zmq)
|
||||
if pending is not None:
|
||||
break
|
||||
if pending is None:
|
||||
raise RuntimeError(
|
||||
f"Could not discover every EC consumer shard at {consumer_zmq}"
|
||||
)
|
||||
|
||||
cancelled = False
|
||||
error: BaseException | None = None
|
||||
for _ in range(_CANCEL_ATTEMPTS):
|
||||
failed = []
|
||||
for addr in pending:
|
||||
try:
|
||||
result = self._control_client.request(
|
||||
addr,
|
||||
make_cancel_request(transfer_id),
|
||||
)
|
||||
except Exception as exc:
|
||||
if error is None:
|
||||
error = exc
|
||||
failed.append(addr)
|
||||
continue
|
||||
cancelled |= isinstance(result, dict) and bool(result.get("cancelled"))
|
||||
if not failed:
|
||||
return cancelled
|
||||
pending = failed
|
||||
if error is not None:
|
||||
raise error
|
||||
return cancelled
|
||||
|
||||
def _note_awaiting_push(
|
||||
self,
|
||||
mm_hash: str,
|
||||
transfer_id: str,
|
||||
request_id: str,
|
||||
now: float,
|
||||
) -> None:
|
||||
record = self._transfers.wait_for_event(
|
||||
transfer_id,
|
||||
request_id,
|
||||
mm_hash,
|
||||
now + self._push_wait_timeout,
|
||||
)
|
||||
if record is None:
|
||||
# The id names another encoding; the request is already failed.
|
||||
return
|
||||
if record.state is not SchedulerTransferState.WAITING_EVENT:
|
||||
return
|
||||
assert record.deadline is not None
|
||||
if now < record.deadline:
|
||||
return
|
||||
elapsed = now - record.deadline + self._push_wait_timeout
|
||||
self._transfers.mark_unavailable(transfer_id, now)
|
||||
logger.warning(
|
||||
"EC Mooncake waited %.1fs for a push of mm_hash=%s "
|
||||
"(transfer_id=%s) that never arrived; "
|
||||
"requests needing it fail with a retryable error.",
|
||||
elapsed,
|
||||
mm_hash,
|
||||
transfer_id,
|
||||
)
|
||||
|
||||
def take_unavailable_requests(self) -> set[str]:
|
||||
failed, self._failed_saves = self._failed_saves, set()
|
||||
return failed | self._transfers.take_unavailable_requests()
|
||||
|
||||
def _poll_pending_cancels(self) -> None:
|
||||
pending = {}
|
||||
for transfer_id, future in self._pending_cancels.items():
|
||||
if not future.done():
|
||||
pending[transfer_id] = future
|
||||
continue
|
||||
try:
|
||||
future.result()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"EC Mooncake reservation cancellation failed", exc_info=True
|
||||
)
|
||||
self._pending_cancels = pending
|
||||
|
||||
def _note_shard_ready(self, data: dict[str, Any]) -> bool:
|
||||
shard_count = self._event_inbox.shard_count
|
||||
if shard_count <= 1:
|
||||
return True
|
||||
transfer_id = str(data["transfer_id"])
|
||||
record = self._transfers.get(transfer_id)
|
||||
if (
|
||||
record is not None
|
||||
and record.state is not SchedulerTransferState.WAITING_EVENT
|
||||
):
|
||||
return False
|
||||
shard = data.get("shard")
|
||||
shards = self._event_ready_shards.setdefault(transfer_id, set())
|
||||
self._event_ready_shards.move_to_end(transfer_id)
|
||||
shards.add(int(shard) if shard is not None else len(shards))
|
||||
if len(shards) < shard_count:
|
||||
while len(self._event_ready_shards) > _MAX_PENDING_EVENTS:
|
||||
self._event_ready_shards.popitem(last=False)
|
||||
return False
|
||||
self._event_ready_shards.pop(transfer_id, None)
|
||||
return True
|
||||
|
||||
def _forget_shard_readiness(self, transfer_id: str) -> None:
|
||||
self._event_ready_shards.pop(transfer_id, None)
|
||||
|
||||
def _store_pushed_spec(self, data: dict[str, Any]) -> bool:
|
||||
transfer_id = str(data["transfer_id"])
|
||||
identifier = str(data["mm_hash"])
|
||||
_, accepted = self._transfers.observe_ready(
|
||||
ECMooncakeLoadSpec(
|
||||
mm_hash=identifier,
|
||||
nbytes=int(data["nbytes"]),
|
||||
shape=tuple(int(value) for value in data["shape"]),
|
||||
dtype=str(data["dtype"]),
|
||||
transfer_id=transfer_id,
|
||||
),
|
||||
time.monotonic() + _RESERVATION_TTL_SECONDS,
|
||||
)
|
||||
return accepted
|
||||
|
||||
def _queue_cancel(
|
||||
self,
|
||||
transfer_id: str,
|
||||
mm_hash: str = "",
|
||||
request_id: str = "",
|
||||
) -> None:
|
||||
if not self._transfers.cancel(
|
||||
transfer_id,
|
||||
time.monotonic(),
|
||||
mm_hash=mm_hash,
|
||||
request_id=request_id,
|
||||
):
|
||||
return
|
||||
self._forget_shard_readiness(transfer_id)
|
||||
self._pending_cancels[transfer_id] = self._control_executor.submit(
|
||||
self._cancel_remote,
|
||||
self._control_addr,
|
||||
transfer_id,
|
||||
)
|
||||
|
||||
def _expire_transfers(self) -> None:
|
||||
now = time.monotonic()
|
||||
expired = self._transfers.expire(now, _MAX_TERMINAL_TRANSFER_RECORDS)
|
||||
for record in expired:
|
||||
self._queue_cancel(record.transfer_id)
|
||||
|
||||
def _drain_push_notifications(self) -> None:
|
||||
now = time.monotonic()
|
||||
if not self._drain_pending and now - self._drained_at < _DRAIN_MIN_INTERVAL:
|
||||
return
|
||||
self._drain_pending = False
|
||||
self._drained_at = now
|
||||
self._poll_pending_cancels()
|
||||
self._expire_transfers()
|
||||
events = self._event_inbox.drain(self._control_addr)
|
||||
for data in events:
|
||||
try:
|
||||
if not isinstance(data, dict):
|
||||
raise TypeError("EC readiness event must be an object")
|
||||
if not data.get("ready"):
|
||||
continue
|
||||
self._accept_ready_event(data)
|
||||
except (KeyError, TypeError, ValueError):
|
||||
# Readiness events cross a plain PULL socket, so a malformed
|
||||
# one costs that event and not the engine.
|
||||
logger.warning(
|
||||
"Discarding a malformed EC readiness event.", exc_info=True
|
||||
)
|
||||
|
||||
def _accept_ready_event(self, data: dict[str, Any]) -> None:
|
||||
transfer_id = str(data["transfer_id"])
|
||||
record = self._transfers.get(transfer_id)
|
||||
if record is not None and record.state in {
|
||||
SchedulerTransferState.CANCELLED,
|
||||
SchedulerTransferState.UNAVAILABLE,
|
||||
SchedulerTransferState.EXPIRED,
|
||||
SchedulerTransferState.FAILED,
|
||||
}:
|
||||
return
|
||||
if not self._note_shard_ready(data):
|
||||
return
|
||||
self._store_pushed_spec(data)
|
||||
|
||||
def has_cache_item(self, identifier: str) -> bool:
|
||||
if not self._is_consumer:
|
||||
return False
|
||||
self._drain_push_notifications()
|
||||
return self._transfers.has_state(
|
||||
identifier,
|
||||
(
|
||||
SchedulerTransferState.READY,
|
||||
SchedulerTransferState.RESIDENT,
|
||||
SchedulerTransferState.AVAILABLE,
|
||||
),
|
||||
)
|
||||
|
||||
def _warn_unresolved_transfer_id(
|
||||
self, request: Any, index: int, where: str
|
||||
) -> None:
|
||||
"""Report an `ec_items` entry that cannot be matched to a feature.
|
||||
|
||||
A caller that cannot resolve a transfer id has to fall back to a
|
||||
locally invented one or skip its bookkeeping entirely, and either way
|
||||
the two sides of a transfer stop agreeing on its name. That desyncs
|
||||
silently -- reservations are never released and only surface minutes
|
||||
later as a full buffer pool -- so say it out loud the first time.
|
||||
|
||||
`mm_hash` in `ec_items` must be a value some engine reported, never one
|
||||
the caller derived itself: `mm_features[i].identifier` folds in the
|
||||
engine's media_io_kwargs and mm_processor_kwargs, so a media uuid alone
|
||||
no longer equals it. Omit the field to match by position instead.
|
||||
"""
|
||||
self._unresolved_transfer_ids += 1
|
||||
count = self._unresolved_transfer_ids
|
||||
if count > _MAX_UNRESOLVED_TRANSFER_ID_WARNINGS and count % 1000:
|
||||
return
|
||||
params = getattr(request, "ec_transfer_params", None)
|
||||
items = (params or {}).get("ec_items") or []
|
||||
logger.warning(
|
||||
"EC Mooncake could not resolve a transfer id at %s for req=%s "
|
||||
"index=%d: feature identifier=%s does not match any of the %d "
|
||||
"ec_items %s (ec_transfer_params present=%s). Occurrence %d. "
|
||||
"ec_items[].mm_hash must echo an engine-reported identifier, or "
|
||||
"be omitted so the entry is matched by position.",
|
||||
where,
|
||||
request.request_id,
|
||||
index,
|
||||
request.mm_features[index].identifier[:16],
|
||||
len(items),
|
||||
[str(item.get("mm_hash"))[:16] for item in items[:4]],
|
||||
params is not None,
|
||||
count,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _request_transfer_id(request: Any, index: int) -> str | None:
|
||||
params = getattr(request, "ec_transfer_params", None) or {}
|
||||
items = params.get("ec_items") or []
|
||||
mm_hash = request.mm_features[index].identifier
|
||||
if index < len(items):
|
||||
item = items[index]
|
||||
if item.get("mm_hash") in (None, mm_hash) and item.get("transfer_id"):
|
||||
return str(item["transfer_id"])
|
||||
for item in items:
|
||||
if item.get("mm_hash") == mm_hash and item.get("transfer_id"):
|
||||
return str(item["transfer_id"])
|
||||
return None
|
||||
|
||||
def ensure_cache_available(self, request: Any, num_computed_tokens: int) -> bool:
|
||||
if self._is_producer:
|
||||
for index, feature in enumerate(request.mm_features):
|
||||
if (
|
||||
feature.mm_position.offset + feature.mm_position.length
|
||||
> num_computed_tokens
|
||||
):
|
||||
self._prepare_push_spec(request, index)
|
||||
if not self._is_consumer:
|
||||
return True
|
||||
|
||||
self._drain_push_notifications()
|
||||
# One timestamp for the whole decision, so the deadlines it hands out
|
||||
# cannot disagree between features.
|
||||
now = time.monotonic()
|
||||
all_ready = True
|
||||
for index, feature in enumerate(request.mm_features):
|
||||
if (
|
||||
feature.mm_position.offset + feature.mm_position.length
|
||||
<= num_computed_tokens
|
||||
):
|
||||
continue
|
||||
mm_hash = feature.identifier
|
||||
transfer_id = self._request_transfer_id(request, index)
|
||||
if transfer_id is not None:
|
||||
self._transfers.touch_available(
|
||||
transfer_id, now + _RESERVATION_TTL_SECONDS
|
||||
)
|
||||
if mm_hash in self._local_cache:
|
||||
continue
|
||||
if self._transfers.has_state(mm_hash, (SchedulerTransferState.READY,)):
|
||||
continue
|
||||
if self._transfers.has_state(mm_hash, (SchedulerTransferState.LOADING,)):
|
||||
all_ready = False
|
||||
continue
|
||||
record = self._transfers.begin_load(
|
||||
mm_hash,
|
||||
transfer_id,
|
||||
request.request_id,
|
||||
now + self._push_wait_timeout,
|
||||
)
|
||||
if record is not None:
|
||||
self._scheduler_pending_work = True
|
||||
all_ready = False
|
||||
else:
|
||||
waiting_id = transfer_id or f"{request.request_id}:{index}"
|
||||
self._note_awaiting_push(mm_hash, waiting_id, request.request_id, now)
|
||||
all_ready = False
|
||||
return all_ready
|
||||
|
||||
def _prepare_push_spec(self, request: Any, index: int) -> None:
|
||||
params = getattr(request, "ec_transfer_params", None) or {}
|
||||
consumer_zmq = params.get("consumer_zmq")
|
||||
mm_hash = request.mm_features[index].identifier
|
||||
transfer_id = self._request_transfer_id(request, index)
|
||||
if transfer_id is None:
|
||||
# Still push: after the proxy has rewritten the item to embeds the
|
||||
# consumer has no media left to fall back on, so a nameless push
|
||||
# beats none. But the consumer knows this transfer by the id it
|
||||
# sent, not by the one invented here, so its cancel will never
|
||||
# reach the reservation this push is about to take.
|
||||
if consumer_zmq:
|
||||
self._warn_unresolved_transfer_id(request, index, "push prepare")
|
||||
transfer_id = f"{request.request_id}:{index}"
|
||||
if not consumer_zmq or transfer_id in self._prepared_push_transfer_ids:
|
||||
return
|
||||
output = self._encoder_output_spec(request, index)
|
||||
self._pushes_to_prepare[transfer_id] = ECMooncakePushSpec(
|
||||
mm_hash=mm_hash,
|
||||
nbytes=output.nbytes,
|
||||
shape=output.shape,
|
||||
dtype=output.dtype.removeprefix("torch."),
|
||||
consumer_zmq=str(consumer_zmq),
|
||||
transfer_id=transfer_id,
|
||||
request_id=request.request_id,
|
||||
)
|
||||
self._prepared_push_transfer_ids.add(transfer_id)
|
||||
|
||||
def _encoder_output_spec(self, request: Any, index: int) -> TensorSpec:
|
||||
dtype = self._model_config.dtype
|
||||
assert isinstance(dtype, torch.dtype)
|
||||
hidden_dim = get_encoder_output_width(
|
||||
self._vllm_config, request.mm_features[index].modality
|
||||
)
|
||||
shape = (request.get_num_encoder_embeds(index), hidden_dim)
|
||||
return TensorSpec(shape, str(dtype), math.prod(shape) * dtype.itemsize)
|
||||
|
||||
def update_state_after_alloc(self, request: Any, index: int) -> None:
|
||||
self._local_cache.add(request.mm_features[index].identifier)
|
||||
if self._is_producer:
|
||||
self._prepare_push_spec(request, index)
|
||||
if (
|
||||
self._cross_encoder_cache
|
||||
and request.mm_features[index].modality == "image"
|
||||
):
|
||||
self._store_candidates.append(
|
||||
(
|
||||
request.request_id,
|
||||
request.mm_features[index].identifier,
|
||||
self._encoder_output_spec(request, index),
|
||||
)
|
||||
)
|
||||
|
||||
def update_state_after_free(self, request: Any, index: int) -> None:
|
||||
if not self._is_consumer:
|
||||
return
|
||||
transfer_id = self._request_transfer_id(request, index)
|
||||
if transfer_id is None:
|
||||
self._warn_unresolved_transfer_id(request, index, "encoder-cache free")
|
||||
return
|
||||
self._queue_cancel(
|
||||
transfer_id,
|
||||
mm_hash=request.mm_features[index].identifier,
|
||||
request_id=request.request_id,
|
||||
)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> ECConnectorMetadata:
|
||||
for mm_hash in scheduler_output.free_encoder_mm_hashes:
|
||||
self._local_cache.discard(mm_hash)
|
||||
self._transfers.release_ready(mm_hash, time.monotonic())
|
||||
for transfer_id in self._transfers.drain_orphaned():
|
||||
self._queue_cancel(transfer_id)
|
||||
meta = ECMooncakeConnectorMetadata(
|
||||
freed=scheduler_output.free_encoder_mm_hashes
|
||||
)
|
||||
preempted = scheduler_output.preempted_req_ids or set()
|
||||
meta.store_candidates = {
|
||||
identifier: output
|
||||
for request_id, identifier, output in self._store_candidates
|
||||
if request_id not in preempted
|
||||
}
|
||||
self._store_candidates.clear()
|
||||
for push_spec in self._pushes_to_prepare.values():
|
||||
meta.pushes.append(push_spec)
|
||||
self._pushes_to_prepare.clear()
|
||||
for record in self._transfers.take_loads_to_dispatch():
|
||||
assert record.spec is not None
|
||||
meta.loads.append(record.spec)
|
||||
self._poll_pending_cancels()
|
||||
self._drain_pending = True
|
||||
return meta
|
||||
|
||||
def update_connector_output(self, connector_output: ECConnectorOutput) -> None:
|
||||
meta = connector_output.ec_connector_worker_meta
|
||||
if not isinstance(meta, ECMooncakeWorkerMetadata):
|
||||
return
|
||||
for mm_hash in meta.loaded:
|
||||
self._transfers.complete_load(mm_hash)
|
||||
for mm_hash in meta.failed_loads:
|
||||
self._transfers.fail_load(mm_hash, time.monotonic())
|
||||
for mm_hash in meta.reclaimed:
|
||||
self._transfers.reclaim(mm_hash, time.monotonic())
|
||||
self._scheduler_pending_work = meta.pending_saves
|
||||
self._failed_saves.update(meta.failed_saves)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
return self._scheduler_pending_work
|
||||
|
||||
def request_finished(self, request: Any) -> tuple[bool, dict[str, Any] | None]:
|
||||
if self._is_consumer:
|
||||
for index in range(len(request.mm_features)):
|
||||
transfer_id = self._request_transfer_id(request, index)
|
||||
if transfer_id is None:
|
||||
self._warn_unresolved_transfer_id(request, index, "request finish")
|
||||
continue
|
||||
self._queue_cancel(
|
||||
transfer_id,
|
||||
mm_hash=request.mm_features[index].identifier,
|
||||
request_id=request.request_id,
|
||||
)
|
||||
if self._is_producer and self._prepared_push_transfer_ids:
|
||||
for index in range(len(request.mm_features)):
|
||||
transfer_id = self._request_transfer_id(request, index)
|
||||
if transfer_id is None:
|
||||
transfer_id = f"{request.request_id}:{index}"
|
||||
self._prepared_push_transfer_ids.discard(transfer_id)
|
||||
if not self._is_producer:
|
||||
return False, None
|
||||
|
||||
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
|
||||
for index, feature in enumerate(request.mm_features):
|
||||
transfer_id = self._request_transfer_id(request, index)
|
||||
if transfer_id is not None:
|
||||
items[feature.identifier]["transfer_id"] = transfer_id
|
||||
|
||||
if not items:
|
||||
return False, None
|
||||
return False, items
|
||||
|
||||
def close(self) -> None:
|
||||
self._control_executor.shutdown(wait=True, cancel_futures=True)
|
||||
self._control_client.close()
|
||||
self._event_inbox.close()
|
||||
@@ -1,448 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Scheduler-owned lifecycle and indexes for Consumer-bound transfers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict, deque
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum, auto
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
|
||||
ECMooncakeLoadSpec,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SchedulerTransferState(Enum):
|
||||
"""Lifecycle states visible to the Scheduler.
|
||||
|
||||
The states cover waiting for a Consumer event, dispatching a Worker load,
|
||||
retaining a locally reusable tensor, and terminal failure conditions.
|
||||
"""
|
||||
|
||||
WAITING_EVENT = auto()
|
||||
AVAILABLE = auto()
|
||||
LOADING = auto()
|
||||
READY = auto()
|
||||
RESIDENT = auto()
|
||||
UNAVAILABLE = auto()
|
||||
EXPIRED = auto()
|
||||
FAILED = auto()
|
||||
CANCELLED = auto()
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchedulerTransfer:
|
||||
"""Track one transfer as observed by the Scheduler."""
|
||||
|
||||
transfer_id: str
|
||||
request_id: str
|
||||
mm_hash: str
|
||||
state: SchedulerTransferState
|
||||
spec: ECMooncakeLoadSpec | None
|
||||
deadline: float | None
|
||||
notified_requests: set[str] = field(default_factory=set, repr=False)
|
||||
|
||||
|
||||
_TERMINAL_STATES = {
|
||||
SchedulerTransferState.UNAVAILABLE,
|
||||
SchedulerTransferState.EXPIRED,
|
||||
SchedulerTransferState.FAILED,
|
||||
SchedulerTransferState.CANCELLED,
|
||||
}
|
||||
|
||||
|
||||
class SchedulerTransferTable:
|
||||
"""Own Scheduler transfer state, lookup indexes, and dispatch queues.
|
||||
|
||||
The Scheduler is single-threaded, so direct transitions are sufficient;
|
||||
the Producer and Consumer managers retain stricter transition matrices
|
||||
because callbacks and control requests can race there.
|
||||
"""
|
||||
|
||||
def __init__(self, resident_capacity: int, tombstone_ttl: float) -> None:
|
||||
self._resident_capacity = resident_capacity
|
||||
self._tombstone_ttl = tombstone_ttl
|
||||
self._records: OrderedDict[str, SchedulerTransfer] = OrderedDict()
|
||||
self._active_ids: dict[str, None] = {}
|
||||
self._terminal_ids: OrderedDict[str, None] = OrderedDict()
|
||||
self._resident_ids: OrderedDict[str, None] = OrderedDict()
|
||||
self._resident_bytes = 0
|
||||
self._hash_index: dict[str, deque[str]] = {}
|
||||
self._loads_to_dispatch: OrderedDict[str, None] = OrderedDict()
|
||||
self._unavailable_requests: set[str] = set()
|
||||
# Records expired straight out of READY/RESIDENT: the consumer worker
|
||||
# still holds their push reservation, and nothing else tells it to let
|
||||
# go, so the destination buffer would sit pinned until the lease TTL.
|
||||
self._orphaned: list[str] = []
|
||||
|
||||
def get(self, transfer_id: str) -> SchedulerTransfer | None:
|
||||
return self._records.get(transfer_id)
|
||||
|
||||
def drain_orphaned(self) -> list[str]:
|
||||
"""Transfer IDs expired out of READY/RESIDENT since the last drain.
|
||||
|
||||
The caller must release the consumer worker's reservation for each:
|
||||
`cancel()` still accepts an EXPIRED record, so the usual cancel path
|
||||
applies.
|
||||
"""
|
||||
orphaned = self._orphaned
|
||||
self._orphaned = []
|
||||
return orphaned
|
||||
|
||||
def records_for_hash(
|
||||
self,
|
||||
mm_hash: str,
|
||||
states: Iterable[SchedulerTransferState],
|
||||
) -> list[SchedulerTransfer]:
|
||||
wanted = set(states)
|
||||
return [
|
||||
record
|
||||
for transfer_id in self._hash_index.get(mm_hash, ())
|
||||
if (record := self._records.get(transfer_id)) is not None
|
||||
and record.state in wanted
|
||||
]
|
||||
|
||||
def first_for_hash(
|
||||
self,
|
||||
mm_hash: str,
|
||||
states: Iterable[SchedulerTransferState],
|
||||
) -> SchedulerTransfer | None:
|
||||
states = tuple(states)
|
||||
if len(states) == 1:
|
||||
wanted_state = states[0]
|
||||
for transfer_id in self._hash_index.get(mm_hash, ()):
|
||||
record = self._records.get(transfer_id)
|
||||
if record is not None and record.state is wanted_state:
|
||||
return record
|
||||
return None
|
||||
wanted_states = set(states)
|
||||
for transfer_id in self._hash_index.get(mm_hash, ()):
|
||||
record = self._records.get(transfer_id)
|
||||
if record is not None and record.state in wanted_states:
|
||||
return record
|
||||
return None
|
||||
|
||||
def has_state(self, mm_hash: str, states: Iterable[SchedulerTransferState]) -> bool:
|
||||
return self.first_for_hash(mm_hash, states) is not None
|
||||
|
||||
def wait_for_event(
|
||||
self,
|
||||
transfer_id: str,
|
||||
request_id: str,
|
||||
mm_hash: str,
|
||||
deadline: float,
|
||||
) -> SchedulerTransfer | None:
|
||||
"""Track a request waiting for a push.
|
||||
|
||||
Returns None when `transfer_id` already names a different encoding.
|
||||
The id comes from the request, so a collision has to fail that one
|
||||
request; re-issuing it re-runs the encode under a fresh id.
|
||||
"""
|
||||
record = self._records.get(transfer_id)
|
||||
if record is None:
|
||||
record = SchedulerTransfer(
|
||||
transfer_id=transfer_id,
|
||||
request_id=request_id,
|
||||
mm_hash=mm_hash,
|
||||
state=SchedulerTransferState.WAITING_EVENT,
|
||||
spec=None,
|
||||
deadline=deadline,
|
||||
)
|
||||
self._insert(record)
|
||||
elif not self._identity_matches(record, mm_hash):
|
||||
self._refuse_colliding_id(record, mm_hash, request_id)
|
||||
return None
|
||||
else:
|
||||
if not record.request_id:
|
||||
record.request_id = request_id
|
||||
if record.state in _TERMINAL_STATES and request_id:
|
||||
self._notify_unavailable(record, request_id)
|
||||
return record
|
||||
|
||||
def observe_ready(
|
||||
self, spec: ECMooncakeLoadSpec, deadline: float
|
||||
) -> tuple[SchedulerTransfer, bool]:
|
||||
transfer_id = spec.transfer_id
|
||||
record = self._records.get(transfer_id)
|
||||
if record is None:
|
||||
record = SchedulerTransfer(
|
||||
transfer_id=transfer_id,
|
||||
request_id="",
|
||||
mm_hash=spec.mm_hash,
|
||||
state=SchedulerTransferState.WAITING_EVENT,
|
||||
spec=None,
|
||||
deadline=None,
|
||||
)
|
||||
self._insert(record)
|
||||
elif not self._identity_matches(record, spec.mm_hash):
|
||||
self._refuse_colliding_id(record, spec.mm_hash, record.request_id)
|
||||
return record, False
|
||||
if record.state is not SchedulerTransferState.WAITING_EVENT:
|
||||
return record, False
|
||||
record.spec = spec
|
||||
record.deadline = deadline
|
||||
self._transition(record, SchedulerTransferState.AVAILABLE)
|
||||
return record, True
|
||||
|
||||
def touch_available(self, transfer_id: str, deadline: float) -> None:
|
||||
record = self._records.get(transfer_id)
|
||||
if record is not None and record.state is SchedulerTransferState.AVAILABLE:
|
||||
record.deadline = deadline
|
||||
|
||||
def begin_load(
|
||||
self,
|
||||
mm_hash: str,
|
||||
transfer_id: str | None = None,
|
||||
request_id: str = "",
|
||||
deadline: float | None = None,
|
||||
) -> SchedulerTransfer | None:
|
||||
record = self._records.get(transfer_id) if transfer_id else None
|
||||
if record is not None and (
|
||||
record.mm_hash != mm_hash
|
||||
or record.state is not SchedulerTransferState.AVAILABLE
|
||||
):
|
||||
record = None
|
||||
if record is None:
|
||||
record = self.first_for_hash(
|
||||
mm_hash,
|
||||
(
|
||||
SchedulerTransferState.AVAILABLE,
|
||||
SchedulerTransferState.RESIDENT,
|
||||
),
|
||||
)
|
||||
if record is None or record.spec is None:
|
||||
return None
|
||||
if not record.request_id:
|
||||
record.request_id = request_id
|
||||
# A dispatched load that never reports back would otherwise hold this
|
||||
# hash in LOADING for good, deferring every later request for it.
|
||||
record.deadline = deadline
|
||||
self._transition(record, SchedulerTransferState.LOADING)
|
||||
self._loads_to_dispatch[record.transfer_id] = None
|
||||
return record
|
||||
|
||||
def take_loads_to_dispatch(self) -> list[SchedulerTransfer]:
|
||||
records = [
|
||||
record
|
||||
for transfer_id in self._loads_to_dispatch
|
||||
if (record := self._records.get(transfer_id)) is not None
|
||||
and record.state is SchedulerTransferState.LOADING
|
||||
]
|
||||
self._loads_to_dispatch.clear()
|
||||
return records
|
||||
|
||||
def complete_load(self, mm_hash: str) -> bool:
|
||||
record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,))
|
||||
if record is None:
|
||||
return self.has_state(mm_hash, (SchedulerTransferState.READY,))
|
||||
self._transition(record, SchedulerTransferState.READY)
|
||||
return True
|
||||
|
||||
def fail_load(self, mm_hash: str, now: float) -> bool:
|
||||
record = self.first_for_hash(mm_hash, (SchedulerTransferState.LOADING,))
|
||||
if record is None:
|
||||
return self.has_state(mm_hash, (SchedulerTransferState.FAILED,))
|
||||
self._transition(record, SchedulerTransferState.FAILED, now=now)
|
||||
return True
|
||||
|
||||
def release_ready(self, mm_hash: str, now: float) -> None:
|
||||
ready = self.records_for_hash(mm_hash, (SchedulerTransferState.READY,))
|
||||
if ready:
|
||||
canonical = ready[-1]
|
||||
for record in self.records_for_hash(
|
||||
mm_hash,
|
||||
(SchedulerTransferState.READY, SchedulerTransferState.RESIDENT),
|
||||
):
|
||||
if record is not canonical:
|
||||
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
|
||||
if canonical.spec is None:
|
||||
self._transition(canonical, SchedulerTransferState.EXPIRED, now=now)
|
||||
else:
|
||||
canonical.spec = replace(canonical.spec, local=True)
|
||||
self._transition(canonical, SchedulerTransferState.RESIDENT)
|
||||
self._evict_residents(now)
|
||||
|
||||
def reclaim(self, mm_hash: str, now: float) -> None:
|
||||
for record in self.records_for_hash(
|
||||
mm_hash,
|
||||
(SchedulerTransferState.READY, SchedulerTransferState.RESIDENT),
|
||||
):
|
||||
if record.state is SchedulerTransferState.READY:
|
||||
record.spec = None
|
||||
else:
|
||||
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
|
||||
|
||||
def mark_unavailable(self, transfer_id: str, now: float) -> None:
|
||||
record = self._records[transfer_id]
|
||||
self._transition(record, SchedulerTransferState.UNAVAILABLE, now=now)
|
||||
if record.request_id:
|
||||
self._notify_unavailable(record, record.request_id)
|
||||
|
||||
def cancel(
|
||||
self,
|
||||
transfer_id: str,
|
||||
now: float,
|
||||
mm_hash: str = "",
|
||||
request_id: str = "",
|
||||
) -> bool:
|
||||
record = self._records.get(transfer_id)
|
||||
if (
|
||||
record is not None
|
||||
and mm_hash
|
||||
and not self._identity_matches(record, mm_hash)
|
||||
):
|
||||
return False
|
||||
if record is None:
|
||||
record = SchedulerTransfer(
|
||||
transfer_id=transfer_id,
|
||||
request_id=request_id,
|
||||
mm_hash=mm_hash,
|
||||
state=SchedulerTransferState.WAITING_EVENT,
|
||||
spec=None,
|
||||
deadline=None,
|
||||
)
|
||||
self._insert(record)
|
||||
if record.state in {
|
||||
SchedulerTransferState.CANCELLED,
|
||||
SchedulerTransferState.READY,
|
||||
SchedulerTransferState.RESIDENT,
|
||||
SchedulerTransferState.FAILED,
|
||||
}:
|
||||
return False
|
||||
self._transition(record, SchedulerTransferState.CANCELLED, now=now)
|
||||
self._loads_to_dispatch.pop(transfer_id, None)
|
||||
return True
|
||||
|
||||
def expire(self, now: float, terminal_limit: int) -> list[SchedulerTransfer]:
|
||||
if terminal_limit < 0:
|
||||
raise ValueError("terminal_limit must be non-negative")
|
||||
expired = []
|
||||
for transfer_id in list(self._active_ids):
|
||||
record = self._records[transfer_id]
|
||||
if record.deadline is None or record.deadline > now:
|
||||
continue
|
||||
if record.state is SchedulerTransferState.AVAILABLE:
|
||||
self._transition(
|
||||
record,
|
||||
SchedulerTransferState.EXPIRED,
|
||||
now=now,
|
||||
)
|
||||
expired.append(record)
|
||||
elif record.state is SchedulerTransferState.LOADING:
|
||||
self._transition(
|
||||
record,
|
||||
SchedulerTransferState.UNAVAILABLE,
|
||||
now=now,
|
||||
)
|
||||
if record.request_id:
|
||||
self._notify_unavailable(record, record.request_id)
|
||||
expired.append(record)
|
||||
while self._terminal_ids:
|
||||
transfer_id = next(iter(self._terminal_ids))
|
||||
record = self._records[transfer_id]
|
||||
if (
|
||||
record.deadline is not None
|
||||
and record.deadline > now
|
||||
and len(self._terminal_ids) <= terminal_limit
|
||||
):
|
||||
break
|
||||
self._remove(transfer_id)
|
||||
return expired
|
||||
|
||||
def take_unavailable_requests(self) -> set[str]:
|
||||
unavailable = self._unavailable_requests
|
||||
self._unavailable_requests = set()
|
||||
return unavailable
|
||||
|
||||
def _notify_unavailable(self, record: SchedulerTransfer, request_id: str) -> None:
|
||||
if request_id not in record.notified_requests:
|
||||
record.notified_requests.add(request_id)
|
||||
self._unavailable_requests.add(request_id)
|
||||
|
||||
def _insert(self, record: SchedulerTransfer) -> None:
|
||||
self._records[record.transfer_id] = record
|
||||
self._active_ids[record.transfer_id] = None
|
||||
if record.mm_hash:
|
||||
self._hash_index.setdefault(record.mm_hash, deque()).append(
|
||||
record.transfer_id
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _identity_matches(record: SchedulerTransfer, mm_hash: str) -> bool:
|
||||
return not record.mm_hash or record.mm_hash == mm_hash
|
||||
|
||||
def _refuse_colliding_id(
|
||||
self, record: SchedulerTransfer, mm_hash: str, request_id: str
|
||||
) -> None:
|
||||
"""Report a transfer id that already names another encoding.
|
||||
|
||||
Transfer ids arrive on the request, so a collision is reachable from
|
||||
outside and must not reach the engine as an exception.
|
||||
"""
|
||||
logger.warning(
|
||||
"EC Mooncake transfer_id=%s already names mm_hash=%s; refusing "
|
||||
"mm_hash=%s for request %s",
|
||||
record.transfer_id,
|
||||
record.mm_hash[:16],
|
||||
mm_hash[:16],
|
||||
request_id or "<unknown>",
|
||||
)
|
||||
if request_id:
|
||||
self._unavailable_requests.add(request_id)
|
||||
|
||||
def _transition(
|
||||
self,
|
||||
record: SchedulerTransfer,
|
||||
state: SchedulerTransferState,
|
||||
now: float | None = None,
|
||||
) -> None:
|
||||
if record.state is SchedulerTransferState.RESIDENT:
|
||||
self._resident_ids.pop(record.transfer_id, None)
|
||||
if record.spec is not None:
|
||||
self._resident_bytes -= record.spec.nbytes
|
||||
if state is SchedulerTransferState.RESIDENT:
|
||||
self._resident_ids[record.transfer_id] = None
|
||||
if record.spec is not None:
|
||||
self._resident_bytes += record.spec.nbytes
|
||||
if state in _TERMINAL_STATES:
|
||||
if now is None:
|
||||
raise ValueError("Terminal transition requires a timestamp")
|
||||
# Keep a bounded tombstone so late events and repeat request IDs
|
||||
# remain idempotent instead of reviving a finished transfer.
|
||||
record.deadline = now + self._tombstone_ttl
|
||||
self._active_ids.pop(record.transfer_id, None)
|
||||
self._terminal_ids[record.transfer_id] = None
|
||||
self._terminal_ids.move_to_end(record.transfer_id)
|
||||
if state is SchedulerTransferState.EXPIRED and record.state in (
|
||||
SchedulerTransferState.READY,
|
||||
SchedulerTransferState.RESIDENT,
|
||||
):
|
||||
self._orphaned.append(record.transfer_id)
|
||||
record.state = state
|
||||
self._records.move_to_end(record.transfer_id)
|
||||
|
||||
def _evict_residents(self, now: float) -> None:
|
||||
while self._resident_bytes > self._resident_capacity and self._resident_ids:
|
||||
record = self._records[next(iter(self._resident_ids))]
|
||||
self._transition(record, SchedulerTransferState.EXPIRED, now=now)
|
||||
|
||||
def _remove(self, transfer_id: str) -> None:
|
||||
record = self._records.pop(transfer_id, None)
|
||||
self._active_ids.pop(transfer_id, None)
|
||||
self._terminal_ids.pop(transfer_id, None)
|
||||
if transfer_id in self._resident_ids:
|
||||
self._resident_ids.pop(transfer_id)
|
||||
if record is not None and record.spec is not None:
|
||||
self._resident_bytes -= record.spec.nbytes
|
||||
self._loads_to_dispatch.pop(transfer_id, None)
|
||||
if record is None or not record.mm_hash:
|
||||
return
|
||||
transfer_ids = self._hash_index[record.mm_hash]
|
||||
transfer_ids.remove(transfer_id)
|
||||
if not transfer_ids:
|
||||
self._hash_index.pop(record.mm_hash)
|
||||
@@ -1,209 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Mooncake data-plane engine and memory-registration ownership.
|
||||
|
||||
The wrapper isolates optional-engine initialization, session addressing,
|
||||
synchronous batched writes, and the lifetime of transient source registrations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_MOONCAKE_IMPORT_ERROR: ImportError | None = None
|
||||
try:
|
||||
from mooncake.engine import TransferEngine
|
||||
except ImportError as error:
|
||||
TransferEngine = None # type: ignore[misc, assignment]
|
||||
_MOONCAKE_IMPORT_ERROR = error
|
||||
|
||||
|
||||
def ensure_mooncake_available() -> None:
|
||||
if _MOONCAKE_IMPORT_ERROR is not None:
|
||||
raise ImportError(
|
||||
"Install mooncake-transfer-engine to use ECMooncakeConnector."
|
||||
) from _MOONCAKE_IMPORT_ERROR
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SourceRegistration:
|
||||
"""Retain one transient source range while batches reference it."""
|
||||
|
||||
tensor: torch.Tensor
|
||||
nbytes: int
|
||||
users: int = 1
|
||||
|
||||
|
||||
class MooncakeTransfer:
|
||||
"""Own a lazy Mooncake engine and transient memory registrations."""
|
||||
|
||||
def __init__(self, hostname: str, protocol: str) -> None:
|
||||
self._hostname = hostname
|
||||
self._protocol = protocol
|
||||
self._engine: TransferEngine | None = None
|
||||
self._engine_lock = threading.Lock()
|
||||
self._source_registrations: dict[int, _SourceRegistration] = {}
|
||||
self._pending_unregister: dict[int, torch.Tensor] = {}
|
||||
self._registration_lock = threading.Lock()
|
||||
self._closed = False
|
||||
|
||||
def _ensure_engine(self) -> TransferEngine:
|
||||
if self._engine is not None:
|
||||
return self._engine
|
||||
with self._engine_lock:
|
||||
if self._engine is not None:
|
||||
return self._engine
|
||||
engine = TransferEngine()
|
||||
ret = engine.initialize(self._hostname, "P2PHANDSHAKE", self._protocol, "")
|
||||
if ret != 0:
|
||||
raise RuntimeError("Mooncake TransferEngine initialization failed.")
|
||||
self._engine = engine
|
||||
logger.info(
|
||||
"ECMooncakeConnector TransferEngine ready at %s:%d",
|
||||
self._hostname,
|
||||
engine.get_rpc_port(),
|
||||
)
|
||||
return self._engine
|
||||
|
||||
def ensure_ready(self) -> None:
|
||||
self._ensure_engine()
|
||||
|
||||
def local_session(self) -> str:
|
||||
engine = self._ensure_engine()
|
||||
return f"{self._hostname}:{engine.get_rpc_port()}"
|
||||
|
||||
def register_memory(self, tensor: torch.Tensor) -> int:
|
||||
return self._ensure_engine().batch_register_memory(
|
||||
[tensor.data_ptr()], [tensor.nbytes]
|
||||
)
|
||||
|
||||
def unregister_memory(self, tensor: torch.Tensor) -> bool:
|
||||
engine = self._ensure_engine()
|
||||
address = tensor.data_ptr()
|
||||
ret = engine.unregister_memory(address)
|
||||
with self._registration_lock:
|
||||
if ret != 0:
|
||||
logger.error(
|
||||
"Mooncake EC memory unregistration failed for address %d: %d",
|
||||
address,
|
||||
ret,
|
||||
)
|
||||
self._pending_unregister[address] = tensor
|
||||
return False
|
||||
self._pending_unregister.pop(address, None)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _source_range(tensor: torch.Tensor) -> tuple[int, int]:
|
||||
# Encoder batches commonly split one storage into sibling tensor views.
|
||||
# Registering each view's exact bytes avoids overlapping memory regions.
|
||||
return tensor.data_ptr(), tensor.nbytes
|
||||
|
||||
def acquire_sources(self, tensors: list[torch.Tensor]) -> list[int]:
|
||||
ranges: dict[int, tuple[int, torch.Tensor]] = {}
|
||||
for tensor in tensors:
|
||||
address, nbytes = self._source_range(tensor)
|
||||
ranges.setdefault(address, (nbytes, tensor))
|
||||
|
||||
engine = self._ensure_engine()
|
||||
acquired: list[int] = []
|
||||
new_addresses: list[int] = []
|
||||
new_lengths: list[int] = []
|
||||
with self._registration_lock:
|
||||
for address, (nbytes, tensor) in ranges.items():
|
||||
entry = self._source_registrations.get(address)
|
||||
if entry is not None:
|
||||
if entry.nbytes != nbytes:
|
||||
raise RuntimeError(
|
||||
"Mooncake EC source storage changed size while registered"
|
||||
)
|
||||
entry.users += 1
|
||||
acquired.append(address)
|
||||
continue
|
||||
new_addresses.append(address)
|
||||
new_lengths.append(nbytes)
|
||||
self._source_registrations[address] = _SourceRegistration(
|
||||
tensor=tensor,
|
||||
nbytes=nbytes,
|
||||
)
|
||||
acquired.append(address)
|
||||
|
||||
if new_addresses:
|
||||
ret = engine.batch_register_memory(new_addresses, new_lengths)
|
||||
if ret != 0:
|
||||
for address in acquired:
|
||||
entry = self._source_registrations[address]
|
||||
entry.users -= 1
|
||||
if entry.users == 0:
|
||||
del self._source_registrations[address]
|
||||
raise RuntimeError("Mooncake EC source registration failed")
|
||||
return acquired
|
||||
|
||||
def release_sources(self, addresses: list[int]) -> bool:
|
||||
if not addresses:
|
||||
return True
|
||||
with self._registration_lock:
|
||||
unused = []
|
||||
for address in addresses:
|
||||
entry = self._source_registrations.get(address)
|
||||
if entry is None:
|
||||
continue
|
||||
entry.users -= 1
|
||||
if entry.users == 0:
|
||||
unused.append(address)
|
||||
if not unused:
|
||||
return True
|
||||
ret = self._ensure_engine().batch_unregister_memory(unused)
|
||||
if ret != 0:
|
||||
logger.warning(
|
||||
"Keeping %d EC source tensors registered after Mooncake "
|
||||
"unregistration failure",
|
||||
len(unused),
|
||||
)
|
||||
return False
|
||||
for address in unused:
|
||||
del self._source_registrations[address]
|
||||
self._pending_unregister.pop(address, None)
|
||||
return True
|
||||
|
||||
def write(
|
||||
self,
|
||||
session: str,
|
||||
sources: list[int],
|
||||
destinations: list[int],
|
||||
lengths: list[int],
|
||||
) -> None:
|
||||
"""Write one synchronous batch, returning only at terminal status."""
|
||||
ret = self._ensure_engine().batch_transfer_sync_write(
|
||||
session, sources, destinations, lengths
|
||||
)
|
||||
if ret != 0:
|
||||
raise RuntimeError(
|
||||
f"Mooncake EC push to {session} failed with status {ret}"
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
engine = self._engine
|
||||
if engine is None:
|
||||
return
|
||||
with self._registration_lock:
|
||||
addresses = list(self._source_registrations)
|
||||
addresses.extend(self._pending_unregister)
|
||||
if not addresses:
|
||||
return
|
||||
ret = engine.batch_unregister_memory(list(dict.fromkeys(addresses)))
|
||||
if ret != 0:
|
||||
logger.error("Mooncake EC batch memory unregistration failed: %d", ret)
|
||||
return
|
||||
self._source_registrations.clear()
|
||||
self._pending_unregister.clear()
|
||||
@@ -1,988 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Worker-side orchestration of Mooncake control, memory, and data planes.
|
||||
|
||||
Consumer Workers expose rank-local reservations and publish received tensors.
|
||||
Producer Workers reserve every destination shard, bind computed sources, run
|
||||
batched Mooncake writes, and report asynchronous completion to the Scheduler.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any, TypeVar, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
|
||||
_RESERVATION_TTL_SECONDS,
|
||||
MooncakeECConfig,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.control import (
|
||||
ConsumerControlServer,
|
||||
ControlClient,
|
||||
make_cancel_request,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.memory import (
|
||||
ConsumerMemoryPool,
|
||||
ProducerMemoryPool,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.metadata import (
|
||||
ECMooncakeConnectorMetadata,
|
||||
ECMooncakePushSpec,
|
||||
ECMooncakeWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.producer import (
|
||||
ProducerPushManager,
|
||||
ProducerPushRecord,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.reservation import (
|
||||
ConsumerReservationManager,
|
||||
ConsumerReservationState,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import (
|
||||
MooncakeTransfer,
|
||||
ensure_mooncake_available,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
|
||||
build_embedding_namespace,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import get_ip
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
from ..mooncake_store_embedding.backend import (
|
||||
MooncakeEmbeddingStoreBackend,
|
||||
)
|
||||
|
||||
_RESERVATION_REFRESH_SECONDS = _RESERVATION_TTL_SECONDS / 2
|
||||
_MAX_CANCELLED_TRANSFER_IDS = 1 << 16
|
||||
_CANCEL_ATTEMPTS = 2
|
||||
_TRANSFER_WORKERS = 4
|
||||
_CONTROL_WORKERS = 8
|
||||
|
||||
|
||||
class _FanoutError(RuntimeError):
|
||||
"""Retain shard outcomes after every started task settles."""
|
||||
|
||||
def __init__(self, error: BaseException, results: list[Any | None]) -> None:
|
||||
self.results = results
|
||||
super().__init__(str(error))
|
||||
|
||||
|
||||
class ECMooncakeWorker:
|
||||
"""Orchestrate consumer reservations and producer push batches.
|
||||
|
||||
Consumers may use TP, PP, and DP. Only the first PP stage receives encoder
|
||||
outputs, and each TP rank exposes a consecutive control port and receives
|
||||
the same source concurrently. Producers remain unsharded and unreplicated.
|
||||
With DP, the caller must route both halves of a request to the same replica
|
||||
and pass that replica's control address to the producer.
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig) -> None:
|
||||
ensure_mooncake_available()
|
||||
config = MooncakeECConfig.from_vllm_config(vllm_config)
|
||||
self._store_config = (
|
||||
(
|
||||
build_embedding_namespace(vllm_config),
|
||||
config.store_max_pending_items,
|
||||
config.store_max_pending_bytes,
|
||||
config.store_read_buffer_bytes,
|
||||
)
|
||||
if config.cross_encoder_cache
|
||||
else None
|
||||
)
|
||||
self._output_store: MooncakeEmbeddingStoreBackend | None = None
|
||||
self.is_producer = config.is_producer
|
||||
self.is_consumer = config.is_consumer
|
||||
self._buffer_device = config.buffer_device
|
||||
self._control_host = config.control_host
|
||||
self._control_port = config.control_port
|
||||
self._transfer = MooncakeTransfer(get_ip(), config.protocol)
|
||||
self._consumer_memory = ConsumerMemoryPool(
|
||||
config.pool_size,
|
||||
self._transfer,
|
||||
)
|
||||
self._reservations = ConsumerReservationManager(
|
||||
self._consumer_memory,
|
||||
_RESERVATION_TTL_SECONDS,
|
||||
_MAX_CANCELLED_TRANSFER_IDS,
|
||||
)
|
||||
self._consumer_rank_resolved = False
|
||||
self._is_receiving_rank = True
|
||||
self._tp_rank = 0
|
||||
self._tp_size = 1
|
||||
self._control_server: ConsumerControlServer | None = None
|
||||
# Worker producer
|
||||
self._producer_memory = ProducerMemoryPool(
|
||||
config.pool_size,
|
||||
self._transfer,
|
||||
)
|
||||
self._control_client = ControlClient(config.control_timeout_ms)
|
||||
self._shutdown_drain_timeout_s = config.control_timeout_ms / 1000
|
||||
self._io_executor = ThreadPoolExecutor(
|
||||
max_workers=_TRANSFER_WORKERS,
|
||||
thread_name_prefix="ec-mooncake-transfer",
|
||||
)
|
||||
self._control_executor = ThreadPoolExecutor(
|
||||
max_workers=_CONTROL_WORKERS,
|
||||
thread_name_prefix="ec-mooncake-control",
|
||||
)
|
||||
self._shard_pool: ThreadPoolExecutor | None = None
|
||||
self._shard_pool_lock = threading.Lock()
|
||||
self._push_ready = threading.Event()
|
||||
self._producer_pushes = ProducerPushManager(self._push_ready.set)
|
||||
self._dispatch_stop = threading.Event()
|
||||
self._dispatcher: threading.Thread | None = None
|
||||
self._failed_saves: set[str] = set()
|
||||
self._collecting_sources = False
|
||||
self._completed_loads: set[str] = set()
|
||||
self._failed_loads: set[str] = set()
|
||||
self._shutdown = False
|
||||
if self.is_producer:
|
||||
self._dispatcher = threading.Thread(
|
||||
target=self._dispatch_pushes, name="ec-mooncake-ready", daemon=True
|
||||
)
|
||||
self._dispatcher.start()
|
||||
|
||||
def _dispatch_pushes(self) -> None:
|
||||
pending_event = False
|
||||
while not self._dispatch_stop.is_set():
|
||||
self._push_ready.wait(timeout=0.001 if pending_event else None)
|
||||
self._push_ready.clear()
|
||||
if self._dispatch_stop.is_set():
|
||||
break
|
||||
if self._collecting_sources:
|
||||
pending_event = False
|
||||
continue
|
||||
pending_event = self._producer_pushes.submit_batches(
|
||||
self._io_executor, self._push_batch
|
||||
)
|
||||
|
||||
def _resolve_consumer_rank(self) -> None:
|
||||
"""Place this worker in the consumer receive topology."""
|
||||
if self._consumer_rank_resolved:
|
||||
return
|
||||
self._consumer_rank_resolved = True
|
||||
try:
|
||||
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
|
||||
|
||||
tp_group = get_tp_group()
|
||||
self._tp_rank = tp_group.rank_in_group
|
||||
self._tp_size = tp_group.world_size
|
||||
self._is_receiving_rank = get_pp_group().is_first_rank
|
||||
except AssertionError:
|
||||
# Groups are only absent outside a distributed run, where this
|
||||
# worker is the whole consumer.
|
||||
self._tp_rank = 0
|
||||
self._tp_size = 1
|
||||
self._is_receiving_rank = True
|
||||
|
||||
def start_services(self) -> None:
|
||||
if self._store_config is not None and self._output_store is None:
|
||||
from ..mooncake_store_embedding.backend import (
|
||||
MooncakeEmbeddingStoreBackend,
|
||||
)
|
||||
from ..mooncake_store_embedding.store_client import (
|
||||
create_mooncake_embedding_store_client,
|
||||
)
|
||||
|
||||
namespace, max_items, max_bytes, read_bytes = self._store_config
|
||||
self._output_store = MooncakeEmbeddingStoreBackend(
|
||||
create_mooncake_embedding_store_client(read_buffer_bytes=read_bytes),
|
||||
namespace,
|
||||
max_pending_items=max_items,
|
||||
max_pending_bytes=max_bytes,
|
||||
)
|
||||
if not self.is_consumer or self._control_server is not None:
|
||||
return
|
||||
self._resolve_consumer_rank()
|
||||
if not self._is_receiving_rank:
|
||||
# Later pipeline stages hold no encoder outputs, so they need
|
||||
# neither a receive pool nor a control channel.
|
||||
return
|
||||
self._consumer_memory.prepare(torch.device(self._buffer_device))
|
||||
consumer_pool = self._consumer_memory.tensor
|
||||
if consumer_pool is None:
|
||||
raise RuntimeError(
|
||||
"Mooncake push mode requires a registered consumer buffer pool."
|
||||
)
|
||||
base_port = self._control_port
|
||||
self._control_server = ConsumerControlServer(
|
||||
# The reservation channel hands out registered-memory addresses
|
||||
# and takes cancellations, so it listens where the Consumer
|
||||
# advertises itself (`ec_ip`), not on every interface.
|
||||
self._control_host,
|
||||
base_port + self._tp_rank,
|
||||
self._reserve_push_destination,
|
||||
self._push_status,
|
||||
self._reservations.complete,
|
||||
self._reservations.cancel,
|
||||
self._reservations.expire,
|
||||
peer_ports=[base_port + rank for rank in range(self._tp_size)],
|
||||
device=consumer_pool.device,
|
||||
drain_ready=self._reservations.drain_ready,
|
||||
)
|
||||
try:
|
||||
self._control_server.start()
|
||||
except Exception:
|
||||
self._control_server.close()
|
||||
self._control_server = None
|
||||
raise
|
||||
|
||||
def _reserve_push_destination(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
transfer_id = str(payload["transfer_id"])
|
||||
mm_hash = str(payload["mm_hash"])
|
||||
nbytes = int(payload["nbytes"])
|
||||
shape = tuple(int(value) for value in payload["shape"])
|
||||
dtype_name = str(payload["dtype"])
|
||||
dtype = getattr(torch, dtype_name, None)
|
||||
if not isinstance(dtype, torch.dtype):
|
||||
raise ValueError(f"Unsupported torch dtype string: {dtype_name!r}")
|
||||
expected_nbytes = math.prod(shape) * dtype.itemsize
|
||||
if expected_nbytes != nbytes:
|
||||
raise ValueError("shape and dtype do not match nbytes")
|
||||
|
||||
self._reservations.expire()
|
||||
reservation, should_write = self._reservations.reserve(
|
||||
transfer_id, mm_hash, nbytes, shape, dtype_name, dtype
|
||||
)
|
||||
if reservation is None:
|
||||
raise RuntimeError("EC consumer buffer pool is full")
|
||||
if reservation.state in {
|
||||
ConsumerReservationState.CANCEL_PENDING,
|
||||
ConsumerReservationState.CANCELLED,
|
||||
}:
|
||||
return {
|
||||
"reservation_id": "",
|
||||
"dst_session": "",
|
||||
"dst_ptr": 0,
|
||||
"nbytes": nbytes,
|
||||
"write": False,
|
||||
"ready": False,
|
||||
"cancelled": True,
|
||||
}
|
||||
assert reservation.allocation is not None
|
||||
|
||||
return {
|
||||
"reservation_id": reservation.reservation_id,
|
||||
"dst_session": self._transfer.local_session(),
|
||||
"dst_ptr": reservation.allocation.tensor.data_ptr(),
|
||||
"nbytes": reservation.allocation.tensor.nbytes,
|
||||
"write": should_write,
|
||||
"ready": reservation.state is ConsumerReservationState.READY,
|
||||
"cached": reservation.lease is not None,
|
||||
}
|
||||
|
||||
def _push_status(self, transfer_id: str) -> dict[str, Any] | None:
|
||||
reservation = self._reservations.status(transfer_id)
|
||||
if reservation is None:
|
||||
return None
|
||||
assert reservation.allocation is not None
|
||||
return {
|
||||
"mm_hash": reservation.mm_hash,
|
||||
"ready": reservation.state is ConsumerReservationState.READY,
|
||||
"reservation_id": reservation.reservation_id,
|
||||
"nbytes": reservation.allocation.tensor.nbytes,
|
||||
"shape": list(reservation.shape),
|
||||
"dtype": reservation.dtype,
|
||||
}
|
||||
|
||||
def _shard_executor(self) -> ThreadPoolExecutor:
|
||||
"""Use a separate pool so nested shard fan-out cannot deadlock."""
|
||||
with self._shard_pool_lock:
|
||||
if self._shard_pool is None:
|
||||
self._shard_pool = ThreadPoolExecutor(
|
||||
max_workers=32, thread_name_prefix="ec-mooncake-shard"
|
||||
)
|
||||
return self._shard_pool
|
||||
|
||||
def _reserve_one(self, addr: str, spec: ECMooncakePushSpec) -> dict[str, Any]:
|
||||
result = self._control_client.request(
|
||||
addr,
|
||||
{
|
||||
"op": "reserve",
|
||||
"transfer_id": spec.transfer_id,
|
||||
"mm_hash": spec.mm_hash,
|
||||
"nbytes": spec.nbytes,
|
||||
"shape": list(spec.shape),
|
||||
"dtype": spec.dtype,
|
||||
},
|
||||
)
|
||||
if not isinstance(result, dict):
|
||||
raise RuntimeError("Invalid EC reservation response")
|
||||
result["_received_at"] = time.monotonic()
|
||||
result["addr"] = addr
|
||||
return result
|
||||
|
||||
def _run_fanout(
|
||||
self,
|
||||
tasks: list[Callable[[], _T]],
|
||||
on_submit: Callable[[int, Future[_T]], None] | None = None,
|
||||
) -> list[_T]:
|
||||
"""Run every started shard task and retain partial results on failure.
|
||||
|
||||
Waiting for all submitted tasks is what makes source-memory release and
|
||||
partial-reservation cleanup safe after one shard fails.
|
||||
"""
|
||||
if not tasks:
|
||||
return []
|
||||
futures: list[tuple[int, Future[_T]]] = []
|
||||
results: list[_T | None] = [None] * len(tasks)
|
||||
error: BaseException | None = None
|
||||
for index, task in enumerate(tasks[1:], 1):
|
||||
try:
|
||||
future = self._shard_executor().submit(task)
|
||||
except Exception as exc:
|
||||
error = exc
|
||||
break
|
||||
futures.append((index, future))
|
||||
if on_submit is not None:
|
||||
on_submit(index, future)
|
||||
if error is None:
|
||||
try:
|
||||
results[0] = tasks[0]()
|
||||
except Exception as exc:
|
||||
error = exc
|
||||
for index, future in futures:
|
||||
try:
|
||||
results[index] = future.result()
|
||||
except Exception as exc:
|
||||
if error is None:
|
||||
error = exc
|
||||
if error is not None:
|
||||
raise _FanoutError(error, results)
|
||||
return cast(list[_T], results)
|
||||
|
||||
def _cancel_reservations(
|
||||
self,
|
||||
spec: ECMooncakePushSpec,
|
||||
reservations: list[dict[str, Any]],
|
||||
*,
|
||||
refresh: bool = False,
|
||||
record: ProducerPushRecord | None = None,
|
||||
) -> None:
|
||||
reservations = [
|
||||
shard for shard in reservations if not shard.get("cancelled", False)
|
||||
]
|
||||
if not reservations:
|
||||
return
|
||||
|
||||
def cancel(shard: dict[str, Any]) -> dict[str, Any]:
|
||||
addr = shard.get("addr")
|
||||
if not isinstance(addr, str) or not addr:
|
||||
raise RuntimeError(
|
||||
"EC reservation is missing a confirmed shard address"
|
||||
)
|
||||
result = self._control_client.request(
|
||||
addr,
|
||||
make_cancel_request(
|
||||
spec.transfer_id,
|
||||
str(shard.get("reservation_id", "")),
|
||||
abandon=True,
|
||||
refresh=refresh,
|
||||
),
|
||||
)
|
||||
if not isinstance(result, dict) or not result.get("cancelled"):
|
||||
raise RuntimeError(
|
||||
f"Could not cancel EC reservation for mm_hash={spec.mm_hash}"
|
||||
)
|
||||
return shard
|
||||
|
||||
def track(_index: int, future: Future[dict[str, Any]]) -> None:
|
||||
if record is not None:
|
||||
self._producer_pushes.track_shard_futures([record], [future])
|
||||
|
||||
self._run_fanout([partial(cancel, shard) for shard in reservations], track)
|
||||
|
||||
def _retry_cancel_reservations(
|
||||
self,
|
||||
spec: ECMooncakePushSpec,
|
||||
reservations: list[dict[str, Any]],
|
||||
*,
|
||||
record: ProducerPushRecord | None = None,
|
||||
) -> None:
|
||||
pending = [shard for shard in reservations if not shard.get("cancelled", False)]
|
||||
error: _FanoutError | None = None
|
||||
for _ in range(_CANCEL_ATTEMPTS):
|
||||
try:
|
||||
self._cancel_reservations(spec, pending, record=record)
|
||||
except _FanoutError as exc:
|
||||
error = exc
|
||||
pending = [
|
||||
shard
|
||||
for index, shard in enumerate(pending)
|
||||
if exc.results[index] is None
|
||||
]
|
||||
continue
|
||||
return
|
||||
assert error is not None
|
||||
raise error
|
||||
|
||||
def _reserve_remote(self, spec: ECMooncakePushSpec) -> list[dict[str, Any]]:
|
||||
"""Reserve a destination on every shard of the consumer."""
|
||||
result = self._reserve_remote_many([spec])[0]
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
return result
|
||||
|
||||
def _refresh_remote_reservations(
|
||||
self,
|
||||
spec: ECMooncakePushSpec,
|
||||
reservations: list[dict[str, Any]],
|
||||
record: ProducerPushRecord | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
stale = [
|
||||
shard
|
||||
for shard in reservations
|
||||
if not shard.get("ready", False)
|
||||
and not shard.get("cached", False)
|
||||
and not shard.get("cancelled", False)
|
||||
]
|
||||
try:
|
||||
self._cancel_reservations(spec, stale, refresh=True, record=record)
|
||||
except _FanoutError as exc:
|
||||
pending = [
|
||||
shard for index, shard in enumerate(stale) if exc.results[index] is None
|
||||
]
|
||||
try:
|
||||
self._retry_cancel_reservations(spec, pending, record=record)
|
||||
except _FanoutError as cleanup_error:
|
||||
raise exc from cleanup_error
|
||||
raise
|
||||
return self._reserve_remote(spec)
|
||||
|
||||
@staticmethod
|
||||
def _validate_push_source(push: ProducerPushRecord) -> None:
|
||||
tensor = push.source_tensor
|
||||
assert tensor is not None
|
||||
spec = push.spec
|
||||
if tuple(tensor.shape) != tuple(spec.shape):
|
||||
raise ValueError(f"EC source shape mismatch for mm_hash={spec.mm_hash}")
|
||||
if str(tensor.dtype).split(".")[-1] != spec.dtype:
|
||||
raise ValueError(f"EC source dtype mismatch for mm_hash={spec.mm_hash}")
|
||||
if not tensor.is_contiguous():
|
||||
raise ValueError(f"EC source must be contiguous for mm_hash={spec.mm_hash}")
|
||||
if tensor.nbytes != spec.nbytes:
|
||||
raise ValueError(f"EC source size mismatch for mm_hash={spec.mm_hash}")
|
||||
|
||||
def start_save_caches(
|
||||
self,
|
||||
metadata: ECMooncakeConnectorMetadata,
|
||||
encoder_cache: dict[str, torch.Tensor] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
self._collecting_sources = True
|
||||
new: dict[str, list[ProducerPushRecord]] = {}
|
||||
for spec in metadata.pushes:
|
||||
try:
|
||||
record, created = self._producer_pushes.reserve(spec, Future)
|
||||
if created:
|
||||
new.setdefault(spec.consumer_zmq, []).append(record)
|
||||
except ValueError:
|
||||
logger.warning("Rejected conflicting EC push", exc_info=True)
|
||||
self._failed_saves.add(spec.request_id)
|
||||
for records in new.values():
|
||||
future = self._control_executor.submit(self._reserve_batch, records)
|
||||
future.add_done_callback(partial(self._reservation_batch_done, records))
|
||||
if not isinstance(encoder_cache, dict):
|
||||
return
|
||||
for mm_hash in dict.fromkeys(spec.mm_hash for spec in metadata.pushes):
|
||||
tensor = encoder_cache.get(mm_hash)
|
||||
if tensor is not None:
|
||||
self._bind_push_source(tensor, mm_hash)
|
||||
if self._output_store is not None:
|
||||
loaded = self._output_store.resolve_inputs(
|
||||
metadata.store_candidates, encoder_cache, self._buffer_device
|
||||
)
|
||||
# Loaded entries precede the runner's new-output snapshot, so they
|
||||
# need explicit P2P source binding here.
|
||||
for mm_hash in loaded:
|
||||
self._bind_push_source(encoder_cache[mm_hash], mm_hash)
|
||||
|
||||
def _reserve_batch(self, records: list[ProducerPushRecord]) -> None:
|
||||
"""Resolve independent item futures with one reserve RPC per shard."""
|
||||
if len(records) == 1:
|
||||
record = records[0]
|
||||
try:
|
||||
record.reservation_future.set_result(self._reserve_remote(record.spec))
|
||||
except Exception as exc:
|
||||
record.reservation_future.set_exception(exc)
|
||||
return
|
||||
outcomes = self._reserve_remote_many([record.spec for record in records])
|
||||
self._producer_pushes.finish_reservations(list(zip(records, outcomes)))
|
||||
|
||||
def _reserve_remote_many(
|
||||
self, specs: list[ECMooncakePushSpec]
|
||||
) -> list[list[dict[str, Any]] | Exception]:
|
||||
shards = None
|
||||
for _ in range(_CANCEL_ATTEMPTS):
|
||||
shards = self._control_client.discover_shards(specs[0].consumer_zmq)
|
||||
if shards is not None:
|
||||
break
|
||||
if shards is None:
|
||||
raise RuntimeError(
|
||||
f"Could not discover every EC consumer shard at {specs[0].consumer_zmq}"
|
||||
)
|
||||
|
||||
def reserve(addr: str) -> list[dict[str, Any]]:
|
||||
if len(specs) == 1:
|
||||
return [{"ok": True, "result": self._reserve_one(addr, specs[0])}]
|
||||
response = self._control_client.request(
|
||||
addr,
|
||||
{
|
||||
"op": "reserve_batch",
|
||||
"items": [
|
||||
{
|
||||
"transfer_id": spec.transfer_id,
|
||||
"mm_hash": spec.mm_hash,
|
||||
"nbytes": spec.nbytes,
|
||||
"shape": list(spec.shape),
|
||||
"dtype": spec.dtype,
|
||||
}
|
||||
for spec in specs
|
||||
],
|
||||
},
|
||||
)
|
||||
items = response["items"]
|
||||
if len(items) != len(specs):
|
||||
raise RuntimeError("Malformed EC reservation batch response")
|
||||
for item in items:
|
||||
if item.get("ok"):
|
||||
item["result"]["_received_at"] = time.monotonic()
|
||||
return items
|
||||
|
||||
fanout_error = None
|
||||
results: list[Any]
|
||||
try:
|
||||
results = self._run_fanout([partial(reserve, addr) for addr in shards])
|
||||
except _FanoutError as exc:
|
||||
results = exc.results
|
||||
fanout_error = exc
|
||||
outcomes: list[list[dict[str, Any]] | Exception] = []
|
||||
for index, spec in enumerate(specs):
|
||||
reservations: list[dict[str, Any]] = []
|
||||
error = None
|
||||
for addr, items in zip(shards, results):
|
||||
item = items[index] if items is not None else {}
|
||||
if item.get("ok"):
|
||||
reservations.append({**item["result"], "addr": addr})
|
||||
else:
|
||||
error = fanout_error or RuntimeError(item["error"])
|
||||
reservations.append({"addr": addr, "reservation_id": ""})
|
||||
if error is None:
|
||||
outcomes.append(reservations)
|
||||
else:
|
||||
failure = _FanoutError(error, list(reservations))
|
||||
try:
|
||||
self._retry_cancel_reservations(spec, reservations)
|
||||
except Exception as cleanup_error:
|
||||
failure.__cause__ = cleanup_error
|
||||
logger.exception("Failed to release partial EC reservation batch")
|
||||
outcomes.append(failure)
|
||||
return outcomes
|
||||
|
||||
@staticmethod
|
||||
def _reservation_batch_done(
|
||||
records: list[ProducerPushRecord], future: Future[None]
|
||||
) -> None:
|
||||
error = future.exception()
|
||||
if error is not None:
|
||||
for record in records:
|
||||
if not record.reservation_future.done():
|
||||
record.reservation_future.set_exception(error)
|
||||
|
||||
def start_load_caches(
|
||||
self,
|
||||
metadata: ECMooncakeConnectorMetadata,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
self._resolve_consumer_rank()
|
||||
if not self._is_receiving_rank:
|
||||
# Later pipeline stages never gather multimodal embeddings.
|
||||
return
|
||||
self._transfer.ensure_ready()
|
||||
if self._buffer_device == "cuda" and not torch.accelerator.is_available():
|
||||
raise RuntimeError(
|
||||
"ECMooncakeConnector requires CUDA for ec_buffer_device=cuda"
|
||||
)
|
||||
self._reservations.retire_stale(encoder_cache, metadata.freed)
|
||||
|
||||
for spec in metadata.loads:
|
||||
if spec.mm_hash in encoder_cache:
|
||||
if not spec.local:
|
||||
# The spec's id is one shard's; cancel by transfer.
|
||||
self._reservations.cancel(spec.transfer_id, "")
|
||||
self._completed_loads.add(spec.mm_hash)
|
||||
continue
|
||||
if spec.local:
|
||||
tensor = self._consumer_memory.take_resident(
|
||||
spec.mm_hash, tuple(spec.shape), spec.dtype
|
||||
)
|
||||
else:
|
||||
try:
|
||||
allocation = self._reservations.take(spec.transfer_id, spec.mm_hash)
|
||||
tensor = allocation.tensor
|
||||
except RuntimeError as e:
|
||||
logger.warning("EC Mooncake pushed load failed: %s", e)
|
||||
tensor = None
|
||||
if tensor is None:
|
||||
self._failed_loads.add(spec.mm_hash)
|
||||
else:
|
||||
encoder_cache[spec.mm_hash] = tensor
|
||||
self._completed_loads.add(spec.mm_hash)
|
||||
|
||||
def _push_batch(self, pushes: list[ProducerPushRecord]) -> None:
|
||||
started_at = time.monotonic()
|
||||
ready: list[tuple[ProducerPushRecord, dict[str, Any]]] = []
|
||||
written_pushes: dict[str, ProducerPushRecord] = {}
|
||||
failure: Exception | None = None
|
||||
try:
|
||||
for push in pushes:
|
||||
self._validate_push_source(push)
|
||||
reservations = self._producer_pushes.resolve_reservations(push)
|
||||
stale = [
|
||||
index
|
||||
for index, shard in enumerate(reservations)
|
||||
if not shard.get("ready", False)
|
||||
and not shard.get("cancelled", False)
|
||||
and time.monotonic() - float(shard.get("_received_at", started_at))
|
||||
>= _RESERVATION_REFRESH_SECONDS
|
||||
]
|
||||
if stale:
|
||||
reservations = self._refresh_remote_reservations(
|
||||
push.spec, reservations, push
|
||||
)
|
||||
self._producer_pushes.replace_reservations(push, reservations)
|
||||
self._producer_pushes.begin_writing(push)
|
||||
writable = [
|
||||
shard
|
||||
for shard in reservations
|
||||
if not shard.get("cached", False)
|
||||
and not shard.get("cancelled", False)
|
||||
and shard.get("write", True)
|
||||
]
|
||||
source = push.source_tensor
|
||||
assert source is not None
|
||||
for shard in writable:
|
||||
if int(shard["nbytes"]) != source.nbytes:
|
||||
raise RuntimeError(
|
||||
"Reserved EC size does not match tensor for "
|
||||
f"mm_hash={push.spec.mm_hash}"
|
||||
)
|
||||
ready.append((push, shard))
|
||||
written_pushes.setdefault(push.spec.transfer_id, push)
|
||||
if ready:
|
||||
# Stage each source once, then write it to every destination.
|
||||
source_index = {
|
||||
push.spec.transfer_id: index
|
||||
for index, push in enumerate(written_pushes.values())
|
||||
}
|
||||
tensors = [
|
||||
cast(torch.Tensor, push.source_tensor)
|
||||
for push in written_pushes.values()
|
||||
if push.source_tensor is not None
|
||||
]
|
||||
lengths = [tensor.nbytes for tensor in tensors]
|
||||
staged = self._producer_memory.stage(tensors)
|
||||
registered_sources: list[int] = []
|
||||
if staged is not None:
|
||||
sources = staged.tensors
|
||||
else:
|
||||
sources = tensors
|
||||
registered_sources = self._transfer.acquire_sources(tensors)
|
||||
addresses = [tensor.data_ptr() for tensor in sources]
|
||||
try:
|
||||
by_session: dict[str, list[tuple[int, int]]] = {}
|
||||
session_records: dict[str, dict[str, ProducerPushRecord]] = {}
|
||||
for push, shard in ready:
|
||||
session = str(shard["dst_session"])
|
||||
by_session.setdefault(session, []).append(
|
||||
(source_index[push.spec.transfer_id], int(shard["dst_ptr"]))
|
||||
)
|
||||
session_records.setdefault(session, {})[
|
||||
push.spec.transfer_id
|
||||
] = push
|
||||
|
||||
def write(session: str, items: list[tuple[int, int]]) -> None:
|
||||
self._transfer.write(
|
||||
session,
|
||||
[addresses[index] for index, _ in items],
|
||||
[dst for _, dst in items],
|
||||
[lengths[index] for index, _ in items],
|
||||
)
|
||||
|
||||
sessions = list(by_session.items())
|
||||
|
||||
# Write shards concurrently to avoid serial TP latency.
|
||||
def track_write(index: int, future: Future[None]) -> None:
|
||||
session = sessions[index][0]
|
||||
self._producer_pushes.track_shard_futures(
|
||||
list(session_records[session].values()), [future]
|
||||
)
|
||||
|
||||
writes: list[Callable[[], None]] = [
|
||||
partial(write, *session) for session in sessions
|
||||
]
|
||||
self._run_fanout(writes, track_write)
|
||||
finally:
|
||||
if staged is not None:
|
||||
self._producer_memory.release(staged)
|
||||
self._transfer.release_sources(registered_sources)
|
||||
|
||||
self._producer_pushes.begin_notifying(pushes)
|
||||
self._notify_completions(ready)
|
||||
self._producer_pushes.complete(pushes)
|
||||
except Exception as exc:
|
||||
# Report asynchronously; raising here would fail EngineCore.
|
||||
failure = exc
|
||||
logger.exception(
|
||||
"EC Mooncake push batch failed for mm_hashes=%s",
|
||||
[push.spec.mm_hash for push in pushes],
|
||||
)
|
||||
self._producer_pushes.settle_all(pushes)
|
||||
self._abandon_pushes(pushes)
|
||||
finally:
|
||||
if failure is not None:
|
||||
self._producer_pushes.fail(pushes, failure)
|
||||
|
||||
def _notify_completions(
|
||||
self, notifications: list[tuple[ProducerPushRecord, dict[str, Any]]]
|
||||
) -> None:
|
||||
"""Tell the consumer, in one message per destination, what landed."""
|
||||
if not notifications:
|
||||
return
|
||||
by_destination: dict[str, list[tuple[ProducerPushRecord, dict[str, Any]]]] = {}
|
||||
for push, reservation in notifications:
|
||||
by_destination.setdefault(
|
||||
str(reservation.get("addr", push.spec.consumer_zmq)), []
|
||||
).append((push, reservation))
|
||||
destinations = list(by_destination.items())
|
||||
|
||||
def notify(
|
||||
consumer_zmq: str,
|
||||
items: list[tuple[ProducerPushRecord, dict[str, Any]]],
|
||||
) -> None:
|
||||
result = self._control_client.request(
|
||||
consumer_zmq,
|
||||
{
|
||||
"op": "complete_batch",
|
||||
"items": [
|
||||
{
|
||||
"transfer_id": push.spec.transfer_id,
|
||||
"reservation_id": reservation["reservation_id"],
|
||||
}
|
||||
for push, reservation in items
|
||||
],
|
||||
},
|
||||
)
|
||||
completions = result.get("items", []) if isinstance(result, dict) else []
|
||||
if len(completions) != len(items):
|
||||
raise RuntimeError("Malformed EC completion response")
|
||||
for (push, _), completion in zip(items, completions):
|
||||
if not completion.get("completed"):
|
||||
raise RuntimeError(
|
||||
f"Unknown EC reservation for mm_hash={push.spec.mm_hash}"
|
||||
)
|
||||
|
||||
def track(index: int, future: Future[None]) -> None:
|
||||
records = {
|
||||
push.spec.transfer_id: push for push, _ in destinations[index][1]
|
||||
}
|
||||
self._producer_pushes.track_shard_futures(list(records.values()), [future])
|
||||
|
||||
self._run_fanout(
|
||||
[
|
||||
partial(notify, destination, items)
|
||||
for destination, items in destinations
|
||||
],
|
||||
track,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _known_reservations(record: ProducerPushRecord) -> list[dict[str, Any]]:
|
||||
if record.reservations:
|
||||
return list(record.reservations)
|
||||
try:
|
||||
return list(record.reservation_future.result())
|
||||
except _FanoutError as exc:
|
||||
return [result for result in exc.results if isinstance(result, dict)]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def _abandon_pushes(self, pushes: list[ProducerPushRecord]) -> None:
|
||||
"""Release the consumer-side reservations of a batch that failed."""
|
||||
for push in pushes:
|
||||
shards = self._known_reservations(push)
|
||||
if not shards:
|
||||
# No confirmed topology means there is no safe address to
|
||||
# cancel. The original push failure remains observable via
|
||||
# ProducerPushManager.fail below.
|
||||
continue
|
||||
try:
|
||||
self._retry_cancel_reservations(push.spec, shards, record=push)
|
||||
except _FanoutError:
|
||||
logger.exception(
|
||||
"Failed to abandon EC reservations for transfer_id=%s",
|
||||
push.spec.transfer_id,
|
||||
)
|
||||
|
||||
def _flush_pending_pushes(self) -> None:
|
||||
self._producer_pushes.submit_batches(
|
||||
self._io_executor,
|
||||
self._push_batch,
|
||||
)
|
||||
|
||||
def _bind_push_source(self, tensor: torch.Tensor, mm_hash: str) -> None:
|
||||
ready_event = None
|
||||
if tensor.device.type == "cuda":
|
||||
ready_event = torch.Event()
|
||||
ready_event.record(torch.accelerator.current_stream(tensor.device))
|
||||
self._producer_pushes.bind_source(mm_hash, tensor, ready_event)
|
||||
|
||||
def _cancel_orphaned_reservation(self, record: ProducerPushRecord) -> None:
|
||||
resolution_error: Exception | None = None
|
||||
try:
|
||||
reservations = self._producer_pushes.resolve_reservations(record)
|
||||
except Exception as exc:
|
||||
resolution_error = exc
|
||||
reservations = self._known_reservations(record)
|
||||
reservations = [
|
||||
shard for shard in reservations if not shard.get("cancelled", False)
|
||||
]
|
||||
if not reservations:
|
||||
if resolution_error is not None:
|
||||
self._producer_pushes.fail([record], resolution_error)
|
||||
else:
|
||||
self._producer_pushes.finish_cancel(record)
|
||||
return
|
||||
try:
|
||||
self._retry_cancel_reservations(record.spec, reservations, record=record)
|
||||
except Exception as cleanup_error:
|
||||
if resolution_error is not None:
|
||||
combined_error = RuntimeError(
|
||||
f"EC reservation resolution failed ({resolution_error}); "
|
||||
f"cleanup also failed ({cleanup_error})"
|
||||
)
|
||||
combined_error.__cause__ = resolution_error
|
||||
cleanup_error = combined_error
|
||||
self._producer_pushes.fail([record], cleanup_error)
|
||||
return
|
||||
if resolution_error is not None:
|
||||
self._producer_pushes.fail([record], resolution_error)
|
||||
else:
|
||||
self._producer_pushes.finish_cancel(record)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
if not self.is_producer:
|
||||
return None, None
|
||||
|
||||
if self._output_store is not None:
|
||||
self._output_store.reap()
|
||||
for record in self._producer_pushes.cancel_requests(finished_req_ids):
|
||||
self._producer_pushes.submit_cancel(
|
||||
record,
|
||||
self._io_executor,
|
||||
self._cancel_orphaned_reservation,
|
||||
)
|
||||
return None, None
|
||||
|
||||
def save_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs: Any
|
||||
) -> None:
|
||||
if not self.is_producer:
|
||||
return
|
||||
tensor = encoder_cache[mm_hash]
|
||||
self._bind_push_source(tensor, mm_hash)
|
||||
if self._output_store is not None:
|
||||
self._output_store.save_output(mm_hash, tensor)
|
||||
|
||||
def build_connector_worker_meta(self) -> ECMooncakeWorkerMetadata | None:
|
||||
if self.is_consumer and not self._is_receiving_rank:
|
||||
# `loaded` is intersected across reporting ranks, so a stage that
|
||||
# never loads must not report at all rather than report nothing.
|
||||
return None
|
||||
|
||||
self._collecting_sources = False
|
||||
self._push_ready.set()
|
||||
self._flush_pending_pushes()
|
||||
failures = self._producer_pushes.poll()
|
||||
for mm_hash, error in failures:
|
||||
logger.error(
|
||||
"EC Mooncake async save failed for mm_hash=%s: %s",
|
||||
mm_hash,
|
||||
error,
|
||||
)
|
||||
reclaimed = self._consumer_memory.drain_reclaimed()
|
||||
meta = ECMooncakeWorkerMetadata(
|
||||
loaded=self._completed_loads,
|
||||
failed_loads=self._failed_loads,
|
||||
reclaimed=reclaimed,
|
||||
pending_saves=self._producer_pushes.pending,
|
||||
failed_saves=self._failed_saves,
|
||||
)
|
||||
self._failed_saves = set()
|
||||
self._completed_loads = set()
|
||||
self._failed_loads = set()
|
||||
return meta
|
||||
|
||||
def close(self) -> None:
|
||||
if self._shutdown:
|
||||
return
|
||||
self._shutdown = True
|
||||
dispatcher = getattr(self, "_dispatcher", None)
|
||||
if dispatcher is not None:
|
||||
self._dispatch_stop.set()
|
||||
self._push_ready.set()
|
||||
dispatcher.join()
|
||||
if self._control_server is not None:
|
||||
self._reservations.begin_shutdown()
|
||||
self._flush_pending_pushes()
|
||||
for record in self._producer_pushes.cancel_requests(None):
|
||||
self._producer_pushes.submit_cancel(
|
||||
record,
|
||||
self._io_executor,
|
||||
self._cancel_orphaned_reservation,
|
||||
)
|
||||
self._control_executor.shutdown(wait=True)
|
||||
self._producer_pushes.submit_batches(
|
||||
self._io_executor, self._push_batch, wait=True
|
||||
)
|
||||
self._io_executor.shutdown(wait=True)
|
||||
if self._shard_pool is not None:
|
||||
self._shard_pool.shutdown(wait=True)
|
||||
# Every producer-side thread that could hold a control socket is stopped.
|
||||
self._control_client.close()
|
||||
drained = True
|
||||
if self._control_server is not None:
|
||||
drained = self._reservations.wait_for_writers(
|
||||
self._shutdown_drain_timeout_s
|
||||
)
|
||||
self._control_server.close()
|
||||
if drained:
|
||||
self._consumer_memory.close()
|
||||
else:
|
||||
logger.error(
|
||||
"Timed out waiting for Mooncake EC writers; keeping the consumer "
|
||||
"receive pool registered"
|
||||
)
|
||||
self._producer_memory.close()
|
||||
self._transfer.close()
|
||||
if self._output_store is not None:
|
||||
self._output_store.shutdown()
|
||||
self._output_store = None
|
||||
@@ -1,3 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Mooncake Store support for cross-Encoder output reuse."""
|
||||
@@ -1,214 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Encoder-side Mooncake Store resolver and publisher."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import traceback
|
||||
from collections.abc import Mapping
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .data import TensorSpec, make_embedding_key
|
||||
from .store_client import (
|
||||
EmbeddingStoreError,
|
||||
EmbeddingStoreOperationError,
|
||||
MooncakeEmbeddingStoreClient,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _StoreSave:
|
||||
tensor: torch.Tensor | None
|
||||
ready_event: torch.Event | None
|
||||
storage_ptr: int
|
||||
|
||||
def release(self) -> None:
|
||||
self.tensor = None
|
||||
self.ready_event = None
|
||||
|
||||
|
||||
class MooncakeEmbeddingStoreBackend:
|
||||
"""Resolve immutable encoder outputs and publish misses asynchronously."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store_client: MooncakeEmbeddingStoreClient,
|
||||
namespace: str,
|
||||
*,
|
||||
max_pending_items: int,
|
||||
max_pending_bytes: int,
|
||||
) -> None:
|
||||
self.store_client = store_client
|
||||
self.namespace = namespace
|
||||
self.max_pending_items = max_pending_items
|
||||
self.max_pending_bytes = max_pending_bytes
|
||||
self._executor = ThreadPoolExecutor(
|
||||
max_workers=1,
|
||||
thread_name_prefix="ec-mooncake-store",
|
||||
)
|
||||
self._pending: dict[str, tuple[Future[None], _StoreSave]] = {}
|
||||
self._pending_storages: dict[int, tuple[torch.UntypedStorage, int]] = {}
|
||||
self._pending_lock = threading.Lock()
|
||||
self._pending_bytes = 0
|
||||
self._step_candidates: set[str] = set()
|
||||
self._closed = False
|
||||
self._fatal_error: BaseException | None = None
|
||||
|
||||
def save_output(self, identifier: str, tensor: torch.Tensor) -> None:
|
||||
if identifier in self._step_candidates:
|
||||
self.reap()
|
||||
self._enqueue_save(identifier, tensor)
|
||||
|
||||
def resolve_inputs(
|
||||
self,
|
||||
expected_tensors: Mapping[str, TensorSpec],
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
device: torch.device | str,
|
||||
) -> set[str]:
|
||||
"""Load outputs before encoding; unresolved items retain normal computation."""
|
||||
self.reap()
|
||||
self._step_candidates = set(expected_tensors)
|
||||
candidates = [key for key in expected_tensors if key not in encoder_cache]
|
||||
if not candidates:
|
||||
return set()
|
||||
|
||||
pool_keys = [
|
||||
make_embedding_key(self.namespace, identifier) for identifier in candidates
|
||||
]
|
||||
try:
|
||||
exists = self.store_client.batch_exists(pool_keys)
|
||||
except (EmbeddingStoreOperationError, OSError):
|
||||
logger.warning(
|
||||
"Mooncake embedding Store lookup failed; falling back to encoder",
|
||||
exc_info=True,
|
||||
)
|
||||
return set()
|
||||
|
||||
hits = {
|
||||
pool_key: expected_tensors[identifier]
|
||||
for identifier, pool_key, hit in zip(
|
||||
candidates, pool_keys, exists, strict=True
|
||||
)
|
||||
if hit
|
||||
}
|
||||
try:
|
||||
tensors = self.store_client.load_tensors(hits, device)
|
||||
except (EmbeddingStoreOperationError, OSError):
|
||||
logger.warning(
|
||||
"Mooncake embedding Store GET failed; falling back to encoder",
|
||||
exc_info=True,
|
||||
)
|
||||
return set()
|
||||
loaded: set[str] = set()
|
||||
for identifier, pool_key in zip(candidates, pool_keys, strict=True):
|
||||
if pool_key in tensors:
|
||||
encoder_cache[identifier] = tensors[pool_key]
|
||||
loaded.add(identifier)
|
||||
return loaded
|
||||
|
||||
def _enqueue_save(self, identifier: str, tensor: torch.Tensor) -> bool:
|
||||
"""Admit contiguous outputs, charging each retained storage once."""
|
||||
if not tensor.is_contiguous():
|
||||
return False
|
||||
storage = tensor.untyped_storage()
|
||||
storage_ptr = storage.data_ptr()
|
||||
with self._pending_lock:
|
||||
if self._fatal_error is not None:
|
||||
raise EmbeddingStoreError(
|
||||
"Store publisher has failed"
|
||||
) from self._fatal_error
|
||||
if self._closed or identifier in self._pending:
|
||||
return False
|
||||
_, ref_count = self._pending_storages.get(storage_ptr, (storage, 0))
|
||||
budget_bytes = 0 if ref_count else storage.nbytes()
|
||||
if (
|
||||
len(self._pending) >= self.max_pending_items
|
||||
or self._pending_bytes + budget_bytes > self.max_pending_bytes
|
||||
):
|
||||
return False
|
||||
# Only admitted work creates an event. Runtime errors propagate.
|
||||
save = _StoreSave(
|
||||
tensor,
|
||||
_record_tensor_ready_event(tensor),
|
||||
storage_ptr,
|
||||
)
|
||||
future = self._executor.submit(
|
||||
self._save, make_embedding_key(self.namespace, identifier), save
|
||||
)
|
||||
self._pending[identifier] = (future, save)
|
||||
# Keep the storage alive until reaping so its address cannot be reused.
|
||||
self._pending_storages[storage_ptr] = (storage, ref_count + 1)
|
||||
self._pending_bytes += budget_bytes
|
||||
return True
|
||||
|
||||
def _save(self, pool_key: str, save: _StoreSave) -> None:
|
||||
try:
|
||||
if self.store_client.exists(pool_key):
|
||||
return
|
||||
tensor = save.tensor
|
||||
assert tensor is not None
|
||||
if save.ready_event is not None:
|
||||
save.ready_event.synchronize()
|
||||
self.store_client.put_tensor(pool_key, tensor)
|
||||
except BaseException as error:
|
||||
# Returned client frames can otherwise keep tensors via a Future's
|
||||
# traceback. Uncertain I/O owners remain retained by the client.
|
||||
traceback.clear_frames(error.__traceback__)
|
||||
raise
|
||||
finally:
|
||||
tensor = None
|
||||
save.release()
|
||||
|
||||
def reap(self) -> None:
|
||||
with self._pending_lock:
|
||||
for identifier, (future, save) in list(self._pending.items()):
|
||||
# Observe once; a racing completion is handled on the next poll.
|
||||
if not future.done():
|
||||
continue
|
||||
try:
|
||||
future.result() # Already done: never waits on Store I/O.
|
||||
except (EmbeddingStoreOperationError, OSError) as error:
|
||||
logger.warning("Store PUT failed for %s: %s", identifier, error)
|
||||
# This failure is consumed; do not retain publisher/caller frames.
|
||||
error.__traceback__ = None
|
||||
except BaseException as error:
|
||||
# The client may still own unsafe native-I/O buffers. Keep
|
||||
# their charge and fatal result until worker termination.
|
||||
if self._fatal_error is None:
|
||||
self._fatal_error = error
|
||||
continue
|
||||
del self._pending[identifier]
|
||||
storage, ref_count = self._pending_storages[save.storage_ptr]
|
||||
if ref_count == 1:
|
||||
del self._pending_storages[save.storage_ptr]
|
||||
self._pending_bytes -= storage.nbytes()
|
||||
else:
|
||||
self._pending_storages[save.storage_ptr] = (storage, ref_count - 1)
|
||||
fatal = self._fatal_error
|
||||
if fatal is not None:
|
||||
raise fatal
|
||||
|
||||
def shutdown(self) -> None:
|
||||
with self._pending_lock:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
self._executor.shutdown(wait=True, cancel_futures=False)
|
||||
self.reap()
|
||||
self.store_client.close()
|
||||
|
||||
|
||||
def _record_tensor_ready_event(tensor: torch.Tensor) -> torch.Event | None:
|
||||
if tensor.device.type != "cuda":
|
||||
return None
|
||||
event = torch.Event()
|
||||
event.record(torch.accelerator.current_stream(tensor.device))
|
||||
return event
|
||||
@@ -1,51 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Content identities and output contracts for the shared Encoder Store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
EMBEDDING_CACHE_KEY_VERSION = "v3"
|
||||
|
||||
|
||||
def make_embedding_key(namespace: str, identifier: str) -> str:
|
||||
return f"{namespace}@id:{identifier}"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TensorSpec:
|
||||
"""Contiguous output contract used by the Encoder and Store codec."""
|
||||
|
||||
shape: tuple[int, ...]
|
||||
dtype: str
|
||||
nbytes: int
|
||||
|
||||
|
||||
def build_embedding_namespace(vllm_config: VllmConfig) -> str:
|
||||
model_config = vllm_config.model_config
|
||||
ec_config = vllm_config.ec_transfer_config
|
||||
assert ec_config is not None
|
||||
extra_config = ec_config.ec_connector_extra_config
|
||||
|
||||
multimodal_config = model_config.multimodal_config
|
||||
encoder_hash = (
|
||||
multimodal_config.compute_hash()
|
||||
if multimodal_config is not None
|
||||
else "encoder:default"
|
||||
)
|
||||
cache_prefix = str(extra_config.get("embedding_cache_prefix", ""))
|
||||
model_name = str(extra_config.get("embedding_model_identity", model_config.model))
|
||||
prefix = f"{cache_prefix}@" if cache_prefix else ""
|
||||
return (
|
||||
f"{prefix}embedding"
|
||||
f"@model:{model_name}"
|
||||
f"@revision:{model_config.revision or 'default'}"
|
||||
f"@encoder:{encoder_hash}"
|
||||
f"@dtype:{model_config.dtype}"
|
||||
f"@protocol:{EMBEDDING_CACHE_KEY_VERSION}"
|
||||
)
|
||||
@@ -1,463 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Thin Mooncake Store client for embedding objects.
|
||||
|
||||
Originally adapted from vLLM PR #47302 (0d1f71f5d36c). The v3 key namespace
|
||||
uses a compact header for complete 2D embeddings, not that PR's tensor format.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import struct
|
||||
import threading
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from enum import IntEnum
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake_store_embedding.data import (
|
||||
TensorSpec,
|
||||
)
|
||||
from vllm.distributed.mooncake_store import MooncakeStoreConfig, setup_mooncake_store
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import get_ip
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# A failed transfer can still use registered memory after the binding returns.
|
||||
# Poisoned clients retain their owners until process exit, including if normal
|
||||
# worker teardown drops the connector. New operations are rejected.
|
||||
_UNSAFE_CLIENTS: list[Any] = []
|
||||
|
||||
|
||||
# Values from Mooncake v0.3.12 / v0.3.12.post1 types.h (ErrorCode).
|
||||
# Python exposes integer returns, not this enum.
|
||||
class _MooncakeErrorCode(IntEnum):
|
||||
NO_AVAILABLE_HANDLE = -200
|
||||
INVALID_PARAMS = -600
|
||||
INVALID_REPLICA = -702
|
||||
REPLICA_IS_NOT_READY = -703
|
||||
OBJECT_NOT_FOUND = -704
|
||||
OBJECT_ALREADY_EXISTS = -705
|
||||
LEASE_EXPIRED = -707
|
||||
RPC_FAIL = -900
|
||||
RPC_TIMEOUT = -901
|
||||
TENANT_QUOTA_EXCEEDED = -1700
|
||||
|
||||
|
||||
# For RAM batch reads and single-replica writes, these rejections occur
|
||||
# before submission or after completed I/O. Transfer timeouts can leave I/O
|
||||
# in flight; all other failures retain registered buffers and poison the client.
|
||||
_SAFE_IO_REJECTIONS = frozenset(
|
||||
{
|
||||
_MooncakeErrorCode.NO_AVAILABLE_HANDLE,
|
||||
_MooncakeErrorCode.REPLICA_IS_NOT_READY,
|
||||
_MooncakeErrorCode.OBJECT_NOT_FOUND,
|
||||
_MooncakeErrorCode.OBJECT_ALREADY_EXISTS,
|
||||
_MooncakeErrorCode.RPC_FAIL,
|
||||
_MooncakeErrorCode.RPC_TIMEOUT,
|
||||
}
|
||||
)
|
||||
# Batch GET validates capacity/replicas before I/O and checks leases afterwards.
|
||||
# PUT rejects tenant quotas before allocating replicas or submitting transfers.
|
||||
_SAFE_GET_REJECTIONS = _SAFE_IO_REJECTIONS | {
|
||||
_MooncakeErrorCode.INVALID_PARAMS,
|
||||
_MooncakeErrorCode.INVALID_REPLICA,
|
||||
_MooncakeErrorCode.LEASE_EXPIRED,
|
||||
}
|
||||
_SAFE_PUT_REJECTIONS = _SAFE_IO_REJECTIONS | {_MooncakeErrorCode.TENANT_QUOTA_EXCEEDED}
|
||||
|
||||
|
||||
_MOONCAKE_TENSOR_OBJECT_MAGIC = 0x4D4F4F4E
|
||||
# Magic, dtype, token count, hidden dimension. Format versioning is in the key.
|
||||
_MOONCAKE_TENSOR_HEADER = struct.Struct("<IIQQ")
|
||||
_MOONCAKE_DTYPE_TO_TORCH_DTYPE = {
|
||||
0: "torch.float32",
|
||||
11: "torch.float16",
|
||||
12: "torch.bfloat16",
|
||||
}
|
||||
_TORCH_DTYPE_TO_MOONCAKE_DTYPE = {
|
||||
value: key for key, value in _MOONCAKE_DTYPE_TO_TORCH_DTYPE.items()
|
||||
}
|
||||
|
||||
|
||||
def create_mooncake_embedding_store_client(
|
||||
read_buffer_bytes: int = 128 * 1024**2,
|
||||
) -> MooncakeEmbeddingStoreClient:
|
||||
try:
|
||||
from mooncake.store import ( # type: ignore
|
||||
MooncakeDistributedStore,
|
||||
ObjectDataType,
|
||||
ReplicateConfig,
|
||||
)
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"Install mooncake with Store support to enable cross_encoder_cache."
|
||||
) from e
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import rdma_utils
|
||||
|
||||
config = MooncakeStoreConfig.load_from_config()
|
||||
if config.enable_offload:
|
||||
raise ValueError(
|
||||
"cross_encoder_cache supports a RAM Store; disable enable_offload"
|
||||
)
|
||||
store = MooncakeDistributedStore()
|
||||
local_ip = get_ip()
|
||||
local_hostname = rdma_utils.get_requester_local_hostname(local_ip)
|
||||
setup_mooncake_store(store, config, local_hostname)
|
||||
|
||||
logger.info(
|
||||
"Initialized embedding Mooncake store mode=%s global_segment_size=%d "
|
||||
"local_buffer_size=%d",
|
||||
config.mode,
|
||||
config.global_segment_size,
|
||||
config.local_buffer_size,
|
||||
)
|
||||
replicate_config = ReplicateConfig()
|
||||
replicate_config.data_type = ObjectDataType.TENSOR
|
||||
return MooncakeEmbeddingStoreClient(
|
||||
store, replicate_config=replicate_config, read_buffer_bytes=read_buffer_bytes
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingStoreError(RuntimeError):
|
||||
"""A Store failure that must propagate to the worker."""
|
||||
|
||||
|
||||
class EmbeddingStoreOperationError(EmbeddingStoreError):
|
||||
"""A rejected or completed operation that permits normal Encoder fallback."""
|
||||
|
||||
|
||||
def _batch_get_completed_safely(results: Any, capacities: list[int]) -> bool:
|
||||
return (
|
||||
isinstance(results, list)
|
||||
and len(results) == len(capacities)
|
||||
and all(
|
||||
type(result) is int
|
||||
and (0 <= result <= capacity or result in _SAFE_GET_REJECTIONS)
|
||||
for result, capacity in zip(results, capacities, strict=True)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class MooncakeEmbeddingStoreClient:
|
||||
"""Wraps Mooncake object and buffer APIs used by embedding transfer."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: Any,
|
||||
replicate_config: Any | None = None,
|
||||
*,
|
||||
read_buffer_bytes: int = 128 * 1024**2,
|
||||
):
|
||||
self._lifetime_lock = threading.Lock()
|
||||
self._unsafe_owners: list[Any] = []
|
||||
self._poisoned = False
|
||||
self.store = store
|
||||
self.replicate_config = replicate_config
|
||||
self._read_buffer_bytes = read_buffer_bytes
|
||||
self._read_buffer: torch.Tensor | None = None
|
||||
self._put_header: ctypes.Array[ctypes.c_char] | None = None
|
||||
|
||||
def _check_healthy(self) -> None:
|
||||
if self._poisoned:
|
||||
raise EmbeddingStoreError(
|
||||
"Mooncake I/O completion is unconfirmed; worker must exit"
|
||||
)
|
||||
|
||||
def _poison(self, owners: list[Any]) -> None:
|
||||
with self._lifetime_lock:
|
||||
self._unsafe_owners.extend(owners)
|
||||
if not self._poisoned:
|
||||
self._poisoned = True
|
||||
_UNSAFE_CLIENTS.append(self)
|
||||
|
||||
@contextmanager
|
||||
def _registered_io(
|
||||
self,
|
||||
buffers: list[tuple[Any, int, int]],
|
||||
) -> Iterator[Callable[[int], None]]:
|
||||
self._check_healthy()
|
||||
registered = []
|
||||
submitted = False
|
||||
confirmed = False
|
||||
|
||||
def confirm(result: int) -> None:
|
||||
nonlocal confirmed
|
||||
if type(result) is not int or (
|
||||
result < 0 and result not in _SAFE_PUT_REJECTIONS
|
||||
):
|
||||
raise EmbeddingStoreError(
|
||||
"Mooncake I/O completion is unconfirmed; retaining buffers"
|
||||
)
|
||||
confirmed = True
|
||||
|
||||
try:
|
||||
for owner, addr, size in buffers:
|
||||
ret = self.store.register_buffer(addr, size)
|
||||
if ret != 0:
|
||||
raise EmbeddingStoreOperationError(
|
||||
f"Failed to register embedding buffer: {ret}"
|
||||
)
|
||||
registered.append(addr)
|
||||
submitted = True
|
||||
yield confirm
|
||||
except BaseException as error:
|
||||
if submitted and not confirmed:
|
||||
self._poison([owner for owner, _, _ in buffers])
|
||||
raise EmbeddingStoreError(
|
||||
"Mooncake I/O completion is unconfirmed; retaining buffers"
|
||||
) from error
|
||||
raise
|
||||
finally:
|
||||
if not submitted or confirmed:
|
||||
try:
|
||||
for addr in reversed(registered):
|
||||
self._unregister_buffer(addr)
|
||||
except BaseException:
|
||||
self._poison([owner for owner, _, _ in buffers])
|
||||
raise
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the supported Mooncake binding after all I/O is drained."""
|
||||
self._check_healthy()
|
||||
if self._read_buffer is not None:
|
||||
try:
|
||||
self._unregister_buffer(self._read_buffer.data_ptr())
|
||||
except BaseException:
|
||||
self._poison([self._read_buffer])
|
||||
raise
|
||||
self._read_buffer = None
|
||||
if self._put_header is not None:
|
||||
try:
|
||||
self._unregister_buffer(ctypes.addressof(self._put_header))
|
||||
except BaseException:
|
||||
self._poison([self._put_header])
|
||||
raise
|
||||
self._put_header = None
|
||||
ret = self.store.close()
|
||||
if ret != 0:
|
||||
raise EmbeddingStoreError(
|
||||
f"failed to close embedding Mooncake Store: {ret}"
|
||||
)
|
||||
|
||||
def exists(self, pool_key: str) -> bool:
|
||||
return self.batch_exists([pool_key])[0]
|
||||
|
||||
def batch_exists(self, keys: list[str]) -> list[bool]:
|
||||
self._check_healthy()
|
||||
if not keys:
|
||||
return []
|
||||
|
||||
states = self.store.batch_is_exist(keys)
|
||||
if len(states) != len(keys):
|
||||
raise RuntimeError(
|
||||
"Mooncake Store returned an unexpected number of lookup results"
|
||||
)
|
||||
return [state == 1 for state in states]
|
||||
|
||||
def load_tensors(
|
||||
self, expected: Mapping[str, TensorSpec], device: torch.device | str
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Synchronously load runner inputs into independently owned tensors."""
|
||||
self._check_healthy()
|
||||
device = torch.device(device)
|
||||
loaded: dict[str, torch.Tensor] = {}
|
||||
chunk: dict[str, TensorSpec] = {}
|
||||
chunk_bytes = 0
|
||||
for key, spec in expected.items():
|
||||
size = _MOONCAKE_TENSOR_HEADER.size + spec.nbytes
|
||||
if size > self._read_buffer_bytes:
|
||||
logger.warning(
|
||||
"Skipping Store GET for %s: object bytes=%d exceed read buffer=%d",
|
||||
key,
|
||||
size,
|
||||
self._read_buffer_bytes,
|
||||
)
|
||||
continue
|
||||
if chunk_bytes + size > self._read_buffer_bytes:
|
||||
loaded.update(self._load_chunk(chunk, device))
|
||||
chunk = {}
|
||||
chunk_bytes = 0
|
||||
chunk[key] = spec
|
||||
chunk_bytes += size
|
||||
if chunk:
|
||||
loaded.update(self._load_chunk(chunk, device))
|
||||
return loaded
|
||||
|
||||
def _load_chunk(
|
||||
self, expected: dict[str, TensorSpec], device: torch.device
|
||||
) -> dict[str, torch.Tensor]:
|
||||
self._check_healthy()
|
||||
if self._read_buffer is None:
|
||||
buffer = torch.empty(
|
||||
self._read_buffer_bytes,
|
||||
dtype=torch.uint8,
|
||||
device="cpu",
|
||||
pin_memory=device.type == "cuda",
|
||||
)
|
||||
ret = self.store.register_buffer(buffer.data_ptr(), buffer.nbytes)
|
||||
if ret != 0:
|
||||
raise EmbeddingStoreOperationError(
|
||||
f"Failed to register embedding read buffer: {ret}"
|
||||
)
|
||||
self._read_buffer = buffer
|
||||
buffer = self._read_buffer
|
||||
base = buffer.data_ptr()
|
||||
sizes = [
|
||||
_MOONCAKE_TENSOR_HEADER.size + spec.nbytes for spec in expected.values()
|
||||
]
|
||||
offsets: list[int] = []
|
||||
end = 0
|
||||
for size in sizes:
|
||||
offsets.append(end)
|
||||
end += size
|
||||
try:
|
||||
results = self.store.batch_get_into(
|
||||
list(expected), [base + offset for offset in offsets], sizes
|
||||
)
|
||||
if not _batch_get_completed_safely(results, sizes):
|
||||
raise EmbeddingStoreError("Unsafe batch GET results")
|
||||
except BaseException as error:
|
||||
self._poison([buffer])
|
||||
raise EmbeddingStoreError(
|
||||
"Mooncake I/O completion is unconfirmed; retaining read buffer"
|
||||
) from error
|
||||
|
||||
valid: list[tuple[str, TensorSpec, int]] = []
|
||||
for (key, spec), offset, size, result in zip(
|
||||
expected.items(), offsets, sizes, results, strict=True
|
||||
):
|
||||
if result != size:
|
||||
logger.warning(
|
||||
"Store GET skipped for %s: result=%d, expected bytes=%d",
|
||||
key,
|
||||
result,
|
||||
size,
|
||||
)
|
||||
continue
|
||||
try:
|
||||
_validate_mooncake_tensor_metadata(
|
||||
key,
|
||||
ctypes.string_at(base + offset, _MOONCAKE_TENSOR_HEADER.size),
|
||||
spec,
|
||||
)
|
||||
except EmbeddingStoreOperationError as error:
|
||||
logger.warning("Store GET skipped for %s: %s", key, error)
|
||||
continue
|
||||
valid.append((key, spec, offset + _MOONCAKE_TENSOR_HEADER.size))
|
||||
if not valid:
|
||||
return {}
|
||||
# Allocate every destination before submitting any asynchronous copy.
|
||||
targets = {
|
||||
key: torch.empty(
|
||||
spec.shape,
|
||||
dtype=getattr(torch, spec.dtype.removeprefix("torch.")),
|
||||
device=device,
|
||||
)
|
||||
for key, spec, _ in valid
|
||||
}
|
||||
ready = torch.Event() if device.type == "cuda" else None
|
||||
try:
|
||||
for key, spec, offset in valid:
|
||||
targets[key].view(torch.uint8).view(-1).copy_(
|
||||
buffer[offset : offset + spec.nbytes],
|
||||
non_blocking=ready is not None,
|
||||
)
|
||||
if ready is not None:
|
||||
ready.record(torch.accelerator.current_stream(device))
|
||||
ready.synchronize()
|
||||
except BaseException as error:
|
||||
if ready is not None:
|
||||
self._poison([buffer, *targets.values(), ready])
|
||||
raise EmbeddingStoreError(
|
||||
"Embedding copy completion is unconfirmed; retaining buffers"
|
||||
) from error
|
||||
raise
|
||||
return targets
|
||||
|
||||
def put_tensor(self, pool_key: str, tensor: torch.Tensor) -> None:
|
||||
self._check_healthy()
|
||||
if not tensor.is_contiguous():
|
||||
raise EmbeddingStoreOperationError("embedding tensor must be contiguous")
|
||||
metadata = _encode_mooncake_tensor_metadata(tensor)
|
||||
# The single publisher reuses this header only after PUT completion.
|
||||
if self._put_header is None:
|
||||
header = ctypes.create_string_buffer(len(metadata))
|
||||
ret = self.store.register_buffer(ctypes.addressof(header), len(metadata))
|
||||
if ret != 0:
|
||||
raise EmbeddingStoreOperationError(
|
||||
f"Failed to register embedding header: {ret}"
|
||||
)
|
||||
self._put_header = header
|
||||
header_ptr = ctypes.addressof(self._put_header)
|
||||
ctypes.memmove(header_ptr, metadata, len(metadata))
|
||||
data_size = tensor.numel() * tensor.element_size()
|
||||
buffers = [
|
||||
(tensor, tensor.data_ptr(), data_size),
|
||||
]
|
||||
with self._registered_io(buffers) as confirm:
|
||||
[result] = self.store.batch_put_from_multi_buffers(
|
||||
[pool_key],
|
||||
[[header_ptr, tensor.data_ptr()]],
|
||||
[[len(metadata), data_size]],
|
||||
self.replicate_config,
|
||||
)
|
||||
confirm(result)
|
||||
if result < 0:
|
||||
raise EmbeddingStoreOperationError(
|
||||
f"failed to put embedding tensor for {pool_key}"
|
||||
)
|
||||
|
||||
def _unregister_buffer(self, addr: int) -> None:
|
||||
try:
|
||||
ret = self.store.unregister_buffer(addr)
|
||||
except Exception as error:
|
||||
raise EmbeddingStoreError(
|
||||
f"could not confirm buffer unregistration addr={addr:#x}"
|
||||
) from error
|
||||
if ret != 0:
|
||||
raise EmbeddingStoreError(
|
||||
f"failed to unregister embedding buffer addr={addr:#x}: {ret}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_mooncake_tensor_metadata(
|
||||
pool_key: str,
|
||||
metadata: bytes,
|
||||
expected: TensorSpec,
|
||||
) -> None:
|
||||
magic, dtype, num_tokens, hidden_dim = _MOONCAKE_TENSOR_HEADER.unpack(metadata)
|
||||
if magic != _MOONCAKE_TENSOR_OBJECT_MAGIC:
|
||||
raise EmbeddingStoreOperationError(
|
||||
f"invalid Mooncake tensor metadata header for {pool_key}"
|
||||
)
|
||||
if _MOONCAKE_DTYPE_TO_TORCH_DTYPE.get(dtype) != expected.dtype:
|
||||
raise EmbeddingStoreOperationError(
|
||||
f"unexpected Mooncake tensor dtype for {pool_key}: {dtype}"
|
||||
)
|
||||
if (num_tokens, hidden_dim) != expected.shape:
|
||||
raise EmbeddingStoreOperationError(
|
||||
f"embedding tensor shape mismatch for {pool_key}: expected={expected.shape}"
|
||||
)
|
||||
|
||||
|
||||
def _encode_mooncake_tensor_metadata(tensor: torch.Tensor) -> bytes:
|
||||
dtype = str(tensor.dtype)
|
||||
if dtype not in _TORCH_DTYPE_TO_MOONCAKE_DTYPE:
|
||||
raise EmbeddingStoreOperationError(
|
||||
f"unsupported embedding tensor dtype: {dtype}"
|
||||
)
|
||||
if tensor.ndim != 2:
|
||||
raise EmbeddingStoreOperationError(
|
||||
f"embedding tensor must be 2D: shape={tuple(tensor.shape)}"
|
||||
)
|
||||
return _MOONCAKE_TENSOR_HEADER.pack(
|
||||
_MOONCAKE_TENSOR_OBJECT_MAGIC,
|
||||
_TORCH_DTYPE_TO_MOONCAKE_DTYPE[dtype],
|
||||
*tensor.shape,
|
||||
)
|
||||
@@ -150,30 +150,12 @@ class KVConnectorFactory:
|
||||
# The registration should not be done in each individual file, as we want to
|
||||
# only load the files corresponding to the current connector.
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"ExampleConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.example_connector",
|
||||
"ExampleConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"ExampleHiddenStatesConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.example_hidden_states_connector",
|
||||
"ExampleHiddenStatesConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"LMCacheConnectorV1",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.lmcache_connector",
|
||||
"LMCacheConnectorV1",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"LMCacheMPConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.lmcache_mp_connector",
|
||||
"LMCacheMPConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"NixlConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl",
|
||||
@@ -222,21 +204,6 @@ KVConnectorFactory.register_connector(
|
||||
"DecodeBenchConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"MooncakeConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector",
|
||||
"MooncakeConnector",
|
||||
)
|
||||
KVConnectorFactory.register_connector(
|
||||
"MooncakeStoreConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.connector",
|
||||
"MooncakeStoreConnector",
|
||||
)
|
||||
KVConnectorFactory.register_connector(
|
||||
"FlexKVConnectorV1",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.flexkv_connector",
|
||||
"FlexKVConnectorV1",
|
||||
)
|
||||
KVConnectorFactory.register_connector(
|
||||
"SimpleCPUOffloadConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector",
|
||||
|
||||
@@ -1,270 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.kv_events import KVCacheEvent
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# FlexKV is a distributed KV Store and multi-level cache management system for
|
||||
# ultra-large-scale LLM inference.
|
||||
# GitHub: https://github.com/taco-project/FlexKV
|
||||
# Install: git clone git@github.com:taco-project/FlexKV.git \
|
||||
# && cd FlexKV && bash build.sh
|
||||
class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
"""KV Connector that offloads KV cache to FlexKV.
|
||||
|
||||
FlexKV is a distributed KV Store and multi-level cache management system
|
||||
designed for ultra-large-scale LLM inference. It supports offloading KV
|
||||
cache to CPU memory, SSD, and remote storage.
|
||||
|
||||
Installation:
|
||||
See https://github.com/taco-project/FlexKV for installation instructions.
|
||||
Quick start::
|
||||
|
||||
git clone git@github.com:taco-project/FlexKV.git
|
||||
cd FlexKV && bash build.sh
|
||||
|
||||
Configuration:
|
||||
Pass ``kv_connector="FlexKVConnectorV1"`` via ``--kv-transfer-config``::
|
||||
|
||||
--kv-transfer-config \
|
||||
'{"kv_connector":"FlexKVConnectorV1","kv_role":"kv_both"}'
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(
|
||||
vllm_config=vllm_config, role=role, kv_cache_config=kv_cache_config
|
||||
)
|
||||
try:
|
||||
from flexkv.integration.vllm.vllm_v1_adapter import FlexKVConnectorV1Impl
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"FlexKV is not installed. Please install it to use "
|
||||
"FlexKVConnectorV1. See https://github.com/taco-project/FlexKV "
|
||||
"for installation instructions."
|
||||
) from e
|
||||
|
||||
self._flexkv_connector = FlexKVConnectorV1Impl(vllm_config, role)
|
||||
|
||||
def shutdown(self):
|
||||
self._flexkv_connector.shutdown()
|
||||
|
||||
# ==============================
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None:
|
||||
"""No-op for FlexKV (currently).
|
||||
|
||||
FlexKV manages all KV transfers on the **scheduler side** via
|
||||
``build_connector_meta`` (which calls ``launch_tasks``) and
|
||||
``update_connector_output`` (which polls ``query_finished_task``).
|
||||
KV blocks are transferred directly between the FlexKV server and
|
||||
vLLM's GPU memory without worker-side intervention during the
|
||||
forward pass — similar to how NIXL operates.
|
||||
|
||||
These worker-side hooks are kept (rather than omitted) to satisfy
|
||||
the ``KVConnectorBase_V1`` interface contract and to serve as
|
||||
extension points for a future worker-side layer-pipelining path.
|
||||
|
||||
Args:
|
||||
forward_context (ForwardContext): the forward context.
|
||||
**kwargs (Any): additional arguments (unused).
|
||||
|
||||
"""
|
||||
self._flexkv_connector.start_load_kv(forward_context, **kwargs)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""No-op for FlexKV (currently).
|
||||
|
||||
FlexKV manages all KV transfers on the scheduler side.
|
||||
This hook is retained for ``KVConnectorBase_V1`` API compatibility.
|
||||
|
||||
Args:
|
||||
layer_name: the name of the layer (unused).
|
||||
|
||||
"""
|
||||
self._flexkv_connector.wait_for_layer_load(layer_name)
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: "AttentionMetadata",
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""No-op for FlexKV (currently).
|
||||
|
||||
FlexKV offloads KV cache asynchronously from the scheduler side
|
||||
after a request finishes (see ``request_finished``). It does not
|
||||
intercept individual layer tensors during the forward pass.
|
||||
|
||||
This hook is retained to satisfy ``KVConnectorBase_V1`` and as an
|
||||
extension point for future per-layer async offload support.
|
||||
|
||||
Args:
|
||||
layer_name (str): the name of the layer (unused).
|
||||
kv_layer (torch.Tensor): the paged KV buffer (unused).
|
||||
attn_metadata (AttentionMetadata): the attention metadata (unused).
|
||||
**kwargs (Any): additional arguments (unused).
|
||||
|
||||
"""
|
||||
self._flexkv_connector.save_kv_layer(
|
||||
layer_name, kv_layer, attn_metadata, **kwargs
|
||||
)
|
||||
|
||||
def wait_for_save(self):
|
||||
"""No-op for FlexKV (currently).
|
||||
|
||||
KV offload tasks are tracked asynchronously by the scheduler
|
||||
connector via ``request_finished`` / ``query_finished_task``.
|
||||
There is no pending worker-side save to wait for at
|
||||
forward-context exit.
|
||||
|
||||
Retained to satisfy ``KVConnectorBase_V1`` and as an extension
|
||||
point for future worker-side save-completion signalling.
|
||||
"""
|
||||
self._flexkv_connector.wait_for_save()
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""Notify worker-side connector of requests that have finished
|
||||
generating tokens.
|
||||
|
||||
Returns:
|
||||
Tuple of (sending/saving ids, recving/loading ids) for requests
|
||||
that have finished asynchronous transfer. The finished saves/sends
|
||||
req ids must belong to a set provided in a call to this method
|
||||
(this call or a prior one).
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.get_finished(finished_req_ids)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Initialize with the KV caches. Useful for pre-registering the
|
||||
KV caches in the KVConnector (e.g. for NIXL).
|
||||
|
||||
Args:
|
||||
kv_caches: dictionary of layer names to kv cache tensors.
|
||||
|
||||
"""
|
||||
self._flexkv_connector.register_kv_caches(kv_caches)
|
||||
|
||||
# ==============================
|
||||
# Scheduler-side methods
|
||||
# ==============================
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int, bool]:
|
||||
"""Get the number of new tokens that can be loaded from the
|
||||
external KV cache beyond ``num_computed_tokens``.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
num_computed_tokens (int): the number of locally computed
|
||||
tokens for this request.
|
||||
|
||||
Returns:
|
||||
Tuple of (num_external_tokens, is_ready) where
|
||||
num_external_tokens is the number of additional tokens that
|
||||
can be loaded from the external KV cache.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""Update KVConnector state after block allocation."""
|
||||
self._flexkv_connector.update_state_after_alloc(
|
||||
request, blocks, num_external_tokens
|
||||
)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.build_connector_meta(scheduler_output)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""Update KVConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
self._flexkv_connector.update_connector_output(connector_output)
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Called when a request has finished, before its blocks are freed.
|
||||
|
||||
Returns:
|
||||
Tuple of (async_save, kv_transfer_params) where async_save is
|
||||
True if the request is being saved/sent asynchronously and blocks
|
||||
should not be freed until the request_id is returned from
|
||||
:meth:`get_finished`. kv_transfer_params is an optional dict of
|
||||
KVTransferParams to be included in the request outputs.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.request_finished(request, block_ids)
|
||||
|
||||
def take_events(self) -> Iterable["KVCacheEvent"]:
|
||||
"""Collect buffered KV cache events.
|
||||
|
||||
Returns:
|
||||
New KV cache events since the last call.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.take_events()
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""Get the KV connector stats collected during the last interval."""
|
||||
return self._flexkv_connector.get_kv_connector_stats()
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Get the block ids that have failed to load."""
|
||||
return self._flexkv_connector.get_block_ids_with_load_errors()
|
||||
@@ -1,358 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_events import (
|
||||
BlockStored,
|
||||
KVCacheEvent,
|
||||
KVConnectorKVEvents,
|
||||
KVEventAggregator,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LMCacheKVEvents(KVConnectorKVEvents):
|
||||
"""Concrete implementation of KVConnectorKVEvents using KVEventAggregator."""
|
||||
|
||||
def __init__(self, num_workers: int) -> None:
|
||||
self._aggregator = KVEventAggregator(num_workers)
|
||||
|
||||
def add_events(self, events: list[KVCacheEvent]) -> None:
|
||||
self._aggregator.add_events(events)
|
||||
|
||||
def aggregate(self) -> "LMCacheKVEvents":
|
||||
"""Aggregate KV events and retain only common events."""
|
||||
common_events = self._aggregator.get_common_events()
|
||||
self._aggregator.clear_events()
|
||||
self._aggregator.add_events(common_events)
|
||||
self._aggregator.reset_workers()
|
||||
return self
|
||||
|
||||
def increment_workers(self, count: int = 1) -> None:
|
||||
self._aggregator.increment_workers(count)
|
||||
|
||||
def get_all_events(self) -> list[KVCacheEvent]:
|
||||
return self._aggregator.get_all_events()
|
||||
|
||||
def get_number_of_workers(self) -> int:
|
||||
return self._aggregator.get_number_of_workers()
|
||||
|
||||
def clear_events(self) -> None:
|
||||
self._aggregator.clear_events()
|
||||
self._aggregator.reset_workers()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<LMCacheKVEvents events={self.get_all_events()}>"
|
||||
|
||||
|
||||
class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
@classmethod
|
||||
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
|
||||
"""LMCache requires PIECEWISE CUDA graph mode when layerwise
|
||||
operations are enabled. The wait_for_layer_load and save_kv_layer
|
||||
methods perform actual async synchronization that cannot be
|
||||
captured in CUDA graphs.
|
||||
"""
|
||||
return extra_config.get("use_layerwise", False)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
):
|
||||
super().__init__(
|
||||
vllm_config=vllm_config, role=role, kv_cache_config=kv_cache_config
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
use_native = vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"use_native", False
|
||||
)
|
||||
if use_native:
|
||||
logger.info("Initializing native LMCache connector")
|
||||
# lazy import
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import lmcache_integration
|
||||
|
||||
_adapter = lmcache_integration.vllm_v1_adapter
|
||||
|
||||
cls = _adapter.LMCacheConnectorV1Impl
|
||||
else:
|
||||
logger.info("Initializing latest dev LMCache connector")
|
||||
# lazy import
|
||||
from lmcache.integration.vllm.vllm_v1_adapter import (
|
||||
LMCacheConnectorV1Impl as LMCacheConnectorLatestImpl,
|
||||
)
|
||||
|
||||
cls = LMCacheConnectorLatestImpl
|
||||
|
||||
self._lmcache_engine = cls(vllm_config, role, self)
|
||||
|
||||
self._kv_cache_events: LMCacheKVEvents | None = None
|
||||
|
||||
# ==============================
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Initialize with the KV caches. Useful for pre-registering the
|
||||
KV Caches in the KVConnector (e.g. for NIXL).
|
||||
|
||||
Args:
|
||||
kv_caches: dictionary of layer names, kv cache
|
||||
|
||||
"""
|
||||
if hasattr(self._lmcache_engine, "register_kv_caches"):
|
||||
self._lmcache_engine.register_kv_caches(kv_caches)
|
||||
else:
|
||||
logger.warning(
|
||||
"LMCache engine does not support register_kv_caches, "
|
||||
"please check and use the latest version"
|
||||
)
|
||||
|
||||
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
|
||||
super().bind_connector_metadata(connector_metadata)
|
||||
engine = self._lmcache_engine
|
||||
if hasattr(engine, "bind_connector_metadata"):
|
||||
engine.bind_connector_metadata(connector_metadata)
|
||||
else:
|
||||
# Compat for engines predating the bind hook: start_load_kv may
|
||||
# run after the forward (SchedulerOutput.has_sync_kv_loads), so
|
||||
# the per-step layerwise state it used to reset must be reset
|
||||
# here. Remove once LMCache ships bind_connector_metadata.
|
||||
if hasattr(engine, "current_layer"):
|
||||
engine.current_layer = 0
|
||||
if hasattr(engine, "layerwise_retrievers"):
|
||||
engine.layerwise_retrievers = []
|
||||
engine.layerwise_storers = []
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""Start loading the KV cache from the connector to vLLM's paged
|
||||
KV buffer. This is called from the forward context before the
|
||||
forward pass to enable async loading during model execution.
|
||||
|
||||
Args:
|
||||
forward_context (ForwardContext): the forward context.
|
||||
**kwargs: additional arguments for the load operation
|
||||
|
||||
Note:
|
||||
The number of elements in kv_caches and layer_names should be
|
||||
the same.
|
||||
|
||||
"""
|
||||
self._lmcache_engine.start_load_kv(forward_context, **kwargs)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""Block until the KV for a specific layer is loaded into vLLM's
|
||||
paged buffer. This is called from within attention layer to ensure
|
||||
async copying from start_load_kv is complete.
|
||||
|
||||
This interface will be useful for layer-by-layer pipelining.
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
self._lmcache_engine.wait_for_layer_load(layer_name)
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Start saving the a layer of KV cache from vLLM's paged buffer
|
||||
to the connector. This is called from within attention layer to
|
||||
enable async copying during execution.
|
||||
|
||||
Args:
|
||||
layer_name (str): the name of the layer.
|
||||
kv_layer (torch.Tensor): the paged KV buffer of the current
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
**kwargs: additional arguments for the save operation.
|
||||
|
||||
"""
|
||||
self._lmcache_engine.save_kv_layer(
|
||||
layer_name, kv_layer, attn_metadata, **kwargs
|
||||
)
|
||||
|
||||
def wait_for_save(self):
|
||||
"""Block until all the save operations is done. This is called
|
||||
as the forward context exits to ensure that the async saving
|
||||
from save_kv_layer is complete before finishing the forward.
|
||||
|
||||
This prevents overwrites of paged KV buffer before saving done.
|
||||
"""
|
||||
self._lmcache_engine.wait_for_save()
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""Notifies worker-side connector ids of requests that have
|
||||
finished generating tokens.
|
||||
|
||||
Returns:
|
||||
ids of requests that have finished asynchronous transfer
|
||||
(requests that previously returned True from request_finished()),
|
||||
tuple of (sending/saving ids, recving/loading ids).
|
||||
The finished saves/sends req ids must belong to a set provided in a
|
||||
call to this method (this call or a prior one).
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.get_finished(finished_req_ids)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Get the set of block IDs that failed to load.
|
||||
|
||||
Returns:
|
||||
Set of block IDs that encountered load errors.
|
||||
Empty set if no load errors occurred.
|
||||
|
||||
"""
|
||||
method = getattr(self._lmcache_engine, "get_block_ids_with_load_errors", None)
|
||||
if callable(method):
|
||||
return method()
|
||||
|
||||
# Fallback for older versions that don't support this method
|
||||
return set()
|
||||
|
||||
def get_kv_connector_kv_cache_events(self) -> LMCacheKVEvents | None:
|
||||
"""Get the KV connector kv cache events collected during the last interval."""
|
||||
events = self._lmcache_engine.get_kv_events() # type: ignore [attr-defined]
|
||||
if not events:
|
||||
return None
|
||||
|
||||
blocks: list[BlockStored] = [
|
||||
BlockStored(
|
||||
block_hashes=e.block_hashes,
|
||||
parent_block_hash=e.parent_block_hash,
|
||||
token_ids=e.token_ids,
|
||||
lora_id=e.lora_id,
|
||||
block_size=e.block_size,
|
||||
medium=e.medium,
|
||||
lora_name=getattr(e, "lora_name", None),
|
||||
)
|
||||
for e in events
|
||||
]
|
||||
|
||||
lmcache_kv_events = LMCacheKVEvents(num_workers=1)
|
||||
lmcache_kv_events.add_events(blocks)
|
||||
return lmcache_kv_events
|
||||
|
||||
# ==============================
|
||||
# Scheduler-side methods
|
||||
# ==============================
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
num_computed_tokens (int): the number of locally
|
||||
computed tokens for this request
|
||||
|
||||
Returns:
|
||||
the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
), False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""Update KVConnector state after block allocation."""
|
||||
self._lmcache_engine.update_state_after_alloc(request, num_external_tokens)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.build_connector_meta(scheduler_output)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""Update KVConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
# Get the KV events
|
||||
kv_cache_events = connector_output.kv_cache_events
|
||||
if not kv_cache_events or not isinstance(kv_cache_events, LMCacheKVEvents):
|
||||
return
|
||||
|
||||
if self._kv_cache_events is None:
|
||||
self._kv_cache_events = kv_cache_events
|
||||
else:
|
||||
self._kv_cache_events.add_events(kv_cache_events.get_all_events())
|
||||
self._kv_cache_events.increment_workers(
|
||||
kv_cache_events.get_number_of_workers()
|
||||
)
|
||||
return
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Called when a request has finished, before its blocks are freed.
|
||||
|
||||
Returns:
|
||||
True if the request is being saved/sent asynchronously and blocks
|
||||
should not be freed until the request_id is returned from
|
||||
get_finished().
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.request_finished(request, block_ids)
|
||||
|
||||
def take_events(self) -> Iterable["KVCacheEvent"]:
|
||||
"""Take the KV cache events from the connector.
|
||||
|
||||
Yields:
|
||||
New KV cache events since the last call.
|
||||
|
||||
"""
|
||||
if self._kv_cache_events is not None:
|
||||
self._kv_cache_events.aggregate()
|
||||
kv_cache_events = self._kv_cache_events.get_all_events()
|
||||
yield from kv_cache_events
|
||||
self._kv_cache_events.clear_events()
|
||||
self._kv_cache_events = None
|
||||
@@ -1,20 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from . import multi_process_adapter, vllm_v1_adapter
|
||||
from .multi_process_adapter import (
|
||||
LMCacheMPSchedulerAdapter,
|
||||
LMCacheMPWorkerAdapter,
|
||||
LoadStoreOp,
|
||||
ParallelStrategy,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"vllm_v1_adapter",
|
||||
"multi_process_adapter",
|
||||
"LMCacheMPSchedulerAdapter",
|
||||
"LMCacheMPWorkerAdapter",
|
||||
"LoadStoreOp",
|
||||
"ParallelStrategy",
|
||||
]
|
||||
-735
@@ -1,735 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from itertools import islice
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import zmq
|
||||
from lmcache.utils import _lmcache_nvtx_annotate, init_logger
|
||||
from lmcache.v1.multiprocess.custom_types import (
|
||||
CudaIPCWrapper,
|
||||
IPCCacheEngineKey,
|
||||
KVCache,
|
||||
)
|
||||
from lmcache.v1.multiprocess.mq import MessageQueueClient, MessagingFuture
|
||||
from lmcache.v1.multiprocess.protocol import RequestType, get_response_class
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def wrap_kv_caches(kv_caches: dict[str, torch.Tensor]) -> KVCache:
|
||||
logger.info("KV caches keys are %s", list(kv_caches.keys()))
|
||||
return [CudaIPCWrapper(tensor) for tensor in kv_caches.values()]
|
||||
|
||||
|
||||
def striding_block_hashes(
|
||||
block_hashes: list[bytes], blocks_in_chunk: int
|
||||
) -> Iterable[bytes]:
|
||||
"""Extract chunk-level hashes from block hashes by striding.
|
||||
|
||||
In hash-based vLLM, each vLLM block has its own hash. LMCache chunks
|
||||
span ``blocks_in_chunk`` consecutive blocks. The representative hash
|
||||
for a chunk is the hash of the **last** block in that chunk (because
|
||||
each block hash already encodes its prefix). So we start at index
|
||||
``blocks_in_chunk - 1`` and stride by ``blocks_in_chunk``.
|
||||
"""
|
||||
return islice(block_hashes, blocks_in_chunk - 1, None, blocks_in_chunk)
|
||||
|
||||
|
||||
def send_lmcache_request(
|
||||
mq_client: MessageQueueClient,
|
||||
request_type: RequestType,
|
||||
payloads: list[Any],
|
||||
) -> MessagingFuture[Any]:
|
||||
"""Helper function to send the request to the LMCache multiprocess server.
|
||||
|
||||
Args:
|
||||
mq_client: The LMCache multiprocess mode message queue client
|
||||
request_type: The request type
|
||||
payloads: The request payloads
|
||||
|
||||
Returns:
|
||||
A messaging future for the request
|
||||
|
||||
"""
|
||||
future = mq_client.submit_request(
|
||||
request_type, payloads, get_response_class(request_type)
|
||||
)
|
||||
return future
|
||||
|
||||
|
||||
def get_lmcache_chunk_size(
|
||||
mq_client: MessageQueueClient,
|
||||
) -> int:
|
||||
"""Helper function to get the LMCache chunk size from the server.
|
||||
|
||||
Args:
|
||||
mq_client: The LMCache multiprocess mode message queue client
|
||||
|
||||
Returns:
|
||||
An integer representing the LMCache chunk size
|
||||
|
||||
"""
|
||||
future = send_lmcache_request(mq_client, RequestType.GET_CHUNK_SIZE, [])
|
||||
chunk_size = future.result()
|
||||
return chunk_size
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParallelStrategy:
|
||||
use_mla: bool
|
||||
"""Whether to use the MLA."""
|
||||
|
||||
kv_world_size: int
|
||||
"""
|
||||
The kv world size, kv_world_size may not be equal to the actual_world_size,
|
||||
in the case of mla, it will 'exclude' the effect of TP, the value is
|
||||
calculated by `extract_world_size_and_kv_rank` in `lmcache_mp_connector.py`.
|
||||
"""
|
||||
|
||||
kv_worker_id: int
|
||||
"""
|
||||
The kv worker id of the sub-process, kv_worker_id may not be equal to the
|
||||
actual_worker_id, in the case of mla, it will 'exclude' the effect of TP,
|
||||
the value is calculated by `extract_world_size_and_kv_rank` in
|
||||
`lmcache_mp_connector.py`.
|
||||
"""
|
||||
|
||||
actual_world_size: int
|
||||
"""The actual world size."""
|
||||
|
||||
actual_worker_id: int
|
||||
"""The actual worker id of the sub-process."""
|
||||
|
||||
tp_size: int
|
||||
"""The tensor parallel size."""
|
||||
|
||||
pp_size: int
|
||||
"""The pipeline parallel size."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadStoreOp:
|
||||
block_ids: list[int]
|
||||
"""Block ids for the load/store operation"""
|
||||
|
||||
token_ids: list[int] | None = None
|
||||
"""Token IDs for the load/store operation (token mode)"""
|
||||
|
||||
block_hashes: list[bytes] | None = None
|
||||
"""Block hashes for the load/store operation (hash mode)"""
|
||||
|
||||
start: int = 0
|
||||
"""Start token index (token mode only)"""
|
||||
|
||||
end: int = 0
|
||||
"""End token index (token mode only)"""
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.block_ids)
|
||||
|
||||
|
||||
StoreResult = bool
|
||||
RetrieveResult = list[bool]
|
||||
LookupResult = int
|
||||
|
||||
|
||||
class LMCacheMPSchedulerAdapter:
|
||||
def __init__(
|
||||
self,
|
||||
server_url: str,
|
||||
context: zmq.Context,
|
||||
model_name: str,
|
||||
vllm_block_size: int,
|
||||
parallel_strategy: ParallelStrategy,
|
||||
):
|
||||
"""Args:
|
||||
server_url: The server URL for the LMCache message queue
|
||||
context: The ZMQ context
|
||||
|
||||
model_name: The model name used for LMCache keys
|
||||
vllm_block_size: The block size used in vLLM
|
||||
parallel_strategy:
|
||||
The parallel strategy, which includes `use_mla`,
|
||||
`world_size`, `worker_id` and so on
|
||||
|
||||
"""
|
||||
self.mq_client = MessageQueueClient(server_url, context)
|
||||
|
||||
# Request futures
|
||||
self.lookup_futures: dict[str, MessagingFuture[LookupResult]] = {}
|
||||
|
||||
self.model_name = model_name
|
||||
self.parallel_strategy = parallel_strategy
|
||||
|
||||
# Read chunk size from lmcache
|
||||
self.chunk_size = get_lmcache_chunk_size(self.mq_client)
|
||||
assert self.chunk_size % vllm_block_size == 0, (
|
||||
"LMCache chunk size should be a multiple of vLLM block size"
|
||||
)
|
||||
self.blocks_in_chunk = self.chunk_size // vllm_block_size
|
||||
|
||||
@property
|
||||
def world_size(self) -> int:
|
||||
"""The world size."""
|
||||
return self.parallel_strategy.kv_world_size
|
||||
|
||||
@property
|
||||
def worker_id(self) -> int:
|
||||
"""The worker id."""
|
||||
return self.parallel_strategy.kv_worker_id
|
||||
|
||||
@property
|
||||
def tp_size(self) -> int:
|
||||
"""The tensor parallel size."""
|
||||
return self.parallel_strategy.tp_size
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def maybe_submit_lookup_request(
|
||||
self,
|
||||
request_id: str,
|
||||
block_hashes: list[bytes] | None = None,
|
||||
token_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
"""Submit a new lookup request to LMCache if there is no ongoing request.
|
||||
|
||||
Supports both token-based and hash-based vLLM:
|
||||
- token_ids: token IDs (token-based vLLM) -> single token-mode key
|
||||
- block_hashes: block hashes (hash-based vLLM) -> strided hash-mode keys
|
||||
|
||||
Exactly one of block_hashes or token_ids must be provided.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the lookup request. The same ID indicates it's
|
||||
from the same request
|
||||
block_hashes: Block hashes to lookup from LMCache (hash mode)
|
||||
token_ids: Token IDs to lookup from LMCache (token mode)
|
||||
|
||||
Returns:
|
||||
None
|
||||
|
||||
Notes:
|
||||
This function will have a side-effect: submitting a look up request to
|
||||
LMCache, which will essentially 'lock' the KV cache chunks in the LMCache
|
||||
for later retrieve operations.
|
||||
In the meantime, this function will record the lookup request, and the
|
||||
status of the look up request can be checked by `check_lookup_result`.
|
||||
|
||||
"""
|
||||
if request_id in self.lookup_futures:
|
||||
# Skip if there is already a lookup request
|
||||
return
|
||||
|
||||
assert (block_hashes is None) != (token_ids is None), (
|
||||
"Exactly one of block_hashes or token_ids must be provided"
|
||||
)
|
||||
|
||||
if block_hashes is not None:
|
||||
# Hash mode: stride block hashes -> N hash-mode keys
|
||||
chunk_hashes = list(
|
||||
striding_block_hashes(block_hashes, self.blocks_in_chunk)
|
||||
)
|
||||
keys = [
|
||||
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
|
||||
]
|
||||
else:
|
||||
# Token mode: truncate to chunk-aligned length
|
||||
assert token_ids is not None
|
||||
aligned_end = (len(token_ids) // self.chunk_size) * self.chunk_size
|
||||
if aligned_end == 0:
|
||||
return
|
||||
keys = [
|
||||
self._create_key(
|
||||
token_ids,
|
||||
start=0,
|
||||
end=aligned_end,
|
||||
request_id=request_id,
|
||||
).no_worker_id_version()
|
||||
]
|
||||
|
||||
future = send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.LOOKUP,
|
||||
[keys],
|
||||
)
|
||||
self.lookup_futures[request_id] = future
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def check_lookup_result(self, request_id: str) -> int | None:
|
||||
"""Check the result of a previously submitted lookup request.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the lookup request submitted in
|
||||
`maybe_submit_lookup_request`
|
||||
|
||||
Returns:
|
||||
An integer representing the total number of tokens matched
|
||||
in LMCache (prefix matching), or
|
||||
None if the lookup request is not finished yet.
|
||||
|
||||
"""
|
||||
assert request_id in self.lookup_futures, (
|
||||
f"Lookup request for request_id={request_id} has not been submitted"
|
||||
)
|
||||
|
||||
future = self.lookup_futures[request_id]
|
||||
if not future.query():
|
||||
return None
|
||||
|
||||
result = future.result()
|
||||
num_chunks = result
|
||||
return num_chunks * self.chunk_size
|
||||
|
||||
def num_blocks_per_chunk(self) -> int:
|
||||
"""Returns:
|
||||
The number of vllm blocks in a LMCache data chunk
|
||||
|
||||
"""
|
||||
return self.blocks_in_chunk
|
||||
|
||||
def cleanup_lookup_result(self, request_id: str) -> None:
|
||||
"""Clean up lookup future for a finished request to prevent memory leak.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the finished request.
|
||||
|
||||
"""
|
||||
self.lookup_futures.pop(request_id, None)
|
||||
|
||||
def end_session(self, request_id: str) -> None:
|
||||
"""Notify LMCache server to remove the session for a finished request.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the finished request.
|
||||
|
||||
"""
|
||||
send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.END_SESSION,
|
||||
[request_id],
|
||||
)
|
||||
|
||||
# Helper functions
|
||||
def _create_key(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
start: int = 0,
|
||||
end: int = 0,
|
||||
request_id: str | None = None,
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Convert token IDs to an IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
worker_id=self.worker_id,
|
||||
token_ids=tuple(token_ids),
|
||||
start=start,
|
||||
end=end,
|
||||
request_id=request_id,
|
||||
tp_size=self.tp_size,
|
||||
)
|
||||
|
||||
def _create_hash_key(
|
||||
self, chunk_hash: bytes, request_id: str | None = None
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Create a hash-mode IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
worker_id=None,
|
||||
chunk_hash=chunk_hash,
|
||||
request_id=request_id,
|
||||
tp_size=self.tp_size,
|
||||
)
|
||||
|
||||
|
||||
class LMCacheMPWorkerAdapter:
|
||||
def __init__(
|
||||
self,
|
||||
server_url: str,
|
||||
context: zmq.Context,
|
||||
model_name: str,
|
||||
vllm_block_size: int,
|
||||
parallel_strategy: ParallelStrategy,
|
||||
):
|
||||
self.mq_client = MessageQueueClient(server_url, context)
|
||||
|
||||
# Instance id for GPU worker
|
||||
self.instance_id = os.getpid()
|
||||
|
||||
# Registered kv caches from vLLM
|
||||
self.kv_caches: dict[str, torch.Tensor] = {}
|
||||
|
||||
# Request futures
|
||||
# request_id -> (future, other merged requests)
|
||||
self.store_futures: dict[
|
||||
str, tuple[MessagingFuture[StoreResult], list[str]]
|
||||
] = {}
|
||||
self.retrieve_futures: dict[
|
||||
str, tuple[MessagingFuture[RetrieveResult], list[str]]
|
||||
] = {}
|
||||
|
||||
# The store requests that have finished execution in LMCache
|
||||
self.finished_stores: set[str] = set()
|
||||
# The finished request ids that are passed via vLLM and also
|
||||
# have corresponding store requests submitted to LMCache before
|
||||
self.previously_finished: set[str] = set()
|
||||
|
||||
self.model_name = model_name
|
||||
self.parallel_strategy = parallel_strategy
|
||||
|
||||
# Read chunk size from lmcache
|
||||
chunk_size = get_lmcache_chunk_size(self.mq_client)
|
||||
assert chunk_size % vllm_block_size == 0, (
|
||||
"LMCache chunk size should be a multiple of vLLM block size"
|
||||
)
|
||||
self.blocks_in_chunk = chunk_size // vllm_block_size
|
||||
|
||||
@property
|
||||
def world_size(self) -> int:
|
||||
"""The world size."""
|
||||
return self.parallel_strategy.kv_world_size
|
||||
|
||||
@property
|
||||
def worker_id(self) -> int:
|
||||
"""The worker id."""
|
||||
return self.parallel_strategy.kv_worker_id
|
||||
|
||||
@property
|
||||
def use_mla(self) -> bool:
|
||||
"""Whether to use MLA."""
|
||||
return self.parallel_strategy.use_mla
|
||||
|
||||
@property
|
||||
def is_first_rank_of_pp_group(self) -> bool:
|
||||
"""Is the first rank of the pipeline parallel group."""
|
||||
return (
|
||||
self.parallel_strategy.actual_worker_id % self.parallel_strategy.tp_size
|
||||
== 0
|
||||
)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Register the kv caches with LMCache server.
|
||||
|
||||
Args:
|
||||
kv_caches: A dict of kv caches to register. The keys are the
|
||||
layer names and the values are the corresponding tensors.
|
||||
|
||||
"""
|
||||
# Register kv cache and send the request
|
||||
self.kv_caches = kv_caches
|
||||
logger.info("Registering kv caches")
|
||||
future = send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.REGISTER_KV_CACHE,
|
||||
[self.instance_id, wrap_kv_caches(kv_caches)],
|
||||
)
|
||||
future.result()
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def submit_store_request(
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
|
||||
):
|
||||
"""Submit a KV cache store request to LMCache.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the request
|
||||
op: The LoadStoreOp describing the store operation.
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
if op.block_hashes is not None:
|
||||
# Hash mode
|
||||
chunk_hashes = list(
|
||||
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
|
||||
)
|
||||
keys = [
|
||||
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
|
||||
]
|
||||
else:
|
||||
# Token mode
|
||||
assert op.token_ids is not None
|
||||
keys = [
|
||||
self._create_key(op.token_ids, op.start, op.end, request_id=request_id)
|
||||
]
|
||||
future = send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.STORE,
|
||||
[keys, self.instance_id, op.block_ids, event.ipc_handle()],
|
||||
).to_cuda_future()
|
||||
self.store_futures[request_id] = (future, [])
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def submit_retrieve_request(
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
|
||||
):
|
||||
"""Submit a KV cache retrieve request to LMCache.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the request
|
||||
op: The LoadStoreOp describing the retrieve operation.
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
if op.block_hashes is not None:
|
||||
# Hash mode
|
||||
chunk_hashes = list(
|
||||
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
|
||||
)
|
||||
keys = [
|
||||
self._create_hash_key(ch, request_id=request_id) for ch in chunk_hashes
|
||||
]
|
||||
else:
|
||||
# Token mode
|
||||
assert op.token_ids is not None
|
||||
keys = [
|
||||
self._create_key(op.token_ids, op.start, op.end, request_id=request_id)
|
||||
]
|
||||
future = send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.RETRIEVE,
|
||||
[keys, self.instance_id, op.block_ids, event.ipc_handle()],
|
||||
).to_cuda_future()
|
||||
self.retrieve_futures[request_id] = (future, [])
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def batched_submit_store_requests(
|
||||
self,
|
||||
request_ids: list[str],
|
||||
ops: list[LoadStoreOp],
|
||||
event: torch.cuda.Event,
|
||||
):
|
||||
"""Submit a batched store request to LMCache.
|
||||
|
||||
Args:
|
||||
request_ids: The IDs of the requests
|
||||
ops: The LoadStoreOps describing the store operations. Should have
|
||||
the same length as request_ids
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
all_keys: list[IPCCacheEngineKey] = []
|
||||
block_ids: list[int] = []
|
||||
for request_id, op in zip(request_ids, ops, strict=False):
|
||||
if op.block_hashes is not None:
|
||||
chunk_hashes = list(
|
||||
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
|
||||
)
|
||||
keys = [
|
||||
self._create_hash_key(ch, request_id=request_id)
|
||||
for ch in chunk_hashes
|
||||
]
|
||||
all_keys.extend(keys)
|
||||
else:
|
||||
assert op.token_ids is not None
|
||||
all_keys.append(
|
||||
self._create_key(
|
||||
op.token_ids, op.start, op.end, request_id=request_id
|
||||
)
|
||||
)
|
||||
block_ids.extend(op.block_ids)
|
||||
future = send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.STORE,
|
||||
[
|
||||
all_keys,
|
||||
self.instance_id,
|
||||
block_ids,
|
||||
event.ipc_handle(),
|
||||
],
|
||||
).to_cuda_future()
|
||||
self.store_futures[request_ids[0]] = (future, list(request_ids[1:]))
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def batched_submit_retrieve_requests(
|
||||
self,
|
||||
request_ids: list[str],
|
||||
ops: list[LoadStoreOp],
|
||||
event: torch.cuda.Event,
|
||||
):
|
||||
"""Submit a batched retrieve request to LMCache.
|
||||
|
||||
Args:
|
||||
request_ids: The IDs of the requests
|
||||
ops: The LoadStoreOps describing the retrieve operations. Should have
|
||||
the same length as request_ids
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
all_keys: list[IPCCacheEngineKey] = []
|
||||
block_ids: list[int] = []
|
||||
for request_id, op in zip(request_ids, ops, strict=False):
|
||||
if op.block_hashes is not None:
|
||||
chunk_hashes = list(
|
||||
striding_block_hashes(op.block_hashes, self.blocks_in_chunk)
|
||||
)
|
||||
keys = [
|
||||
self._create_hash_key(ch, request_id=request_id)
|
||||
for ch in chunk_hashes
|
||||
]
|
||||
all_keys.extend(keys)
|
||||
else:
|
||||
assert op.token_ids is not None
|
||||
all_keys.append(
|
||||
self._create_key(
|
||||
op.token_ids, op.start, op.end, request_id=request_id
|
||||
)
|
||||
)
|
||||
block_ids.extend(op.block_ids)
|
||||
future = send_lmcache_request(
|
||||
self.mq_client,
|
||||
RequestType.RETRIEVE,
|
||||
[
|
||||
all_keys,
|
||||
self.instance_id,
|
||||
block_ids,
|
||||
event.ipc_handle(),
|
||||
],
|
||||
).to_cuda_future()
|
||||
self.retrieve_futures[request_ids[0]] = (future, list(request_ids[1:]))
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def get_finished(
|
||||
self, finished_req_ids_from_engine: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""Check and get the finished store and retrieve requests.
|
||||
|
||||
Args:
|
||||
finished_req_ids_from_engine: the set of request ids that are
|
||||
reported as finished from the vLLM engine side.
|
||||
|
||||
Returns:
|
||||
A tuple of two sets:
|
||||
- The first set contains the finished store request ids. The returned
|
||||
store request ids MUST be seen before in the
|
||||
`finished_req_ids_from_engine`.
|
||||
- The second set contains the finished retrieve request ids.
|
||||
|
||||
Notes:
|
||||
When enabling async scheduling in vLLM, the same request ID may appear
|
||||
multiple times in `finished_req_ids_from_engine`. The adapter should
|
||||
take care of deduplicating the request IDs and only return the request
|
||||
IDs that have not been returned before.
|
||||
|
||||
"""
|
||||
finished_stores = set()
|
||||
finished_retrieves = set()
|
||||
for request_id, (s_future, other_reqs) in self.store_futures.items():
|
||||
if not s_future.query():
|
||||
continue
|
||||
|
||||
s_result = s_future.result()
|
||||
finished_stores.add(request_id)
|
||||
finished_stores.update(other_reqs)
|
||||
|
||||
if not s_result:
|
||||
# TODO: add error handling here
|
||||
logger.error(
|
||||
"Something went wrong when processing the "
|
||||
"store request for request_id=%s",
|
||||
request_id,
|
||||
)
|
||||
|
||||
for request_id, (r_future, other_reqs) in self.retrieve_futures.items():
|
||||
if not r_future.query():
|
||||
continue
|
||||
|
||||
r_result = r_future.result()
|
||||
finished_retrieves.add(request_id)
|
||||
finished_retrieves.update(other_reqs)
|
||||
|
||||
if not all(r_result):
|
||||
# TODO: add error handing here
|
||||
logger.error(
|
||||
"Something went wrong when processing the "
|
||||
"retrieve request for request_id=%s, result=%s",
|
||||
request_id,
|
||||
r_result,
|
||||
)
|
||||
|
||||
# Remove the finished requests from the tracking dicts
|
||||
for request_id in finished_stores:
|
||||
self.store_futures.pop(request_id, None)
|
||||
for request_id in finished_retrieves:
|
||||
self.retrieve_futures.pop(request_id, None)
|
||||
|
||||
# Update the internal states
|
||||
self.finished_stores.update(finished_stores)
|
||||
|
||||
ret_stores = set()
|
||||
for req_id in finished_req_ids_from_engine:
|
||||
if req_id in self.finished_stores or req_id in self.store_futures:
|
||||
self.previously_finished.add(req_id)
|
||||
else:
|
||||
ret_stores.add(req_id)
|
||||
|
||||
# Calculate the final finished stores
|
||||
ret_stores.update(self._update_and_get_finished_store())
|
||||
|
||||
return ret_stores, finished_retrieves
|
||||
|
||||
def num_blocks_per_chunk(self) -> int:
|
||||
"""Returns:
|
||||
The number of vllm blocks in a LMCache data chunk
|
||||
|
||||
"""
|
||||
return self.blocks_in_chunk
|
||||
|
||||
def shutdown(self):
|
||||
"""Shutdown the LMCache MP worker adapter."""
|
||||
logger.info("Unregistering kv caches")
|
||||
send_lmcache_request(
|
||||
self.mq_client, RequestType.UNREGISTER_KV_CACHE, [self.instance_id]
|
||||
).result()
|
||||
|
||||
self.mq_client.close()
|
||||
|
||||
# Helper functions
|
||||
def _update_and_get_finished_store(
|
||||
self,
|
||||
) -> set[str]:
|
||||
"""Converge the internal states about finished stores
|
||||
and returns the 'safe finished store request ids' back
|
||||
"""
|
||||
safe_finished_s = self.finished_stores.intersection(self.previously_finished)
|
||||
self.finished_stores.difference_update(self.previously_finished)
|
||||
self.previously_finished.difference_update(safe_finished_s)
|
||||
|
||||
return safe_finished_s
|
||||
|
||||
def _create_key(
|
||||
self,
|
||||
token_ids: list[int],
|
||||
start: int = 0,
|
||||
end: int = 0,
|
||||
request_id: str | None = None,
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Convert token IDs to an IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
worker_id=self.worker_id,
|
||||
token_ids=tuple(token_ids),
|
||||
start=start,
|
||||
end=end,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
def _create_hash_key(
|
||||
self, chunk_hash: bytes, request_id: str | None = None
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Create a hash-mode IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
worker_id=self.worker_id,
|
||||
chunk_hash=chunk_hash,
|
||||
request_id=request_id,
|
||||
)
|
||||
@@ -1,134 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Standard
|
||||
import os
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Union
|
||||
|
||||
import torch
|
||||
from lmcache.logging import init_logger
|
||||
from lmcache.v1.config import LMCacheEngineConfig as V1Config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal.inputs import PlaceholderRange
|
||||
from vllm.v1.core.sched.output import NewRequestData
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
ENGINE_NAME = "vllm-instance"
|
||||
|
||||
# Thread-safe singleton storage
|
||||
_config_instance: V1Config | None = None
|
||||
_config_lock = threading.Lock()
|
||||
|
||||
|
||||
def lmcache_get_or_create_config() -> V1Config:
|
||||
"""Get the LMCache configuration from the environment variable
|
||||
`LMCACHE_CONFIG_FILE`. If the environment variable is not set, this
|
||||
function will return the default configuration.
|
||||
|
||||
This function is thread-safe and implements singleton pattern,
|
||||
ensuring the configuration is loaded only once.
|
||||
"""
|
||||
global _config_instance
|
||||
|
||||
# Double-checked locking for thread-safe singleton
|
||||
if _config_instance is None:
|
||||
with _config_lock:
|
||||
if _config_instance is None: # Check again within lock
|
||||
LMCacheEngineConfig = V1Config # type: ignore[assignment]
|
||||
|
||||
if "LMCACHE_CONFIG_FILE" not in os.environ:
|
||||
logger.warning(
|
||||
"No LMCache configuration file is set. Trying to read"
|
||||
" configurations from the environment variables."
|
||||
)
|
||||
logger.warning(
|
||||
"You can set the configuration file through "
|
||||
"the environment variable: LMCACHE_CONFIG_FILE"
|
||||
)
|
||||
_config_instance = LMCacheEngineConfig.from_env()
|
||||
else:
|
||||
config_file = os.environ["LMCACHE_CONFIG_FILE"]
|
||||
logger.info("Loading LMCache config file %s", config_file)
|
||||
_config_instance = LMCacheEngineConfig.from_file(config_file)
|
||||
# Update config from environment variables
|
||||
_config_instance.update_config_from_env()
|
||||
return _config_instance
|
||||
|
||||
|
||||
def hex_hash_to_int16(s: str) -> int:
|
||||
"""Convert a hex hash string to a 16-bit integer."""
|
||||
return int(s, 16) & 0xFFFF
|
||||
|
||||
|
||||
def apply_mm_hashes_to_token_ids(
|
||||
token_ids: torch.Tensor,
|
||||
mm_hashes: list[str],
|
||||
mm_positions: list["PlaceholderRange"],
|
||||
) -> torch.Tensor:
|
||||
"""Overwrite token_ids in-place for multimodal placeholders using
|
||||
efficient slice assignments.
|
||||
"""
|
||||
n = token_ids.size(0)
|
||||
for hash_str, placeholder in zip(mm_hashes, mm_positions):
|
||||
start, length = placeholder.offset, placeholder.length
|
||||
if start >= n:
|
||||
continue
|
||||
end = min(start + length, n)
|
||||
token_ids[start:end] = hex_hash_to_int16(hash_str)
|
||||
return token_ids
|
||||
|
||||
|
||||
def mla_enabled(model_config: "ModelConfig") -> bool:
|
||||
return (
|
||||
hasattr(model_config, "use_mla")
|
||||
and isinstance(model_config.use_mla, bool)
|
||||
and model_config.use_mla
|
||||
)
|
||||
|
||||
|
||||
def extract_mm_features(
|
||||
request: Union["Request", "NewRequestData"], modify: bool = False
|
||||
) -> tuple[list[str], list["PlaceholderRange"]]:
|
||||
"""Normalize multimodal information from a Request into parallel lists.
|
||||
|
||||
This helper reads either:
|
||||
1) `request.mm_features` (objects each exposing `.identifier` and
|
||||
`.mm_position`), or
|
||||
2) legacy fields `request.mm_hashes` and `request.mm_positions`.
|
||||
|
||||
It returns two equally sized lists: the multimodal hash identifiers and
|
||||
their corresponding positions. If the request contains no multimodal info,
|
||||
it returns `([], [])`.
|
||||
|
||||
Args:
|
||||
request (Request): The source object.
|
||||
modify (bool):
|
||||
Controls copy semantics for the legacy-path return values.
|
||||
- If True and legacy fields are used, shallow-copies are returned so
|
||||
the caller can mutate the lists without affecting `request`.
|
||||
- If False, the original legacy sequences are returned as-is
|
||||
(zero-copy); treat them as read-only.
|
||||
|
||||
Returns:
|
||||
tuple[list[str], list[PlaceholderRange]]: (`mm_hashes`, `mm_positions`).
|
||||
May be `([], [])` when no multimodal data is present.
|
||||
|
||||
"""
|
||||
if getattr(request, "mm_features", None):
|
||||
mm_hashes, mm_positions = zip(
|
||||
*((f.identifier, f.mm_position) for f in request.mm_features)
|
||||
)
|
||||
return (list(mm_hashes), list(mm_positions))
|
||||
elif getattr(request, "mm_hashes", None):
|
||||
if modify:
|
||||
return (
|
||||
request.mm_hashes.copy(), # type: ignore
|
||||
request.mm_positions.copy(), # type: ignore
|
||||
)
|
||||
else:
|
||||
return (request.mm_hashes, request.mm_positions) # type: ignore
|
||||
else:
|
||||
return ([], [])
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,178 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import EngineId
|
||||
from vllm.logger import init_logger
|
||||
|
||||
WorkerAddr = str
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class RegisterWorkerPayload(BaseModel):
|
||||
engine_id: EngineId
|
||||
dp_rank: int
|
||||
tp_rank: int
|
||||
pp_rank: int
|
||||
addr: WorkerAddr
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineEntry:
|
||||
engine_id: EngineId
|
||||
# {tp_rank: {pp_rank: worker_addr}}
|
||||
worker_addr: dict[int, dict[int, WorkerAddr]]
|
||||
|
||||
|
||||
class MooncakeBootstrapServer:
|
||||
"""A centralized registry for prefiller connection info (IP, port, ranks)."""
|
||||
|
||||
def __init__(self, host: str, port: int):
|
||||
self.workers: dict[int, EngineEntry] = {}
|
||||
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.app = FastAPI()
|
||||
self._register_routes()
|
||||
self.server_thread: threading.Thread | None = None
|
||||
self.server: uvicorn.Server | None = None
|
||||
self._socket: socket.socket | None = None
|
||||
self._startup_error: BaseException | None = None
|
||||
|
||||
def __del__(self):
|
||||
self.shutdown()
|
||||
|
||||
def _register_routes(self):
|
||||
# All methods are async. No need to use lock to protect data.
|
||||
self.app.post("/register")(self.register_worker)
|
||||
self.app.get("/query", response_model=dict[int, EngineEntry])(self.query)
|
||||
|
||||
def start(self, *, timeout: float = 30.0):
|
||||
if self.server_thread:
|
||||
if (
|
||||
self.server_thread.is_alive()
|
||||
and self.server is not None
|
||||
and self.server.started
|
||||
and not self.server.should_exit
|
||||
and self._socket is not None
|
||||
and self._socket.fileno() >= 0
|
||||
):
|
||||
return
|
||||
raise RuntimeError(
|
||||
"Mooncake bootstrap server is still starting or stopping"
|
||||
)
|
||||
|
||||
try:
|
||||
# Bind in the caller so errors propagate, and retain the listener
|
||||
# when handing it to Uvicorn, including for automatically chosen ports.
|
||||
family = socket.AF_INET6 if ":" in self.host else socket.AF_INET
|
||||
self._socket = socket.create_server((self.host, self.port), family=family)
|
||||
self.port = self._socket.getsockname()[1]
|
||||
config = uvicorn.Config(app=self.app, host=self.host, port=self.port)
|
||||
server = self.server = uvicorn.Server(config=config)
|
||||
listener = self._socket
|
||||
self._startup_error = None
|
||||
|
||||
def run():
|
||||
try:
|
||||
server.run(sockets=[listener])
|
||||
except BaseException as exc:
|
||||
self._startup_error = exc
|
||||
|
||||
self.server_thread = threading.Thread(
|
||||
target=run, name="mooncake_bootstrap_server", daemon=True
|
||||
)
|
||||
self.server_thread.start()
|
||||
deadline = time.monotonic() + timeout
|
||||
while not server.started:
|
||||
if not self.server_thread.is_alive():
|
||||
raise RuntimeError(
|
||||
"Mooncake bootstrap server exited during startup"
|
||||
) from self._startup_error
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError(
|
||||
"Mooncake bootstrap server did not start in time"
|
||||
)
|
||||
time.sleep(0.01)
|
||||
except BaseException:
|
||||
self.shutdown()
|
||||
raise
|
||||
logger.info("Mooncake Bootstrap Server started at %s:%d", self.host, self.port)
|
||||
|
||||
def shutdown(self):
|
||||
was_started = self.server is not None and self.server.started
|
||||
if self.server is not None:
|
||||
self.server.should_exit = True
|
||||
if self.server_thread is not None and self.server_thread.ident is not None:
|
||||
self.server_thread.join(timeout=5)
|
||||
if self._socket is not None:
|
||||
self._socket.close()
|
||||
self._socket = None
|
||||
if self.server_thread is not None and self.server_thread.is_alive():
|
||||
logger.warning("Mooncake bootstrap server did not stop in time")
|
||||
else:
|
||||
self.server_thread = None
|
||||
self.server = None
|
||||
if was_started:
|
||||
logger.info("Mooncake Bootstrap Server stopped.")
|
||||
|
||||
async def register_worker(self, payload: RegisterWorkerPayload):
|
||||
"""Handles registration of a prefiller worker."""
|
||||
if payload.dp_rank not in self.workers:
|
||||
self.workers[payload.dp_rank] = EngineEntry(
|
||||
engine_id=payload.engine_id,
|
||||
worker_addr={},
|
||||
)
|
||||
|
||||
dp_entry = self.workers[payload.dp_rank]
|
||||
if dp_entry.engine_id != payload.engine_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Engine ID mismatch for dp_rank={payload.dp_rank}: "
|
||||
f"expected {dp_entry.engine_id}, got {payload.engine_id}"
|
||||
),
|
||||
)
|
||||
if payload.tp_rank not in dp_entry.worker_addr:
|
||||
dp_entry.worker_addr[payload.tp_rank] = {}
|
||||
|
||||
tp_entry = dp_entry.worker_addr[payload.tp_rank]
|
||||
existing = tp_entry.get(payload.pp_rank)
|
||||
if existing is not None:
|
||||
# A client timeout can fire after the server recorded the
|
||||
# registration so an identical retry must not be an error.
|
||||
if existing == payload.addr:
|
||||
return {"status": "ok"}
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(
|
||||
f"Worker with dp_rank={payload.dp_rank}, "
|
||||
f"tp_rank={payload.tp_rank}, pp_rank={payload.pp_rank} "
|
||||
f"is already registered at "
|
||||
f"{existing}, "
|
||||
f"but still want to register at {payload.addr}"
|
||||
),
|
||||
)
|
||||
|
||||
tp_entry[payload.pp_rank] = payload.addr
|
||||
logger.debug(
|
||||
"Registered worker: engine_id=%s, dp_rank=%d, tp_rank=%d, pp_rank=%d at %s",
|
||||
payload.engine_id,
|
||||
payload.dp_rank,
|
||||
payload.tp_rank,
|
||||
payload.pp_rank,
|
||||
payload.addr,
|
||||
)
|
||||
|
||||
return {"status": "ok"}
|
||||
|
||||
async def query(self) -> dict[int, EngineEntry]:
|
||||
return self.workers
|
||||
@@ -1,46 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Mooncake requester config helpers."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def normalize_string_override(value: Any) -> str | None:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
normalized = value.strip()
|
||||
return normalized or None
|
||||
|
||||
|
||||
def get_requester_local_hostname(local_ip: str) -> str:
|
||||
override = normalize_string_override(envs.MOONCAKE_REQUESTER_LOCAL_HOSTNAME)
|
||||
if override is not None:
|
||||
return override
|
||||
return local_ip
|
||||
|
||||
|
||||
def get_configured_preferred_segment(
|
||||
extra_config: Mapping[str, Any],
|
||||
) -> str | None:
|
||||
preferred_segment = normalize_string_override(extra_config.get("preferred_segment"))
|
||||
if preferred_segment is not None:
|
||||
return preferred_segment
|
||||
if extra_config.get("preferred_segment") is not None:
|
||||
raise ValueError(
|
||||
"Mooncake preferred_segment override must be a non-empty string"
|
||||
)
|
||||
|
||||
env_value = normalize_string_override(envs.MOONCAKE_PREFERRED_SEGMENT)
|
||||
if env_value is not None:
|
||||
logger.info(
|
||||
"Mooncake preferred_segment from MOONCAKE_PREFERRED_SEGMENT: %s",
|
||||
env_value,
|
||||
)
|
||||
return env_value
|
||||
return None
|
||||
@@ -1,146 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stats container for the Mooncake connector."""
|
||||
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorStats,
|
||||
)
|
||||
|
||||
# TODO(mooncake-stats): add MooncakePromMetrics (mirror NixlPromMetrics)
|
||||
# and wire it via MooncakeConnector.build_prom_metrics in a follow-up PR.
|
||||
|
||||
|
||||
@dataclass
|
||||
class MooncakeKVConnectorStats(KVConnectorStats):
|
||||
"""Container for Mooncake KV transfer performance metrics.
|
||||
|
||||
`_lock` serializes record_* against clone_and_reset so each row's
|
||||
appends are atomic and column lengths stay aligned. Writers run on
|
||||
the sender pool / receiver loop / sender loop; reader runs on the
|
||||
main worker thread.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
self._lock = threading.Lock()
|
||||
if not self.data:
|
||||
self.reset()
|
||||
|
||||
# threading.Lock is not picklable; strip it from the wire form and
|
||||
# rebuild a fresh per-process lock on the receiver side.
|
||||
def __getstate__(self) -> dict[str, Any]:
|
||||
state = self.__dict__.copy()
|
||||
state.pop("_lock", None)
|
||||
return state
|
||||
|
||||
def __setstate__(self, state: dict[str, Any]) -> None:
|
||||
self.__dict__.update(state)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def reset(self):
|
||||
self.data: dict[str, list[float | int]] = {
|
||||
"transfer_duration": [],
|
||||
"bytes_transferred": [],
|
||||
"num_descriptors": [],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_recvs": [],
|
||||
"num_kv_expired_reqs": [],
|
||||
}
|
||||
|
||||
def record_transfer(self, duration_s: float, total_bytes: int, num_descs: int):
|
||||
with self._lock:
|
||||
self.data["transfer_duration"].append(duration_s)
|
||||
self.data["bytes_transferred"].append(total_bytes)
|
||||
self.data["num_descriptors"].append(num_descs)
|
||||
|
||||
# Failure counters store a list of 1s so a future Prom counter can iterate
|
||||
# with .inc(list_item), mirroring NIXL's NixlPromMetrics.observe.
|
||||
def record_failed_transfer(self):
|
||||
with self._lock:
|
||||
self.data["num_failed_transfers"].append(1)
|
||||
|
||||
def record_failed_recv(self):
|
||||
with self._lock:
|
||||
self.data["num_failed_recvs"].append(1)
|
||||
|
||||
def record_kv_expired_req(self):
|
||||
with self._lock:
|
||||
self.data["num_kv_expired_reqs"].append(1)
|
||||
|
||||
def clone_and_reset(self) -> "MooncakeKVConnectorStats":
|
||||
# Copy lists under the lock for length alignment; return a fresh
|
||||
# instance so the snapshot has its own _lock.
|
||||
with self._lock:
|
||||
snapshot_data: dict[str, list[float | int]] = {
|
||||
k: list(v) for k, v in self.data.items()
|
||||
}
|
||||
self.reset()
|
||||
return MooncakeKVConnectorStats(data=snapshot_data)
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return (
|
||||
self.num_successful_transfers == 0
|
||||
and len(self.data["num_failed_transfers"]) == 0
|
||||
and len(self.data["num_failed_recvs"]) == 0
|
||||
and len(self.data["num_kv_expired_reqs"]) == 0
|
||||
)
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if not other.is_empty():
|
||||
for k, v in other.data.items():
|
||||
accumulator = self.data[k]
|
||||
assert isinstance(accumulator, list)
|
||||
accumulator.extend(v)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
num_failed_transfers = len(self.data["num_failed_transfers"])
|
||||
num_failed_recvs = len(self.data["num_failed_recvs"])
|
||||
num_kv_expired_reqs = len(self.data["num_kv_expired_reqs"])
|
||||
|
||||
if self.num_successful_transfers == 0:
|
||||
return {
|
||||
"Num successful transfers": 0,
|
||||
"Avg xfer time (ms)": 0,
|
||||
"P90 xfer time (ms)": 0,
|
||||
"Avg MB per transfer": 0,
|
||||
"Throughput (MB/s)": 0,
|
||||
"Avg number of descriptors": 0,
|
||||
"Num failed transfers": num_failed_transfers,
|
||||
"Num failed recvs": num_failed_recvs,
|
||||
"Num KV expired reqs": num_kv_expired_reqs,
|
||||
}
|
||||
|
||||
xfer_time = np.asarray(self.data["transfer_duration"])
|
||||
mb = np.asarray(self.data["bytes_transferred"]) / 2**20
|
||||
descs = np.asarray(self.data["num_descriptors"], dtype=np.uint32)
|
||||
n = len(descs)
|
||||
assert n == self.num_successful_transfers
|
||||
|
||||
total_mb = mb.sum()
|
||||
avg_mb = total_mb / n
|
||||
total_time_seconds = xfer_time.sum()
|
||||
throughput_mb_s = (
|
||||
(total_mb / total_time_seconds).item() if total_time_seconds > 0 else 0.0
|
||||
)
|
||||
|
||||
return {
|
||||
"Num successful transfers": n,
|
||||
"Avg xfer time (ms)": round(xfer_time.mean().item() * 1e3, 3),
|
||||
"P90 xfer time (ms)": round(np.percentile(xfer_time, 90).item() * 1e3, 3),
|
||||
"Avg MB per transfer": round(avg_mb.item(), 3),
|
||||
"Throughput (MB/s)": round(throughput_mb_s, 3),
|
||||
"Avg number of descriptors": round(descs.mean().item(), 1),
|
||||
"Num failed transfers": num_failed_transfers,
|
||||
"Num failed recvs": num_failed_recvs,
|
||||
"Num KV expired reqs": num_kv_expired_reqs,
|
||||
}
|
||||
|
||||
@property
|
||||
def num_successful_transfers(self) -> int:
|
||||
return len(self.data["transfer_duration"])
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,436 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Adapted from vllm-project/vllm-ascend
|
||||
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
|
||||
"""MooncakeStoreConnector - KV cache connector using MooncakeDistributedStore.
|
||||
|
||||
Unlike MooncakeConnector which does direct P2P transfer, this connector
|
||||
uses MooncakeDistributedStore as a shared KV cache pool. Both producer
|
||||
and consumer instances read/write KV to/from the store independently,
|
||||
enabling prefix caching via hash-based deduplication.
|
||||
"""
|
||||
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Sequence
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import KVTransferConfig, VllmConfig
|
||||
from vllm.distributed.kv_events import (
|
||||
BlockStored,
|
||||
KVCacheEvent,
|
||||
KVConnectorKVEvents,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
KVConnectorWorkerMetadata,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
from .data import MooncakeStoreConnectorMetadata
|
||||
from .metrics import MooncakeStoreConnectorStats, MooncakeStorePromMetrics
|
||||
from .scheduler import MooncakeStoreScheduler
|
||||
from .worker import MooncakeStoreWorker
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MooncakeStoreKVEvents(KVConnectorKVEvents):
|
||||
"""KV event aggregation for MooncakeStoreConnector."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_workers: int,
|
||||
group_tp_replication_factors: Sequence[int] = (1,),
|
||||
) -> None:
|
||||
if num_workers <= 0:
|
||||
raise ValueError("num_workers must be greater than zero.")
|
||||
if any(factor <= 0 for factor in group_tp_replication_factors):
|
||||
raise ValueError("TP replication factors must be greater than zero.")
|
||||
self._event_counter: Counter[KVCacheEvent] = Counter()
|
||||
self._num_workers = num_workers
|
||||
self._group_tp_replication_factors = tuple(group_tp_replication_factors)
|
||||
|
||||
def add_events(self, events: list[KVCacheEvent]) -> None:
|
||||
if not isinstance(events, list):
|
||||
raise TypeError("events must be a list of KVCacheEvent.")
|
||||
self._event_counter.update(events)
|
||||
|
||||
def _replication_factor(self, event: KVCacheEvent) -> int:
|
||||
if not isinstance(event, BlockStored) or event.group_idx is None:
|
||||
return 1
|
||||
return self._group_tp_replication_factors[event.group_idx]
|
||||
|
||||
def _is_common_event(self, event: KVCacheEvent, count: int) -> bool:
|
||||
return count * self._replication_factor(event) >= self._num_workers
|
||||
|
||||
def aggregate(self) -> "MooncakeStoreKVEvents":
|
||||
common_events = self.pop_common_events()
|
||||
self._event_counter.clear()
|
||||
self._event_counter.update(common_events)
|
||||
self._num_workers = 1
|
||||
return self
|
||||
|
||||
def pop_common_events(self) -> list[KVCacheEvent]:
|
||||
common_events = [
|
||||
event
|
||||
for event, count in self._event_counter.items()
|
||||
if self._is_common_event(event, count)
|
||||
]
|
||||
for event in common_events:
|
||||
del self._event_counter[event]
|
||||
return common_events
|
||||
|
||||
def has_events(self) -> bool:
|
||||
return bool(self._event_counter)
|
||||
|
||||
def increment_workers(self, count: int = 1) -> None:
|
||||
if count <= 0:
|
||||
raise ValueError("count must be positive.")
|
||||
self._num_workers += count
|
||||
|
||||
def get_all_events(self) -> list[KVCacheEvent]:
|
||||
return list(self._event_counter.elements())
|
||||
|
||||
def get_number_of_workers(self) -> int:
|
||||
return self._num_workers
|
||||
|
||||
def clear_events(self) -> None:
|
||||
self._event_counter.clear()
|
||||
self._num_workers = 1
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<MooncakeStoreKVEvents events={self.get_all_events()}>"
|
||||
|
||||
|
||||
class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""KV connector using MooncakeDistributedStore as shared KV pool."""
|
||||
|
||||
@classmethod
|
||||
def supports_sleep_mode(cls, kv_transfer_config: KVTransferConfig) -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _validate_kv_cache_config(
|
||||
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||
) -> None:
|
||||
from vllm.v1.kv_cache_interface import CrossAttentionSpec, MambaSpec
|
||||
|
||||
unsupported: list[str] = []
|
||||
store_group_ids = kv_cache_config.prefix_cacheable_group_ids
|
||||
if not store_group_ids:
|
||||
raise ValueError(
|
||||
"MooncakeStore requires at least one prefix-cacheable KV cache group"
|
||||
)
|
||||
for group_id in store_group_ids:
|
||||
spec = kv_cache_config.kv_cache_groups[group_id].kv_cache_spec
|
||||
if isinstance(spec, CrossAttentionSpec):
|
||||
unsupported.append(f"group {group_id}: CrossAttentionSpec")
|
||||
if isinstance(spec, MambaSpec) and spec.mamba_cache_mode != "align":
|
||||
unsupported.append(
|
||||
f"group {group_id}: mamba_cache_mode="
|
||||
f"{spec.mamba_cache_mode!r} != 'align'"
|
||||
)
|
||||
pcp = vllm_config.parallel_config.prefill_context_parallel_size
|
||||
if len(store_group_ids) > 1 and pcp > 1:
|
||||
unsupported.append(f"PCP > 1 (pcp={pcp}) with hybrid attention")
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
"MooncakeStoreConnector does not support: " + "; ".join(unsupported)
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: KVCacheConfig | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
vllm_config=vllm_config,
|
||||
role=role,
|
||||
kv_cache_config=kv_cache_config, # type: ignore[arg-type]
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
assert kv_cache_config is not None, "kv_cache_config is required"
|
||||
self.kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
|
||||
save_decode_cache = extra_config.get("save_decode_cache", False)
|
||||
# Capacity-only: contributes its segment to the store pool but transfers
|
||||
# no KV, so the KV-cache-shape invariants below cannot be reached.
|
||||
self._capacity_only = (
|
||||
self.kv_role == "kv_consumer"
|
||||
and not extra_config.get("enable_lookup", True)
|
||||
and not save_decode_cache
|
||||
)
|
||||
if not self._capacity_only:
|
||||
self._validate_kv_cache_config(vllm_config, kv_cache_config)
|
||||
self._kv_cache_config = kv_cache_config
|
||||
self._kv_cache_events: MooncakeStoreKVEvents | None = None
|
||||
|
||||
self.connector_scheduler: MooncakeStoreScheduler | None = None
|
||||
self.connector_worker: MooncakeStoreWorker | None = None
|
||||
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = MooncakeStoreScheduler(
|
||||
vllm_config, kv_cache_config
|
||||
)
|
||||
else:
|
||||
self.connector_worker = MooncakeStoreWorker(vllm_config, kv_cache_config)
|
||||
|
||||
def shutdown(self):
|
||||
"""Release connector resources on teardown.
|
||||
|
||||
Closes the worker's MooncakeDistributedStore handle so its
|
||||
TransferEngine and RDMA registrations are released. Invoked from the
|
||||
engine's explicit shutdown path and as a backstop from ``__del__``;
|
||||
a no-op on the scheduler role, which holds no store handle.
|
||||
"""
|
||||
worker = getattr(self, "connector_worker", None)
|
||||
if worker is not None:
|
||||
worker.close()
|
||||
|
||||
def __del__(self):
|
||||
self.shutdown()
|
||||
|
||||
# ============================================================
|
||||
# Scheduler-side methods
|
||||
# ============================================================
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: Request,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: Request,
|
||||
blocks: KVCacheBlocks,
|
||||
num_external_tokens: int,
|
||||
):
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.update_state_after_alloc(
|
||||
request, blocks, num_external_tokens
|
||||
)
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.has_pending_push_work()
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> KVConnectorMetadata:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.build_connector_worker_meta()
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
return self.request_finished_all_groups(request, (block_ids,))
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
# An in-flight store job holds its own reference on the blocks it reads,
|
||||
# so a finishing request never has to defer freeing them.
|
||||
return False, None
|
||||
|
||||
def register_finished_partial_tail(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
partial_tail_offloads: list[tuple[int, int, int]],
|
||||
) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.register_finished_partial_tail(
|
||||
request, block_ids, partial_tail_offloads
|
||||
)
|
||||
|
||||
def reset_cache(self) -> bool | None:
|
||||
"""Reset the external Mooncake store on prefix-cache reset.
|
||||
|
||||
Drains the worker send queue, then runs ``remove_all`` on the
|
||||
Mooncake master. Caller must first pause generation (e.g.
|
||||
``pause_generation``) so no new puts are enqueued during drain.
|
||||
|
||||
Returns True on ack, False on failure, None for the worker role.
|
||||
"""
|
||||
if self.role == KVConnectorRole.SCHEDULER:
|
||||
assert self.connector_scheduler is not None
|
||||
# Clear local references to keys we're about to wipe.
|
||||
self.connector_scheduler.load_specs.clear()
|
||||
self._kv_cache_events = None
|
||||
return self.connector_scheduler.reset_store()
|
||||
return None
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
kv_cache_events = connector_output.kv_cache_events
|
||||
if not kv_cache_events or not isinstance(
|
||||
kv_cache_events, MooncakeStoreKVEvents
|
||||
):
|
||||
return
|
||||
|
||||
if self._kv_cache_events is None:
|
||||
self._kv_cache_events = kv_cache_events
|
||||
else:
|
||||
self._kv_cache_events.add_events(kv_cache_events.get_all_events())
|
||||
|
||||
def take_events(self) -> Iterable[KVCacheEvent]:
|
||||
if self._kv_cache_events is not None:
|
||||
events = self._kv_cache_events.pop_common_events()
|
||||
if not self._kv_cache_events.has_events():
|
||||
self._kv_cache_events = None
|
||||
yield from events
|
||||
|
||||
# ============================================================
|
||||
# Worker-side methods
|
||||
# ============================================================
|
||||
|
||||
def get_mem_pool_context(self) -> AbstractContextManager | None:
|
||||
"""Return a context manager for the custom MemPool, or None.
|
||||
|
||||
Called by the Worker before ``initialize_kv_cache`` so that KV
|
||||
cache is allocated from the Mooncake-managed pool when
|
||||
``custom_mem_pool`` is set in ``kv_connector_extra_config``.
|
||||
"""
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_mem_pool_context()
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
self.connector_worker.start_load_kv(metadata)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
# No layerwise support - no-op
|
||||
return
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
# No layerwise support - no-op
|
||||
return
|
||||
|
||||
def wait_for_save(self):
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
self.connector_worker.wait_for_save(metadata)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
return self.connector_worker.get_finished(finished_req_ids, metadata)
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> KVConnectorTransferResults:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, MooncakeStoreConnectorMetadata)
|
||||
return self.connector_worker.get_transfer_results(finished_req_ids, metadata)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_block_ids_with_load_errors()
|
||||
|
||||
def get_kv_connector_kv_cache_events(
|
||||
self,
|
||||
) -> MooncakeStoreKVEvents | None:
|
||||
assert self.connector_worker is not None
|
||||
if (
|
||||
not self.connector_worker.enable_kv_events
|
||||
or self.connector_worker.kv_send_thread is None
|
||||
):
|
||||
return None
|
||||
events = self.connector_worker.get_kv_events()
|
||||
# Empty containers still count this worker toward the poll's quorum.
|
||||
kv_events = MooncakeStoreKVEvents(
|
||||
num_workers=1,
|
||||
group_tp_replication_factors=(
|
||||
self.connector_worker.group_tp_replication_factors
|
||||
),
|
||||
)
|
||||
kv_events.add_events(events)
|
||||
return kv_events
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
if self.connector_worker is None:
|
||||
return None
|
||||
return self.connector_worker.get_kv_connector_stats()
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> KVConnectorStats | None:
|
||||
return (
|
||||
MooncakeStoreConnectorStats(data=data)
|
||||
if data is not None
|
||||
else MooncakeStoreConnectorStats()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_prom_metrics(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> KVConnectorPromMetrics:
|
||||
return MooncakeStorePromMetrics(
|
||||
vllm_config, metric_types, labelnames, per_engine_labelvalues
|
||||
)
|
||||
@@ -1,515 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""External-store cache-hit coordinator for MooncakeStoreConnector."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, NamedTuple, cast
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
chunk_hashes_for_block_size,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_coordinator import replay_boundaries
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
KVCacheBlock,
|
||||
eagle_proof_margin,
|
||||
resolve_dcp_kv_cache_spec,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
SingleTypeKVCacheManager,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
class StoreSpecGroup(NamedTuple):
|
||||
spec: KVCacheSpec
|
||||
group_ids: list[int]
|
||||
manager_cls: type[SingleTypeKVCacheManager]
|
||||
use_eagle: bool
|
||||
|
||||
|
||||
class ExternalCachedBlockPool:
|
||||
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hash_block_size: int,
|
||||
exists: set[tuple[int, bytes]] | None = None,
|
||||
) -> None:
|
||||
# ``exists=None`` is used on the recv side where hit_length is already
|
||||
# determined and we just want each spec's manager to apply its own mask.
|
||||
self._exists = exists
|
||||
self.hash_block_size = hash_block_size
|
||||
self.null_block = KVCacheBlock(block_id=0)
|
||||
# Dummy ID 1 for present block for duck-typing.
|
||||
self._present_block = KVCacheBlock(block_id=1)
|
||||
|
||||
def get_cached_block(
|
||||
self,
|
||||
block_hash: BlockHash,
|
||||
group_ids: list[int],
|
||||
) -> list[KVCacheBlock] | None:
|
||||
# Mirrors BlockPool.get_cached_block: hit only when every group_id
|
||||
# (groups sharing a spec) has the hash cached.
|
||||
if self._exists is None:
|
||||
return [self._present_block] * len(group_ids)
|
||||
h = bytes(block_hash)
|
||||
if all((g, h) in self._exists for g in group_ids):
|
||||
return [self._present_block] * len(group_ids)
|
||||
return None
|
||||
|
||||
def contains(self, group_id: int, block_hash: BlockHash) -> bool:
|
||||
"""Return whether a group has a loadable key for this hash."""
|
||||
return (
|
||||
self._exists is not None and (group_id, bytes(block_hash)) in self._exists
|
||||
)
|
||||
|
||||
|
||||
class MooncakeStoreCoordinator:
|
||||
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
|
||||
``ExternalCachedBlockPool``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
scheduler_block_size: int,
|
||||
hash_block_size: int,
|
||||
use_eagle: bool = False,
|
||||
retention_interval: int | None = None,
|
||||
enable_partial_hash_hits: bool = False,
|
||||
) -> None:
|
||||
# Mirrors core's resolve_kv_cache_block_sizes: the hash unit only has
|
||||
# to divide groups that participate in prefix caching. Non-shareable
|
||||
# scratch groups (e.g. GLM-5.3-Flash's kpool tail) are skipped by
|
||||
# _verify_and_split_kv_cache_groups and never probed for hits.
|
||||
assert all(
|
||||
g.kv_cache_spec.block_size % hash_block_size == 0
|
||||
for g in kv_cache_groups
|
||||
if g.kv_cache_spec.prefix_cacheable
|
||||
), "block_size must be divisible by hash_block_size"
|
||||
assert scheduler_block_size % hash_block_size == 0, (
|
||||
f"scheduler_block_size ({scheduler_block_size}) must be a multiple of "
|
||||
f"hash_block_size ({hash_block_size})"
|
||||
)
|
||||
assert all(
|
||||
scheduler_block_size % g.kv_cache_spec.block_size == 0
|
||||
for g in kv_cache_groups
|
||||
if g.kv_cache_spec.prefix_cacheable
|
||||
), "scheduler_block_size must be a multiple of each group's block_size"
|
||||
self.kv_cache_groups = kv_cache_groups
|
||||
self.mamba_group_ids = {
|
||||
group_id
|
||||
for group_id, group in enumerate(kv_cache_groups)
|
||||
if isinstance(_unwrap_spec(group.kv_cache_spec), MambaSpec)
|
||||
}
|
||||
self.hash_block_size = hash_block_size
|
||||
self.lcm_block_size = scheduler_block_size
|
||||
# Core's decision: whether hits land on the hash unit, below the block.
|
||||
self.enable_partial_hash_hits = enable_partial_hash_hits
|
||||
self.use_eagle = use_eagle
|
||||
# Mirror vLLM core's KVCacheCoordinator.retention_interval.
|
||||
self.retention_interval = retention_interval
|
||||
self._verify_and_split_kv_cache_groups()
|
||||
|
||||
@classmethod
|
||||
def from_kv_cache_config(
|
||||
cls,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
vllm_config: "VllmConfig",
|
||||
scheduler_block_size: int,
|
||||
hash_block_size: int,
|
||||
) -> "MooncakeStoreCoordinator":
|
||||
"""Build the coordinator the connector's scheduler and workers share.
|
||||
|
||||
Specs are DCP-resolved, so a block spans the tokens of one block-table
|
||||
entry. Partial hits follow core's hit alignment, which checks every KV
|
||||
cache group and its manager, not only the groups the store transfers;
|
||||
unset (a config the engine core did not resolve) keeps hits on the
|
||||
block.
|
||||
"""
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
spec_config = vllm_config.speculative_config
|
||||
alignment = kv_cache_config.cache_hit_alignment_tokens
|
||||
return cls(
|
||||
[
|
||||
replace(
|
||||
group,
|
||||
kv_cache_spec=resolve_dcp_kv_cache_spec(
|
||||
group.kv_cache_spec, dcp_world_size
|
||||
),
|
||||
)
|
||||
for group in kv_cache_config.prefix_cacheable_groups
|
||||
],
|
||||
scheduler_block_size,
|
||||
hash_block_size,
|
||||
use_eagle=spec_config is not None and spec_config.use_eagle_block_drop(),
|
||||
retention_interval=kv_cache_config.prefix_cache_retention_interval,
|
||||
enable_partial_hash_hits=(
|
||||
alignment is not None and alignment < scheduler_block_size
|
||||
),
|
||||
)
|
||||
|
||||
def align_lookup_length(self, length: int) -> int:
|
||||
alignment = (
|
||||
self.hash_block_size
|
||||
if self.enable_partial_hash_hits
|
||||
else self.lcm_block_size
|
||||
)
|
||||
return length // alignment * alignment
|
||||
|
||||
def _verify_and_split_kv_cache_groups(self) -> None:
|
||||
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
|
||||
dispatches via spec_manager_map (we don't allocate managers).
|
||||
"""
|
||||
attention_groups: list[StoreSpecGroup] = []
|
||||
for i, g in enumerate(self.kv_cache_groups):
|
||||
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
|
||||
# kpool tail): per-request scratch, never shareable, so they must
|
||||
# not participate in hit lookup. Mirrors core's
|
||||
# KVCacheCoordinator.verify_and_split_kv_cache_groups.
|
||||
if not g.kv_cache_spec.prefix_cacheable:
|
||||
continue
|
||||
spec = _unwrap_spec(g.kv_cache_spec)
|
||||
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
|
||||
assert manager_cls is not None, (
|
||||
f"No manager registered for KVCacheSpec {spec}"
|
||||
)
|
||||
for idx, group in enumerate(attention_groups):
|
||||
if group.spec == spec:
|
||||
assert manager_cls is group.manager_cls
|
||||
group.group_ids.append(i)
|
||||
if g.is_eagle_group and not group.use_eagle:
|
||||
attention_groups[idx] = group._replace(use_eagle=True)
|
||||
break
|
||||
else:
|
||||
attention_groups.append(
|
||||
StoreSpecGroup(spec, [i], manager_cls, g.is_eagle_group)
|
||||
)
|
||||
# Full attention first (matches upstream convergence ordering).
|
||||
attention_groups.sort(key=lambda g: not isinstance(g.spec, FullAttentionSpec))
|
||||
# Conservatively flag all groups when use_eagle is set but none is flagged.
|
||||
if self.use_eagle and not any(g.use_eagle for g in attention_groups):
|
||||
attention_groups = [g._replace(use_eagle=True) for g in attention_groups]
|
||||
self.attention_groups = attention_groups
|
||||
# Per-group eagle bits. SpecGroup carries use_eagle for the whole
|
||||
# merged spec group, so the per-group store/lookup masks agree with
|
||||
# the merged-group hit check, which applies the eagle drop to every
|
||||
# group sharing the spec.
|
||||
self.eagle_group_ids = {
|
||||
gid for g in attention_groups if g.use_eagle for gid in g.group_ids
|
||||
}
|
||||
self.eagle_proof_margin_by_group = {
|
||||
gid: eagle_proof_margin(
|
||||
group.spec.block_size,
|
||||
self.hash_block_size,
|
||||
self.enable_partial_hash_hits
|
||||
and group.manager_cls.supports_fine_grained_hash_lookup,
|
||||
)
|
||||
for group in attention_groups
|
||||
if group.use_eagle and not isinstance(group.spec, MambaSpec)
|
||||
for gid in group.group_ids
|
||||
}
|
||||
|
||||
def find_longest_cache_hit(
|
||||
self,
|
||||
block_hashes: Sequence[BlockHash],
|
||||
max_length: int,
|
||||
cached_block_pool: ExternalCachedBlockPool,
|
||||
*,
|
||||
apply_eagle: bool = True,
|
||||
) -> tuple[tuple[list[bool], ...], int]:
|
||||
"""Returns ``(load_mask_per_group, hit_length)``. ``mask[g][i]`` is True iff
|
||||
group ``g`` populates chunk ``i`` locally (e.g. SWA and Mamba tail-only);
|
||||
recv-side callers skip False slots.
|
||||
|
||||
``apply_eagle`` controls whether the per-spec ``use_eagle`` last-block
|
||||
pop is applied. Lookup callers want it (the drafter requires recomputing
|
||||
the last block); per-chunk mask callers must not, because ``token_len``
|
||||
already reflects the eagle-pruned hit length and a second pop would
|
||||
leave the trailing block unloaded.
|
||||
"""
|
||||
blocks_per_group, hit_length = self._find_hit_blocks(
|
||||
block_hashes, max_length, cached_block_pool, apply_eagle=apply_eagle
|
||||
)
|
||||
masks = tuple(
|
||||
[blk is not cached_block_pool.null_block for blk in blocks]
|
||||
for blocks in blocks_per_group
|
||||
)
|
||||
return masks, hit_length
|
||||
|
||||
def load_mask(
|
||||
self,
|
||||
block_hashes: Sequence[BlockHash],
|
||||
token_len: int,
|
||||
) -> tuple[list[bool], ...]:
|
||||
"""Per-group load masks: ``mask[g][i]`` is True iff group ``g``'s
|
||||
spec would populate chunk ``i`` locally at length ``token_len``
|
||||
(e.g. SWA / Mamba tail-only).
|
||||
"""
|
||||
# ``apply_eagle=False`` because ``token_len`` is already the
|
||||
# eagle-pruned hit length returned by ``client.lookup``. Re-applying
|
||||
# the pop here would shorten the mask by one extra block; the recv
|
||||
# thread would then silently skip the trailing chunk yielded by
|
||||
# ``db.process_tokens`` and leave that block uninitialized in the
|
||||
# local KV pool.
|
||||
masks, _ = self.find_longest_cache_hit(
|
||||
block_hashes,
|
||||
token_len,
|
||||
ExternalCachedBlockPool(self.hash_block_size),
|
||||
apply_eagle=False,
|
||||
)
|
||||
return masks
|
||||
|
||||
def store_mask(
|
||||
self,
|
||||
aligned_token_len: int,
|
||||
start_token: int = 0,
|
||||
num_prompt_tokens: int | None = None,
|
||||
) -> tuple[list[bool] | None, ...]:
|
||||
"""Per-group store masks for the suffix starting at ``start_token``.
|
||||
|
||||
``mask[g][i]`` is True iff the i-th chunk of group ``g`` *after*
|
||||
``start_token`` should be written to the store so a future cache hit
|
||||
can consume it. ``None`` is the all-True sentinel for the suffix.
|
||||
|
||||
Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask``
|
||||
so the store retains exactly the blocks the local prefix cache would.
|
||||
|
||||
Mamba groups are always all-False: the normal save resolves blocks
|
||||
positionally from the connector's append-only block-ID snapshot, but
|
||||
an align-mode mamba block table is not append-only (interior state
|
||||
blocks are nulled/freed, and speculative decoding relocates the spec
|
||||
blocks in place), so a positional read may hit a null, freed, or live
|
||||
speculative-state block and persist wrong bytes under a valid prefix
|
||||
hash. Mamba state is persisted only through the connector-pinned exact
|
||||
block hand-off path
|
||||
(``SchedulerOutput.kv_connector_block_state.boundary_state_offloads``).
|
||||
"""
|
||||
return self._reachable_masks(
|
||||
aligned_token_len,
|
||||
start_token,
|
||||
retention_interval=self.retention_interval,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
exclude_mamba=True,
|
||||
)
|
||||
|
||||
def lookup_mask(
|
||||
self,
|
||||
aligned_token_len: int,
|
||||
) -> tuple[list[bool] | None, ...]:
|
||||
"""Per-group lookup masks.
|
||||
|
||||
``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be
|
||||
looked up as an aligned hit boundary. ``None`` is the all-True
|
||||
sentinel.
|
||||
"""
|
||||
return self._reachable_masks(
|
||||
aligned_token_len,
|
||||
0,
|
||||
retention_interval=None,
|
||||
num_prompt_tokens=None,
|
||||
)
|
||||
|
||||
def _reachable_masks(
|
||||
self,
|
||||
aligned_token_len: int,
|
||||
start_token: int,
|
||||
*,
|
||||
retention_interval: int | None,
|
||||
num_prompt_tokens: int | None,
|
||||
exclude_mamba: bool = False,
|
||||
) -> tuple[list[bool] | None, ...]:
|
||||
mask_alignment = (
|
||||
self.hash_block_size
|
||||
if self.enable_partial_hash_hits
|
||||
else self.lcm_block_size
|
||||
)
|
||||
assert aligned_token_len % mask_alignment == 0, (
|
||||
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
|
||||
f"{mask_alignment}"
|
||||
)
|
||||
reachable_boundaries = (
|
||||
()
|
||||
if num_prompt_tokens is None
|
||||
else replay_boundaries(
|
||||
num_prompt_tokens,
|
||||
self.lcm_block_size,
|
||||
use_eagle=bool(self.eagle_group_ids),
|
||||
)
|
||||
)
|
||||
masks: list[list[bool] | None] = []
|
||||
for g_idx, g in enumerate(self.kv_cache_groups):
|
||||
spec = _unwrap_spec(g.kv_cache_spec)
|
||||
end_chunk = aligned_token_len // spec.block_size
|
||||
start_chunk = min(end_chunk, max(0, cdiv(start_token, spec.block_size)))
|
||||
if exclude_mamba and isinstance(spec, MambaSpec):
|
||||
masks.append([False] * (end_chunk - start_chunk))
|
||||
continue
|
||||
manager_cls = KVCacheSpecRegistry.get_manager_class(spec)
|
||||
assert manager_cls is not None
|
||||
use_eagle = g_idx in self.eagle_group_ids
|
||||
mask = manager_cls.reachable_block_mask(
|
||||
start_block=start_chunk,
|
||||
end_block=end_chunk,
|
||||
alignment_tokens=self.lcm_block_size,
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=use_eagle,
|
||||
retention_interval=retention_interval,
|
||||
reachable_boundaries=reachable_boundaries,
|
||||
# ``spec`` is already DCP-resolved (worker.py applies
|
||||
# resolve_dcp_kv_cache_spec) and ``end_chunk`` is indexed in
|
||||
# that scaled block size, so the mask must not scale again.
|
||||
dcp_world_size=1,
|
||||
)
|
||||
if mask is not None:
|
||||
assert len(mask) == end_chunk - start_chunk
|
||||
masks.append(mask)
|
||||
return tuple(masks)
|
||||
|
||||
def block_hashes_for_spec(
|
||||
self, block_hashes: Sequence[BlockHash], spec: KVCacheSpec
|
||||
) -> Sequence[BlockHash]:
|
||||
return chunk_hashes_for_block_size(
|
||||
block_hashes, self.hash_block_size, spec.block_size
|
||||
)
|
||||
|
||||
def _find_hit_blocks(
|
||||
self,
|
||||
block_hashes: Sequence[BlockHash],
|
||||
max_length: int,
|
||||
cached_block_pool: ExternalCachedBlockPool,
|
||||
*,
|
||||
apply_eagle: bool = True,
|
||||
) -> tuple[tuple[list[KVCacheBlock], ...], int]:
|
||||
"""Mirrors HybridKVCacheCoordinator.find_longest_cache_hit but
|
||||
dispatches via spec_manager_map (we don't allocate managers).
|
||||
|
||||
When ``apply_eagle`` is False, ignore each group's ``use_eagle`` —
|
||||
used by ``load_mask`` to avoid popping a second block on top of the
|
||||
one already removed by the lookup.
|
||||
"""
|
||||
alignment_tokens = (
|
||||
self.hash_block_size
|
||||
if self.enable_partial_hash_hits
|
||||
else self.lcm_block_size
|
||||
)
|
||||
if len(self.attention_groups) == 1:
|
||||
spec, group_ids, manager_cls, group_eagle = self.attention_groups[0]
|
||||
hit_blocks, hit_length = manager_cls.find_longest_cache_hit(
|
||||
block_hashes=block_hashes, # type: ignore[arg-type]
|
||||
max_length=max_length,
|
||||
kv_cache_group_ids=group_ids,
|
||||
block_pool=cast(BlockPool, cached_block_pool),
|
||||
kv_cache_spec=spec,
|
||||
drop_eagle_block=apply_eagle and group_eagle,
|
||||
alignment_tokens=alignment_tokens,
|
||||
)
|
||||
num_groups = len(self.kv_cache_groups)
|
||||
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
|
||||
for gid, blks in zip(group_ids, hit_blocks, strict=True):
|
||||
blocks_by_group[gid] = blks
|
||||
return tuple(blocks_by_group), hit_length
|
||||
|
||||
num_groups = len(self.kv_cache_groups)
|
||||
hit_length = max_length
|
||||
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
|
||||
hit_length_by_group: list[int] = [0] * num_groups
|
||||
|
||||
is_simple_hybrid = len(self.attention_groups) == 2 and isinstance(
|
||||
self.attention_groups[0].spec, FullAttentionSpec
|
||||
)
|
||||
eagle_verified: set[int] = set()
|
||||
|
||||
while True:
|
||||
curr_hit_length = hit_length
|
||||
|
||||
for idx, (spec, group_ids, manager_cls, group_eagle) in enumerate(
|
||||
self.attention_groups
|
||||
):
|
||||
first_group_id = group_ids[0]
|
||||
cached = hit_blocks_by_group[first_group_id]
|
||||
if isinstance(spec, FullAttentionSpec) and cached is not None:
|
||||
curr_hit_length = min(
|
||||
curr_hit_length, hit_length_by_group[first_group_id]
|
||||
)
|
||||
continue
|
||||
|
||||
drop_eagle_block = (
|
||||
apply_eagle and group_eagle and idx not in eagle_verified
|
||||
)
|
||||
_max_length = curr_hit_length
|
||||
# No eagle peek margin for a recurrent (Mamba) group: its finder
|
||||
# never drops a block, so a widened bound would match past the
|
||||
# attention-verified hit and resume from speculative state (#43559).
|
||||
if drop_eagle_block and not isinstance(spec, MambaSpec):
|
||||
eagle_margin = eagle_proof_margin(
|
||||
spec.block_size,
|
||||
self.hash_block_size,
|
||||
self.enable_partial_hash_hits
|
||||
and manager_cls.supports_fine_grained_hash_lookup,
|
||||
)
|
||||
_max_length = min(
|
||||
curr_hit_length + eagle_margin,
|
||||
len(block_hashes) * self.hash_block_size,
|
||||
)
|
||||
hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit(
|
||||
block_hashes=block_hashes, # type: ignore[arg-type]
|
||||
max_length=_max_length,
|
||||
kv_cache_group_ids=group_ids,
|
||||
block_pool=cast(BlockPool, cached_block_pool),
|
||||
kv_cache_spec=spec,
|
||||
drop_eagle_block=drop_eagle_block,
|
||||
alignment_tokens=alignment_tokens,
|
||||
)
|
||||
if drop_eagle_block:
|
||||
eagle_verified.add(idx)
|
||||
elif _new_hit_length < curr_hit_length:
|
||||
eagle_verified.clear()
|
||||
curr_hit_length = _new_hit_length
|
||||
for gid, blocks in zip(group_ids, hit_blocks, strict=True):
|
||||
hit_blocks_by_group[gid] = blocks
|
||||
hit_length_by_group[gid] = _new_hit_length
|
||||
|
||||
if curr_hit_length >= hit_length:
|
||||
break
|
||||
hit_length = curr_hit_length
|
||||
if is_simple_hybrid:
|
||||
break
|
||||
|
||||
# Truncate full-attention hit_blocks to final converged length;
|
||||
# other specs already trim themselves inside their hit logic. cdiv keeps
|
||||
# the partial tail block when hit_length is not block-aligned.
|
||||
for group in self.attention_groups:
|
||||
if not isinstance(group.spec, FullAttentionSpec):
|
||||
continue
|
||||
num_blocks = cdiv(hit_length, group.spec.block_size)
|
||||
for group_id in group.group_ids:
|
||||
full_blks = hit_blocks_by_group[group_id]
|
||||
assert full_blks is not None
|
||||
del full_blks[num_blocks:]
|
||||
hit_length_by_group[group_id] = hit_length
|
||||
|
||||
return (
|
||||
tuple(blks if blks is not None else [] for blks in hit_blocks_by_group),
|
||||
hit_length,
|
||||
)
|
||||
|
||||
|
||||
def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
return next(iter(spec.kv_cache_specs.values()))
|
||||
return spec
|
||||
@@ -1,898 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Adapted from vllm-project/vllm-ascend
|
||||
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
|
||||
"""Data classes for MooncakeStoreConnector."""
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import NamedTuple, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
KVConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import is_non_overlapping_and_dense
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
BlockHashListWithBlockSize,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class BlobBlockHashes(Sequence[BlockHash]):
|
||||
"""Lazy view over a flat buffer of fixed-size block hashes to avoid the overhead
|
||||
of materializing all hashes upfront.
|
||||
"""
|
||||
|
||||
def __init__(self, blob: memoryview, hash_len: int):
|
||||
self._blob = blob
|
||||
self._hash_len = hash_len
|
||||
self._n = len(blob) // hash_len if hash_len else 0
|
||||
|
||||
def __len__(self) -> int:
|
||||
return self._n
|
||||
|
||||
def __getitem__(self, idx):
|
||||
if isinstance(idx, slice):
|
||||
return [self[i] for i in range(*idx.indices(self._n))]
|
||||
if idx < 0:
|
||||
idx += self._n
|
||||
if not 0 <= idx < self._n:
|
||||
raise IndexError(idx)
|
||||
off = idx * self._hash_len
|
||||
return BlockHash(self._blob[off : off + self._hash_len])
|
||||
|
||||
|
||||
class _CompactChunkHashList(BlockHashListWithBlockSize):
|
||||
"""View that keys each ``block_size`` chunk by the last constituent
|
||||
``hash_block_size`` hash instead of concatenating all of them.
|
||||
|
||||
The engine chains block hashes (each hash folds in the previous one), so the
|
||||
final sub-block hash of a chunk already uniquely identifies the whole chunk
|
||||
and its prefix. Using it keeps a Mooncake key at a single hash digest
|
||||
regardless of the ``block_size`` / ``hash_block_size`` ratio, instead of
|
||||
growing the key linearly with it (e.g. 64x for ``block_size=256``,
|
||||
``hash_block_size=4``).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
block_hashes: Sequence[BlockHash],
|
||||
hash_block_size: int,
|
||||
target_block_size: int,
|
||||
):
|
||||
# Accept any indexable sequence (e.g. the lazy ``BlobBlockHashes``), not
|
||||
# just ``list``; the base only indexes/sizes it.
|
||||
assert target_block_size % hash_block_size == 0
|
||||
self.block_hashes = block_hashes # type: ignore[assignment]
|
||||
self.scale_factor = target_block_size // hash_block_size
|
||||
|
||||
def _get_value_at(self, idx: int) -> BlockHash:
|
||||
return self.block_hashes[idx * self.scale_factor + self.scale_factor - 1]
|
||||
|
||||
|
||||
def chunk_hashes_for_block_size(
|
||||
block_hashes: Sequence[BlockHash],
|
||||
hash_block_size: int,
|
||||
block_size: int,
|
||||
) -> Sequence[BlockHash]:
|
||||
"""Map ``hash_block_size``-granular block hashes to one compact hash per
|
||||
``block_size`` chunk (the chunk's last sub-hash). Returns ``block_hashes``
|
||||
unchanged when the two sizes are equal.
|
||||
"""
|
||||
if block_size == hash_block_size:
|
||||
return block_hashes
|
||||
# Structurally a Sequence[BlockHash] (indexable + sized); the base class
|
||||
# just isn't declared as one.
|
||||
return cast(
|
||||
"Sequence[BlockHash]",
|
||||
_CompactChunkHashList(block_hashes, hash_block_size, block_size),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KeyMetadata:
|
||||
"""Metadata for constructing pool keys."""
|
||||
|
||||
model_name: str
|
||||
tp_rank: int
|
||||
pcp_rank: int
|
||||
dcp_rank: int
|
||||
pp_rank: int
|
||||
group_id: int = 0
|
||||
# Optional namespace prepended to every key. Lets separate deployments
|
||||
# share one Mooncake master without colliding on identical block hashes.
|
||||
# Empty (the default) keeps keys byte-identical to the unprefixed format.
|
||||
cache_prefix: str = ""
|
||||
# Complete namespace for an opt-in Store payload format. Empty keeps
|
||||
# historical keys byte-identical.
|
||||
store_namespace: str = ""
|
||||
|
||||
|
||||
@dataclass(order=True)
|
||||
class PoolKey:
|
||||
"""Key for addressing KV cache blocks in the distributed store."""
|
||||
|
||||
key_metadata: KeyMetadata
|
||||
chunk_hash: str
|
||||
|
||||
def __hash__(self):
|
||||
return hash(
|
||||
(
|
||||
self.key_metadata.cache_prefix,
|
||||
self.key_metadata.model_name,
|
||||
self.key_metadata.store_namespace,
|
||||
self.key_metadata.tp_rank,
|
||||
self.key_metadata.pcp_rank,
|
||||
self.key_metadata.dcp_rank,
|
||||
self.key_metadata.pp_rank,
|
||||
self.key_metadata.group_id,
|
||||
self.chunk_hash,
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def build_prefix(
|
||||
key_metadata: KeyMetadata,
|
||||
*,
|
||||
tp_rank: int | None = None,
|
||||
pcp_rank: int | None = None,
|
||||
dcp_rank: int | None = None,
|
||||
pp_rank: int | None = None,
|
||||
) -> str:
|
||||
"""Return the stable prefix for a Mooncake pool key."""
|
||||
prefix = f"{key_metadata.cache_prefix}@" if key_metadata.cache_prefix else ""
|
||||
return (
|
||||
f"{prefix}"
|
||||
f"{key_metadata.model_name}"
|
||||
f"{key_metadata.store_namespace}"
|
||||
f"@tp_rank:{key_metadata.tp_rank if tp_rank is None else tp_rank}"
|
||||
f"@pcp{key_metadata.pcp_rank if pcp_rank is None else pcp_rank}"
|
||||
f"@dcp{key_metadata.dcp_rank if dcp_rank is None else dcp_rank}"
|
||||
f"@pp_rank:{key_metadata.pp_rank if pp_rank is None else pp_rank}"
|
||||
f"@group:{key_metadata.group_id}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def build_key_string(key_prefix: str, chunk_hash: str) -> str:
|
||||
return f"{key_prefix}@{chunk_hash}"
|
||||
|
||||
def to_string(self) -> str:
|
||||
return self.build_key_string(
|
||||
self.build_prefix(self.key_metadata), self.chunk_hash
|
||||
)
|
||||
|
||||
|
||||
StoreShardId = int
|
||||
# Physical ranks encoded in a rank-local key: (TP, PCP, DCP, PP).
|
||||
RankNamespace = tuple[int, int, int, int]
|
||||
|
||||
|
||||
class StoreLayout:
|
||||
"""Store payload layout for one KV-cache group on one local TP rank."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int,
|
||||
) -> None:
|
||||
self.metadata = metadata
|
||||
self.block_size = block_size
|
||||
self.hash_block_size = hash_block_size
|
||||
|
||||
@property
|
||||
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
|
||||
"""Store objects contributed or consumed by this local rank."""
|
||||
raise NotImplementedError
|
||||
|
||||
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def lookup_key_prefixes(
|
||||
self,
|
||||
rank_namespaces: Sequence[RankNamespace],
|
||||
) -> tuple[str, ...]:
|
||||
"""Prefixes that must exist for a logical block to be reusable."""
|
||||
raise NotImplementedError
|
||||
|
||||
def register_kv_caches(
|
||||
self,
|
||||
kv_caches: Sequence[torch.Tensor],
|
||||
num_blocks: int,
|
||||
) -> None:
|
||||
"""Build descriptors from the local KV cache tensors."""
|
||||
raise NotImplementedError
|
||||
|
||||
def prepare_values(
|
||||
self,
|
||||
chunks: Sequence[tuple[int, int]],
|
||||
block_ids: list[int],
|
||||
shard_ids: Sequence[StoreShardId],
|
||||
) -> tuple[list[list[int]], list[list[int]], list[int]]:
|
||||
"""Build Store multi-buffer descriptors for logical chunks."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class RankLocalStoreLayout(StoreLayout):
|
||||
"""Historical rank-local Store payload layout."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int,
|
||||
) -> None:
|
||||
super().__init__(metadata, block_size, hash_block_size)
|
||||
self._key_prefix = PoolKey.build_prefix(metadata)
|
||||
self.kv_caches_base_addr: list[int] = []
|
||||
self.block_len: list[int] = []
|
||||
|
||||
@property
|
||||
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
|
||||
return (0,)
|
||||
|
||||
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
|
||||
assert shard_id == 0
|
||||
return PoolKey.build_key_string(self._key_prefix, chunk_hash.hex())
|
||||
|
||||
def lookup_key_prefixes(
|
||||
self,
|
||||
rank_namespaces: Sequence[RankNamespace],
|
||||
) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
PoolKey.build_prefix(
|
||||
self.metadata,
|
||||
tp_rank=tp_rank,
|
||||
pcp_rank=pcp_rank,
|
||||
dcp_rank=dcp_rank,
|
||||
pp_rank=pp_rank,
|
||||
)
|
||||
for tp_rank, pcp_rank, dcp_rank, pp_rank in rank_namespaces
|
||||
)
|
||||
|
||||
def set_kv_caches_base_addr(self, base_addrs: list[int]) -> None:
|
||||
self.kv_caches_base_addr = base_addrs
|
||||
|
||||
def set_block_len(self, block_lens: list[int]) -> None:
|
||||
self.block_len = block_lens
|
||||
|
||||
def register_kv_caches(
|
||||
self,
|
||||
kv_caches: Sequence[torch.Tensor],
|
||||
num_blocks: int,
|
||||
) -> None:
|
||||
seen_region_ptrs: set[int] = set()
|
||||
base_addrs: list[int] = []
|
||||
block_lens: list[int] = []
|
||||
for cache in kv_caches:
|
||||
storage = cache.untyped_storage()
|
||||
storage_addr = storage.data_ptr()
|
||||
storage_len = storage.nbytes()
|
||||
if not is_non_overlapping_and_dense(cache[0]):
|
||||
for head_idx in range(cache.shape[1]):
|
||||
head_cache = cache[:, head_idx]
|
||||
assert is_non_overlapping_and_dense(head_cache[0])
|
||||
region_addr = head_cache.data_ptr()
|
||||
if region_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(region_addr)
|
||||
base_addrs.append(region_addr)
|
||||
block_lens.append(head_cache.stride(0) * head_cache.element_size())
|
||||
elif cache.stride(0) * cache.element_size() * num_blocks == storage_len:
|
||||
if storage_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(storage_addr)
|
||||
base_addrs.append(storage_addr)
|
||||
block_lens.append(storage_len // num_blocks)
|
||||
else:
|
||||
region_addr = cache.data_ptr()
|
||||
if region_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(region_addr)
|
||||
base_addrs.append(region_addr)
|
||||
block_lens.append(cache.stride(0) * cache.element_size())
|
||||
self.kv_caches_base_addr = base_addrs
|
||||
self.block_len = block_lens
|
||||
|
||||
def prepare_values(
|
||||
self,
|
||||
chunks: Sequence[tuple[int, int]],
|
||||
block_ids: list[int],
|
||||
shard_ids: Sequence[StoreShardId],
|
||||
) -> tuple[list[list[int]], list[list[int]], list[int]]:
|
||||
if not chunks:
|
||||
return [], [], []
|
||||
assert len(chunks) == len(shard_ids) and all(
|
||||
shard_id == 0 for shard_id in shard_ids
|
||||
)
|
||||
base = np.asarray(self.kv_caches_base_addr, dtype=np.int64)
|
||||
length = len(self.block_len)
|
||||
blen = np.asarray(
|
||||
[self.block_len[i % length] for i in range(base.shape[0])],
|
||||
dtype=np.int64,
|
||||
)
|
||||
n = len(chunks)
|
||||
starts = np.fromiter((c[0] for c in chunks), dtype=np.int64, count=n)
|
||||
spans = np.fromiter((c[1] for c in chunks), dtype=np.int64, count=n) - starts
|
||||
assert np.all(spans % self.hash_block_size == 0)
|
||||
bids = np.fromiter(
|
||||
(block_ids[i] for i in (starts // self.block_size).tolist()),
|
||||
dtype=np.int64,
|
||||
count=n,
|
||||
)
|
||||
addrs = base[None, :] + bids[:, None] * blen[None, :]
|
||||
block_counts = (spans + self.block_size - 1) // self.block_size
|
||||
sizes = blen[None, :] * block_counts[:, None]
|
||||
return addrs.tolist(), sizes.tolist(), bids.tolist()
|
||||
|
||||
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
|
||||
length = len(self.block_len)
|
||||
return (
|
||||
[
|
||||
base_addr + block_id * self.block_len[index % length]
|
||||
for index, base_addr in enumerate(self.kv_caches_base_addr)
|
||||
],
|
||||
[
|
||||
self.block_len[index % length]
|
||||
for index in range(len(self.kv_caches_base_addr))
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class TPShardedStoreLayout(StoreLayout):
|
||||
"""Store layout shared by divisible TP sizes."""
|
||||
|
||||
store_format: str
|
||||
|
||||
@classmethod
|
||||
def shared_namespace(cls, store_tp_size: int, pp_size: int) -> str:
|
||||
return (
|
||||
f"@store_tp:{store_tp_size}@store_pp:{pp_size}"
|
||||
f"@store_format:{cls.store_format}"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int,
|
||||
local_tp_size: int,
|
||||
store_tp_size: int,
|
||||
tp_rank: int,
|
||||
num_kv_heads: int,
|
||||
) -> None:
|
||||
super().__init__(metadata, block_size, hash_block_size)
|
||||
self.shards_per_rank = store_tp_size // local_tp_size
|
||||
first_shard = tp_rank * self.shards_per_rank
|
||||
self.store_shard_ids = tuple(
|
||||
range(first_shard, first_shard + self.shards_per_rank)
|
||||
)
|
||||
self.store_tp_size = store_tp_size
|
||||
self.heads_per_store_shard = num_kv_heads // store_tp_size
|
||||
self.local_num_kv_heads = num_kv_heads // local_tp_size
|
||||
self._shard_key_prefixes = {
|
||||
shard_id: PoolKey.build_prefix(metadata, tp_rank=shard_id)
|
||||
for shard_id in self.store_shard_ids
|
||||
}
|
||||
self._shard_addr_bases = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
|
||||
self._shard_block_strides = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
|
||||
self._shard_sizes = np.empty((self.shards_per_rank, 0), dtype=np.uint64)
|
||||
|
||||
@property
|
||||
def local_shard_ids(self) -> tuple[StoreShardId, ...]:
|
||||
return self.store_shard_ids
|
||||
|
||||
def key_for(self, shard_id: StoreShardId, chunk_hash: BlockHash) -> str:
|
||||
return PoolKey.build_key_string(
|
||||
self._shard_key_prefixes[shard_id], chunk_hash.hex()
|
||||
)
|
||||
|
||||
def lookup_key_prefixes(
|
||||
self,
|
||||
rank_namespaces: Sequence[RankNamespace],
|
||||
) -> tuple[str, ...]:
|
||||
pp_ranks = sorted({namespace[3] for namespace in rank_namespaces})
|
||||
return tuple(
|
||||
PoolKey.build_prefix(self.metadata, tp_rank=shard_id, pp_rank=pp_rank)
|
||||
for shard_id in range(self.store_tp_size)
|
||||
for pp_rank in pp_ranks
|
||||
)
|
||||
|
||||
def _set_segment_templates(
|
||||
self, templates: Sequence[tuple[list[int], list[int], list[int]]]
|
||||
) -> None:
|
||||
self._shard_addr_bases = np.asarray(
|
||||
[template[0] for template in templates], dtype=np.uint64
|
||||
)
|
||||
self._shard_block_strides = np.asarray(
|
||||
[template[1] for template in templates], dtype=np.uint64
|
||||
)
|
||||
self._shard_sizes = np.asarray(
|
||||
[template[2] for template in templates], dtype=np.uint64
|
||||
)
|
||||
|
||||
def _has_packed_strides(
|
||||
self, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
def _shard_segments(
|
||||
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> Iterable[tuple[int, int]]:
|
||||
raise NotImplementedError
|
||||
|
||||
def register_kv_caches(
|
||||
self,
|
||||
kv_caches: Sequence[torch.Tensor],
|
||||
num_blocks: int,
|
||||
) -> None:
|
||||
templates: list[tuple[list[int], list[int], list[int]]] = [
|
||||
([], [], []) for _ in range(self.shards_per_rank)
|
||||
]
|
||||
for cache in kv_caches:
|
||||
if cache.ndim != 4 or tuple(cache.shape[:3]) != (
|
||||
num_blocks,
|
||||
self.local_num_kv_heads,
|
||||
self.block_size,
|
||||
):
|
||||
raise ValueError(
|
||||
"TP-shared Mooncake store requires packed KV caches with "
|
||||
"logical shape (num_blocks, local_kv_heads, block_size, content)"
|
||||
)
|
||||
|
||||
element_size = cache.element_size()
|
||||
block_stride, head_stride, token_stride, content_stride = (
|
||||
stride * element_size for stride in cache.stride()
|
||||
)
|
||||
content_bytes = cache.shape[3] * element_size
|
||||
if content_stride != element_size or not self._has_packed_strides(
|
||||
head_stride, token_stride, content_bytes
|
||||
):
|
||||
raise ValueError(
|
||||
"TP-shared Mooncake store requires packed "
|
||||
f"{self.store_format} KV layout"
|
||||
)
|
||||
|
||||
for local_shard, (addr_bases, block_strides, sizes) in enumerate(templates):
|
||||
head_start = local_shard * self.heads_per_store_shard
|
||||
for offset, size in self._shard_segments(
|
||||
head_start, head_stride, token_stride, content_bytes
|
||||
):
|
||||
addr_bases.append(cache.data_ptr() + offset)
|
||||
block_strides.append(block_stride)
|
||||
sizes.append(size)
|
||||
|
||||
self._set_segment_templates(templates)
|
||||
|
||||
def prepare_values(
|
||||
self,
|
||||
chunks: Sequence[tuple[int, int]],
|
||||
block_ids: list[int],
|
||||
shard_ids: Sequence[StoreShardId],
|
||||
) -> tuple[list[list[int]], list[list[int]], list[int]]:
|
||||
if not chunks:
|
||||
return [], [], []
|
||||
if len(chunks) != len(shard_ids):
|
||||
raise ValueError("Each Store chunk must have one shard ID")
|
||||
|
||||
count = len(chunks)
|
||||
starts = np.fromiter(
|
||||
(chunk[0] for chunk in chunks), dtype=np.int64, count=count
|
||||
)
|
||||
ends = np.fromiter((chunk[1] for chunk in chunks), dtype=np.int64, count=count)
|
||||
if np.any(ends - starts != self.block_size):
|
||||
raise ValueError("TP-shared Mooncake store requires full KV blocks")
|
||||
|
||||
first_shard = self.store_shard_ids[0]
|
||||
local_shards = np.fromiter(
|
||||
(shard_id - first_shard for shard_id in shard_ids),
|
||||
dtype=np.int64,
|
||||
count=count,
|
||||
)
|
||||
if np.any(local_shards < 0) or np.any(local_shards >= self.shards_per_rank):
|
||||
raise ValueError("Store shard is not owned by this TP rank")
|
||||
chunk_block_ids = np.fromiter(
|
||||
(block_ids[index] for index in (starts // self.block_size).tolist()),
|
||||
dtype=np.uint64,
|
||||
count=count,
|
||||
)
|
||||
addr_bases = self._shard_addr_bases[local_shards]
|
||||
block_strides = self._shard_block_strides[local_shards]
|
||||
addrs = addr_bases + chunk_block_ids[:, None] * block_strides
|
||||
sizes = self._shard_sizes[local_shards]
|
||||
return addrs.tolist(), sizes.tolist(), chunk_block_ids.tolist()
|
||||
|
||||
|
||||
class LBHNCStoreLayout(TPShardedStoreLayout):
|
||||
"""Native head-major layout shared by divisible TP sizes."""
|
||||
|
||||
store_format = "tp_shared_lbhnc"
|
||||
|
||||
def _has_packed_strides(
|
||||
self, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> bool:
|
||||
return (
|
||||
token_stride == content_bytes
|
||||
and head_stride == self.block_size * content_bytes
|
||||
)
|
||||
|
||||
def _shard_segments(
|
||||
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> Iterable[tuple[int, int]]:
|
||||
yield head_start * head_stride, self.heads_per_store_shard * head_stride
|
||||
|
||||
|
||||
class LBNHCStoreLayout(TPShardedStoreLayout):
|
||||
"""Native token-major layout shared by divisible TP sizes."""
|
||||
|
||||
store_format = "tp_shared_lbnhc"
|
||||
|
||||
def _has_packed_strides(
|
||||
self, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> bool:
|
||||
return (
|
||||
head_stride == content_bytes
|
||||
and token_stride == self.local_num_kv_heads * content_bytes
|
||||
)
|
||||
|
||||
def _shard_segments(
|
||||
self, head_start: int, head_stride: int, token_stride: int, content_bytes: int
|
||||
) -> Iterable[tuple[int, int]]:
|
||||
for token_idx in range(self.block_size):
|
||||
yield (
|
||||
token_idx * token_stride + head_start * head_stride,
|
||||
self.heads_per_store_shard * content_bytes,
|
||||
)
|
||||
|
||||
|
||||
class ChunkedTokenDatabase:
|
||||
"""Enumerates logical token chunks and their hashes."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
metadata: KeyMetadata,
|
||||
block_size: int,
|
||||
hash_block_size: int | None = None,
|
||||
store_layout: StoreLayout | None = None,
|
||||
):
|
||||
self.metadata = metadata
|
||||
self.block_size = block_size
|
||||
self.hash_block_size = hash_block_size or block_size
|
||||
if self.block_size % self.hash_block_size != 0:
|
||||
raise ValueError(
|
||||
f"block_size ({self.block_size}) must be a multiple of "
|
||||
f"hash_block_size ({self.hash_block_size})"
|
||||
)
|
||||
self.store_layout = store_layout or RankLocalStoreLayout(
|
||||
metadata, block_size, self.hash_block_size
|
||||
)
|
||||
|
||||
@property
|
||||
def kv_caches_base_addr(self) -> list[int]:
|
||||
return self._rank_local_layout().kv_caches_base_addr
|
||||
|
||||
@property
|
||||
def block_len(self) -> list[int]:
|
||||
return self._rank_local_layout().block_len
|
||||
|
||||
def _rank_local_layout(self) -> RankLocalStoreLayout:
|
||||
if not isinstance(self.store_layout, RankLocalStoreLayout):
|
||||
raise RuntimeError("This operation requires a rank-local Store layout")
|
||||
return self.store_layout
|
||||
|
||||
def key_for(self, chunk_hash: BlockHash) -> str:
|
||||
return self._rank_local_layout().key_for(0, chunk_hash)
|
||||
|
||||
def set_kv_caches_base_addr(self, kv_caches_base_addr: list[int]):
|
||||
self._rank_local_layout().set_kv_caches_base_addr(kv_caches_base_addr)
|
||||
|
||||
def set_block_len(self, block_len: list[int]):
|
||||
self._rank_local_layout().set_block_len(block_len)
|
||||
|
||||
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
|
||||
"""Return addresses and sizes for one physical block slot."""
|
||||
return self._rank_local_layout().prepare_value_for_block(block_id)
|
||||
|
||||
def process_tokens(
|
||||
self,
|
||||
token_len: int,
|
||||
block_hashes: list[BlockHash],
|
||||
mask_num: int = 0,
|
||||
*,
|
||||
chunk_mask: list[bool] | None = None,
|
||||
put_step: int = 1,
|
||||
put_step_rank: int = 0,
|
||||
) -> Iterable[tuple[int, int, BlockHash]]:
|
||||
"""Process tokens and yield (start_idx, end_idx, block_hash) tuples.
|
||||
|
||||
When there are fewer KV heads than TP ranks, chunks are distributed
|
||||
across TP ranks to avoid duplicate load/store. The assignment keys off
|
||||
the absolute ``chunk_id`` so a given chunk always lands on the same
|
||||
rank regardless of where the processed suffix begins.
|
||||
|
||||
Args:
|
||||
token_len: Total number of tokens. Must be hash-block aligned and
|
||||
covered by ``block_hashes`` when hashes are present.
|
||||
block_hashes: Block hashes computed at ``hash_block_size`` granularity.
|
||||
When ``block_size > hash_block_size`` each group's ``block_size`` chunk
|
||||
is keyed by its last sub-hash via ``chunk_hashes_for_block_size``.
|
||||
mask_num: Number of tokens to skip from the beginning.
|
||||
chunk_mask: Optional mask relative to the first chunk after
|
||||
``mask_num``. False entries are skipped before hash access.
|
||||
put_step: Stride for distributing chunks across ranks.
|
||||
put_step_rank: ``chunk_id % put_step`` value this rank stores.
|
||||
|
||||
"""
|
||||
assert put_step > 0
|
||||
if not block_hashes:
|
||||
return
|
||||
assert token_len % self.hash_block_size == 0
|
||||
assert token_len // self.hash_block_size <= len(block_hashes)
|
||||
start_chunk = max(0, cdiv(mask_num, self.block_size))
|
||||
max_chunks = cdiv(token_len, self.block_size)
|
||||
if chunk_mask is not None:
|
||||
max_chunks = min(max_chunks, start_chunk + len(chunk_mask))
|
||||
for chunk_id in range(start_chunk, max_chunks):
|
||||
if chunk_mask is not None and not chunk_mask[chunk_id - start_chunk]:
|
||||
continue
|
||||
if chunk_id % put_step != put_step_rank:
|
||||
continue
|
||||
start_idx = chunk_id * self.block_size
|
||||
end_idx = min(start_idx + self.block_size, token_len)
|
||||
h = block_hashes[end_idx // self.hash_block_size - 1]
|
||||
yield start_idx, end_idx, h
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TailKeyBoundary:
|
||||
"""Hash boundary used to key a group's tail block in the store.
|
||||
|
||||
Attributes:
|
||||
group_id: KV-cache group containing the tail block.
|
||||
num_tokens: Token boundary whose prefix hash identifies the matched
|
||||
stored block. The loader uses
|
||||
``block_hashes[num_tokens // hash_block_size - 1]`` instead of the
|
||||
hash implied by ``MooncakeLookupResult.hit_length``. This changes
|
||||
only the load key, not the reusable prefix.
|
||||
|
||||
"""
|
||||
|
||||
group_id: int
|
||||
num_tokens: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class MooncakeLookupResult:
|
||||
"""Lookup result used to build the subsequent load request.
|
||||
|
||||
Attributes:
|
||||
hit_length: Longest prefix that every KV-cache group can reuse after
|
||||
their individual cache hits converge.
|
||||
tail_key_boundaries: Hash boundary used to store each cache group's
|
||||
tail block when ``hit_length`` does not identify its store key.
|
||||
There is one entry per group for every nonzero hit.
|
||||
|
||||
"""
|
||||
|
||||
hit_length: int
|
||||
tail_key_boundaries: tuple[TailKeyBoundary, ...] = ()
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadSpec:
|
||||
"""Specification for loading KV cache from external store."""
|
||||
|
||||
vllm_cached_tokens: int
|
||||
kvpool_cached_tokens: int
|
||||
can_load: bool
|
||||
token_len: int = 0
|
||||
tail_key_boundaries: tuple[TailKeyBoundary, ...] = ()
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequestTracker:
|
||||
"""Tracks per-request state across scheduler ticks."""
|
||||
|
||||
req_id: str
|
||||
token_len: int
|
||||
allocated_block_ids: tuple[list[int], ...]
|
||||
num_saved_tokens: int = 0
|
||||
token_ids: list[int] | None = None
|
||||
has_pending_offload: bool = False
|
||||
# Snapshot of the prefill range length at tracker creation time.
|
||||
# For a fresh request this is len(prompt). For a resumed-from-preemption
|
||||
# request it includes previously-generated tokens, which are re-prefilled.
|
||||
prefill_end_tokens: int = 0
|
||||
partial_tail_sent: bool = False
|
||||
|
||||
def reset(self) -> None:
|
||||
self.token_len = 0
|
||||
self.allocated_block_ids = ()
|
||||
self.num_saved_tokens = 0
|
||||
self.token_ids = None
|
||||
self.has_pending_offload = False
|
||||
self.prefill_end_tokens = 0
|
||||
self.partial_tail_sent = False
|
||||
|
||||
def update(
|
||||
self,
|
||||
new_block_ids: tuple[list[int], ...] | list[int],
|
||||
) -> None:
|
||||
# Backward-compat: accept a single list (broadcast to single group).
|
||||
if isinstance(new_block_ids, list):
|
||||
new_block_ids = (new_block_ids,)
|
||||
if len(new_block_ids) != len(self.allocated_block_ids):
|
||||
raise ValueError(
|
||||
f"Group count mismatch: tracker has "
|
||||
f"{len(self.allocated_block_ids)} groups, update has "
|
||||
f"{len(new_block_ids)}"
|
||||
)
|
||||
for existing, new in zip(self.allocated_block_ids, new_block_ids, strict=True):
|
||||
if new:
|
||||
existing.extend(new)
|
||||
|
||||
|
||||
class BoundaryPut(NamedTuple):
|
||||
"""A block stored under the hash of the prefix ending at ``num_tokens``."""
|
||||
|
||||
group_id: int
|
||||
block_id: int
|
||||
num_tokens: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReqMeta:
|
||||
"""Per-request metadata for store put/get operations."""
|
||||
|
||||
req_id: str
|
||||
token_len_chunk: int
|
||||
block_ids: tuple[list[int], ...]
|
||||
block_hashes: list[BlockHash]
|
||||
|
||||
can_save: bool | None = None
|
||||
load_spec: LoadSpec | None = None
|
||||
current_event: torch.cuda.Event | None = None
|
||||
|
||||
token_ids: list[int] | None = None
|
||||
# Absolute request offset represented by token_ids[0].
|
||||
token_ids_start: int = 0
|
||||
num_prompt_tokens: int | None = None
|
||||
# Total prefill tokens, including generated tokens replayed after preemption.
|
||||
prefill_end_tokens: int | None = None
|
||||
# Identifies this store job for the engine's lifetime. A request id cannot
|
||||
# serve that purpose: it is reused once a preempted request resumes, so it
|
||||
# would release the wrong job's blocks.
|
||||
store_job_id: int | None = None
|
||||
# Blocks stored under an explicit prefix hash rather than by position:
|
||||
# core-provided mamba "align" states (a block-aligned entry is a committed
|
||||
# snapshot, a non-aligned one the sub-block CoW prompt tail) and the
|
||||
# scheduler-resolved non-Mamba blocks of the prompt's partial tail. The
|
||||
# store-job reference keeps each block alive until every worker rank
|
||||
# finishes the job.
|
||||
boundary_puts: list[BoundaryPut] | None = None
|
||||
# Total computed prefix length at the end of this step.
|
||||
completed_token_len: int | None = None
|
||||
# Set on the save that first covers the whole prompt, so the scheduler
|
||||
# publishes the prompt's partial tail once rather than on every save.
|
||||
publish_partial_tail: bool = False
|
||||
|
||||
@staticmethod
|
||||
def from_request_tracker(
|
||||
tracker: RequestTracker,
|
||||
block_size: int,
|
||||
load_spec: LoadSpec | None = None,
|
||||
skip_save: bool | None = False,
|
||||
block_hashes: list[BlockHash] | None = None,
|
||||
save_partial_tail: bool = False,
|
||||
num_prompt_tokens: int | None = None,
|
||||
) -> "ReqMeta | None":
|
||||
"""Create ReqMeta from a RequestTracker."""
|
||||
if save_partial_tail:
|
||||
assert num_prompt_tokens is not None
|
||||
if block_hashes is None:
|
||||
block_hashes = []
|
||||
input_token_len = tracker.token_len
|
||||
|
||||
token_ids_start = tracker.num_saved_tokens
|
||||
chunk_boundary = cdiv(token_ids_start + 1, block_size) * block_size
|
||||
num_tokens_to_save = input_token_len // block_size * block_size
|
||||
|
||||
publish_tail = (
|
||||
save_partial_tail
|
||||
and not tracker.partial_tail_sent
|
||||
and num_prompt_tokens is not None
|
||||
and 0 < num_prompt_tokens <= input_token_len
|
||||
)
|
||||
skip_save = skip_save or (
|
||||
num_tokens_to_save < chunk_boundary and not publish_tail
|
||||
)
|
||||
# A ReqMeta must never carry both a save AND a load.
|
||||
# The save would also be wasted work — the bytes are being looked up
|
||||
# in the store right now. Later cached_reqs steps save new tokens
|
||||
# normally.
|
||||
if load_spec is not None and load_spec.can_load:
|
||||
skip_save = True
|
||||
if skip_save and load_spec is None:
|
||||
return None
|
||||
|
||||
if not skip_save:
|
||||
tracker.num_saved_tokens = num_tokens_to_save
|
||||
tracker.partial_tail_sent |= publish_tail
|
||||
|
||||
token_ids = None
|
||||
if tracker.token_ids and not skip_save:
|
||||
# Scheduler tracking continues while this job is handled by an
|
||||
# asynchronous worker, so metadata must own a stable snapshot.
|
||||
token_ids = tracker.token_ids[token_ids_start:num_tokens_to_save]
|
||||
|
||||
if load_spec is not None and load_spec.can_load:
|
||||
logger.debug(
|
||||
"Scheduled to load %d tokens for request %s",
|
||||
load_spec.kvpool_cached_tokens,
|
||||
tracker.req_id,
|
||||
)
|
||||
else:
|
||||
load_spec = None
|
||||
|
||||
logger.debug(
|
||||
"request:%s, meta save spec:%s, meta load spec:%s",
|
||||
tracker.req_id,
|
||||
not skip_save,
|
||||
load_spec,
|
||||
)
|
||||
return ReqMeta(
|
||||
req_id=tracker.req_id,
|
||||
token_len_chunk=num_tokens_to_save,
|
||||
block_ids=tracker.allocated_block_ids,
|
||||
can_save=not skip_save,
|
||||
load_spec=load_spec,
|
||||
block_hashes=block_hashes,
|
||||
token_ids=token_ids,
|
||||
token_ids_start=token_ids_start,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
prefill_end_tokens=tracker.prefill_end_tokens,
|
||||
completed_token_len=input_token_len,
|
||||
publish_partial_tail=publish_tail and not skip_save,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MooncakeStoreWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
"""Maps ``ReqMeta.store_job_id`` to the number of ranks done with that job."""
|
||||
|
||||
completed_saves: dict[int, int] = field(default_factory=dict)
|
||||
|
||||
def aggregate(
|
||||
self, other: "KVConnectorWorkerMetadata"
|
||||
) -> "MooncakeStoreWorkerMetadata":
|
||||
assert isinstance(other, MooncakeStoreWorkerMetadata)
|
||||
for store_job_id, count in other.completed_saves.items():
|
||||
self.completed_saves[store_job_id] = (
|
||||
self.completed_saves.get(store_job_id, 0) + count
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class MooncakeStoreConnectorMetadata(KVConnectorMetadata):
|
||||
"""Metadata passed from scheduler to worker."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
unfinished_request_ids: set[str],
|
||||
preempted_req_ids: set[str],
|
||||
):
|
||||
self.requests: list[ReqMeta] = []
|
||||
self.unfinished_request_ids = unfinished_request_ids
|
||||
self.preempted_req_ids = preempted_req_ids
|
||||
|
||||
def add_request(self, req_meta: ReqMeta) -> None:
|
||||
self.requests.append(req_meta)
|
||||
@@ -1,189 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Per-operation telemetry for MooncakeStoreConnector.
|
||||
|
||||
Records one row per Mooncake RPC (``save_exists``, ``save_put``, ``load_get``,
|
||||
``lookup_exists``) with duration, key/byte counts, status, and failed-key
|
||||
count. Exposed to the logger via ``KVConnectorLogging`` and to Prometheus
|
||||
via ``MooncakeStorePromMetrics``.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from statistics import fmean
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorPromMetrics,
|
||||
KVConnectorStats,
|
||||
PromMetric,
|
||||
PromMetricT,
|
||||
)
|
||||
|
||||
|
||||
def _nearest_rank_percentile(values: list[float], percentile: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
sorted_values = sorted(values)
|
||||
rank = max(
|
||||
0, min(len(sorted_values) - 1, int(percentile * len(sorted_values) - 1e-12))
|
||||
)
|
||||
return sorted_values[rank]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MooncakeStoreConnectorStats(KVConnectorStats):
|
||||
"""Serializable Mooncake store communication telemetry."""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
self.data: dict[str, list[dict[str, int | float | str]]] = {}
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return not self.data
|
||||
|
||||
def aggregate(self, other: KVConnectorStats) -> KVConnectorStats:
|
||||
if other.is_empty():
|
||||
return self
|
||||
for operation, records in other.data.items():
|
||||
self.data.setdefault(operation, []).extend(records)
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
reduced: dict[str, int | float] = {}
|
||||
for operation, records in sorted(self.data.items()):
|
||||
if not records:
|
||||
continue
|
||||
durations = [float(record["duration_seconds"]) for record in records]
|
||||
reduced[f"{operation}_count"] = len(records)
|
||||
reduced[f"{operation}_avg_ms"] = round(fmean(durations) * 1e3, 3)
|
||||
reduced[f"{operation}_p90_ms"] = round(
|
||||
_nearest_rank_percentile(durations, 0.9) * 1e3, 3
|
||||
)
|
||||
reduced[f"{operation}_total_keys"] = sum(
|
||||
int(record["num_keys"]) for record in records
|
||||
)
|
||||
reduced[f"{operation}_total_bytes"] = sum(
|
||||
int(record["num_bytes"]) for record in records
|
||||
)
|
||||
reduced[f"{operation}_failed_keys"] = sum(
|
||||
int(record["num_failed_keys"]) for record in records
|
||||
)
|
||||
reduced[f"{operation}_error_count"] = sum(
|
||||
1 for record in records if record["status"] == "error"
|
||||
)
|
||||
return reduced
|
||||
|
||||
def record_operation(
|
||||
self,
|
||||
operation: str,
|
||||
duration_seconds: float,
|
||||
num_keys: int,
|
||||
*,
|
||||
num_bytes: int = 0,
|
||||
status: str = "ok",
|
||||
num_failed_keys: int = 0,
|
||||
) -> None:
|
||||
self.data.setdefault(operation, []).append(
|
||||
{
|
||||
"duration_seconds": duration_seconds,
|
||||
"num_keys": num_keys,
|
||||
"num_bytes": num_bytes,
|
||||
"status": status,
|
||||
"num_failed_keys": num_failed_keys,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class MooncakeStorePromMetrics(KVConnectorPromMetrics):
|
||||
"""Prometheus metrics for Mooncake store communication."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
metric_types: dict[type[PromMetric], type[PromMetricT]],
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
):
|
||||
super().__init__(vllm_config, metric_types, labelnames, per_engine_labelvalues)
|
||||
metric_labelnames = labelnames + ["operation", "status"]
|
||||
self._metric_cache: dict[tuple[int, str, str], dict[str, PromMetric]] = {}
|
||||
|
||||
self._histogram_operation_time = self._histogram_cls(
|
||||
name="vllm:mooncake_store_operation_time_seconds",
|
||||
documentation="Histogram of Mooncake store communication time.",
|
||||
buckets=[
|
||||
1e-3,
|
||||
5e-3,
|
||||
1e-2,
|
||||
5e-2,
|
||||
1e-1,
|
||||
2e-1,
|
||||
3e-1,
|
||||
4e-1,
|
||||
5e-1,
|
||||
7.5e-1,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
],
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
self._counter_operation_calls = self._counter_cls(
|
||||
name="vllm:mooncake_store_operation_total",
|
||||
documentation="Number of Mooncake store communication operations.",
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
self._counter_operation_keys = self._counter_cls(
|
||||
name="vllm:mooncake_store_operation_keys_total",
|
||||
documentation="Number of Mooncake store keys touched by operations.",
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
self._counter_operation_bytes = self._counter_cls(
|
||||
name="vllm:mooncake_store_operation_bytes_total",
|
||||
documentation="Number of bytes transferred by Mooncake store operations.",
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
self._counter_failed_keys = self._counter_cls(
|
||||
name="vllm:mooncake_store_operation_failed_keys_total",
|
||||
documentation="Number of Mooncake store keys that failed in operations.",
|
||||
labelnames=metric_labelnames,
|
||||
)
|
||||
|
||||
def _get_metrics(
|
||||
self,
|
||||
engine_idx: int,
|
||||
operation: str,
|
||||
status: str,
|
||||
) -> dict[str, PromMetric]:
|
||||
cache_key = (engine_idx, operation, status)
|
||||
if cache_key not in self._metric_cache:
|
||||
label_values = self.per_engine_labelvalues[engine_idx] + [operation, status]
|
||||
self._metric_cache[cache_key] = {
|
||||
"time": self._histogram_operation_time.labels(*label_values),
|
||||
"calls": self._counter_operation_calls.labels(*label_values),
|
||||
"keys": self._counter_operation_keys.labels(*label_values),
|
||||
"bytes": self._counter_operation_bytes.labels(*label_values),
|
||||
"failed_keys": self._counter_failed_keys.labels(*label_values),
|
||||
}
|
||||
return self._metric_cache[cache_key]
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any] | None, engine_idx: int = 0):
|
||||
if not transfer_stats_data:
|
||||
return
|
||||
for operation, records in transfer_stats_data.items():
|
||||
assert isinstance(records, list)
|
||||
for record in records:
|
||||
assert isinstance(record, dict)
|
||||
status = str(record["status"])
|
||||
metrics = self._get_metrics(engine_idx, operation, status)
|
||||
metrics["time"].observe(float(record["duration_seconds"]))
|
||||
metrics["calls"].inc()
|
||||
metrics["keys"].inc(int(record["num_keys"]))
|
||||
metrics["bytes"].inc(int(record["num_bytes"]))
|
||||
metrics["failed_keys"].inc(int(record["num_failed_keys"]))
|
||||
@@ -1,81 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Wire-format constants for the LookupKey ZMQ admin channel.
|
||||
|
||||
This is the single source of truth shared by ``LookupKeyClient`` and
|
||||
``LookupKeyServer`` on the scheduler<->worker rank-0 admin channel.
|
||||
|
||||
Wire format (REQ/REP over IPC):
|
||||
|
||||
Request: [msg_type: bytes] [payload_frames...]
|
||||
|
||||
msg_type == LOOKUP_MSG:
|
||||
frame 1: num_tokens (u32 big-endian, 4 bytes); the worker derives
|
||||
the aligned lookup length
|
||||
frame 2: hash_len (u16 big-endian, 2 bytes) — byte length of each
|
||||
fixed-size block hash (0 when there are no hashes)
|
||||
frame 3: raw block hashes concatenated back-to-back (each hash_len
|
||||
bytes); the server splits on hash_len
|
||||
Response: hit_length (u32 big-endian, first 4 bytes), followed by zero
|
||||
or more 8-byte tail-key boundaries. Each entry is group_id
|
||||
(u32), then boundary_tokens (u32).
|
||||
|
||||
msg_type == RESET_MSG:
|
||||
(no payload frames)
|
||||
Response: [RESP_OK] or [RESP_ERR]
|
||||
|
||||
The first frame of every request is a named bytes tag (not a numeric
|
||||
sentinel that aliases the data field) so the protocol stays
|
||||
self-describing and extensible: adding new admin commands requires
|
||||
only a new tag and a new dispatch branch.
|
||||
|
||||
Mirrors the named-tag convention used by the NIXL connector (see
|
||||
``vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py``).
|
||||
"""
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
MooncakeLookupResult,
|
||||
TailKeyBoundary,
|
||||
)
|
||||
|
||||
# Request message-type tags. Frame 0 of every request.
|
||||
LOOKUP_MSG: bytes = b"lookup"
|
||||
RESET_MSG: bytes = b"reset"
|
||||
|
||||
# Single-byte response status codes for admin commands.
|
||||
RESP_OK: bytes = b"\x01"
|
||||
RESP_ERR: bytes = b"\x00"
|
||||
|
||||
# group_id (u32), boundary_tokens (u32).
|
||||
TAIL_KEY_BOUNDARY_ENTRY_SIZE: int = 8
|
||||
|
||||
|
||||
def encode_lookup_response(result: MooncakeLookupResult) -> bytes:
|
||||
hit_length = result.hit_length.to_bytes(4, "big")
|
||||
if not result.tail_key_boundaries:
|
||||
return hit_length
|
||||
|
||||
payload = bytearray(hit_length)
|
||||
for boundary in result.tail_key_boundaries:
|
||||
payload.extend(boundary.group_id.to_bytes(4, "big"))
|
||||
payload.extend(boundary.num_tokens.to_bytes(4, "big"))
|
||||
return bytes(payload)
|
||||
|
||||
|
||||
def decode_lookup_response(payload: bytes) -> MooncakeLookupResult:
|
||||
if len(payload) < 4 or (len(payload) - 4) % TAIL_KEY_BOUNDARY_ENTRY_SIZE:
|
||||
raise ValueError("Invalid Mooncake lookup response")
|
||||
|
||||
hit_length = int.from_bytes(payload[:4], "big")
|
||||
if len(payload) == 4:
|
||||
return MooncakeLookupResult(hit_length)
|
||||
|
||||
boundaries = []
|
||||
for offset in range(4, len(payload), TAIL_KEY_BOUNDARY_ENTRY_SIZE):
|
||||
boundaries.append(
|
||||
TailKeyBoundary(
|
||||
group_id=int.from_bytes(payload[offset : offset + 4], "big"),
|
||||
num_tokens=int.from_bytes(payload[offset + 4 : offset + 8], "big"),
|
||||
)
|
||||
)
|
||||
return MooncakeLookupResult(hit_length, tuple(boundaries))
|
||||
@@ -1,788 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Adapted from vllm-project/vllm-ascend
|
||||
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
|
||||
"""Scheduler-side logic for MooncakeStoreConnector."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import (
|
||||
MooncakeStoreCoordinator,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
|
||||
BoundaryPut,
|
||||
LoadSpec,
|
||||
MooncakeStoreConnectorMetadata,
|
||||
MooncakeStoreWorkerMetadata,
|
||||
ReqMeta,
|
||||
RequestTracker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( # noqa: E501
|
||||
LookupKeyClient,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
MambaSpec,
|
||||
get_mamba_prefill_checkpoint_position,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _new_req_prefill_tokens(request: NewRequestData) -> list[int]:
|
||||
"""Tokens this prefill will compute KV for.
|
||||
|
||||
Under the v2 model runner, resumed-from-preemption requests appear in
|
||||
``scheduled_new_reqs`` with ``prefill_token_ids`` set to the request's full
|
||||
token list (prompt + previously-generated). For all other cases this falls
|
||||
back to the original prompt.
|
||||
"""
|
||||
if request.prefill_token_ids is not None:
|
||||
return request.prefill_token_ids
|
||||
assert request.prompt_token_ids is not None
|
||||
return request.prompt_token_ids
|
||||
|
||||
|
||||
def _partial_tail_non_mamba_puts(
|
||||
coord: MooncakeStoreCoordinator,
|
||||
req_meta: ReqMeta,
|
||||
block_sizes: Sequence[int],
|
||||
) -> list[BoundaryPut]:
|
||||
"""Locate the non-Mamba blocks a partial-tail save publishes.
|
||||
|
||||
A later request resumes at the prompt's Mamba checkpoint (``boundary``) only
|
||||
if both pieces of KV are stored:
|
||||
|
||||
- Mamba groups: the state at ``boundary``, handed off by the core as a
|
||||
boundary put of its own, so Mamba groups are not listed here.
|
||||
- Other groups: the KV from the last LCM-aligned normal save up to
|
||||
``boundary`` plus the group's EAGLE proof margin (``proof_end``). Each
|
||||
block is keyed by the hash at its end, clipped to ``proof_end``.
|
||||
|
||||
Groups whose proof is not computed yet are omitted. For example, with LCM
|
||||
16, blocks of 4 tokens, and a checkpoint at 44, the full-attention blocks
|
||||
are 8-10, keyed at 36, 40 and 44.
|
||||
|
||||
A Mamba state handed off at any other position, i.e. a shared-prefix
|
||||
junction (``--enable-mamba-shared-prefix-checkpoint``), adds no puts here.
|
||||
|
||||
``block_sizes`` are the per-group block sizes, indexed like
|
||||
``req_meta.block_ids``.
|
||||
"""
|
||||
mamba_tails = [
|
||||
put.num_tokens
|
||||
for put in req_meta.boundary_puts or []
|
||||
if put.group_id in coord.mamba_group_ids
|
||||
and put.num_tokens % block_sizes[put.group_id]
|
||||
]
|
||||
# The tail is due on the save that first covers the prompt, or when Mamba
|
||||
# hands off its checkpoint state.
|
||||
if not mamba_tails and not req_meta.publish_partial_tail:
|
||||
return []
|
||||
prompt_tokens = req_meta.num_prompt_tokens
|
||||
completed = req_meta.completed_token_len
|
||||
assert prompt_tokens is not None and completed is not None
|
||||
hash_block_size = coord.hash_block_size
|
||||
# Core drops the Mamba checkpoint whenever EAGLE block drop is on, whichever
|
||||
# groups carry the eagle flag.
|
||||
boundary = get_mamba_prefill_checkpoint_position(
|
||||
prompt_tokens, hash_block_size, coord.use_eagle
|
||||
)
|
||||
# Any other sub-block Mamba hand-off is a shared-prefix junction. Its
|
||||
# attention KV, EAGLE proof included, ends on an attention block end no later
|
||||
# than this checkpoint's proof, so the normal and prompt-tail saves store it.
|
||||
if not req_meta.publish_partial_tail and boundary not in mamba_tails:
|
||||
return []
|
||||
num_hashes = len(req_meta.block_hashes)
|
||||
if boundary == 0 or boundary // hash_block_size > num_hashes:
|
||||
return []
|
||||
start = boundary // coord.lcm_block_size * coord.lcm_block_size
|
||||
puts: list[BoundaryPut] = []
|
||||
for group_id, block_size in enumerate(block_sizes):
|
||||
if group_id in coord.mamba_group_ids:
|
||||
continue
|
||||
proof_end = boundary + coord.eagle_proof_margin_by_group.get(group_id, 0)
|
||||
if proof_end > completed or proof_end // hash_block_size > num_hashes:
|
||||
continue
|
||||
group_blocks = req_meta.block_ids[group_id]
|
||||
for block_idx in range(start // block_size, cdiv(proof_end, block_size)):
|
||||
if block_idx >= len(group_blocks):
|
||||
break
|
||||
block_id = group_blocks[block_idx]
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
continue
|
||||
num_tokens = min((block_idx + 1) * block_size, proof_end)
|
||||
puts.append(BoundaryPut(group_id, block_id, num_tokens))
|
||||
return puts
|
||||
|
||||
|
||||
class MooncakeStoreScheduler:
|
||||
"""Scheduler-side component for MooncakeStoreConnector."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
self.kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
kvc_extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
|
||||
self.load_async = kvc_extra_config.get("load_async", True)
|
||||
self.lookup_async = kvc_extra_config.get("lookup_async", False)
|
||||
# Skips lookup CPU cost on instances that never load KV from the store.
|
||||
self.enable_lookup = kvc_extra_config.get("enable_lookup", True)
|
||||
self.save_decode_cache = kvc_extra_config.get("save_decode_cache", False)
|
||||
kv_event_config = vllm_config.kv_events_config
|
||||
self.enable_kv_events = bool(
|
||||
kv_event_config and kv_event_config.enable_kv_cache_events
|
||||
)
|
||||
self.client = LookupKeyClient(vllm_config)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self._store_group_ids = kv_cache_config.prefix_cacheable_group_ids
|
||||
# Map scheduler group IDs to store group indices. Groups outside the
|
||||
# store projection, such as the QSA ring, map to None and are skipped.
|
||||
self._store_group_id_by_kv_cache_group_id = {
|
||||
group_id: store_group_id
|
||||
for store_group_id, group_id in enumerate(self._store_group_ids)
|
||||
}
|
||||
store_groups = kv_cache_config.prefix_cacheable_groups
|
||||
|
||||
# Align with the engine's own scheduler_block_size and hash_block_size.
|
||||
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
mamba_groups = {
|
||||
group_id: group.kv_cache_spec
|
||||
for group_id, group in enumerate(store_groups)
|
||||
if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
}
|
||||
self._boundary_state_group_ids = frozenset(mamba_groups)
|
||||
|
||||
# Mirrors MooncakeStoreConnector._capacity_only.
|
||||
self._capacity_only = (
|
||||
self.kv_role == "kv_consumer"
|
||||
and not self.enable_lookup
|
||||
and not self.save_decode_cache
|
||||
)
|
||||
self._store_coord: MooncakeStoreCoordinator | None = None
|
||||
self.enable_partial_hash_hits = False
|
||||
if not self._capacity_only:
|
||||
assert all(
|
||||
spec.mamba_cache_mode == "align" for spec in mamba_groups.values()
|
||||
), "MooncakeStoreScheduler requires mamba_cache_mode='align'"
|
||||
self._store_coord = MooncakeStoreCoordinator.from_kv_cache_config(
|
||||
kv_cache_config, vllm_config, self._block_size, self._hash_block_size
|
||||
)
|
||||
self.enable_partial_hash_hits = self._store_coord.enable_partial_hash_hits
|
||||
|
||||
self._gpu_block_pool: BlockPool | None = None
|
||||
self._num_workers = vllm_config.parallel_config.world_size
|
||||
self._next_store_job_id = 0
|
||||
# store_job_id -> (referenced block ids, ranks yet to report completion)
|
||||
self._pinned_saves: dict[int, tuple[list[int], int]] = {}
|
||||
|
||||
# Per-request state
|
||||
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
|
||||
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
|
||||
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
|
||||
self._unfinished_request_ids: set[str] = set()
|
||||
self._finished_partial_tail_metas: dict[str, ReqMeta] = {}
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
|
||||
self._gpu_block_pool = gpu_block_pool
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: Request,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""Check for external KV cache hit.
|
||||
|
||||
Returns ``(None, False)`` when an async lookup is still in flight,
|
||||
signaling the scheduler to retry this request on a later step.
|
||||
"""
|
||||
if not self.enable_lookup:
|
||||
return 0, False
|
||||
|
||||
# Fine-grained hits may land on a hash boundary inside a block; without
|
||||
# partial hits, prefixes shorter than one physical block are skipped.
|
||||
align = (
|
||||
self._hash_block_size if self.enable_partial_hash_hits else self._block_size
|
||||
)
|
||||
if request.num_tokens < align:
|
||||
return 0, False
|
||||
|
||||
lookup_result = self.client.lookup(
|
||||
request.request_id,
|
||||
request.num_tokens,
|
||||
request.block_hashes,
|
||||
non_block=self.lookup_async,
|
||||
)
|
||||
if lookup_result is None:
|
||||
# Lookup not ready yet; scheduler will retry on a later step.
|
||||
return None, False
|
||||
num_external_hit_tokens = lookup_result.hit_length
|
||||
|
||||
if num_external_hit_tokens < num_computed_tokens:
|
||||
need_to_allocate = 0
|
||||
else:
|
||||
need_to_allocate = num_external_hit_tokens - num_computed_tokens
|
||||
|
||||
logger.debug(
|
||||
"Reqid: %s, Total tokens %d, kvpool hit tokens: %d, need to load: %d",
|
||||
request.request_id,
|
||||
request.num_tokens,
|
||||
num_external_hit_tokens,
|
||||
need_to_allocate,
|
||||
)
|
||||
|
||||
if need_to_allocate <= 0:
|
||||
return 0, False
|
||||
|
||||
self.load_specs[request.request_id] = LoadSpec(
|
||||
vllm_cached_tokens=num_computed_tokens,
|
||||
kvpool_cached_tokens=num_external_hit_tokens,
|
||||
can_load=False,
|
||||
tail_key_boundaries=lookup_result.tail_key_boundaries,
|
||||
)
|
||||
|
||||
return need_to_allocate, self.load_async
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: Request,
|
||||
blocks: KVCacheBlocks,
|
||||
num_external_tokens: int,
|
||||
):
|
||||
"""Update state after block allocation."""
|
||||
local_block_ids: tuple[list[int], ...] = ()
|
||||
if num_external_tokens > 0:
|
||||
local_block_ids = blocks.get_block_ids(
|
||||
group_ids=self._store_group_ids,
|
||||
)
|
||||
|
||||
self._unfinished_requests[request.request_id] = (request, local_block_ids)
|
||||
self._unfinished_request_ids.add(request.request_id)
|
||||
|
||||
if request.request_id not in self.load_specs:
|
||||
return
|
||||
|
||||
if num_external_tokens == 0:
|
||||
self.load_specs[request.request_id].can_load = False
|
||||
return
|
||||
|
||||
assert (
|
||||
num_external_tokens > 0
|
||||
and num_external_tokens
|
||||
== self.load_specs[request.request_id].kvpool_cached_tokens
|
||||
- self.load_specs[request.request_id].vllm_cached_tokens
|
||||
), (
|
||||
f"Mismatch in number of tokens: {num_external_tokens} vs "
|
||||
f"{self.load_specs[request.request_id].kvpool_cached_tokens} - "
|
||||
f"{self.load_specs[request.request_id].vllm_cached_tokens}"
|
||||
f" for request {request.request_id}"
|
||||
)
|
||||
|
||||
self.load_specs[request.request_id].can_load = True
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
"""Build connector metadata for this scheduler step."""
|
||||
is_consumer = self.kv_role == "kv_consumer"
|
||||
can_process_cached = not is_consumer or self.save_decode_cache
|
||||
|
||||
for finished_req_id in scheduler_output.finished_req_ids:
|
||||
self.client.discard(finished_req_id)
|
||||
self.load_specs.pop(finished_req_id, None)
|
||||
self._request_trackers.pop(finished_req_id, None)
|
||||
self._unfinished_requests.pop(finished_req_id, None)
|
||||
self._unfinished_request_ids.discard(finished_req_id)
|
||||
|
||||
preempted_ids = scheduler_output.preempted_req_ids or set()
|
||||
for req_id in preempted_ids:
|
||||
self.load_specs.pop(req_id, None)
|
||||
if request_tracker := self._request_trackers.get(req_id):
|
||||
request_tracker.reset()
|
||||
self._unfinished_requests.pop(req_id, None)
|
||||
|
||||
meta = MooncakeStoreConnectorMetadata(
|
||||
self._unfinished_request_ids,
|
||||
preempted_ids,
|
||||
)
|
||||
|
||||
# Handle new requests
|
||||
for request in scheduler_output.scheduled_new_reqs:
|
||||
load_spec = self.load_specs.pop(request.req_id, None)
|
||||
num_tokens_to_compute = (
|
||||
request.num_computed_tokens
|
||||
+ scheduler_output.num_scheduled_tokens[request.req_id]
|
||||
)
|
||||
assert request.req_id in self._unfinished_requests
|
||||
request_tuple = self._unfinished_requests.get(request.req_id)
|
||||
request_real = request_tuple[0] # type: ignore[index]
|
||||
|
||||
unfolded_block_ids = tuple(
|
||||
request.block_ids[group_id].copy() for group_id in self._store_group_ids
|
||||
)
|
||||
|
||||
prefill_tokens = _new_req_prefill_tokens(request)
|
||||
request_tracker = RequestTracker(
|
||||
req_id=request.req_id,
|
||||
token_len=num_tokens_to_compute,
|
||||
allocated_block_ids=unfolded_block_ids,
|
||||
num_saved_tokens=0,
|
||||
token_ids=(
|
||||
prefill_tokens[:num_tokens_to_compute]
|
||||
if self.enable_kv_events
|
||||
else None
|
||||
),
|
||||
prefill_end_tokens=len(prefill_tokens),
|
||||
)
|
||||
self._request_trackers[request.req_id] = request_tracker
|
||||
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
request_tracker,
|
||||
self._block_size,
|
||||
load_spec=load_spec,
|
||||
# A consumer may write decode KV without becoming a prefill
|
||||
# producer. Loads are still carried by the same metadata.
|
||||
skip_save=is_consumer,
|
||||
block_hashes=request_real.block_hashes,
|
||||
num_prompt_tokens=request_real.num_prompt_tokens,
|
||||
save_partial_tail=self.enable_partial_hash_hits,
|
||||
)
|
||||
if req_meta is not None:
|
||||
meta.add_request(req_meta)
|
||||
|
||||
# Handle cached (running, or MRV1 resumed-from-preemption) requests
|
||||
cached_reqs = scheduler_output.scheduled_cached_reqs
|
||||
if can_process_cached:
|
||||
for i, req_id in enumerate(cached_reqs.req_ids):
|
||||
new_block_ids = cached_reqs.new_block_ids[i]
|
||||
if new_block_ids:
|
||||
new_block_ids = tuple(
|
||||
new_block_ids[group_id] for group_id in self._store_group_ids
|
||||
)
|
||||
|
||||
req_meta = None
|
||||
if req_id in cached_reqs.resumed_req_ids:
|
||||
# Resumed after preemption
|
||||
if not new_block_ids:
|
||||
continue
|
||||
new_block_ids = tuple(b.copy() for b in new_block_ids)
|
||||
load_spec = self.load_specs.pop(req_id, None)
|
||||
request_tuple = self._unfinished_requests.get(req_id)
|
||||
request_real = request_tuple[0] # type: ignore[index]
|
||||
num_tokens_to_compute = (
|
||||
request_real.num_computed_tokens
|
||||
+ scheduler_output.num_scheduled_tokens[req_id]
|
||||
)
|
||||
# On resume, the request re-prefills prompt + previously
|
||||
# generated tokens (all_token_ids).
|
||||
prefill_tokens = list(request_real.all_token_ids)
|
||||
request_tracker = RequestTracker(
|
||||
req_id=req_id,
|
||||
token_len=num_tokens_to_compute,
|
||||
allocated_block_ids=new_block_ids,
|
||||
num_saved_tokens=0,
|
||||
token_ids=(
|
||||
prefill_tokens[:num_tokens_to_compute]
|
||||
if self.enable_kv_events
|
||||
else None
|
||||
),
|
||||
prefill_end_tokens=len(prefill_tokens),
|
||||
)
|
||||
self._request_trackers[req_id] = request_tracker
|
||||
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
request_tracker,
|
||||
self._block_size,
|
||||
load_spec=load_spec,
|
||||
skip_save=is_consumer,
|
||||
block_hashes=request_real.block_hashes,
|
||||
num_prompt_tokens=request_real.num_prompt_tokens,
|
||||
save_partial_tail=self.enable_partial_hash_hits,
|
||||
)
|
||||
else:
|
||||
# Decode/chunked request
|
||||
request_tracker = self._request_trackers[req_id]
|
||||
num_computed_token = cached_reqs.num_computed_tokens[i]
|
||||
# Use the tracker's snapshot of the prefill range so resumed
|
||||
# requests keep saving past the original prompt boundary.
|
||||
prefill_end = request_tracker.prefill_end_tokens
|
||||
is_decode = num_computed_token >= prefill_end
|
||||
if is_decode and not self.save_decode_cache:
|
||||
continue
|
||||
|
||||
num_new_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
req_tuple = self._unfinished_requests.get(req_id)
|
||||
if req_tuple:
|
||||
unfinished_req = req_tuple[0]
|
||||
num_current_tokens = request_tracker.token_len
|
||||
new_token_ids = unfinished_req.all_token_ids[
|
||||
num_current_tokens : num_current_tokens + num_new_tokens
|
||||
]
|
||||
request_tracker.token_len += len(new_token_ids)
|
||||
if request_tracker.token_ids is not None:
|
||||
request_tracker.token_ids.extend(new_token_ids)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Request {req_id} is not in _unfinished_requests"
|
||||
)
|
||||
# A block is usually allocated before the step that fills
|
||||
# it, so reaching a save boundary does not imply that this
|
||||
# step has new block ids.
|
||||
if new_block_ids:
|
||||
request_tracker.update(new_block_ids)
|
||||
if is_consumer and not is_decode:
|
||||
continue
|
||||
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
request_tracker,
|
||||
self._block_size,
|
||||
load_spec=None,
|
||||
skip_save=False,
|
||||
block_hashes=unfinished_req.block_hashes,
|
||||
num_prompt_tokens=unfinished_req.num_prompt_tokens,
|
||||
save_partial_tail=self.enable_partial_hash_hits,
|
||||
)
|
||||
|
||||
if req_meta is not None:
|
||||
meta.add_request(req_meta)
|
||||
|
||||
# Handle requests with pending load specs not yet scheduled
|
||||
request_ids = [req.req_id for req in scheduler_output.scheduled_new_reqs]
|
||||
for request_id, (
|
||||
unfinished_req,
|
||||
block_ids,
|
||||
) in self._unfinished_requests.items():
|
||||
if request_id not in request_ids and request_id not in cached_reqs.req_ids:
|
||||
load_spec = self.load_specs.pop(request_id, None)
|
||||
# A load spec may have been proposed by this connector's
|
||||
# lookup but rejected by MultiConnector in favor of another
|
||||
# connector. Only the chosen connector may issue the pending
|
||||
# load; the normal store path gets its blocks later from
|
||||
# SchedulerOutput once the request is actually scheduled.
|
||||
if load_spec is None or not load_spec.can_load:
|
||||
continue
|
||||
num_tokens_to_compute = load_spec.kvpool_cached_tokens
|
||||
request_tracker = RequestTracker(
|
||||
req_id=request_id,
|
||||
token_len=num_tokens_to_compute,
|
||||
allocated_block_ids=block_ids,
|
||||
num_saved_tokens=0,
|
||||
)
|
||||
self._request_trackers[request_id] = request_tracker
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
request_tracker,
|
||||
self._block_size,
|
||||
load_spec=load_spec,
|
||||
skip_save=None,
|
||||
block_hashes=unfinished_req.block_hashes,
|
||||
num_prompt_tokens=unfinished_req.num_prompt_tokens,
|
||||
)
|
||||
if req_meta is not None:
|
||||
meta.add_request(req_meta)
|
||||
|
||||
block_state = getattr(scheduler_output, "kv_connector_block_state", None)
|
||||
if (
|
||||
block_state is not None
|
||||
and block_state.boundary_state_offloads
|
||||
and not is_consumer
|
||||
):
|
||||
self._handle_boundary_state_offloads(
|
||||
block_state.boundary_state_offloads, meta
|
||||
)
|
||||
|
||||
self._apply_current_save_block_ids(meta, scheduler_output)
|
||||
|
||||
# Finish-time handoffs arrive after the producing step's metadata was
|
||||
# built. Their exact blocks were pinned when they were registered, so
|
||||
# they remain valid after request cleanup and need no current snapshot.
|
||||
for req_meta in self._finished_partial_tail_metas.values():
|
||||
meta.add_request(req_meta)
|
||||
self._finished_partial_tail_metas.clear()
|
||||
|
||||
if not self._capacity_only:
|
||||
self._reference_save_blocks(meta)
|
||||
return meta
|
||||
|
||||
def _apply_current_save_block_ids(
|
||||
self,
|
||||
meta: MooncakeStoreConnectorMetadata,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> None:
|
||||
"""Replace append-only mirrors with the core's current block tables."""
|
||||
save_metas = [req_meta for req_meta in meta.requests if req_meta.can_save]
|
||||
if not save_metas:
|
||||
return
|
||||
|
||||
block_state = scheduler_output.kv_connector_block_state
|
||||
assert block_state is not None, (
|
||||
"Current block tables are required for Mooncake store jobs"
|
||||
)
|
||||
for req_meta in save_metas:
|
||||
block_ids = block_state.get_block_ids(req_meta.req_id)
|
||||
assert block_ids is not None, (
|
||||
f"Missing current block table for store request {req_meta.req_id}"
|
||||
)
|
||||
req_meta.block_ids = tuple(
|
||||
block_ids[group_id] for group_id in self._store_group_ids
|
||||
)
|
||||
|
||||
def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None:
|
||||
"""Take a GPU block reference for every store job this step emits.
|
||||
|
||||
The worker DMAs out of these blocks after the step that scheduled them,
|
||||
so a reference keeps them out of the free queue even once the request
|
||||
itself is freed, until every rank reports the job done.
|
||||
"""
|
||||
pool = self._gpu_block_pool
|
||||
coord = self._store_coord
|
||||
assert coord is not None
|
||||
block_sizes = [g.kv_cache_spec.block_size for g in coord.kv_cache_groups]
|
||||
for req_meta in meta.requests:
|
||||
if not req_meta.can_save:
|
||||
continue
|
||||
assert pool is not None, (
|
||||
"GPU block pool must be bound before any store job is emitted"
|
||||
)
|
||||
if req_meta.store_job_id is not None:
|
||||
assert req_meta.store_job_id in self._pinned_saves
|
||||
continue
|
||||
if tail := _partial_tail_non_mamba_puts(coord, req_meta, block_sizes):
|
||||
req_meta.boundary_puts = [*tail, *(req_meta.boundary_puts or [])]
|
||||
req_meta.store_job_id = store_job_id = self._next_store_job_id
|
||||
self._next_store_job_id += 1
|
||||
block_ids = [put.block_id for put in req_meta.boundary_puts or []]
|
||||
assert NULL_BLOCK_ID not in block_ids, (
|
||||
"A null block cannot back a boundary put"
|
||||
)
|
||||
if req_meta.token_len_chunk:
|
||||
# Normal prefix save: pin all non-Mamba sources for retries.
|
||||
# Every allocated block is referenced, not just the ones covering
|
||||
# this job's token range: a rank resumes from its own last
|
||||
# successful offset, which lags the scheduler's whenever a save was
|
||||
# skipped or failed, so it may read anywhere below the range.
|
||||
block_ids.extend(
|
||||
block_id
|
||||
for group_id, group in enumerate(req_meta.block_ids)
|
||||
if group_id not in self._boundary_state_group_ids
|
||||
for block_id in group
|
||||
if block_id != NULL_BLOCK_ID
|
||||
)
|
||||
# An aligned boundary block may also be present in the request's
|
||||
# block table. Take and release exactly one reference per block.
|
||||
block_ids = list(dict.fromkeys(block_ids))
|
||||
assert NULL_BLOCK_ID not in block_ids
|
||||
if not block_ids:
|
||||
continue
|
||||
self._pinned_saves[store_job_id] = (block_ids, self._num_workers)
|
||||
pool.touch([pool.blocks[block_id] for block_id in block_ids])
|
||||
|
||||
def register_finished_partial_tail(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
partial_tail_offloads: list[tuple[int, int, int]],
|
||||
) -> bool:
|
||||
"""Queue and pin a finish-time tail for the next connector step."""
|
||||
if self.kv_role == "kv_consumer" or not partial_tail_offloads:
|
||||
return False
|
||||
tracker = self._request_trackers.get(request.request_id)
|
||||
if tracker is None or not any(block_ids):
|
||||
return False
|
||||
boundaries = {boundary for _, _, boundary in partial_tail_offloads}
|
||||
if len(boundaries) != 1:
|
||||
raise ValueError(
|
||||
"Partial-tail offloads for one request must share a boundary"
|
||||
)
|
||||
boundary_tokens = next(iter(boundaries))
|
||||
if boundary_tokens > tracker.prefill_end_tokens:
|
||||
return False
|
||||
|
||||
mamba_puts: list[BoundaryPut] = []
|
||||
for group_id, block_id, boundary in partial_tail_offloads:
|
||||
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
|
||||
if store_group_id not in self._boundary_state_group_ids:
|
||||
return False
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
return False
|
||||
mamba_puts.append(BoundaryPut(store_group_id, block_id, boundary))
|
||||
req_meta = ReqMeta(
|
||||
req_id=request.request_id,
|
||||
token_len_chunk=0,
|
||||
block_ids=tuple(
|
||||
block_ids[group_id].copy() for group_id in self._store_group_ids
|
||||
),
|
||||
block_hashes=list(request.block_hashes),
|
||||
can_save=True,
|
||||
num_prompt_tokens=request.num_prompt_tokens,
|
||||
prefill_end_tokens=tracker.prefill_end_tokens,
|
||||
boundary_puts=mamba_puts,
|
||||
completed_token_len=request.num_computed_tokens,
|
||||
)
|
||||
coord = self._store_coord
|
||||
assert coord is not None
|
||||
req_meta.boundary_puts = [
|
||||
*_partial_tail_non_mamba_puts(
|
||||
coord,
|
||||
req_meta,
|
||||
[g.kv_cache_spec.block_size for g in coord.kv_cache_groups],
|
||||
),
|
||||
*mamba_puts,
|
||||
]
|
||||
pinned_block_ids = list(
|
||||
dict.fromkeys(put.block_id for put in req_meta.boundary_puts)
|
||||
)
|
||||
|
||||
pool = self._gpu_block_pool
|
||||
assert pool is not None, (
|
||||
"GPU block pool must be bound before a finish-time handoff"
|
||||
)
|
||||
assert request.request_id not in self._finished_partial_tail_metas
|
||||
store_job_id = self._next_store_job_id
|
||||
self._next_store_job_id += 1
|
||||
self._pinned_saves[store_job_id] = (pinned_block_ids, self._num_workers)
|
||||
pool.touch([pool.blocks[block_id] for block_id in pinned_block_ids])
|
||||
|
||||
req_meta.store_job_id = store_job_id
|
||||
self._finished_partial_tail_metas[request.request_id] = req_meta
|
||||
tracker.has_pending_offload = True
|
||||
# The store job owns exact block refs, so request cleanup need not wait.
|
||||
return False
|
||||
|
||||
def _handle_boundary_state_offloads(
|
||||
self,
|
||||
offloads: dict[str, list[tuple[int, int, int]]],
|
||||
meta: MooncakeStoreConnectorMetadata,
|
||||
) -> None:
|
||||
"""Attach exact boundary-state blocks to store jobs for this step.
|
||||
|
||||
Flushed in the step they arrive: the CoW copy is enqueued before the
|
||||
connector event records, so this step's event fences the exact block.
|
||||
Entries ride the request's save meta when present, else an
|
||||
offload-only ReqMeta (``token_len_chunk=0`` skips the normal save,
|
||||
``can_save=True`` takes the normal enqueue and store-job pinning path).
|
||||
"""
|
||||
save_metas = {m.req_id: m for m in meta.requests if m.can_save}
|
||||
for req_id, entries in offloads.items():
|
||||
tracker = self._request_trackers.get(req_id)
|
||||
req_tuple = self._unfinished_requests.get(req_id)
|
||||
if tracker is None or req_tuple is None:
|
||||
# Request finished/preempted within this step; its blocks are
|
||||
# going away, so the offload is conservatively dropped.
|
||||
logger.debug("Dropping boundary-state offload for request %s", req_id)
|
||||
continue
|
||||
accepted: list[BoundaryPut] = []
|
||||
for group_id, block_id, boundary_tokens in entries:
|
||||
# Every other group stops saving at the end of this prefill, so
|
||||
# a mamba-only key past it can never complete a joint hybrid
|
||||
# hit. `prefill_end_tokens` — not the original prompt length —
|
||||
# is the boundary: a resumed request re-prefills and re-saves
|
||||
# its previously generated tokens for every group.
|
||||
if boundary_tokens > tracker.prefill_end_tokens:
|
||||
continue
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
continue
|
||||
store_group_id = self._store_group_id_by_kv_cache_group_id.get(group_id)
|
||||
if store_group_id not in self._boundary_state_group_ids:
|
||||
continue
|
||||
accepted.append(BoundaryPut(store_group_id, block_id, boundary_tokens))
|
||||
if not accepted:
|
||||
continue
|
||||
tracker.has_pending_offload = True
|
||||
if (req_meta := save_metas.get(req_id)) is not None:
|
||||
req_meta.boundary_puts = accepted
|
||||
continue
|
||||
meta.add_request(
|
||||
ReqMeta(
|
||||
req_id=req_id,
|
||||
token_len_chunk=0,
|
||||
block_ids=tracker.allocated_block_ids,
|
||||
block_hashes=req_tuple[0].block_hashes,
|
||||
can_save=True,
|
||||
num_prompt_tokens=req_tuple[0].num_prompt_tokens,
|
||||
prefill_end_tokens=tracker.prefill_end_tokens,
|
||||
boundary_puts=accepted,
|
||||
completed_token_len=tracker.token_len,
|
||||
)
|
||||
)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
"""Drop the block references of store jobs every rank has finished."""
|
||||
meta = connector_output.kv_connector_worker_meta
|
||||
if not isinstance(meta, MooncakeStoreWorkerMetadata):
|
||||
return
|
||||
pool = self._gpu_block_pool
|
||||
assert pool is not None
|
||||
for store_job_id, count in meta.completed_saves.items():
|
||||
pinned = self._pinned_saves.get(store_job_id)
|
||||
if pinned is None:
|
||||
# The job referenced no blocks, so nothing was recorded for it.
|
||||
continue
|
||||
block_ids, remaining = pinned
|
||||
remaining -= count
|
||||
if remaining > 0:
|
||||
self._pinned_saves[store_job_id] = (block_ids, remaining)
|
||||
continue
|
||||
assert remaining == 0, (
|
||||
f"store job {store_job_id} reported by too many ranks"
|
||||
)
|
||||
del self._pinned_saves[store_job_id]
|
||||
# Tail-first, as elsewhere, so the shared prefix is evicted last.
|
||||
pool.free_blocks(pool.blocks[bid] for bid in reversed(block_ids))
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
"""Keep the engine stepping while any store job still holds block refs.
|
||||
|
||||
Completions only reach the scheduler as worker metadata on a step, so an
|
||||
engine that quiesced with jobs in flight would leave those references
|
||||
held indefinitely. Nothing else keeps it alive now that a finishing
|
||||
request no longer defers its own free.
|
||||
"""
|
||||
return bool(self._pinned_saves)
|
||||
|
||||
def reset_store(self) -> bool:
|
||||
"""Trigger a global ``remove_all(force=True)`` on the Mooncake master.
|
||||
|
||||
Routes through the existing LookupKey ZMQ admin channel to worker
|
||||
rank 0, which owns the ``MooncakeDistributedStore`` handle.
|
||||
|
||||
Ordering assumption: caller (typically
|
||||
``Scheduler.reset_connector_cache``, invoked via
|
||||
``reset_prefix_cache(reset_connector=True)``) MUST ensure no
|
||||
in-flight Mooncake lookups or transfers. For RL workflows this is
|
||||
satisfied at the step boundary after weight updates and rollout
|
||||
drain. Violating this can allow stale KV to be served on the next
|
||||
request, defeating the hard-reset guarantee.
|
||||
|
||||
Returns True on ACK from worker, False on NACK or RPC error.
|
||||
"""
|
||||
try:
|
||||
ok = self.client.reset()
|
||||
if ok:
|
||||
logger.info("Mooncake store reset via remove_all succeeded.")
|
||||
else:
|
||||
logger.warning("Mooncake store reset returned NACK from worker.")
|
||||
return ok
|
||||
except Exception as e:
|
||||
logger.error("Mooncake reset_store RPC failed: %s", e)
|
||||
return False
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,41 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .connector import UMBPStoreConnector
|
||||
from .data import (
|
||||
BlockIdentityCodec,
|
||||
BlockTransferPlan,
|
||||
KVLayoutDescriptor,
|
||||
KVLayoutPlanner,
|
||||
KVRange,
|
||||
KVRegion,
|
||||
LoadSpec,
|
||||
RankTopology,
|
||||
RequestTracker,
|
||||
TransferJobState,
|
||||
TransferJobStatus,
|
||||
UMBPConnectorMetadata,
|
||||
UMBPConnectorWorkerMetadata,
|
||||
UMBPNamespace,
|
||||
)
|
||||
from .runtime import UMBPRuntimeConfig, UMBPRuntimeFactory
|
||||
|
||||
__all__ = [
|
||||
"UMBPStoreConnector",
|
||||
"BlockIdentityCodec",
|
||||
"BlockTransferPlan",
|
||||
"KVLayoutPlanner",
|
||||
"KVLayoutDescriptor",
|
||||
"KVRange",
|
||||
"KVRegion",
|
||||
"LoadSpec",
|
||||
"RequestTracker",
|
||||
"RankTopology",
|
||||
"TransferJobState",
|
||||
"TransferJobStatus",
|
||||
"UMBPConnectorMetadata",
|
||||
"UMBPConnectorWorkerMetadata",
|
||||
"UMBPNamespace",
|
||||
"UMBPRuntimeConfig",
|
||||
"UMBPRuntimeFactory",
|
||||
]
|
||||
@@ -1,220 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vLLM-facing shared UMBP connector."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
from .data import (
|
||||
BlockIdentityCodec,
|
||||
KVLayoutPlanner,
|
||||
RankTopology,
|
||||
UMBPConnectorMetadata,
|
||||
UMBPConnectorWorkerMetadata,
|
||||
UMBPNamespace,
|
||||
)
|
||||
from .runtime import UMBPRuntimeConfig, UMBPRuntimeFactory
|
||||
from .scheduler import UMBPStoreConnectorScheduler
|
||||
from .worker import UMBPStoreConnectorWorker
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
|
||||
class UMBPStoreConnector(KVConnectorBase_V1):
|
||||
"""vLLM connector lifecycle shared by every UMBP runtime mode."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
topology = RankTopology.from_vllm_config(vllm_config)
|
||||
runtime_config = UMBPRuntimeConfig.from_vllm(
|
||||
vllm_config
|
||||
).resolve_for_rank_count(topology.rank_count)
|
||||
if topology.pp_size > 1:
|
||||
raise NotImplementedError(
|
||||
f"{runtime_config.mode} UMBP does not support pipeline parallelism"
|
||||
)
|
||||
runtime = UMBPRuntimeFactory.build(runtime_config)
|
||||
namespace = UMBPNamespace.from_vllm_config(vllm_config, kv_cache_config)
|
||||
codec = BlockIdentityCodec(
|
||||
namespace=namespace,
|
||||
tp_rank=topology.tp_rank,
|
||||
pp_rank=topology.pp_rank,
|
||||
pcp_rank=topology.pcp_rank,
|
||||
dcp_rank=topology.dcp_rank,
|
||||
)
|
||||
layout = KVLayoutPlanner.from_kv_cache_config(kv_cache_config)
|
||||
layout_descriptor = layout.describe(topology)
|
||||
self._runtime = runtime
|
||||
self._store_submitted = False
|
||||
self.connector_scheduler: UMBPStoreConnectorScheduler | None = None
|
||||
self.connector_worker: UMBPStoreConnectorWorker | None = None
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = UMBPStoreConnectorScheduler(
|
||||
vllm_config,
|
||||
kv_cache_config,
|
||||
runtime.create_scheduler_handle(
|
||||
namespace.value, topology, layout_descriptor
|
||||
),
|
||||
codec,
|
||||
topology,
|
||||
)
|
||||
else:
|
||||
self.connector_worker = UMBPStoreConnectorWorker(
|
||||
runtime.create_worker_handle(
|
||||
namespace.value, topology, layout_descriptor
|
||||
),
|
||||
layout,
|
||||
codec=codec,
|
||||
)
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: Request, num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: Request,
|
||||
blocks: KVCacheBlocks,
|
||||
num_external_tokens: int,
|
||||
) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_state_after_alloc(
|
||||
request, blocks, num_external_tokens
|
||||
)
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: Any) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def request_finished(
|
||||
self, request: Request, block_ids: list[int]
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, (block_ids,))
|
||||
|
||||
def update_connector_output(self, connector_output: Any) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.has_pending_push_work()
|
||||
|
||||
def reset_cache(self) -> bool | None:
|
||||
if self.connector_scheduler is None:
|
||||
return None
|
||||
return self.connector_scheduler.reset_store()
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def bind_connector_metadata(self, connector_metadata: KVConnectorMetadata) -> None:
|
||||
super().bind_connector_metadata(connector_metadata)
|
||||
self._store_submitted = False
|
||||
|
||||
def clear_connector_metadata(self) -> None:
|
||||
# No-forward steps skip the model runner's wait_for_save hook.
|
||||
if self.connector_worker is not None and self.has_connector_metadata():
|
||||
self.wait_for_save()
|
||||
super().clear_connector_metadata()
|
||||
|
||||
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata) -> None:
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(kv_connector_metadata, UMBPConnectorMetadata)
|
||||
self.connector_worker.handle_preemptions(kv_connector_metadata)
|
||||
|
||||
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
|
||||
del kwargs
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, UMBPConnectorMetadata)
|
||||
self.connector_worker.start_load_kv(forward_context, metadata)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.wait_for_layer_load(layer_name)
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
# Stores are submitted once per step, from wait_for_save.
|
||||
return
|
||||
|
||||
def wait_for_save(self) -> None:
|
||||
if self._store_submitted:
|
||||
return
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, UMBPConnectorMetadata)
|
||||
self.connector_worker.enqueue_stores(metadata)
|
||||
self.connector_worker.wait_for_save()
|
||||
self._store_submitted = True
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_finished(finished_req_ids)
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> KVConnectorTransferResults:
|
||||
assert self.connector_worker is not None
|
||||
finished_sending, finished_recving = self.connector_worker.get_finished(
|
||||
finished_req_ids
|
||||
)
|
||||
return KVConnectorTransferResults(
|
||||
finished_sending=set(finished_sending or ()),
|
||||
finished_recving=set(finished_recving or ()),
|
||||
)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_block_ids_with_load_errors()
|
||||
|
||||
def build_connector_worker_meta(self) -> UMBPConnectorWorkerMetadata:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.build_connector_worker_meta()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.connector_scheduler is not None:
|
||||
self.connector_scheduler.close()
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.close()
|
||||
@@ -1,620 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared data types for the UMBP KV connector.
|
||||
|
||||
The types in this module deliberately do not depend on a UMBP client. They
|
||||
describe vLLM's logical KV objects and the physical ranges a runtime adapter
|
||||
will transfer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field, replace
|
||||
from enum import Enum
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
KVConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RankTopology:
|
||||
"""Model-parallel ranks that participate in one KV object."""
|
||||
|
||||
tp_rank: int = 0
|
||||
tp_size: int = 1
|
||||
pp_rank: int = 0
|
||||
pp_size: int = 1
|
||||
pcp_rank: int = 0
|
||||
pcp_size: int = 1
|
||||
dcp_rank: int = 0
|
||||
dcp_size: int = 1
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for name in ("tp_size", "pp_size", "pcp_size", "dcp_size"):
|
||||
if getattr(self, name) <= 0:
|
||||
raise ValueError(f"{name} must be positive")
|
||||
for rank, size, name in (
|
||||
(self.tp_rank, self.tp_size, "tp_rank"),
|
||||
(self.pp_rank, self.pp_size, "pp_rank"),
|
||||
(self.pcp_rank, self.pcp_size, "pcp_rank"),
|
||||
(self.dcp_rank, self.dcp_size, "dcp_rank"),
|
||||
):
|
||||
if rank < 0 or rank >= size:
|
||||
raise ValueError(f"{name}={rank} is outside [0, {size})")
|
||||
|
||||
@classmethod
|
||||
def from_vllm_config(cls, vllm_config: Any) -> RankTopology:
|
||||
parallel = vllm_config.parallel_config
|
||||
tp_size = getattr(parallel, "tensor_parallel_size", 1)
|
||||
pp_size = getattr(parallel, "pipeline_parallel_size", 1)
|
||||
pcp_size = getattr(parallel, "prefill_context_parallel_size", 1)
|
||||
# An engine's workers are laid out as [PP, PCP, TP], TP innermost
|
||||
# (initialize_model_parallel).
|
||||
rank = getattr(parallel, "rank", 0)
|
||||
tp_rank = rank % tp_size
|
||||
dcp_size = getattr(parallel, "decode_context_parallel_size", 1)
|
||||
return cls(
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
pp_rank=rank // (tp_size * pcp_size) % pp_size,
|
||||
pp_size=pp_size,
|
||||
pcp_rank=rank // tp_size % pcp_size,
|
||||
pcp_size=pcp_size,
|
||||
# DCP partitions the TP workers; it does not create a second
|
||||
# Cartesian worker dimension. ParallelConfig is constructed before
|
||||
# process groups, so derive this stable identity from TP rank.
|
||||
dcp_rank=tp_rank % dcp_size,
|
||||
dcp_size=dcp_size,
|
||||
)
|
||||
|
||||
@property
|
||||
def local_namespace(self) -> tuple[int, int, int, int]:
|
||||
return (self.tp_rank, self.pcp_rank, self.dcp_rank, self.pp_rank)
|
||||
|
||||
@property
|
||||
def rank_count(self) -> int:
|
||||
return len(self.all_namespaces())
|
||||
|
||||
def all_namespaces(self) -> tuple[tuple[int, int, int, int], ...]:
|
||||
if self.dcp_size > 1:
|
||||
if self.tp_size % self.dcp_size:
|
||||
raise ValueError("dcp_size must divide tp_size")
|
||||
return tuple(
|
||||
(tp, pcp, tp % self.dcp_size, pp)
|
||||
for pp in range(self.pp_size)
|
||||
for pcp in range(self.pcp_size)
|
||||
for tp in range(self.tp_size)
|
||||
)
|
||||
return tuple(
|
||||
(tp, pcp, dcp, pp)
|
||||
for pp in range(self.pp_size)
|
||||
for dcp in range(self.dcp_size)
|
||||
for pcp in range(self.pcp_size)
|
||||
for tp in range(self.tp_size)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UMBPNamespace:
|
||||
"""Compatibility namespace for objects stored outside the engine."""
|
||||
|
||||
value: str
|
||||
|
||||
@classmethod
|
||||
def from_vllm_config(
|
||||
cls,
|
||||
vllm_config: Any,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> UMBPNamespace:
|
||||
extra = vllm_config.kv_transfer_config.kv_connector_extra_config
|
||||
configured = extra.get("key_namespace", "auto")
|
||||
if configured != "auto":
|
||||
if not isinstance(configured, str) or not configured:
|
||||
raise ValueError("key_namespace must be a non-empty string or 'auto'")
|
||||
return cls(configured)
|
||||
|
||||
model_config = vllm_config.model_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
# Draft-model KV (e.g. DSpark/EAGLE layers) is stored alongside the
|
||||
# target's, so a different drafter must not reuse it.
|
||||
speculative_config = getattr(vllm_config, "speculative_config", None)
|
||||
draft_model_config = getattr(speculative_config, "draft_model_config", None)
|
||||
speculative = (
|
||||
None
|
||||
if speculative_config is None
|
||||
else {
|
||||
"method": getattr(speculative_config, "method", None),
|
||||
"model": getattr(draft_model_config, "model", None),
|
||||
"revision": getattr(draft_model_config, "revision", None),
|
||||
}
|
||||
)
|
||||
fields = {
|
||||
"model": getattr(model_config, "model", ""),
|
||||
"revision": getattr(model_config, "revision", None),
|
||||
"block_size": vllm_config.cache_config.block_size,
|
||||
"cache_layout": getattr(kv_cache_config, "kv_cache_layout", None),
|
||||
"groups": [
|
||||
{
|
||||
"layers": group.layer_names,
|
||||
"block_size": group.kv_cache_spec.block_size,
|
||||
"kind": type(group.kv_cache_spec).__name__,
|
||||
"dtype": str(getattr(group.kv_cache_spec, "dtype", None)),
|
||||
}
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
],
|
||||
"tp": parallel_config.tensor_parallel_size,
|
||||
"pp": parallel_config.pipeline_parallel_size,
|
||||
"pcp": getattr(parallel_config, "prefill_context_parallel_size", 1),
|
||||
"dcp": getattr(parallel_config, "decode_context_parallel_size", 1),
|
||||
"layout_version": 1,
|
||||
}
|
||||
if speculative is not None:
|
||||
# Added only when set, so namespaces without a drafter are unchanged.
|
||||
fields["speculative"] = speculative
|
||||
payload = json.dumps(fields, sort_keys=True, separators=(",", ":")).encode()
|
||||
return cls(hashlib.sha256(payload).hexdigest()[:32])
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BlockIdentityCodec:
|
||||
"""Build stable rank-local keys from vLLM block hashes."""
|
||||
|
||||
namespace: UMBPNamespace
|
||||
tp_rank: int = 0
|
||||
pp_rank: int = 0
|
||||
pcp_rank: int = 0
|
||||
dcp_rank: int = 0
|
||||
|
||||
def key(self, block_hash: bytes, group_id: int = 0) -> str:
|
||||
return self.key_for_namespace(
|
||||
block_hash,
|
||||
group_id,
|
||||
(self.tp_rank, self.pcp_rank, self.dcp_rank, self.pp_rank),
|
||||
)
|
||||
|
||||
def key_for_namespace(
|
||||
self,
|
||||
block_hash: bytes,
|
||||
group_id: int,
|
||||
namespace: tuple[int, int, int, int],
|
||||
) -> str:
|
||||
if not isinstance(block_hash, bytes) or not block_hash:
|
||||
raise ValueError("block_hash must be non-empty bytes")
|
||||
if group_id < 0:
|
||||
raise ValueError("group_id must be non-negative")
|
||||
tp_rank, pcp_rank, dcp_rank, pp_rank = namespace
|
||||
return (
|
||||
f"umbp:vllm:v1:{self.namespace.value}:"
|
||||
f"tp{tp_rank}:pcp{pcp_rank}:dcp{dcp_rank}:"
|
||||
f"pp{pp_rank}:g{group_id}:"
|
||||
f"{block_hash.hex()}"
|
||||
)
|
||||
|
||||
def keys_for_topology(
|
||||
self,
|
||||
block_hash: bytes,
|
||||
topology: RankTopology,
|
||||
group_ids: Sequence[int],
|
||||
) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
self.key_for_namespace(block_hash, group_id, namespace)
|
||||
for namespace in topology.all_namespaces()
|
||||
for group_id in group_ids
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KVRegion:
|
||||
"""One deterministic region in a rank-local KV object."""
|
||||
|
||||
layer_name: str
|
||||
group_id: int
|
||||
block_stride: int
|
||||
block_bytes: int
|
||||
object_offset: int
|
||||
block_size: int = 1
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.block_stride <= 0 or self.block_bytes <= 0:
|
||||
raise ValueError("KV region sizes must be positive")
|
||||
if self.block_size <= 0:
|
||||
raise ValueError("KV region block_size must be positive")
|
||||
if self.block_bytes > self.block_stride:
|
||||
raise ValueError(
|
||||
f"KV region bytes ({self.block_bytes}) exceed its stride "
|
||||
f"({self.block_stride}) for {self.layer_name!r}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KVLayoutDescriptor:
|
||||
"""Backend-neutral description of one rank-local KV object layout."""
|
||||
|
||||
regions: tuple[KVRegion, ...]
|
||||
topology: RankTopology
|
||||
layout_format: str = "rank_local"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.layout_format:
|
||||
raise ValueError("layout_format must not be empty")
|
||||
names = [region.layer_name for region in self.regions]
|
||||
if len(names) != len(set(names)):
|
||||
raise ValueError("KV layout contains duplicate layer regions")
|
||||
|
||||
@property
|
||||
def object_size(self) -> int:
|
||||
return sum(region.block_bytes for region in self.regions)
|
||||
|
||||
|
||||
class KVLayoutPlanner:
|
||||
"""Create a deterministic all-layer object layout from KVCacheConfig."""
|
||||
|
||||
def __init__(self, regions: Sequence[KVRegion]) -> None:
|
||||
self.regions = tuple(regions)
|
||||
self._regions_by_group = {
|
||||
group_id: tuple(r for r in self.regions if r.group_id == group_id)
|
||||
for group_id in {r.group_id for r in self.regions}
|
||||
}
|
||||
self._base_addresses: dict[str, int] = {}
|
||||
self._physical_strides: dict[str, int] = {}
|
||||
|
||||
@classmethod
|
||||
def from_kv_cache_config(cls, config: KVCacheConfig) -> KVLayoutPlanner:
|
||||
group_by_layer = {
|
||||
layer: group_id
|
||||
for group_id, group in enumerate(config.kv_cache_groups)
|
||||
for layer in group.layer_names
|
||||
if group.enable_kv_transfer
|
||||
}
|
||||
regions: list[KVRegion] = []
|
||||
offset = 0
|
||||
for tensor in config.kv_cache_tensors:
|
||||
for layer_name in tensor.layers:
|
||||
group_id = group_by_layer.get(layer_name)
|
||||
if group_id is None:
|
||||
continue
|
||||
spec = config.kv_cache_groups[group_id].kv_cache_spec
|
||||
block_bytes = spec.page_size_bytes
|
||||
regions.append(
|
||||
KVRegion(
|
||||
layer_name=layer_name,
|
||||
group_id=group_id,
|
||||
block_stride=tensor.block_stride,
|
||||
block_bytes=block_bytes,
|
||||
object_offset=offset,
|
||||
block_size=spec.block_size,
|
||||
)
|
||||
)
|
||||
offset += block_bytes
|
||||
return cls(regions)
|
||||
|
||||
@property
|
||||
def object_size(self) -> int:
|
||||
return sum(region.block_bytes for region in self.regions)
|
||||
|
||||
def describe(
|
||||
self,
|
||||
topology: RankTopology,
|
||||
layout_format: str = "rank_local",
|
||||
) -> KVLayoutDescriptor:
|
||||
return KVLayoutDescriptor(
|
||||
regions=self.regions,
|
||||
topology=topology,
|
||||
layout_format=layout_format,
|
||||
)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
"""Capture stable base addresses for the planned KV regions."""
|
||||
required = {region.layer_name for region in self.regions}
|
||||
missing = required - kv_caches.keys()
|
||||
if missing:
|
||||
raise ValueError(f"missing KV cache layers: {sorted(missing)}")
|
||||
addresses: dict[str, int] = {}
|
||||
for layer_name in required:
|
||||
cache = kv_caches[layer_name]
|
||||
if cache.ndim == 0:
|
||||
raise ValueError(f"KV cache {layer_name!r} must be a tensor")
|
||||
block_stride = cache.stride(0) * cache.element_size()
|
||||
addresses[layer_name] = cache.data_ptr()
|
||||
for region in self.regions:
|
||||
if region.layer_name != layer_name:
|
||||
continue
|
||||
if (
|
||||
block_stride > region.block_stride
|
||||
or region.block_stride % block_stride != 0
|
||||
or (
|
||||
region.block_bytes > block_stride
|
||||
and region.block_bytes % block_stride != 0
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
f"KV block stride mismatch for {layer_name!r}: "
|
||||
f"config={region.block_stride}, tensor={block_stride}"
|
||||
)
|
||||
self._physical_strides[layer_name] = block_stride
|
||||
self._base_addresses = addresses
|
||||
|
||||
def plan_registered_block(
|
||||
self,
|
||||
key: str,
|
||||
block_id: int,
|
||||
*,
|
||||
group_id: int | None = None,
|
||||
request_id: str | None = None,
|
||||
) -> BlockTransferPlan:
|
||||
if not self._base_addresses:
|
||||
raise RuntimeError("KV caches must be registered before planning")
|
||||
return self.plan_for_block(
|
||||
key,
|
||||
block_id,
|
||||
self._base_addresses,
|
||||
group_id=group_id,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
def plan_for_block(
|
||||
self,
|
||||
key: str,
|
||||
block_id: int,
|
||||
base_addresses: dict[str, int],
|
||||
*,
|
||||
group_id: int | None = None,
|
||||
request_id: str | None = None,
|
||||
) -> BlockTransferPlan:
|
||||
"""Lower one GPU block into a deterministic scatter/gather plan."""
|
||||
return BlockTransferPlan(
|
||||
key=key,
|
||||
block_id=block_id,
|
||||
ranges=self._ranges_for_block(block_id, base_addresses, group_id),
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
def materialize(self, plan: BlockTransferPlan) -> BlockTransferPlan:
|
||||
if not self._base_addresses:
|
||||
raise RuntimeError("KV caches must be registered before planning")
|
||||
return replace(
|
||||
plan,
|
||||
ranges=self._ranges_for_block(
|
||||
plan.block_id, self._base_addresses, plan.group_id
|
||||
),
|
||||
)
|
||||
|
||||
def _ranges_for_block(
|
||||
self,
|
||||
block_id: int,
|
||||
base_addresses: dict[str, int],
|
||||
group_id: int | None,
|
||||
) -> tuple[KVRange, ...]:
|
||||
if block_id < 0:
|
||||
raise ValueError("block_id must be non-negative")
|
||||
regions = (
|
||||
self.regions
|
||||
if group_id is None
|
||||
else self._regions_by_group.get(group_id, ())
|
||||
)
|
||||
if group_id is not None and not regions:
|
||||
raise ValueError(f"KV layout does not contain cache group {group_id}")
|
||||
ranges: list[KVRange] = []
|
||||
object_offset = 0
|
||||
for region in regions:
|
||||
try:
|
||||
base_address = base_addresses[region.layer_name]
|
||||
except KeyError as exc:
|
||||
raise ValueError(
|
||||
f"missing base address for layer {region.layer_name!r}"
|
||||
) from exc
|
||||
physical_stride = self._physical_strides.get(
|
||||
region.layer_name, region.block_stride
|
||||
)
|
||||
if region.block_bytes <= physical_stride:
|
||||
ranges.append(
|
||||
KVRange(
|
||||
layer_name=region.layer_name,
|
||||
group_id=region.group_id,
|
||||
block_id=block_id,
|
||||
base_address=base_address + block_id * physical_stride,
|
||||
stride=physical_stride,
|
||||
length=region.block_bytes,
|
||||
object_offset=(
|
||||
object_offset
|
||||
if group_id is not None
|
||||
else region.object_offset
|
||||
),
|
||||
)
|
||||
)
|
||||
object_offset += region.block_bytes
|
||||
continue
|
||||
physical_parts = max(1, region.block_bytes // physical_stride)
|
||||
for part in range(physical_parts):
|
||||
part_start = part * physical_stride
|
||||
part_end = min(part_start + physical_stride, region.block_bytes)
|
||||
physical_block_id = block_id * physical_parts + part
|
||||
ranges.append(
|
||||
KVRange(
|
||||
layer_name=region.layer_name,
|
||||
group_id=region.group_id,
|
||||
block_id=block_id,
|
||||
base_address=base_address + physical_block_id * physical_stride,
|
||||
stride=physical_stride,
|
||||
length=part_end - part_start,
|
||||
object_offset=(
|
||||
object_offset + part_start
|
||||
if group_id is not None
|
||||
else region.object_offset + part * physical_stride
|
||||
),
|
||||
)
|
||||
)
|
||||
object_offset += region.block_bytes
|
||||
return tuple(ranges)
|
||||
|
||||
|
||||
class KVRange(NamedTuple):
|
||||
"""A source/destination range in a runtime transfer."""
|
||||
|
||||
layer_name: str
|
||||
group_id: int
|
||||
block_id: int
|
||||
base_address: int
|
||||
stride: int
|
||||
length: int
|
||||
object_offset: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BlockTransferPlan:
|
||||
"""One complete logical block transfer."""
|
||||
|
||||
key: str
|
||||
block_id: int
|
||||
ranges: tuple[KVRange, ...] = ()
|
||||
request_id: str | None = None
|
||||
group_id: int | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoadSpec:
|
||||
"""Scheduler decision for one externally loaded prefix."""
|
||||
|
||||
local_tokens: int
|
||||
external_tokens: int
|
||||
|
||||
@property
|
||||
def num_tokens_to_load(self) -> int:
|
||||
return max(self.external_tokens - self.local_tokens, 0)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequestTracker:
|
||||
"""Core request state shared by every runtime mode."""
|
||||
|
||||
saved_tokens: int = 0
|
||||
block_ids: tuple[list[int], ...] = ()
|
||||
load_spec: LoadSpec | None = None
|
||||
|
||||
def reset(self) -> None:
|
||||
self.saved_tokens = 0
|
||||
self.block_ids = ()
|
||||
self.load_spec = None
|
||||
|
||||
def update_blocks(self, block_ids: tuple[list[int], ...]) -> None:
|
||||
if not self.block_ids:
|
||||
self.block_ids = tuple(list(group) for group in block_ids)
|
||||
return
|
||||
if len(self.block_ids) != len(block_ids):
|
||||
raise ValueError("KV cache group count changed for a request")
|
||||
for current, new in zip(self.block_ids, block_ids, strict=True):
|
||||
if new[: len(current)] == current:
|
||||
current.extend(new[len(current) :])
|
||||
elif not new or current[-len(new) :] == new:
|
||||
continue
|
||||
else:
|
||||
current.extend(new)
|
||||
|
||||
def replace_blocks(self, block_ids: tuple[list[int], ...]) -> None:
|
||||
"""Replace stale pre-preemption tables with resumed full tables."""
|
||||
self.block_ids = tuple(list(group) for group in block_ids)
|
||||
self.saved_tokens = 0
|
||||
|
||||
def mark_saved(self, token_count: int, block_size: int) -> int:
|
||||
"""Advance the save watermark only to a complete block boundary."""
|
||||
complete_tokens = token_count // block_size * block_size
|
||||
if complete_tokens < self.saved_tokens:
|
||||
return self.saved_tokens
|
||||
self.saved_tokens = complete_tokens
|
||||
return complete_tokens
|
||||
|
||||
|
||||
class TransferJobStatus(str, Enum):
|
||||
PENDING = "pending"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransferJobState:
|
||||
"""Per-key completion state; one failed key does not fail its siblings."""
|
||||
|
||||
plans: tuple[BlockTransferPlan, ...]
|
||||
status: TransferJobStatus = TransferJobStatus.PENDING
|
||||
completed_keys: set[str] = field(default_factory=set)
|
||||
failed_keys: set[str] = field(default_factory=set)
|
||||
error: str | None = None
|
||||
|
||||
@property
|
||||
def keys(self) -> Sequence[str]:
|
||||
return tuple(plan.key for plan in self.plans)
|
||||
|
||||
def start(self) -> None:
|
||||
if self.status != TransferJobStatus.PENDING:
|
||||
raise RuntimeError(f"cannot start job in state {self.status}")
|
||||
self.status = TransferJobStatus.RUNNING
|
||||
|
||||
def complete(self, keys: Sequence[str] = ()) -> None:
|
||||
self.completed_keys.update(keys or self.keys)
|
||||
self._finish_if_done()
|
||||
|
||||
def fail(self, keys: Sequence[str], error: str | None = None) -> None:
|
||||
self.failed_keys.update(keys)
|
||||
self.error = error
|
||||
self._finish_if_done()
|
||||
|
||||
@property
|
||||
def failed_block_ids(self) -> set[int]:
|
||||
return {plan.block_id for plan in self.plans if plan.key in self.failed_keys}
|
||||
|
||||
def _finish_if_done(self) -> None:
|
||||
keys = set(self.keys)
|
||||
if self.completed_keys | self.failed_keys >= keys:
|
||||
self.status = (
|
||||
TransferJobStatus.FAILED
|
||||
if self.failed_keys
|
||||
else TransferJobStatus.COMPLETED
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class UMBPConnectorMetadata(KVConnectorMetadata):
|
||||
"""Scheduler-to-worker metadata for one engine step."""
|
||||
|
||||
async_load: bool = False
|
||||
store_plans: list[BlockTransferPlan] = field(default_factory=list)
|
||||
load_requests: dict[str, list[BlockTransferPlan]] = field(default_factory=dict)
|
||||
store_requests: dict[str, list[BlockTransferPlan]] = field(default_factory=dict)
|
||||
preempted_request_ids: set[str] = field(default_factory=set)
|
||||
store_event: int = -1
|
||||
|
||||
|
||||
@dataclass
|
||||
class StoreEventResult:
|
||||
"""Number of workers that finished one store event."""
|
||||
|
||||
completed_workers: int = 0
|
||||
|
||||
def merge(self, other: StoreEventResult) -> None:
|
||||
self.completed_workers += other.completed_workers
|
||||
|
||||
|
||||
@dataclass
|
||||
class UMBPConnectorWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
"""Worker completion metadata aggregated across ranks."""
|
||||
|
||||
store_events: dict[int, StoreEventResult] = field(default_factory=dict)
|
||||
|
||||
def aggregate(
|
||||
self, other: KVConnectorWorkerMetadata
|
||||
) -> UMBPConnectorWorkerMetadata:
|
||||
if not isinstance(other, UMBPConnectorWorkerMetadata):
|
||||
raise TypeError("cannot aggregate incompatible UMBP worker metadata")
|
||||
for event, result in other.store_events.items():
|
||||
self.store_events.setdefault(event, StoreEventResult()).merge(result)
|
||||
return self
|
||||
@@ -1,13 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .base import IUMBPRuntime, UMBPSchedulerHandle, UMBPWorkerHandle
|
||||
from .factory import UMBPRuntimeConfig, UMBPRuntimeFactory
|
||||
|
||||
__all__ = [
|
||||
"IUMBPRuntime",
|
||||
"UMBPSchedulerHandle",
|
||||
"UMBPWorkerHandle",
|
||||
"UMBPRuntimeConfig",
|
||||
"UMBPRuntimeFactory",
|
||||
]
|
||||
@@ -1,69 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Backend-neutral runtime contract for UMBP."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Protocol
|
||||
|
||||
import torch
|
||||
|
||||
from ..data import (
|
||||
BlockTransferPlan,
|
||||
KVLayoutDescriptor,
|
||||
RankTopology,
|
||||
TransferJobState,
|
||||
)
|
||||
|
||||
|
||||
class UMBPSchedulerHandle(Protocol):
|
||||
def lookup(self, keys: Sequence[str]) -> Sequence[bool]:
|
||||
"""Return authoritative existence for each key."""
|
||||
|
||||
def clear(self) -> bool:
|
||||
"""Clear all objects visible to this scheduler handle."""
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release scheduler-side resources."""
|
||||
|
||||
|
||||
class UMBPWorkerHandle(Protocol):
|
||||
def register_buffers(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
"""Register each unique KV allocation with the runtime."""
|
||||
|
||||
def load(self, plans: Sequence[BlockTransferPlan]) -> TransferJobState:
|
||||
"""Start loading the requested plans."""
|
||||
|
||||
def store(self, plans: Sequence[BlockTransferPlan]) -> TransferJobState:
|
||||
"""Start storing the requested plans."""
|
||||
|
||||
def wait(self, job: TransferJobState) -> TransferJobState:
|
||||
"""Block until the job is final and its buffers are safe to reuse."""
|
||||
|
||||
def poll(self, job: TransferJobState) -> TransferJobState | None:
|
||||
"""Return a finished job without blocking, or None if still pending."""
|
||||
|
||||
def publish(self, job: TransferJobState) -> None:
|
||||
"""Accept a completed store; a runtime may defer visibility until then."""
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release worker-side resources."""
|
||||
|
||||
|
||||
class IUMBPRuntime(Protocol):
|
||||
def create_scheduler_handle(
|
||||
self,
|
||||
namespace: str,
|
||||
topology: RankTopology,
|
||||
layout: KVLayoutDescriptor,
|
||||
) -> UMBPSchedulerHandle:
|
||||
"""Create a scheduler-side handle."""
|
||||
|
||||
def create_worker_handle(
|
||||
self,
|
||||
namespace: str,
|
||||
topology: RankTopology,
|
||||
layout: KVLayoutDescriptor,
|
||||
) -> UMBPWorkerHandle:
|
||||
"""Create a worker-side handle."""
|
||||
@@ -1,76 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Runtime configuration and adapter factory for UMBP."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
from .base import IUMBPRuntime
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UMBPRuntimeConfig:
|
||||
"""Shared connector options passed to the selected runtime adapter."""
|
||||
|
||||
mode: str
|
||||
options: dict[str, Any]
|
||||
rank_count: int = 1
|
||||
dp_index: int = 0
|
||||
|
||||
@classmethod
|
||||
def from_vllm(cls, vllm_config: VllmConfig) -> UMBPRuntimeConfig:
|
||||
transfer_config = vllm_config.kv_transfer_config
|
||||
if transfer_config is None:
|
||||
raise ValueError("UMBP requires kv_transfer_config")
|
||||
options = dict(transfer_config.kv_connector_extra_config)
|
||||
mode = options.get("mode", "embedded")
|
||||
if mode not in {"embedded"}:
|
||||
raise ValueError(f"unknown UMBP mode: {mode!r}")
|
||||
namespace = options.get("key_namespace", "auto")
|
||||
if namespace != "auto" and (not isinstance(namespace, str) or not namespace):
|
||||
raise ValueError("key_namespace must be a non-empty string or 'auto'")
|
||||
return cls(
|
||||
mode=mode,
|
||||
options=options,
|
||||
dp_index=int(
|
||||
getattr(vllm_config.parallel_config, "data_parallel_index", 0) or 0
|
||||
),
|
||||
)
|
||||
|
||||
def resolve_for_rank_count(self, rank_count: int) -> UMBPRuntimeConfig:
|
||||
"""Pass topology to the adapter without interpreting its capacity options."""
|
||||
if rank_count <= 0:
|
||||
raise ValueError("rank_count must be positive")
|
||||
return replace(self, rank_count=rank_count)
|
||||
|
||||
|
||||
RuntimeBuilder = Callable[[UMBPRuntimeConfig], IUMBPRuntime]
|
||||
|
||||
|
||||
class UMBPRuntimeFactory:
|
||||
"""Registry used by UMBP adapters without importing MORI in vLLM."""
|
||||
|
||||
_builders: dict[str, RuntimeBuilder] = {}
|
||||
|
||||
@classmethod
|
||||
def register(cls, mode: str, builder: RuntimeBuilder) -> None:
|
||||
if mode not in {"embedded"}:
|
||||
raise ValueError(f"unknown UMBP mode: {mode!r}")
|
||||
if mode in cls._builders:
|
||||
raise ValueError(f"UMBP runtime mode {mode!r} is already registered")
|
||||
cls._builders[mode] = builder
|
||||
|
||||
@classmethod
|
||||
def build(cls, config: UMBPRuntimeConfig) -> IUMBPRuntime:
|
||||
try:
|
||||
builder = cls._builders[config.mode]
|
||||
except KeyError as exc:
|
||||
raise RuntimeError(
|
||||
f"no UMBP runtime adapter is registered for mode {config.mode!r}"
|
||||
) from exc
|
||||
return builder(config)
|
||||
@@ -1,518 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared scheduler-side UMBP connector behavior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
resolve_dcp_kv_block_size,
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
from .data import (
|
||||
BlockIdentityCodec,
|
||||
BlockTransferPlan,
|
||||
LoadSpec,
|
||||
RankTopology,
|
||||
RequestTracker,
|
||||
StoreEventResult,
|
||||
UMBPConnectorMetadata,
|
||||
UMBPConnectorWorkerMetadata,
|
||||
)
|
||||
from .runtime import UMBPSchedulerHandle
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@lru_cache(maxsize=8192)
|
||||
def _rank_keys(
|
||||
codec: BlockIdentityCodec,
|
||||
topology: RankTopology,
|
||||
block_hash: bytes,
|
||||
group_id: int,
|
||||
) -> tuple[str, ...]:
|
||||
"""Reuse immutable object identities, not runtime residency."""
|
||||
return codec.keys_for_topology(block_hash, topology, (group_id,))
|
||||
|
||||
|
||||
@dataclass
|
||||
class _StoreEvent:
|
||||
plans: tuple[BlockTransferPlan, ...]
|
||||
result: StoreEventResult = field(default_factory=StoreEventResult)
|
||||
|
||||
|
||||
class UMBPStoreConnectorScheduler:
|
||||
"""Own vLLM prefix matching while the runtime owns lookup transport."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
runtime: UMBPSchedulerHandle,
|
||||
codec: BlockIdentityCodec,
|
||||
topology: RankTopology | None = None,
|
||||
) -> None:
|
||||
self.block_size, resolved_hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
dcp_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
self.group_block_sizes = {
|
||||
group_id: resolve_dcp_kv_block_size(group.kv_cache_spec, dcp_size)
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
}
|
||||
self.runtime = runtime
|
||||
self.codec = codec
|
||||
self.topology = topology or RankTopology()
|
||||
transfer_config = vllm_config.kv_transfer_config
|
||||
assert transfer_config is not None
|
||||
extra = transfer_config.kv_connector_extra_config
|
||||
self.load_async = bool(extra.get("load_async", True))
|
||||
self.enable_lookup = bool(extra.get("enable_lookup", True))
|
||||
self.save_decode_cache = bool(extra.get("save_decode_cache", False))
|
||||
# A kv_consumer only restores from the pool, as with other store
|
||||
# connectors; it never offloads its own KV.
|
||||
self.store_enabled = getattr(transfer_config, "kv_role", None) != "kv_consumer"
|
||||
# Request.block_hashes are computed at this size; keys index into them.
|
||||
self.hash_block_size = resolved_hash_block_size
|
||||
speculative_config = getattr(vllm_config, "speculative_config", None)
|
||||
self.use_eagle = bool(
|
||||
speculative_config is not None and speculative_config.use_eagle_block_drop()
|
||||
)
|
||||
self._pending_loads: dict[str, list[BlockTransferPlan]] = {}
|
||||
self._load_specs: dict[str, LoadSpec] = {}
|
||||
self._lookup_units = {
|
||||
group_id: self.group_block_sizes[group_id]
|
||||
for group_id in kv_cache_config.prefix_cacheable_group_ids
|
||||
}
|
||||
self._requests: dict[str, Request] = {}
|
||||
self._request_trackers: dict[str, RequestTracker] = {}
|
||||
self._gpu_block_pool: BlockPool | None = None
|
||||
self._store_events: dict[int, _StoreEvent] = {}
|
||||
self._num_workers = getattr(vllm_config.parallel_config, "world_size", 1)
|
||||
self._store_event_counter = 0
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
|
||||
self._gpu_block_pool = gpu_block_pool
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: Request, num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
if not self.enable_lookup:
|
||||
return 0, False
|
||||
hashes = request.block_hashes
|
||||
if not hashes or request.num_tokens < self.block_size:
|
||||
return 0, False
|
||||
if num_computed_tokens % self.block_size != 0:
|
||||
return 0, False
|
||||
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
|
||||
# vLLM must execute at least the final prompt token. Returning a hit
|
||||
# for the entire prompt leaves the scheduler with no new token to run.
|
||||
max_external_token = request.num_tokens - 1
|
||||
if max_external_token <= num_computed_tokens:
|
||||
self._load_specs.pop(request.request_id, None)
|
||||
return 0, False
|
||||
logical_objects_by_group, keys = self._lookup_objects(
|
||||
request, num_computed_tokens, hashes
|
||||
)
|
||||
try:
|
||||
hits = list(self.runtime.lookup(keys))
|
||||
except Exception as exc:
|
||||
logger.debug("UMBP lookup failed request=%s: %s", request.request_id, exc)
|
||||
return 0, False
|
||||
if len(hits) != len(keys):
|
||||
logger.debug(
|
||||
"UMBP lookup returned an invalid result length request=%s",
|
||||
request.request_id,
|
||||
)
|
||||
return 0, False
|
||||
per_rank = self.topology.rank_count
|
||||
offset = 0
|
||||
logical_hits_by_group: dict[int, list[bool]] = {}
|
||||
for group_id in group_ids:
|
||||
group_size = len(logical_objects_by_group[group_id]) * per_rank
|
||||
group_hits = hits[offset : offset + group_size]
|
||||
offset += group_size
|
||||
logical_hits_by_group[group_id] = (
|
||||
group_hits
|
||||
if per_rank == 1
|
||||
else [
|
||||
all(group_hits[index : index + per_rank])
|
||||
for index in range(0, len(group_hits), per_rank)
|
||||
]
|
||||
)
|
||||
group_id = group_ids[0]
|
||||
matched_blocks = 0
|
||||
for hit in logical_hits_by_group[group_id]:
|
||||
if not hit:
|
||||
break
|
||||
matched_blocks += 1
|
||||
if self.use_eagle:
|
||||
# As core's coordinator: the drafter recomputes the last block.
|
||||
matched_blocks = max(0, matched_blocks - 1)
|
||||
need_to_load = matched_blocks * self._lookup_units[group_id]
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(
|
||||
"UMBP lookup result request=%s keys=%s group_hits=%s load_tokens=%d",
|
||||
request.request_id,
|
||||
keys,
|
||||
logical_hits_by_group,
|
||||
need_to_load,
|
||||
)
|
||||
matched_tokens = num_computed_tokens + need_to_load
|
||||
if need_to_load <= 0:
|
||||
return 0, False
|
||||
self._load_specs[request.request_id] = LoadSpec(
|
||||
local_tokens=num_computed_tokens,
|
||||
external_tokens=matched_tokens,
|
||||
)
|
||||
return need_to_load, self.load_async
|
||||
|
||||
def _lookup_objects(
|
||||
self, request: Request, local_tokens: int, hashes: Sequence[bytes]
|
||||
) -> tuple[dict[int, list[tuple[bytes, tuple[str, ...]]]], list[str]]:
|
||||
objects: dict[int, list[tuple[bytes, tuple[str, ...]]]] = {}
|
||||
keys: list[str] = []
|
||||
for group_id, unit in self._lookup_units.items():
|
||||
group_objects = []
|
||||
for token_end in range(local_tokens + unit, request.num_tokens, unit):
|
||||
block_hash = self._object_hash_at_token_end(hashes, token_end)
|
||||
rank_keys = _rank_keys(self.codec, self.topology, block_hash, group_id)
|
||||
group_objects.append((block_hash, rank_keys))
|
||||
keys.extend(rank_keys)
|
||||
objects[group_id] = group_objects
|
||||
return objects, keys
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: Request,
|
||||
blocks: KVCacheBlocks,
|
||||
num_external_tokens: int,
|
||||
) -> None:
|
||||
self._requests[request.request_id] = request
|
||||
block_groups = blocks.get_block_ids(
|
||||
group_ids=self.kv_cache_config.prefix_cacheable_group_ids
|
||||
)
|
||||
if not block_groups:
|
||||
return
|
||||
tracker = self._tracker_for_request(request.request_id)
|
||||
# Core passes the request's whole table on every admission. A request
|
||||
# whose load failed is recomputed into newly allocated blocks.
|
||||
tracker.replace_blocks(tuple(list(group) for group in block_groups))
|
||||
if num_external_tokens <= 0:
|
||||
self._pending_loads.pop(request.request_id, None)
|
||||
self._load_specs.pop(request.request_id, None)
|
||||
return
|
||||
spec = self._load_specs.get(request.request_id)
|
||||
if spec is not None:
|
||||
tracker.load_spec = spec
|
||||
# A request admitted without its load would wait for it forever.
|
||||
assert num_external_tokens == spec.num_tokens_to_load, (
|
||||
f"UMBP matched {spec.num_tokens_to_load} external tokens for "
|
||||
f"request {request.request_id} but {num_external_tokens} were "
|
||||
"allocated"
|
||||
)
|
||||
plans = self._load_plans_for_external_tokens(
|
||||
request,
|
||||
tracker,
|
||||
block_groups,
|
||||
num_external_tokens,
|
||||
)
|
||||
self._pending_loads[request.request_id] = plans
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
meta = UMBPConnectorMetadata(async_load=self.load_async)
|
||||
for request_id in scheduler_output.finished_req_ids:
|
||||
self._pending_loads.pop(request_id, None)
|
||||
self._load_specs.pop(request_id, None)
|
||||
self._request_trackers.pop(request_id, None)
|
||||
self._requests.pop(request_id, None)
|
||||
for request in scheduler_output.scheduled_new_reqs:
|
||||
load_plans = self._pending_loads.pop(request.req_id, [])
|
||||
self._load_specs.pop(request.req_id, None)
|
||||
if load_plans:
|
||||
meta.load_requests[request.req_id] = load_plans
|
||||
else:
|
||||
tracker = self._tracker_for_request(request.req_id)
|
||||
total_tokens = (
|
||||
request.num_computed_tokens
|
||||
+ scheduler_output.num_scheduled_tokens[request.req_id]
|
||||
)
|
||||
tracked_request = self._requests.get(request.req_id)
|
||||
selected_block_ids = tuple(
|
||||
request.block_ids[group_id]
|
||||
for group_id in (self.kv_cache_config.prefix_cacheable_group_ids)
|
||||
)
|
||||
store_plans = (
|
||||
[]
|
||||
if not self.store_enabled or tracked_request is None
|
||||
else self._store_plans(
|
||||
tracked_request,
|
||||
tracker,
|
||||
total_tokens,
|
||||
block_ids_override=selected_block_ids,
|
||||
)
|
||||
)
|
||||
logger.debug(
|
||||
"UMBP store planning request=%s tokens=%d plans=%d keys=%s",
|
||||
request.req_id,
|
||||
total_tokens,
|
||||
len(store_plans),
|
||||
tuple(plan.key for plan in store_plans),
|
||||
)
|
||||
meta.store_plans.extend(store_plans)
|
||||
if store_plans:
|
||||
meta.store_requests[request.req_id] = store_plans
|
||||
|
||||
block_state = getattr(scheduler_output, "kv_connector_block_state", None)
|
||||
cached_reqs = scheduler_output.scheduled_cached_reqs
|
||||
resumed_ids: set[str] = set(getattr(cached_reqs, "resumed_req_ids", ()))
|
||||
for request_id in cached_reqs.req_ids:
|
||||
load_plans = self._pending_loads.pop(request_id, [])
|
||||
self._load_specs.pop(request_id, None)
|
||||
if load_plans:
|
||||
meta.load_requests[request_id] = load_plans
|
||||
elif self.store_enabled:
|
||||
cached_request = self._requests.get(request_id)
|
||||
cached_tracker = self._request_trackers.get(request_id)
|
||||
if cached_request is not None and cached_tracker is not None:
|
||||
request_index = cached_reqs.req_ids.index(request_id)
|
||||
num_computed_tokens = cached_reqs.num_computed_tokens[request_index]
|
||||
# num_tokens counts sampled tokens, so a decode step is still
|
||||
# short of it; only the prompt bounds the prefill.
|
||||
is_prefill = num_computed_tokens < cached_request.num_prompt_tokens
|
||||
if not is_prefill and not self.save_decode_cache:
|
||||
continue
|
||||
new_block_ids = cached_reqs.new_block_ids[request_index]
|
||||
if new_block_ids:
|
||||
selected = tuple(
|
||||
new_block_ids[group_id]
|
||||
for group_id in (
|
||||
self.kv_cache_config.prefix_cacheable_group_ids
|
||||
)
|
||||
)
|
||||
if request_id in resumed_ids:
|
||||
cached_tracker.replace_blocks(selected)
|
||||
else:
|
||||
cached_tracker.update_blocks(selected)
|
||||
# The appended table keeps blocks core has since freed, such
|
||||
# as sliding-window blocks out of the window, which a store
|
||||
# would read; core's current table does not.
|
||||
current = (
|
||||
block_state.get_block_ids(request_id)
|
||||
if block_state is not None
|
||||
else None
|
||||
)
|
||||
if current is not None:
|
||||
cached_tracker.block_ids = tuple(
|
||||
list(current[group_id])
|
||||
for group_id in (
|
||||
self.kv_cache_config.prefix_cacheable_group_ids
|
||||
)
|
||||
)
|
||||
total_tokens = (
|
||||
num_computed_tokens
|
||||
+ scheduler_output.num_scheduled_tokens[request_id]
|
||||
)
|
||||
if not self.save_decode_cache:
|
||||
total_tokens = min(
|
||||
total_tokens, cached_request.num_prompt_tokens
|
||||
)
|
||||
store_plans = self._store_plans(
|
||||
cached_request,
|
||||
cached_tracker,
|
||||
total_tokens,
|
||||
block_ids_override=cached_tracker.block_ids,
|
||||
)
|
||||
meta.store_plans.extend(store_plans)
|
||||
if store_plans:
|
||||
meta.store_requests[request_id] = store_plans
|
||||
|
||||
# Async loads are admitted without scheduling model tokens, so those
|
||||
# requests are absent from both scheduled request collections above.
|
||||
# Forward their plans explicitly so workers can start the transfer and
|
||||
# eventually move the request out of WAITING_FOR_REMOTE_KVS.
|
||||
for request_id, load_plans in list(self._pending_loads.items()):
|
||||
self._pending_loads.pop(request_id, None)
|
||||
self._load_specs.pop(request_id, None)
|
||||
if load_plans:
|
||||
meta.load_requests[request_id] = load_plans
|
||||
|
||||
for request_id in scheduler_output.preempted_req_ids or set():
|
||||
meta.preempted_request_ids.add(request_id)
|
||||
self._load_specs.pop(request_id, None)
|
||||
if preempted_tracker := self._request_trackers.get(request_id):
|
||||
preempted_tracker.reset()
|
||||
|
||||
event_plans = tuple(meta.store_plans)
|
||||
if event_plans:
|
||||
event = self._store_event_counter
|
||||
self._store_event_counter += 1
|
||||
meta.store_event = event
|
||||
self._store_events[event] = _StoreEvent(event_plans)
|
||||
if self._gpu_block_pool is not None:
|
||||
self._gpu_block_pool.touch(
|
||||
[
|
||||
self._gpu_block_pool.blocks[block_id]
|
||||
for block_id in dict.fromkeys(
|
||||
plan.block_id for plan in event_plans
|
||||
)
|
||||
]
|
||||
)
|
||||
return meta
|
||||
|
||||
def _tracker_for_request(self, request_id: str) -> RequestTracker:
|
||||
return self._request_trackers.setdefault(request_id, RequestTracker())
|
||||
|
||||
def _load_plans_for_external_tokens(
|
||||
self,
|
||||
request: Any,
|
||||
tracker: RequestTracker,
|
||||
block_groups: tuple[list[int], ...],
|
||||
num_external_tokens: int,
|
||||
) -> list[BlockTransferPlan]:
|
||||
"""Plan the logical objects selected by the external hit window."""
|
||||
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
|
||||
spec = tracker.load_spec
|
||||
local_tokens = spec.local_tokens if spec is not None else 0
|
||||
end_tokens = local_tokens + num_external_tokens
|
||||
plans: list[BlockTransferPlan] = []
|
||||
for group_index, group_id in enumerate(group_ids):
|
||||
group_block_size = self.group_block_sizes[group_id]
|
||||
start_block = local_tokens // group_block_size
|
||||
num_full_blocks = end_tokens // group_block_size
|
||||
for block_index in range(start_block, num_full_blocks):
|
||||
token_end = (block_index + 1) * group_block_size
|
||||
block_hash = self._object_hash_at_token_end(
|
||||
request.block_hashes, token_end
|
||||
)
|
||||
plans.append(
|
||||
BlockTransferPlan(
|
||||
key=self.codec.key(block_hash, group_id=group_id),
|
||||
block_id=block_groups[group_index][block_index],
|
||||
request_id=request.request_id,
|
||||
group_id=group_id,
|
||||
)
|
||||
)
|
||||
return plans
|
||||
|
||||
def _object_hash_at_token_end(
|
||||
self, hashes: Sequence[bytes], token_end: int
|
||||
) -> bytes:
|
||||
hash_index = token_end // self.hash_block_size - 1
|
||||
if hash_index < 0 or hash_index >= len(hashes):
|
||||
raise ValueError(
|
||||
f"request has {len(hashes)} hashes, needs index {hash_index}"
|
||||
)
|
||||
return hashes[hash_index]
|
||||
|
||||
def _store_plans(
|
||||
self,
|
||||
request: Any,
|
||||
tracker: RequestTracker,
|
||||
token_count: int,
|
||||
block_ids_override: tuple[list[int], ...] | None = None,
|
||||
) -> list[BlockTransferPlan]:
|
||||
"""Describe full blocks produced by a scheduled prefill."""
|
||||
request_id = getattr(request, "request_id", None) or request.req_id
|
||||
group_ids = self.kv_cache_config.prefix_cacheable_group_ids
|
||||
# Scheduled speculative tokens are not yet verified and have no block
|
||||
# hash; a block is storable only once its hash exists.
|
||||
token_count = min(token_count, len(request.block_hashes) * self.hash_block_size)
|
||||
previous_saved_tokens = tracker.saved_tokens
|
||||
# Hash-aligned rather than block-aligned: a group with smaller blocks
|
||||
# than the largest must not wait for that group's block to fill.
|
||||
save_to = tracker.mark_saved(token_count, self.hash_block_size)
|
||||
plans: list[BlockTransferPlan] = []
|
||||
block_groups = (
|
||||
block_ids_override
|
||||
if block_ids_override is not None
|
||||
else tuple(request.block_ids[group_id] for group_id in group_ids)
|
||||
)
|
||||
block_ranges = {
|
||||
group_id: range(
|
||||
previous_saved_tokens // self.group_block_sizes[group_id],
|
||||
min(save_to // self.group_block_sizes[group_id], len(block_ids)),
|
||||
)
|
||||
for group_id, block_ids in zip(group_ids, block_groups)
|
||||
}
|
||||
for group_id, block_ids in zip(group_ids, block_groups):
|
||||
if group_id not in block_ranges:
|
||||
continue
|
||||
group_block_size = self.group_block_sizes[group_id]
|
||||
for index in block_ranges[group_id]:
|
||||
if block_ids[index] == NULL_BLOCK_ID:
|
||||
continue
|
||||
token_end = (index + 1) * group_block_size
|
||||
block_hash = self._object_hash_at_token_end(
|
||||
request.block_hashes, token_end
|
||||
)
|
||||
plans.append(
|
||||
BlockTransferPlan(
|
||||
key=self.codec.key(block_hash, group_id=group_id),
|
||||
block_id=block_ids[index],
|
||||
request_id=request_id,
|
||||
group_id=group_id,
|
||||
)
|
||||
)
|
||||
return plans
|
||||
|
||||
def request_finished(
|
||||
self, request: Request, block_ids: tuple[list[int], ...]
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
self._pending_loads.pop(request.request_id, None)
|
||||
self._load_specs.pop(request.request_id, None)
|
||||
self._requests.pop(request.request_id, None)
|
||||
return False, None
|
||||
|
||||
def update_connector_output(self, output: KVConnectorOutput) -> None:
|
||||
metadata = output.kv_connector_worker_meta
|
||||
if not isinstance(metadata, UMBPConnectorWorkerMetadata):
|
||||
return
|
||||
pool = self._gpu_block_pool
|
||||
for event_id, result in metadata.store_events.items():
|
||||
event = self._store_events.get(event_id)
|
||||
if event is None:
|
||||
continue
|
||||
event.result.merge(result)
|
||||
if event.result.completed_workers < self._num_workers:
|
||||
continue
|
||||
del self._store_events[event_id]
|
||||
if pool is not None:
|
||||
block_ids = dict.fromkeys(plan.block_id for plan in event.plans)
|
||||
pool.free_blocks(
|
||||
pool.blocks[block_id] for block_id in reversed(block_ids)
|
||||
)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
return bool(self._store_events)
|
||||
|
||||
def reset_store(self) -> bool:
|
||||
if self._store_events:
|
||||
return False
|
||||
self._load_specs.clear()
|
||||
self._pending_loads.clear()
|
||||
return self.runtime.clear()
|
||||
|
||||
def close(self) -> None:
|
||||
self.runtime.close()
|
||||
@@ -1,240 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared worker-side UMBP connector behavior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .data import (
|
||||
BlockIdentityCodec,
|
||||
BlockTransferPlan,
|
||||
KVLayoutPlanner,
|
||||
StoreEventResult,
|
||||
TransferJobState,
|
||||
TransferJobStatus,
|
||||
UMBPConnectorMetadata,
|
||||
UMBPConnectorWorkerMetadata,
|
||||
)
|
||||
from .runtime import UMBPWorkerHandle
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _StoreBatch:
|
||||
event_id: int
|
||||
jobs: dict[str, TransferJobState]
|
||||
|
||||
|
||||
class UMBPStoreConnectorWorker:
|
||||
"""Translate shared metadata into runtime load/store jobs."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
runtime: UMBPWorkerHandle,
|
||||
layout: KVLayoutPlanner | None = None,
|
||||
*,
|
||||
codec: BlockIdentityCodec | None = None,
|
||||
) -> None:
|
||||
self.runtime = runtime
|
||||
self.layout = layout
|
||||
self.codec = codec
|
||||
self._load_jobs: dict[str, dict[str | None, TransferJobState]] = {}
|
||||
self._store_jobs: dict[str, TransferJobState] = {}
|
||||
self._pending_stores: list[_StoreBatch] = []
|
||||
self._worker_meta = UMBPConnectorWorkerMetadata()
|
||||
self._finished_recving: set[str] = set()
|
||||
self._load_error_block_ids: set[int] = set()
|
||||
self._report_load_completions = False
|
||||
self._active_store_event = -1
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
if self.layout is not None and self.layout.regions:
|
||||
self.layout.register_kv_caches(kv_caches)
|
||||
self.runtime.register_buffers(kv_caches)
|
||||
|
||||
def _materialize_plans(
|
||||
self, plans: list[BlockTransferPlan]
|
||||
) -> list[BlockTransferPlan]:
|
||||
plans = [self._localize_plan(plan) for plan in plans]
|
||||
if self.layout is None or not self.layout.regions:
|
||||
return plans
|
||||
materialized: list[BlockTransferPlan] = []
|
||||
for plan in plans:
|
||||
if plan.ranges:
|
||||
materialized.append(plan)
|
||||
continue
|
||||
materialized.append(self.layout.materialize(plan))
|
||||
return materialized
|
||||
|
||||
def _localize_plan(self, plan: BlockTransferPlan) -> BlockTransferPlan:
|
||||
if self.codec is None or plan.group_id is None:
|
||||
return plan
|
||||
key = self._localize_key(plan.key, plan.group_id)
|
||||
if key == plan.key:
|
||||
return plan
|
||||
return replace(plan, key=key)
|
||||
|
||||
def _localize_key(self, key: str, group_id: int) -> str:
|
||||
if self.codec is None:
|
||||
return key
|
||||
try:
|
||||
block_hash = bytes.fromhex(key.rsplit(":", 1)[1])
|
||||
except (IndexError, ValueError):
|
||||
return key
|
||||
return self.codec.key(block_hash, group_id)
|
||||
|
||||
def start_load_kv(
|
||||
self, forward_context: ForwardContext, metadata: UMBPConnectorMetadata
|
||||
) -> None:
|
||||
del forward_context
|
||||
self._report_load_completions = metadata.async_load
|
||||
for request_id, plans in metadata.load_requests.items():
|
||||
if plans:
|
||||
self._submit_loads(request_id, plans)
|
||||
|
||||
def _submit_loads(self, request_id: str, plans: list[BlockTransferPlan]) -> None:
|
||||
materialized = self._materialize_plans(plans)
|
||||
logger.debug(
|
||||
"UMBP load submitted request=%s plans=%d ranges=%d bytes=%d",
|
||||
request_id,
|
||||
len(materialized),
|
||||
sum(len(plan.ranges) for plan in materialized),
|
||||
sum(item.length for plan in materialized for item in plan.ranges),
|
||||
)
|
||||
self._load_jobs[request_id] = {None: self.runtime.load(materialized)}
|
||||
|
||||
def _finish_job(
|
||||
self, request_id: str, result: TransferJobState, *, is_load: bool
|
||||
) -> bool:
|
||||
operation = "load" if is_load else "store"
|
||||
succeeded = result.status == TransferJobStatus.COMPLETED
|
||||
if not succeeded:
|
||||
logger.debug(
|
||||
"UMBP %s failed request=%s status=%s error=%s",
|
||||
operation,
|
||||
request_id,
|
||||
result.status.value,
|
||||
result.error,
|
||||
)
|
||||
if is_load:
|
||||
self._load_error_block_ids.update(result.failed_block_ids)
|
||||
elif succeeded:
|
||||
self.runtime.publish(result)
|
||||
return succeeded
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
if self._report_load_completions:
|
||||
# Asynchronous loads fill blocks of requests that do not run until
|
||||
# get_finished reports them, so no forward pass reads those blocks.
|
||||
return
|
||||
self._drain_load_jobs(wait=True)
|
||||
|
||||
def wait_for_save(self) -> None:
|
||||
if self._store_jobs or self._active_store_event >= 0:
|
||||
self._pending_stores.append(
|
||||
_StoreBatch(self._active_store_event, self._store_jobs)
|
||||
)
|
||||
self._store_jobs = {}
|
||||
self._active_store_event = -1
|
||||
self._drain_store_jobs(wait=False)
|
||||
|
||||
def _drain_store_jobs(self, *, wait: bool) -> None:
|
||||
for batch in list(self._pending_stores):
|
||||
for request_id, job in list(batch.jobs.items()):
|
||||
result = self.runtime.wait(job) if wait else self.runtime.poll(job)
|
||||
if result is None:
|
||||
continue
|
||||
# A partly failed job is not published, including its stored keys.
|
||||
self._finish_job(request_id, result, is_load=False)
|
||||
del batch.jobs[request_id]
|
||||
if not batch.jobs:
|
||||
if batch.event_id >= 0:
|
||||
self._worker_meta.store_events[batch.event_id] = StoreEventResult(
|
||||
completed_workers=1
|
||||
)
|
||||
self._pending_stores.remove(batch)
|
||||
|
||||
def _submit_store_plans(
|
||||
self, plans: list[BlockTransferPlan], request_id: str
|
||||
) -> None:
|
||||
if not plans:
|
||||
return
|
||||
materialized = self._materialize_plans(plans)
|
||||
self._store_jobs[request_id] = self.runtime.store(materialized)
|
||||
|
||||
def enqueue_stores(self, metadata: UMBPConnectorMetadata) -> None:
|
||||
self._active_store_event = metadata.store_event
|
||||
store_requests = {
|
||||
request_id: list(plans)
|
||||
for request_id, plans in metadata.store_requests.items()
|
||||
}
|
||||
grouped = {plan for plans in store_requests.values() for plan in plans}
|
||||
for plan in metadata.store_plans:
|
||||
if plan not in grouped:
|
||||
store_requests.setdefault(
|
||||
plan.request_id or "__umbp_batch__", []
|
||||
).append(plan)
|
||||
grouped.add(plan)
|
||||
for request_id, plans in store_requests.items():
|
||||
self._submit_store_plans(plans, request_id)
|
||||
|
||||
def handle_preemptions(self, metadata: UMBPConnectorMetadata) -> None:
|
||||
"""Finish request-local jobs before vLLM reuses their GPU blocks."""
|
||||
preempted = metadata.preempted_request_ids
|
||||
if not preempted:
|
||||
return
|
||||
for request_id in preempted:
|
||||
for job in self._load_jobs.pop(request_id, {}).values():
|
||||
self.runtime.wait(job)
|
||||
for jobs in [self._store_jobs, *(batch.jobs for batch in self._pending_stores)]:
|
||||
for request_id, job in jobs.items():
|
||||
if request_id in preempted or any(
|
||||
plan.request_id in preempted for plan in job.plans
|
||||
):
|
||||
jobs[request_id] = self.runtime.wait(job)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
del finished_req_ids
|
||||
self._drain_load_jobs(wait=False)
|
||||
self._drain_store_jobs(wait=False)
|
||||
recving = set(self._finished_recving)
|
||||
self._finished_recving.difference_update(recving)
|
||||
return None, recving or None
|
||||
|
||||
def _drain_load_jobs(self, *, wait: bool) -> None:
|
||||
for request_id, jobs in list(self._load_jobs.items()):
|
||||
for layer, job in list(jobs.items()):
|
||||
result = self.runtime.wait(job) if wait else self.runtime.poll(job)
|
||||
if result is None:
|
||||
continue
|
||||
del jobs[layer]
|
||||
self._finish_job(request_id, result, is_load=True)
|
||||
if not jobs:
|
||||
del self._load_jobs[request_id]
|
||||
if self._report_load_completions:
|
||||
self._finished_recving.add(request_id)
|
||||
|
||||
def build_connector_worker_meta(self) -> UMBPConnectorWorkerMetadata:
|
||||
result = self._worker_meta
|
||||
self._worker_meta = UMBPConnectorWorkerMetadata()
|
||||
return result
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
failed = self._load_error_block_ids
|
||||
self._load_error_block_ids = set()
|
||||
return failed
|
||||
|
||||
def close(self) -> None:
|
||||
self.wait_for_save()
|
||||
self._drain_store_jobs(wait=True)
|
||||
self._drain_load_jobs(wait=True)
|
||||
self.runtime.close()
|
||||
@@ -1,144 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared configuration and setup for EC and KV Mooncake Store clients."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal
|
||||
|
||||
import regex as re
|
||||
|
||||
DEFAULT_GLOBAL_SEGMENT_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
|
||||
DEFAULT_LOCAL_BUFFER_SIZE = 4 * 1024 * 1024 * 1024 # 4 GiB
|
||||
DEFAULT_TENANT_ID = "default"
|
||||
|
||||
|
||||
MooncakeMode = Literal["embedded", "standalone-store"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MooncakeStoreConfig:
|
||||
"""Configuration for MooncakeDistributedStore.
|
||||
|
||||
``mode`` selects the topology: ``embedded`` (each rank contributes
|
||||
``global_segment_size`` in-process) or ``standalone-store`` (rank
|
||||
contributes 0; an external ``mooncake_client`` process owns the pool
|
||||
and the SSD tier).
|
||||
"""
|
||||
|
||||
metadata_server: str
|
||||
master_server_address: str
|
||||
protocol: str
|
||||
device_name: str
|
||||
mode: MooncakeMode = "embedded"
|
||||
global_segment_size: int = DEFAULT_GLOBAL_SEGMENT_SIZE
|
||||
local_buffer_size: int = DEFAULT_LOCAL_BUFFER_SIZE
|
||||
enable_offload: bool = False
|
||||
tenant_id: str = DEFAULT_TENANT_ID
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.mode not in ("embedded", "standalone-store"):
|
||||
raise ValueError(f"unknown Mooncake mode: {self.mode!r}")
|
||||
if self.local_buffer_size <= 0:
|
||||
raise ValueError("local_buffer_size must be > 0")
|
||||
if self.mode == "embedded" and self.global_segment_size == 0:
|
||||
raise ValueError("embedded mode requires global_segment_size > 0")
|
||||
if self.mode == "standalone-store" and self.global_segment_size != 0:
|
||||
raise ValueError("standalone-store mode requires global_segment_size == 0")
|
||||
|
||||
@staticmethod
|
||||
def from_file(file_path: str) -> "MooncakeStoreConfig":
|
||||
with open(file_path) as file:
|
||||
config = json.load(file)
|
||||
return MooncakeStoreConfig(
|
||||
metadata_server=config.get("metadata_server", ""),
|
||||
master_server_address=config.get("master_server_address", ""),
|
||||
protocol=config.get("protocol", "rdma"),
|
||||
device_name=config.get("device_name", ""),
|
||||
mode=config.get("mode", "embedded"),
|
||||
global_segment_size=_parse_size(
|
||||
config.get("global_segment_size", DEFAULT_GLOBAL_SEGMENT_SIZE)
|
||||
),
|
||||
local_buffer_size=_parse_size(
|
||||
config.get("local_buffer_size", DEFAULT_LOCAL_BUFFER_SIZE)
|
||||
),
|
||||
enable_offload=bool(config.get("enable_offload", False)),
|
||||
tenant_id=_normalize_tenant_id(config.get("tenant_id", DEFAULT_TENANT_ID)),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def load_from_config() -> "MooncakeStoreConfig":
|
||||
config_path = os.getenv("MOONCAKE_CONFIG_PATH")
|
||||
if not config_path:
|
||||
raise ValueError(
|
||||
"The environment variable 'MOONCAKE_CONFIG_PATH' is not set."
|
||||
)
|
||||
return MooncakeStoreConfig.from_file(config_path)
|
||||
|
||||
|
||||
def _normalize_tenant_id(value: Any) -> str:
|
||||
if value is None:
|
||||
return DEFAULT_TENANT_ID
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(
|
||||
f"tenant_id must be a string or null, got {type(value).__name__}: {value!r}"
|
||||
)
|
||||
tenant_id = value.strip()
|
||||
return tenant_id if tenant_id else DEFAULT_TENANT_ID
|
||||
|
||||
|
||||
def _parse_size(value: Any) -> int:
|
||||
"""Parse storage size strings with units: GB, MB, KB, B."""
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if not isinstance(value, str):
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError) as e:
|
||||
raise TypeError(f"Unsupported type for size: {type(value)}") from e
|
||||
|
||||
cleaned = value.strip().lower()
|
||||
if not cleaned:
|
||||
raise ValueError("Size cannot be empty.")
|
||||
|
||||
unit_multipliers = {
|
||||
"gb": 1024**3,
|
||||
"mb": 1024**2,
|
||||
"kb": 1024,
|
||||
"b": 1,
|
||||
}
|
||||
match = re.match(r"^\s*([\d.]+)\s*(gb|mb|kb|b)?\s*$", cleaned)
|
||||
if not match:
|
||||
raise ValueError(f"Invalid format: '{value}'")
|
||||
|
||||
number_str = match.group(1)
|
||||
unit = match.group(2) or "b"
|
||||
multiplier = unit_multipliers[unit]
|
||||
|
||||
try:
|
||||
numeric_value = float(number_str)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"Invalid numeric value '{number_str}' in: '{value}'") from exc
|
||||
return int(numeric_value * multiplier)
|
||||
|
||||
|
||||
def setup_mooncake_store(
|
||||
store: Any, config: MooncakeStoreConfig, local_hostname: str
|
||||
) -> None:
|
||||
"""Initialize a native Store, preserving explicit tenant configuration."""
|
||||
setup_kwargs: dict[str, str] = {}
|
||||
if config.tenant_id != DEFAULT_TENANT_ID:
|
||||
setup_kwargs["tenant_id"] = config.tenant_id
|
||||
ret = store.setup(
|
||||
local_hostname,
|
||||
config.metadata_server,
|
||||
config.global_segment_size,
|
||||
config.local_buffer_size,
|
||||
config.protocol,
|
||||
config.device_name,
|
||||
config.master_server_address,
|
||||
**setup_kwargs,
|
||||
)
|
||||
if ret != 0:
|
||||
raise RuntimeError("Initialize MooncakeDistributedStore failed.")
|
||||
Reference in New Issue
Block a user