Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
@@ -22,9 +22,7 @@ class AiterCustomAllreduce:
@classmethod
def effective_max_size(cls) -> int:
"""
Max input byte size eligible for AITER custom allreduce.
"""
"""Max input byte size eligible for AITER custom allreduce."""
return cls.MAX_SIZE // 2
def __init__(
@@ -44,8 +44,7 @@ logger = init_logger(__name__)
class AgRsAll2AllManager(All2AllManagerBase):
"""
An implementation of all2all communication based on
"""An implementation of all2all communication based on
all-gather (dispatch) and reduce-scatter (combine).
"""
@@ -79,9 +78,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Gather hidden_states and router_logits from all dp ranks.
"""
"""Gather hidden_states and router_logits from all dp ranks."""
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
@@ -111,9 +108,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Gather hidden_states and router_logits from all dp ranks.
"""
"""Gather hidden_states and router_logits from all dp ranks."""
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
@@ -140,9 +135,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Reduce-scatter hidden_states across all dp ranks.
"""
"""Reduce-scatter hidden_states across all dp ranks."""
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(
hidden_states.shape[0] // dist_group.world_size,
@@ -156,9 +149,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
class DeepEPAll2AllManagerBase(All2AllManagerBase):
"""
All2All communication based on DeepEP High-Throughput kernels.
"""
"""All2All communication based on DeepEP High-Throughput kernels."""
def __init__(self, cpu_group, tcp_store_group=None):
assert has_deep_ep(), (
@@ -210,9 +201,7 @@ class DeepEPAll2AllManagerBase(All2AllManagerBase):
class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
"""
All2All communication based on DeepEP High-Throughput kernels.
"""
"""All2All communication based on DeepEP High-Throughput kernels."""
def __init__(self, cpu_group, tcp_store_group=None):
super().__init__(cpu_group, tcp_store_group)
@@ -271,9 +260,7 @@ class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
"""
All2All communication based on DeepEP Low-Latency kernels.
"""
"""All2All communication based on DeepEP Low-Latency kernels."""
_buffer: Any = None
_mask: torch.Tensor | None = None
@@ -293,8 +280,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
num_global_experts: int,
num_local_experts: int,
) -> dict[Any, Any]:
"""
max_num_tokens_per_dp_rank : the maximum number of tokens a DP rank
"""max_num_tokens_per_dp_rank : the maximum number of tokens a DP rank
can dispatch all the ranks must hold the same value.
token_hidden_size: the hidden dimension of each token.
num_ep_ranks: the number of EP group ranks.
@@ -330,8 +316,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
return kwargs
def get_handle(self, kwargs):
"""
The kwargs for DeepEPLLAll2AllManager is dictated by
"""The kwargs for DeepEPLLAll2AllManager is dictated by
_make_all2all_kwargs.
"""
import deep_ep # type: ignore[import-not-found]
@@ -385,8 +370,7 @@ class _NixlEPBufferState:
class NixlEPAll2AllManager(All2AllManagerBase):
"""
All2All communication based on NIXL EP kernels.
"""All2All communication based on NIXL EP kernels.
This backend supports elastic EP with dynamic rank connection/disconnection.
"""
@@ -612,9 +596,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
"""
All2All communication based on flashinfer all2allv/two-sided NVLink kernels.
"""
"""All2All communication based on flashinfer all2allv/two-sided NVLink kernels."""
# This type lint could be removed after all of the work in
# https://github.com/vllm-project/vllm/issues/26533 done.
@@ -640,7 +622,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
rank: int,
gpus_per_node: int,
):
"""Initialize workspace"""
"""Initialize workspace."""
if self.initialized:
return
@@ -681,7 +663,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
)
def ensure_alltoall_workspace_initialized(self):
"""Ensure workspace is initialized"""
"""Ensure workspace is initialized."""
if not has_flashinfer_nvlink_two_sided():
return False
@@ -700,7 +682,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
return self
def cleanup(self):
"""Clean up workspace"""
"""Clean up workspace."""
if (
self.initialized
and self.workspace_tensor is not None
@@ -719,8 +701,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
"""
All2All communication based on FlashInfer's MoeAlltoAll/One-sided NVLink kernel.
"""All2All communication based on FlashInfer's MoeAlltoAll/One-sided NVLink kernel.
This is a newer kernel from trtllm that should perform better than the kernel
used by flashinfer_nvlink_two_sided.
"""
@@ -1023,8 +1004,7 @@ class MoriAll2AllManager(All2AllManagerBase):
class DeepEPV2All2AllManager(All2AllManagerBase):
"""
All2All communication based on DeepEP v2 ElasticBuffer (unified API).
"""All2All communication based on DeepEP v2 ElasticBuffer (unified API).
Uses NCCL Gin backend with analytical SM calculation.
"""
@@ -119,8 +119,7 @@ NCCL_SYMM_MEM_ALL_REDUCE_CONFIG: dict[str, Any] = {
def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor) -> bool:
"""
Determine if NCCL symmetric memory allreduce should be used.
"""Determine if NCCL symmetric memory allreduce should be used.
Based on H100 and GB200 benchmarks, NCCL symm_mem is preferred for:
- Small tensors (≤16K): Lower latency than custom_AR
@@ -157,7 +156,8 @@ def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor)
def should_nccl_symm_mem_ag_rs() -> bool:
"""Check whether NCCL symmetric memory should be used for
AllGather / ReduceScatter collectives."""
AllGather / ReduceScatter collectives.
"""
from vllm.distributed.device_communicators.pynccl_allocator import (
is_symmetric_memory_enabled,
)
@@ -344,7 +344,6 @@ _gpu_p2p_access_cache: dict[str, bool] | None = None
def gpu_p2p_access_check(src: int, tgt: int) -> bool:
"""Check if GPU src can access GPU tgt."""
# if the cache variable is already calculated,
# read from the cache instead of checking it again
global _gpu_p2p_access_cache
@@ -114,6 +114,7 @@ class All2AllManagerBase:
Returns:
An int32 device tensor where 0 marks a live rank and 1 marks a
masked (dead/unreachable) rank.
"""
raise NotImplementedError
@@ -171,8 +172,7 @@ class All2AllManagerBase:
class DeviceCommunicatorBase:
"""
Base class for device-specific communicator.
"""Base class for device-specific communicator.
It can use the `cpu_group` to initialize the communicator.
If the device has PyTorch integration (PyTorch can recognize its
communication backend), the `device_group` will also be given.
@@ -318,8 +318,7 @@ class DeviceCommunicatorBase:
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> torch.Tensor | None:
"""
NOTE: We assume that the input tensor is on the same device across
"""NOTE: We assume that the input tensor is on the same device across
all the ranks.
NOTE: `dst` is the local rank of the destination rank.
"""
@@ -347,7 +346,7 @@ class DeviceCommunicatorBase:
return output_tensor
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
"""Sends a tensor to the destination rank in a blocking way"""
"""Sends a tensor to the destination rank in a blocking way."""
"""NOTE: `dst` is the local rank of the destination rank."""
if dst is None:
dst = (self.rank_in_group + 1) % self.world_size
@@ -385,8 +384,7 @@ class DeviceCommunicatorBase:
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and router logits to the appropriate device.
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
if extra_tensors is not None:
@@ -404,8 +402,7 @@ class DeviceCommunicatorBase:
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and topk weights/ids to the appropriate device.
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
if extra_tensors is not None:
@@ -415,8 +412,7 @@ class DeviceCommunicatorBase:
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Combine the hidden states and router logits from the appropriate device.
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
return hidden_states
@@ -69,8 +69,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
logger.info("Using allgather_reducescatter all2all manager.")
def _all_group_ranks_share_shm_group_name(self) -> bool:
"""
CPUSHM requires all ranks in this group to agree on one SHM group name.
"""CPUSHM requires all ranks in this group to agree on one SHM group name.
This is a lightweight consistency check for VLLM_DIST_IDENT/name inputs.
"""
local_name = _CPUSHMDistributed.make_group_name(self)
@@ -89,8 +88,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> torch.Tensor | None:
"""
NOTE: We assume that the input tensor is on the same device across
"""NOTE: We assume that the input tensor is on the same device across
all the ranks.
NOTE: `dst` is the local rank of the destination rank.
"""
@@ -180,11 +178,9 @@ class CpuCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and router logits to the appropriate device.
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch_router_logits(
hidden_states,
@@ -204,8 +200,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and topk weights/ids to the appropriate device.
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -220,8 +215,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Combine the hidden states and router logits from the appropriate device.
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -611,7 +611,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
return symm_output
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
"""Sends a tensor to the destination rank in a blocking way"""
"""Sends a tensor to the destination rank in a blocking way."""
"""NOTE: `dst` is the local rank of the destination rank."""
if dst is None:
dst = (self.rank_in_group + 1) % self.world_size
@@ -851,11 +851,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and router logits to the appropriate device.
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch_router_logits(
hidden_states,
@@ -875,8 +873,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and topk weights/ids to the appropriate device.
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -891,8 +888,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Combine the hidden states and router logits from the appropriate device.
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -129,8 +129,7 @@ class CustomAllreduce:
max_mnnvl_reduce_scatter_size=_DEFAULT_MNNVL_REDUCE_SCATTER_MAX_SIZE,
symm_mem_enabled=False,
) -> None:
"""
Args:
"""Args:
group: the process group to work on. If None, it will use the
default process group.
device: the device to bind the CustomAllreduce to. If None,
@@ -138,6 +137,7 @@ class CustomAllreduce:
It is the caller's responsibility to make sure each communicator
is bind to a unique device, and all communicators in this group
are in the same node.
"""
self._IS_CAPTURING = False
self._ptr = 0
@@ -370,8 +370,7 @@ class CustomAllreduce:
@contextmanager
def capture(self):
"""
The main responsibility of this context manager is the
"""The main responsibility of this context manager is the
`register_graph_buffers` call at the end of the context.
It records all the buffer addresses used in the CUDA graph.
"""
@@ -140,6 +140,7 @@ def _resolve_fi_ar_backend() -> tuple[str, bool]:
is True only when ``auto`` selects mnnvl for a single node, so that
workspace creation can fall back to trtllm on single-node topologies
without NVSwitch multicast support (where mnnvl is unavailable).
"""
backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND
if backend != "auto":
@@ -168,8 +169,7 @@ def get_fi_ar_workspace(
dtype: torch.dtype,
group: ProcessGroup,
):
"""
Return the allreduce workspace for non-quant patterns, initializing if needed.
"""Return the allreduce workspace for non-quant patterns, initializing if needed.
Used by AllReduceFusionPass (non-quant patterns) and FlashInferAllReduce
for standalone allreduce. Backend is controlled by
@@ -234,8 +234,7 @@ def get_fi_ar_quant_workspace(
dtype: torch.dtype,
group: ProcessGroup,
):
"""
Return the allreduce workspace for quant patterns, initializing if needed.
"""Return the allreduce workspace for quant patterns, initializing if needed.
Backend is controlled by VLLM_FLASHINFER_ALLREDUCE_BACKEND env var, matching
non-quant fusion. With ``auto`` this prefers mnnvl and falls back to trtllm
@@ -98,8 +98,7 @@ class PyNcclCommunicator:
device: int | str | torch.device,
library_path: str | None = None,
):
"""
Args:
"""Args:
group: the process group to work on. If None, it will use the
default process group.
device: the device to bind the PyNcclCommunicator to. If None,
@@ -108,6 +107,7 @@ class PyNcclCommunicator:
use the default library path.
It is the caller's responsibility to make sure each communicator
is bind to a unique device.
"""
if not isinstance(group, StatelessProcessGroup):
assert dist.is_initialized()
@@ -62,8 +62,7 @@ class QuickAllReduce:
}
def __init__(self, group: ProcessGroup, device: int | str | torch.device) -> None:
"""
Custom allreduce provides non-destructive acceleration and is
"""Custom allreduce provides non-destructive acceleration and is
available for CUDA and ROCm MI300 series.
Custom quick allreduce leverages quantization for further
@@ -86,6 +85,7 @@ class QuickAllReduce:
It is the caller's responsibility to make sure each communicator
is bind to a unique device, and all communicators in this group
are in the same node.
"""
self.disabled = True
if not self._rocm_arch_available():
@@ -301,8 +301,7 @@ class QuickAllReduce:
return False
def create_shared_buffer(self):
"""
Creates a shared buffer for quickreduce.
"""Creates a shared buffer for quickreduce.
Has to be called after init_custom_qr
"""
handle = ops.qr_get_handle(self._ptr)
@@ -312,9 +311,7 @@ class QuickAllReduce:
ops.qr_open_handles(self._ptr, handles)
def should_quick_allreduce(self, inp: torch.Tensor):
"""
Check if quickreduce is available
"""
"""Check if quickreduce is available."""
if self.disabled:
return False
if inp.dtype not in self._SUPPORTED_DTYPES:
@@ -20,8 +20,7 @@ logger = init_logger(__name__)
class RayPPCommunicator(Communicator):
"""
Communicator to be used for pipeline parallelism in Ray Compiled Graph.
"""Communicator to be used for pipeline parallelism in Ray Compiled Graph.
This is wraps around the vLLM _PP GroupCoordinator.
This class is not thread-safe.
@@ -38,8 +37,7 @@ class RayPPCommunicator(Communicator):
cuda_stream: torch.cuda.Stream | None,
use_communication_streams: bool = False,
):
"""
Initialize a RayPPCommunicator that can be used to communicate with
"""Initialize a RayPPCommunicator that can be used to communicate with
other Ray Compiled Graph actors for pipeline parallelism.
Args:
@@ -54,6 +52,7 @@ class RayPPCommunicator(Communicator):
is not supported.
use_communication_streams: Whether to use communication streams.
This is not supported.
"""
self._world_size = world_size
self._rank: int | None = None
@@ -87,8 +86,7 @@ class RayPPCommunicator(Communicator):
self._closed = False
def _build_actor_rank_mapping(self):
"""
Use collective communication to build a mapping from actor IDs to ranks.
"""Use collective communication to build a mapping from actor IDs to ranks.
This should be called once during initialization.
"""
if self._comm is None:
@@ -128,9 +126,7 @@ class RayPPCommunicator(Communicator):
return self._actor_handles
def get_rank(self, actor: ray.actor.ActorHandle) -> int:
"""
Return the given actor's rank using device communicator collective ops.
"""
"""Return the given actor's rank using device communicator collective ops."""
assert hasattr(self, "_actor_id_to_rank"), (
"Actor rank mapping not built. "
"This should have been done during initialization."
@@ -144,20 +140,15 @@ class RayPPCommunicator(Communicator):
raise ValueError(f"Actor {actor} not found in communicator group")
def get_self_rank(self) -> int | None:
"""
Return this actor's rank.
"""
"""Return this actor's rank."""
return self._rank
def get_world_size(self) -> int:
"""
Return the number of ranks in the RayPPCommunicator group.
"""
"""Return the number of ranks in the RayPPCommunicator group."""
return self._world_size
def send(self, buf: "torch.Tensor", peer_rank: int) -> None:
"""
Send a torch.Tensor to a peer.
"""Send a torch.Tensor to a peer.
This returns when the send kernel has been queued, but the kernel may
not have completed. Therefore, the caller should ensure that there are
@@ -170,6 +161,7 @@ class RayPPCommunicator(Communicator):
buf: The torch.Tensor to send. It should already be on this
actor's default device.
peer_rank: The rank of the actor to send to.
"""
if self._closed:
raise RayChannelError("RayPPCommunicator has been destroyed.")
@@ -184,8 +176,7 @@ class RayPPCommunicator(Communicator):
peer_rank: int,
allocator: TorchTensorAllocator,
) -> "torch.Tensor":
"""
Receive a torch.Tensor from a peer and synchronize the current stream.
"""Receive a torch.Tensor from a peer and synchronize the current stream.
After this call returns, the receive buffer is safe to read from
any stream. An RayChannelError will be raised if an error occurred
@@ -197,6 +188,7 @@ class RayPPCommunicator(Communicator):
peer_rank: The rank of the actor to receive from.
allocator: The allocator to use to create the received tensor.
This is ignored for this implementation.
"""
if self._closed:
raise RayChannelError("RayPPCommunicator has been destroyed.")
@@ -77,8 +77,7 @@ _memory_fence_lock = threading.Lock()
def memory_fence():
"""
Full memory barrier for shared memory synchronization.
"""Full memory barrier for shared memory synchronization.
Ensures all prior memory writes are visible to other processes before
any subsequent reads. This is critical for lock-free producer-consumer
@@ -111,8 +110,7 @@ LONG_WAIT_TIME_LOG_MSG = (
class SpinCondition:
"""
This class implements an interface similar to a threading.Condition. It
"""This class implements an interface similar to a threading.Condition. It
allows a writer to notify readers to wake up and read from the shared memory
buffer. This notification is done over a zmq socket.
@@ -241,6 +239,7 @@ def check_shm_free_space(
Raises:
RuntimeError: If the SHM filesystem has insufficient space.
"""
if os.path.isdir(shm_path):
free_bytes = shutil.disk_usage(shm_path).free
@@ -651,7 +650,8 @@ class MessageQueue:
def shutdown(self):
"""If this is an idle reader, wakes it up so it can clean up and shut
down"""
down
"""
self.shutting_down = True
if self._spin_condition is not None:
self._spin_condition.cancel()
@@ -761,7 +761,8 @@ class MessageQueue:
def should_warn(self) -> bool:
"""Returns true if it's time to log a warning for a timeout that is not
indefinite"""
indefinite
"""
if self._should_warn:
elapsed = time.monotonic() - self.started
if elapsed >= VLLM_RINGBUFFER_WARNING_INTERVAL * self.n_warning:
@@ -941,8 +942,7 @@ class MessageQueue:
reader_rank: int = 0,
blocking: bool = False,
) -> tuple["MessageQueue", list[Handle]]:
"""
Creates a MessageQueue for a process group with a single reader.
"""Creates a MessageQueue for a process group with a single reader.
This method is designed for scenarios where only one process (the reader)
will consume messages, and all other processes are writers. It sets up
@@ -962,6 +962,7 @@ class MessageQueue:
tuple[MessageQueue, list[Handle]]:
The MessageQueue instance for the calling process,
and a list of handles (only non-empty for the reader process).
"""
from vllm.platforms.interface import get_assigned_physical_gpu_ids
@@ -994,8 +995,7 @@ class MessageQueue:
external_writer_handle=None,
blocking: bool = True,
) -> "MessageQueue":
"""
Creates a MessageQueue for a distributed process group with one writer and
"""Creates a MessageQueue for a distributed process group with one writer and
multiple readers.
This method is designed for scenarios where one process (the writer) sends
@@ -20,8 +20,7 @@ logger = init_logger(__name__)
class SingleWriterShmRingBuffer:
"""
A single-writer, multiple-reader ring buffer implementation using shared
"""A single-writer, multiple-reader ring buffer implementation using shared
memory. This class provides a thread-safe ring buffer where one process
can write data while multiple processes/threads can read from it.
@@ -190,8 +189,7 @@ class SingleWriterShmRingBuffer:
return int.from_bytes(byte_data, "little", signed=True)
def allocate_buf(self, size: int) -> tuple[int, int]:
"""
Allocate a buffer `MD_SIZE` + `size` bytes in the shared memory.
"""Allocate a buffer `MD_SIZE` + `size` bytes in the shared memory.
Memory layout:
`[4-byte monotonic_id][4-byte size][buffer data...]`
"""
@@ -260,8 +258,7 @@ class SingleWriterShmRingBuffer:
is_free_fn: Callable[[int, memoryview], bool],
nbytes: int | None = None,
) -> Iterable[int]:
"""
Free a buffer of the given size. This is a no-op in shared memory,
"""Free a buffer of the given size. This is a no-op in shared memory,
but we need to keep track of the metadata.
If freed memory spreads across the end and start of the ring buffer,
@@ -269,10 +266,13 @@ class SingleWriterShmRingBuffer:
still might not be a contiguous space of `nbytes` available.
Args:
is_free_fn (Callable[[int, memoryview], bool]): Predicate called with
a monotonic id and the buffer, returning True when that buffer
can be reclaimed.
nbytes (int, optional): The size of the buffer to free. If None,
frees the maximum size of the ring buffer.
"""
"""
assert self.is_writer, "Only the writer can free buffers."
logger.debug(
"Freeing up space in the ring buffer, "
@@ -412,8 +412,7 @@ class ShmObjectStorageHandle:
class SingleWriterShmObjectStorage:
"""
A single-writer, multiple-reader object storage system built on top of a
"""A single-writer, multiple-reader object storage system built on top of a
shared memory ring buffer. Provides key-value storage with automatic memory
management and cross-process serialization support.
@@ -458,8 +457,7 @@ class SingleWriterShmObjectStorage:
serde_class: type[ObjectSerde] = MsgpackSerde,
reader_lock: LockType | None = None,
):
"""
Initialize the object storage.
"""Initialize the object storage.
Args:
max_object_size: Maximum size for a single object in bytes.
@@ -467,10 +465,11 @@ class SingleWriterShmObjectStorage:
ring_buffer: The shared memory ring buffer for storing objects.
serde_class: Serializer/deserializer for objects.
reader_lock: Optional lock for synchronizing reader access.
Raises:
ValueError: If reader_lock is None for readers.
"""
"""
self.max_object_size = max_object_size
self.n_readers = n_readers
self.serde_class = serde_class
@@ -547,22 +546,17 @@ class SingleWriterShmObjectStorage:
del self.writer_flag[freed_id]
def is_cached(self, key: str) -> bool:
"""
Check if the object with the given key is cached.
"""
"""Check if the object with the given key is cached."""
return key in self.key_index
def get_cached(self, key: str) -> tuple[int, int]:
"""
Get the cached object by key if it exists.
"""
"""Get the cached object by key if it exists."""
address, monotonic_id = self.key_index[key]
self.increment_writer_flag(monotonic_id)
return address, monotonic_id
def put(self, key: str, value: Any) -> tuple[int, int]:
"""
Store a key-value pair in the object storage.
"""Store a key-value pair in the object storage.
Attempts to free max_object_size bytes using FIFO order
when the ring buffer runs out of space during a put() operation.
@@ -574,6 +568,7 @@ class SingleWriterShmObjectStorage:
MemoryError: If there's not enough space in the buffer
ValueError: If the serialized object is too large
ValueError: If the key already exists in the storage
"""
if key in self.key_index:
raise ValueError(f"Key '{key}' already exists in the storage.")
@@ -639,8 +634,7 @@ class SingleWriterShmObjectStorage:
address: int = 0,
monotonic_id: int = 0,
) -> None:
"""
Touch an existing cached item to update its eviction status.
"""Touch an existing cached item to update its eviction status.
For writers (ShmObjectStoreSenderCache): Increment writer_flag
For readers (ShmObjectStoreReceiverCache): Increment reader_count
@@ -700,8 +694,7 @@ class SingleWriterShmObjectStorage:
)
def default_is_free_check(self, id: int, buf: memoryview) -> bool:
"""
Default is_free function that checks if the first 4 bytes are zero.
"""Default is_free function that checks if the first 4 bytes are zero.
This indicates that the buffer is free.
"""
reader_count = int.from_bytes(buf[0:4], "little", signed=True)
@@ -205,11 +205,9 @@ class XpuCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and router logits to the appropriate device.
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch_router_logits(
hidden_states,
@@ -229,8 +227,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and topk weights/ids to the appropriate device.
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -245,8 +242,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Combine the hidden states and router logits from the appropriate device.
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
ECConnectorBase Class for Distributed Encoder Cache &
"""ECConnectorBase Class for Distributed Encoder Cache &
P2P Encoder cache communication in V1
The class provides the following primitives:
@@ -50,8 +49,7 @@ class ECConnectorRole(enum.Enum):
class ECConnectorMetadata(ABC): # noqa: B024
"""
Abstract Metadata used to communicate between the
"""Abstract Metadata used to communicate between the
Scheduler ECConnector and Worker ECConnector.
"""
@@ -59,8 +57,7 @@ class ECConnectorMetadata(ABC): # noqa: B024
class ECConnectorWorkerMetadata(ABC):
"""
Abstract Metadata used to communicate back
"""Abstract Metadata used to communicate back
Worker ECConnector -> Scheduler ECConnector.
Each worker can output its own metadata.
@@ -73,9 +70,7 @@ class ECConnectorWorkerMetadata(ABC):
def aggregate(
self, other: "ECConnectorWorkerMetadata"
) -> "ECConnectorWorkerMetadata":
"""
Aggregate metadata with another `ECConnectorWorkerMetadata` object.
"""
"""Aggregate metadata with another `ECConnectorWorkerMetadata` object."""
pass
@@ -103,8 +98,7 @@ class ECConnectorBase(ABC):
return self._is_consumer
def shutdown(self) -> None:
"""
Shutdown the connector. This is called when the process
"""Shutdown the connector. This is called when the process
is shutting down to ensure that all the async operations are
completed and the connector is cleaned up properly.
"""
@@ -123,6 +117,7 @@ class ECConnectorBase(ABC):
Args:
connector_metadata (dict): the connector metadata.
"""
self._connector_metadata = connector_metadata
@@ -141,8 +136,8 @@ class ECConnectorBase(ABC):
Returns:
ConnectorMetadata: the connector metadata.
"""
"""
# Should only be called while set to valid metadata.
assert self._connector_metadata is not None
return self._connector_metadata
@@ -151,10 +146,11 @@ class ECConnectorBase(ABC):
self,
ec_caches: dict[str, torch.Tensor],
):
"""
Initialize with the EC caches.
"""Initialize with the EC caches.
Args:
ec_caches: dictionary of encoder cache
"""
# TODO: Implement this later for P2P feature
return
@@ -163,8 +159,7 @@ class ECConnectorBase(ABC):
def start_load_caches(
self, encoder_cache: dict[str, torch.Tensor], **kwargs
) -> None:
"""
Start loading the cache from the connector into vLLM's encoder cache.
"""Start loading the cache from the connector into vLLM's encoder cache.
This method loads the encoder cache based on metadata provided by the scheduler.
It is called before `_gather_mm_embeddings` for the EC Connector. For EC,
@@ -174,6 +169,7 @@ class ECConnectorBase(ABC):
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
data hashes (`mm_hash`) to encoder cache tensors.
kwargs (dict): Additional keyword arguments for the connector.
"""
pass
@@ -181,8 +177,7 @@ class ECConnectorBase(ABC):
def save_caches(
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs
) -> None:
"""
Save the encoder cache to the connector.
"""Save the encoder cache to the connector.
This method saves the encoder cache from the worker's local storage
to shared storage or another external connector.
@@ -192,14 +187,14 @@ class ECConnectorBase(ABC):
data hashes (`mm_hash`) to encoder cache tensors.
mm_hash (str): The hash of the multimodal data whose cache is being saved.
kwargs (dict): Additional keyword arguments for the connector.
"""
pass
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
"""Notifies worker-side connector ids of requests that have
finished generating tokens on the worker.
The scheduler process (via the Executors) will use this output
to track which workers are done.
@@ -210,16 +205,17 @@ class ECConnectorBase(ABC):
tuple of (sending/saving ids, recving/loading ids).
The finished saves/sends req ids must belong to a set provided in a
call to this method (this call or a prior one).
"""
return None, None
def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None:
"""
Build the ECConnector worker metadata for this engine step.
"""Build the ECConnector worker metadata for this engine step.
Returns:
ECConnectorWorkerMetadata: the worker metadata.
None if no worker metadata is available.
"""
return None
@@ -232,8 +228,7 @@ class ECConnectorBase(ABC):
self,
identifier: str,
) -> bool:
"""
Check if a single encoder cache exists
"""Check if a single encoder cache exists.
Args:
identifier (str): the identifier of the media.
@@ -241,14 +236,14 @@ class ECConnectorBase(ABC):
Returns:
A bool where value is True if cache exist for
the media
"""
pass
def ensure_cache_available(
self, request: "Request", num_computed_tokens: int
) -> bool:
"""
Ensure encoder cache items are available for the given request.
"""Ensure encoder cache items are available for the given request.
May initiate asynchronous transfers for items not yet local.
Args:
@@ -258,6 +253,7 @@ class ECConnectorBase(ABC):
Returns:
True if all items are ready or no transfer is needed.
False if any items are still in transit (request should be deferred).
"""
return True
@@ -282,11 +278,12 @@ class ECConnectorBase(ABC):
@abstractmethod
def update_state_after_alloc(self, request: "Request", index: int):
"""
Update ECConnector state to decide allocate cache for requests
"""Update ECConnector state to decide allocate cache for requests.
Args:
request (Request): the request object.
index (int): index of the encoder cache entry being allocated.
"""
pass
@@ -294,37 +291,37 @@ class ECConnectorBase(ABC):
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> ECConnectorMetadata:
"""
Build the connector metadata for this step.
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
pass
def update_connector_output(self, connector_output: ECConnectorOutput):
"""
Update ECConnector state from worker-side connectors output.
"""Update ECConnector state from worker-side connectors output.
Args:
connector_output (ECConnectorOutput): the worker-side
connectors output.
"""
return
def request_finished(
self, request: "Request"
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its encoder cache is freed.
"""Called when a request has finished, before its encoder cache is freed.
Returns:
True if the request is being saved/sent asynchronously and cached
should not be freed until the request_id is returned from
get_finished().
"""
return False, None
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Lightweight mmap-backed shared memory region for encoder cache (EC) data.
"""Lightweight mmap-backed shared memory region for encoder cache (EC) data.
Modeled after SharedOffloadRegion (vllm/v1/kv_offload/cpu/) but simplified
for EC: flat shared layout, no multi-tensor cursor, no block_size_factor.
@@ -229,6 +229,7 @@ class ECCPUScheduler:
Returns:
True if no item needs to wait for a remote encoding. False if any
item is pending or the request must be aborted.
"""
if not request.ec_transfer_params:
return True
@@ -259,6 +260,7 @@ class ECCPUScheduler:
Raises:
_RemoteUnavailable: The remote source is invalid, the read failed,
or the item's total wait budget expired.
"""
pos = feature.mm_position
mm_hash = feature.identifier
@@ -334,6 +336,7 @@ class ECCPUScheduler:
item's remote announcement so subsequent steps do not retry it.
False if only placeholder metadata (or no input) is available,
recording the request for take_unavailable_requests() to drain.
"""
mm_hash = feature.identifier
self._deferred_since.pop((request.request_id, mm_hash), None)
@@ -15,7 +15,8 @@ from collections import OrderedDict
class CacheEntry:
"""A single cache entry. Read `.block_ids` and `.ready` freely;
mutations only through EmbeddingCache methods."""
mutations only through EmbeddingCache methods.
"""
__slots__ = ("block_ids", "_pin_count")
@@ -67,8 +67,7 @@ class ECExampleConnector(ECConnectorBase):
raise ValueError("ec_transfer_config must be set for ECConnectorBase")
def start_load_caches(self, encoder_cache, **kwargs) -> None:
"""
Start loading the cache from the connector into vLLM's encoder cache.
"""Start loading the cache from the connector into vLLM's encoder cache.
This method loads the encoder cache based on metadata provided by the scheduler.
It is called before `_gather_mm_embeddings` for the EC Connector. For EC,
@@ -78,6 +77,7 @@ class ECExampleConnector(ECConnectorBase):
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
data hashes (`mm_hash`) to encoder cache tensors.
kwargs (dict): Additional keyword arguments for the connector.
"""
from vllm.platforms import current_platform
@@ -108,8 +108,7 @@ class ECExampleConnector(ECConnectorBase):
logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash)
def save_caches(self, encoder_cache, mm_hash, **kwargs) -> None:
"""
Save the encoder cache to the connector.
"""Save the encoder cache to the connector.
This method saves the encoder cache from the worker's local storage
to shared storage or another external connector.
@@ -119,6 +118,7 @@ class ECExampleConnector(ECConnectorBase):
data hashes (`mm_hash`) to encoder cache tensors.
mm_hash (str): The hash of the multimodal data whose cache is being saved.
kwargs (dict): Additional keyword arguments for the connector.
"""
# Return if it is PD Instance
if not self.is_producer:
@@ -134,14 +134,14 @@ class ECExampleConnector(ECConnectorBase):
self,
identifier: str,
) -> bool:
"""
Check if cache exist externally for the media
"""Check if cache exist externally for the media.
Args:
identifier (str): the identifier of the media.
Returns:
Bool indicate that media exists in cache or not
"""
return self._found_match_for_mm_data(identifier)
@@ -150,9 +150,7 @@ class ECExampleConnector(ECConnectorBase):
request: "Request",
index: int,
) -> None:
"""
Update ECConnector state after encoder cache allocation.
"""
"""Update ECConnector state after encoder cache allocation."""
mm_hash = request.mm_features[index].identifier
# Only load cache if it is consumer and cache exists
if not self.is_consumer or not self.has_cache_item(mm_hash):
@@ -212,8 +210,7 @@ class ECExampleConnector(ECConnectorBase):
mm_hash: str,
create_folder: bool = True, # <- now defaults to True
) -> str:
"""
Return the folder in which the cache for this mm_hash lives.
"""Return the folder in which the cache for this mm_hash lives.
If `create_folder` is True (default) the directory is created
recursively the first time it is needed.
"""
@@ -223,8 +220,7 @@ class ECExampleConnector(ECConnectorBase):
return foldername
def _generate_filename_debug(self, mm_hash: str) -> str:
"""
Return the full path of the safetensors file for this mm_hash.
"""Return the full path of the safetensors file for this mm_hash.
Ensures the parent directory exists because
`_generate_foldername_debug` is called with its default
(`create_folder=True`).
@@ -24,10 +24,7 @@ def has_ec_transfer() -> bool:
def ensure_ec_transfer_initialized(vllm_config: "VllmConfig") -> None:
"""
Initialize EC cache connector.
"""
"""Initialize EC cache connector."""
global _EC_CONNECTOR_AGENT
if vllm_config.ec_transfer_config is None:
+1 -3
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
The async worker that transfers experts in the background.
"""
"""The async worker that transfers experts in the background."""
import threading
from typing import TYPE_CHECKING
+7 -5
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
EPLB communicator implementations and factory.
"""
"""EPLB communicator implementations and factory."""
import contextlib
import time
@@ -86,7 +84,8 @@ class EplbCommunicator(ABC):
@property
def needs_profile_buffer_reservation(self) -> bool:
"""Whether the profile path must run a dummy collective operation to reserve
communication buffers."""
communication buffers.
"""
return True
def set_stream(self, cuda_stream: torch.cuda.Stream | None) -> None:
@@ -258,6 +257,7 @@ class NixlEplbCommunicator(EplbCommunicator):
cpu_group: CPU process group for metadata exchange.
all_expert_weights: Expert weight tensors for all MoE layers.
expert_buffer: Pre-allocated receive buffer tensors.
"""
assert all_expert_weights, (
"NixlEplbCommunicator requires non-empty all_expert_weights."
@@ -443,7 +443,8 @@ class NixlEplbCommunicator(EplbCommunicator):
def _exchange_remote_send_meta(self) -> None:
"""Exchange per-layer per-tensor metadata so receivers can compute
remote RDMA addresses at transfer time."""
remote RDMA addresses at transfer time.
"""
local_meta: dict[tuple[int, int], tuple[int, int, int]] = {}
for layer_idx, layer_tensors in enumerate(self._all_expert_weights):
for t_idx, t in enumerate(layer_tensors):
@@ -666,6 +667,7 @@ def create_eplb_communicator(
zero-copy RDMA reads.
expert_buffer: Pre-allocated receive buffer tensors (one per
weight tensor in a single layer).
"""
first_layer = expert_weights[0] if expert_weights else []
tensor_device_type = first_layer[0].device.type if first_layer else "cpu"
+18 -33
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Expert parallelism load balancer (EPLB) metrics and states.
"""Expert parallelism load balancer (EPLB) metrics and states.
# Glossary
@@ -74,9 +73,7 @@ def _compute_eplb_load_stats(
@dataclass
class EplbStats:
"""
Model stats used in EPLB rebalancing algorithm.
"""
"""Model stats used in EPLB rebalancing algorithm."""
global_expert_load_window: torch.Tensor
"""
@@ -233,9 +230,7 @@ class EplbModelState:
class EplbState:
"""
EplbState of each expert parallel model. Key is the model config hash.
"""
"""EplbState of each expert parallel model. Key is the model config hash."""
def __init__(self, parallel_config: ParallelConfig, device: torch.device):
self.parallel_config = parallel_config
@@ -305,14 +300,14 @@ class EplbState:
num_routed_experts: int,
num_redundant_experts: int,
) -> Sequence[int]:
"""
Build an initial expert arrangement using the following structure:
"""Build an initial expert arrangement using the following structure:
[original routed experts, redundant experts]
Returns:
physical_to_logical_map (Sequence[int]): A list of integers,
where each integer is the index of the logical expert
that the corresponding physical expert maps to.
"""
global_physical_to_logical_map = list(range(num_routed_experts))
global_physical_to_logical_map += [
@@ -321,8 +316,7 @@ class EplbState:
return global_physical_to_logical_map
def validate_ep_configuration(self, new_model: MixtureOfExperts):
"""
Validate that the expert parallel configuration of
"""Validate that the expert parallel configuration of
the new model is the same as the existing models.
"""
if len(self.model_states) > 0:
@@ -361,9 +355,7 @@ class EplbState:
model: MixtureOfExperts,
model_config: ModelConfig,
):
"""
Build the initial EPLB state.
"""
"""Build the initial EPLB state."""
self.validate_ep_configuration(model)
self.is_async = self.parallel_config.eplb_config.use_async
@@ -534,6 +526,7 @@ class EplbState:
ubatch_slices: When DBO is active, a list of
``UBatchSlice`` objects describing each micro-batch's
token range. When ``None``, only ``tensors[0]`` is filled.
"""
model_state = self.model_states.get(compute_hash_cached(model_config))
if model_state is None or model_state.num_unpadded_tokens_tensors is None:
@@ -556,8 +549,7 @@ class EplbState:
is_profile: bool = False,
log_stats: bool = False,
) -> None:
"""
Step the EPLB state.
"""Step the EPLB state.
Args:
is_dummy (bool): If `True`, this is a dummy step and the load
@@ -574,6 +566,7 @@ class EplbState:
- `avg_tokens`: The average load across ranks.
- `max_tokens`: The maximum load across ranks.
- `balancedness`: The ratio of average load to maximum load.
"""
ep_group = get_ep_group().device_group
if is_profile:
@@ -752,8 +745,7 @@ class EplbState:
is_profile: bool = False,
rank_mapping: dict[int, int] | None = None,
) -> torch.Tensor | None:
"""
Rearrange the experts according to the current load.
"""Rearrange the experts according to the current load.
Args:
is_profile (bool): If `True`, perform a dummy rearrangement.
@@ -761,8 +753,8 @@ class EplbState:
no memory movement will be performed. Default is False.
rank_mapping (dict[int, int] | None): The rank mapping
when scaling is done in EEP.
"""
"""
ep_group = get_ep_group().device_group
ep_rank = ep_group.rank()
@@ -1033,9 +1025,7 @@ class EplbState:
return int(flag.item()) == device_group.size()
def _allreduce_list(self, tensor_list: list[torch.Tensor]) -> list[torch.Tensor]:
"""
All-reduce a list of tensors.
"""
"""All-reduce a list of tensors."""
ep_group = get_ep_group().device_group
if len(tensor_list) == 1:
all_reduce(tensor_list[0], group=ep_group)
@@ -1060,8 +1050,7 @@ class EplbState:
return all_reduce_list
def _sync_load_pass(self) -> list[torch.Tensor]:
"""
Sync the expert load pass across all ranks for log stats.
"""Sync the expert load pass across all ranks for log stats.
Doesn't update the expert load pass in eplb_model_state.
"""
load_pass_list = []
@@ -1250,8 +1239,7 @@ def compute_logical_maps(
physical_to_logical_map: torch.Tensor,
num_logical_experts: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Derive logical_to_physical_map and logical_replica_count from
"""Derive logical_to_physical_map and logical_replica_count from
physical_to_logical_map.
Args:
@@ -1264,6 +1252,7 @@ def compute_logical_maps(
physical slots per logical expert; -1 where unused
logical_replica_count: [num_layers, num_logical_experts], number of
physical replicas per logical expert
"""
device = physical_to_logical_map.device
assert physical_to_logical_map.device.type == "cpu"
@@ -1339,13 +1328,11 @@ def _commit_eplb_maps_for_layer(
new_physical_to_logical_map: torch.Tensor,
layer: int,
) -> None:
"""
Per-layer version of _commit_eplb_maps that's used by the sync portion of EPLB
"""Per-layer version of _commit_eplb_maps that's used by the sync portion of EPLB
when running async EPLB. Copies all of the new_* maps into model_state. After this
function completes, the new mappings will become the current mappings and will be
visible to the model.
"""
# Commit physical_to_logical_map
src = new_physical_to_logical_map
dst = model_state.physical_to_logical_map[layer]
@@ -1376,12 +1363,10 @@ def _commit_eplb_maps(
model_state: EplbModelState,
new_physical_to_logical_map: torch.Tensor,
) -> None:
"""
Copies all of the new_* maps into model_state. After this function completes,
"""Copies all of the new_* maps into model_state. After this function completes,
the new mappings will become the current mappings and will be visible to the
model.
"""
# Commit physical_to_logical_map
src = new_physical_to_logical_map
dst = model_state.physical_to_logical_map
+5 -8
View File
@@ -14,8 +14,7 @@ logger = init_logger(__name__)
class CpuGpuEvent:
"""
Combines a CUDA event with a CPU threading event to enforce record->wait
"""Combines a CUDA event with a CPU threading event to enforce record->wait
ordering across two threads.
This class is designed for exactly two threads: one producer that calls
@@ -35,8 +34,7 @@ class CpuGpuEvent:
self._recorded = threading.Event()
def wait(self, stream: torch.cuda.Stream | None = None):
"""
Blocks the calling thread until record finishes. Used to guarantee that the
"""Blocks the calling thread until record finishes. Used to guarantee that the
record kernel is called before wait.
Should only be called by the Async Eplb thread.
@@ -46,8 +44,7 @@ class CpuGpuEvent:
self._recorded.clear()
def record(self, stream: torch.cuda.Stream | None = None):
"""
Unblocks the waiting thread after calling event.record().
"""Unblocks the waiting thread after calling event.record().
Should only be called by the main thread.
"""
@@ -65,12 +62,12 @@ def override_envs_for_eplb(
parallel_config: ParallelConfig,
moe_backend: str | None = None,
) -> None:
"""
Override environment variables for EPLB when specific conditions are met.
"""Override environment variables for EPLB when specific conditions are met.
Args:
parallel_config: The parallel configuration object.
moe_backend: The configured MoE backend (e.g. ``deep_gemm_mega_moe``).
"""
is_data_parallel = parallel_config.data_parallel_size > 1
is_eplb_enabled = parallel_config.enable_eplb
+4 -3
View File
@@ -18,10 +18,9 @@ class AbstractEplbPolicy(ABC):
num_ranks: int,
old_global_expert_indices: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Entry point for expert-parallelism load balancer.
"""Entry point for expert-parallelism load balancer.
Parameters:
Args:
weight: [layers, num_logical_experts], the load statistics
for all logical experts
num_replicas: number of physical experts, must be a multiple of
@@ -32,8 +31,10 @@ class AbstractEplbPolicy(ABC):
old_global_expert_indices: [layers, num_logical_experts], the old global
expert indices. Used to avoid unnecessary weight copying
for experts moving within one rank.
Returns:
physical_to_logical_map: [layers, num_replicas], the expert
index of each replica
"""
raise NotImplementedError
+15 -15
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Expert parallelism load balancer (EPLB) for vLLM.
"""Expert parallelism load balancer (EPLB) for vLLM.
This module implements the core rearrangement algorithm.
@@ -23,17 +22,17 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
def balanced_packing(
cls, weight: np.ndarray, num_packs: int
) -> tuple[np.ndarray, np.ndarray]:
"""
Pack n weighted objects to m packs, such that each bin contains exactly
"""Pack n weighted objects to m packs, such that each bin contains exactly
n/m objects and the weights of all packs are as balanced as possible.
Parameters:
Args:
weight: [X, n], the weight of each item
num_packs: number of packs
Returns:
pack_index: [X, n], the pack index of each item
rank_in_pack: [X, n], the rank of the item in the pack
"""
num_layers, num_groups = weight.shape
assert num_groups % num_packs == 0
@@ -76,17 +75,17 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
def replicate_experts(
cls, weight: np.ndarray, num_phy: int
) -> tuple[np.ndarray, np.ndarray]:
"""
Replicate `num_log` experts to `num_phy` replicas, such that the maximum
"""Replicate `num_log` experts to `num_phy` replicas, such that the maximum
load of all replicas is minimized.
Parameters:
Args:
weight: [X, num_log]
num_phy: total number of experts after replication
Returns:
phy2log: [X, num_phy], logical expert id of each physical expert
logcnt: [X, num_log], number of replicas for each logical expert
"""
n, num_log = weight.shape
num_redundant = num_phy - num_log
@@ -109,8 +108,7 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
num_nodes: int,
num_gpus: int,
) -> np.ndarray:
"""
Parameters:
"""Parameters
weight: [num_moe_layers, num_logical_experts]
num_physical_experts: number of physical experts after replication
num_groups: number of expert groups
@@ -121,6 +119,7 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
Returns:
phy2log: [layers, num_replicas], the expert
index of each replica
"""
num_layers, num_logical_experts = weight.shape
assert num_logical_experts % num_groups == 0
@@ -195,8 +194,7 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
num_ranks: int,
old_phy2log: np.ndarray,
) -> np.ndarray:
"""
Reorder the new mapping per GPU so that experts that remain on the same GPU
"""Reorder the new mapping per GPU so that experts that remain on the same GPU
keep their previous slot positions when possible. Incoming experts to that GPU
fill any remaining available slots. This is applied only when the number of GPUs
is unchanged and the slots per GPU remain the same between
@@ -281,10 +279,9 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
num_ranks: int,
old_global_expert_indices: torch.Tensor | None = None,
) -> torch.Tensor:
"""
Entry point for expert-parallelism load balancer.
"""Entry point for expert-parallelism load balancer.
Parameters:
Args:
weight: [layers, num_logical_experts], the load statistics for all
logical experts
num_replicas: number of physical experts, must be a multiple of
@@ -296,9 +293,12 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
old_global_expert_indices: [layers, num_logical_experts], the old global
expert indices. Used to avoid unnecessary weight copying
for experts moving within one rank.
Returns:
-------
phy2log: [layers, num_replicas], the expert
index of each replica
"""
weight_np = weight.float().cpu().numpy()
old_phy2log_np = (
+15 -18
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
The actual execution of the rearrangement.
"""The actual execution of the rearrangement.
This involves the exchange of expert weights between GPUs.
"""
@@ -41,9 +40,7 @@ class TransferMetadata:
@dataclass
class AsyncEplbLayerResult:
"""
The result of one completed async EPLB layer transfer.
"""
"""The result of one completed async EPLB layer transfer."""
layer_idx: int
"""Index of the MoE layer that was transferred."""
@@ -69,8 +66,7 @@ def get_ep_ranks_with_experts_batch(
old_indices: np.ndarray,
new_indices: np.ndarray,
) -> tuple[dict[int, list[int]], dict[int, list[int]]]:
"""
Get the ranks of the experts that need to be exchanged.
"""Get the ranks of the experts that need to be exchanged.
Args:
expert_ids: 1D array of expert indices to query.
@@ -82,6 +78,7 @@ def get_ep_ranks_with_experts_batch(
A tuple of two dictionaries mapping expert_id to:
- ranks_to_send: The ranks that have this expert and need to send.
- ranks_to_recv: The ranks that need to receive this expert.
"""
ranks_to_send_map: dict[int, list[int]] = {}
ranks_to_recv_map: dict[int, list[int]] = {}
@@ -181,8 +178,7 @@ def move_to_buffer(
communicator: EplbCommunicator,
layer_idx: int = 0,
) -> TransferMetadata:
"""
Rearranges expert weights during EPLB rebalancing.
"""Rearranges expert weights during EPLB rebalancing.
Args:
num_local_experts: Number of local experts.
@@ -199,6 +195,7 @@ def move_to_buffer(
Returns:
TransferMetadata: Metadata needed for completing remote weight transfers.
"""
assert old_indices.shape == new_indices.shape
recv_primary_mask = np.zeros((num_local_experts,), dtype=np.bool_)
@@ -355,9 +352,8 @@ def move_from_buffer(
new_indices: np.ndarray,
ep_rank: int,
) -> None:
"""
Copies expert weights from communication buffers back to the target weight tensors
after EPLB rebalancing.
"""Copies expert weights from communication buffers back to the target weight
tensors after EPLB rebalancing.
Args:
expert_weights: List of the actual MoE layer weights used in the execution.
@@ -367,6 +363,7 @@ def move_from_buffer(
new_indices: (num_experts_total,) mapping from local rows to desired
(possibly global) expert id, after rebalance.
ep_rank: Rank of the process in the expert parallel group.
"""
is_unchanged = transfer_metadata.is_unchanged
is_received_locally = transfer_metadata.is_received_locally
@@ -437,8 +434,7 @@ def transfer_layer(
rank_mapping: dict[int, int] | None = None,
layer_idx: int = 0,
) -> TransferMetadata:
"""
Rearranges the expert weights in place according to the new expert indices.
"""Rearranges the expert weights in place according to the new expert indices.
The value of the indices arguments are logical indices of the experts,
while keys are physical.
@@ -462,6 +458,7 @@ def transfer_layer(
Returns:
TransferMetadata: Metadata needed for completing remote weight transfers,
including is_unchanged and is_received_locally masks.
"""
ep_size = ep_group.size()
if rank_mapping is not None:
@@ -519,8 +516,7 @@ def rearrange_expert_weights_inplace(
is_profile: bool = False,
rank_mapping: dict[int, int] | None = None,
) -> None:
"""
Rearranges the expert weights in place according to the new expert indices.
"""Rearranges the expert weights in place according to the new expert indices.
The value of the indices arguments are logical indices of the experts,
while keys are physical.
@@ -540,6 +536,7 @@ def rearrange_expert_weights_inplace(
This is used during profile run, where we only perform dummy
communications to reserve enough memory for the buffers.
rank_mapping: A dictionary mapping old rank to new rank.
"""
if rank_mapping is not None:
if len(rank_mapping) == ep_group.size():
@@ -624,8 +621,7 @@ def _map_old_expert_indices_with_rank_mapping(
rank_mapping: dict[int, int],
new_ep_size: int,
) -> torch.Tensor:
"""
Map the old global expert indices to the new global expert indices.
"""Map the old global expert indices to the new global expert indices.
Args:
old_global_expert_indices:
@@ -636,6 +632,7 @@ def _map_old_expert_indices_with_rank_mapping(
Returns:
Mapped expert indices with shape
(num_layers, new_ep_size * num_local_physical_experts).
"""
num_layers, old_num_physical_experts = old_global_expert_indices.shape
assert rank_mapping, "Rank mapping is required"
+17 -21
View File
@@ -39,7 +39,7 @@ class KVCacheEvent(
gc=False, # type: ignore[call-arg]
tag=True,
):
"""Base class for all KV cache-related events"""
"""Base class for all KV cache-related events."""
MEDIUM_GPU = "GPU"
@@ -135,8 +135,7 @@ class KVEventBatch(EventBatch):
class KVEventAggregator:
"""
Aggregates KV events across multiple workers.
"""Aggregates KV events across multiple workers.
Tracks how many times each event appears and returns only those
that were emitted by all workers.
"""
@@ -150,22 +149,22 @@ class KVEventAggregator:
self._num_workers: int = num_workers
def add_events(self, events: list[KVCacheEvent]) -> None:
"""
Add events from a worker batch.
"""Add events from a worker batch.
Args:
events: List of KVCacheEvent objects.
"""
if not isinstance(events, list):
raise TypeError("events must be a list of KVCacheEvent.")
self._event_counter.update(events)
def get_common_events(self) -> list[KVCacheEvent]:
"""
Return events that appeared in all workers.
"""Return events that appeared in all workers.
Returns:
List of events present in all workers.
"""
return [
event
@@ -174,43 +173,39 @@ class KVEventAggregator:
]
def get_all_events(self) -> list[KVCacheEvent]:
"""
Return all events for all workers.
"""Return all events for all workers.
Returns:
List of events for all workers.
"""
return list(self._event_counter.elements())
def clear_events(self) -> None:
"""
Clear all tracked events.
"""
"""Clear all tracked events."""
self._event_counter.clear()
def increment_workers(self, count: int = 1) -> None:
"""
Increment the number of workers contributing events.
"""Increment the number of workers contributing events.
Args:
count: Number to increment the workers by.
"""
if count <= 0:
raise ValueError("count must be positive.")
self._num_workers += count
def reset_workers(self) -> None:
"""
Reset the number of workers to 1.
"""
"""Reset the number of workers to 1."""
self._num_workers = 1
def get_number_of_workers(self) -> int:
"""
Return the number of workers.
"""Return the number of workers.
Returns:
int number of workers.
"""
return self._num_workers
@@ -222,8 +217,7 @@ class KVEventAggregator:
class KVConnectorKVEvents(ABC):
"""
Abstract base class for KV events.
"""Abstract base class for KV events.
Acts as a container for KV events from the connector.
"""
@@ -323,6 +317,7 @@ class ZmqEventPublisher(EventPublisher):
Maximum number of events to buffer in memory.
topic:
Topic to publish events to.
"""
SHUTDOWN_TIMEOUT: float = 1.0
@@ -520,6 +515,7 @@ class ZmqEventPublisher(EventPublisher):
Returns:
The endpoint with the port offset by data_parallel_rank
or suffix appended
"""
# Do nothing if input is None or data_parallel_rank is 0
if not endpoint or data_parallel_rank == 0:
@@ -87,6 +87,7 @@ class KVConnectorFactory:
Returns:
The connector class.
"""
if connector_name not in cls._registry:
raise ValueError(f"Connector '{connector_name}' is not registered.")
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
KV cache helper for store.
"""
"""KV cache helper for store."""
from collections.abc import Iterator
from dataclasses import dataclass
@@ -53,7 +51,8 @@ def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
class KVOutputAggregator:
"""Utility class to aggregate the output of all workers into a single
output corresponding to Rank 0 for scheduler."""
output corresponding to Rank 0 for scheduler.
"""
def __init__(self, expected_finished_count: int):
# Complete transfer tracker. Used to track finished requests
@@ -233,11 +232,10 @@ def copy_kv_blocks(
def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio):
"""
Transforms the layout of received KV cache blocks to the local block_size.
"""Transforms the layout of received KV cache blocks to the local block_size.
(Only works for local blocksize > remote blocksize)
example:
Example:
local blocksize = 16 tokens, remote blocksize = 4 tokens
local block[0] = remote block[0, 1, 2, 3]
remote is |h0-b0|h1-b0|h2-b0|h3-b0|h0-b1|h1-b1|h2-b1|h3-b1|...
@@ -246,6 +244,7 @@ def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio):
1. view => view remote as n_blocks * remote_shape(H,remoteN,D)
2. permute => (H, nblocks, remoteN, D)
3. flatten => (H, localN, D)
"""
blocks_to_update = cache.index_select(0, indices)
# use physical order
@@ -293,8 +292,7 @@ def kv_postprocess_layout_on_receive(cache, indices):
def kv_postprocess_blksize_and_layout_on_receive(cache, indices, block_size_ratio):
"""
Transforms the layout of received KV cache to the local block_size and LBHNC.
"""Transforms the layout of received KV cache to the local block_size and LBHNC.
(Only works for local blocksize > remote blocksize)
prefill is LBHNC, smaller block_size
@@ -317,9 +315,9 @@ def kv_postprocess_blksize_and_layout_on_receive(cache, indices, block_size_rati
def yield_req_data(
scheduler_output,
) -> Iterator[tuple[str, tuple[list[int], ...] | None, bool]]:
"""
Yields:
(req_id, new_block_id_groups, preempted)
"""Yields:
(req_id, new_block_id_groups, preempted)
"""
# new requests
for req_data in scheduler_output.scheduled_new_reqs:
@@ -346,6 +344,7 @@ def get_current_attn_backends(
Returns:
Deduplicated list of attention backend classes.
"""
layer_type = cast(type[Any], AttentionLayerBase)
layers = get_layers_from_vllm_config(vllm_config, layer_type, layer_names)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
KVConnectorBase_V1 Class for Distributed KV Cache & Hidden State
"""KVConnectorBase_V1 Class for Distributed KV Cache & Hidden State
communication in vLLM v1
The class provides the following primitives:
@@ -97,8 +96,7 @@ class KVConnectorTransferResults:
class SupportsHMA(ABC):
"""
The class that indicates the corresponding connector supports hybrid memory
"""The class that indicates the corresponding connector supports hybrid memory
allocator (HMA).
This is required to use the connector together with hybrid memory allocator.
"""
@@ -109,8 +107,7 @@ class SupportsHMA(ABC):
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
"""
Called exactly once when a request has finished for all kv cache groups,
"""Called exactly once when a request has finished for all kv cache groups,
before its blocks are freed for each group.
NOTE(Kuntai): This function is only supported by connectors that support HMA.
@@ -124,6 +121,7 @@ class SupportsHMA(ABC):
get_finished().
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
raise NotImplementedError
@@ -144,8 +142,7 @@ class KVConnectorRole(enum.Enum):
class KVConnectorHandshakeMetadata(ABC): # noqa: B024
"""
Metadata used for out of band connector handshake between
"""Metadata used for out of band connector handshake between
P/D workers. This needs to serializable.
"""
@@ -153,8 +150,7 @@ class KVConnectorHandshakeMetadata(ABC): # noqa: B024
class KVConnectorMetadata(ABC): # noqa: B024
"""
Abstract Metadata used to communicate
"""Abstract Metadata used to communicate
Scheduler KVConnector -> Worker KVConnector.
"""
@@ -162,8 +158,7 @@ class KVConnectorMetadata(ABC): # noqa: B024
class KVConnectorWorkerMetadata(ABC):
"""
Abstract Metadata used to communicate back
"""Abstract Metadata used to communicate back
Worker KVConnector -> Scheduler KVConnector.
Each worker can output its own metadata.
@@ -176,16 +171,12 @@ class KVConnectorWorkerMetadata(ABC):
def aggregate(
self, other: "KVConnectorWorkerMetadata"
) -> "KVConnectorWorkerMetadata":
"""
Aggregate metadata with another `KVConnectorWorkerMetadata` object.
"""
"""Aggregate metadata with another `KVConnectorWorkerMetadata` object."""
pass
class KVConnectorBase_V1(ABC):
"""
Base class for KV connectors.
"""
"""Base class for KV connectors."""
@property
def supports_divergent_local_hybrid_hits(self) -> bool:
@@ -245,6 +236,7 @@ class KVConnectorBase_V1(ABC):
Args:
connector_metadata (dict): the connector metadata.
"""
self._connector_metadata = connector_metadata
@@ -263,6 +255,7 @@ class KVConnectorBase_V1(ABC):
Returns:
ConnectorMetadata: the connector metadata.
"""
# Should only be called while set to valid metadata.
assert self._connector_metadata is not None
@@ -273,22 +266,22 @@ class KVConnectorBase_V1(ABC):
Returns:
bool: True if connector metadata exists, False otherwise.
"""
return self._connector_metadata is not None
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""
Initialize with the KV caches. Useful for pre-registering the
"""Initialize with the KV caches. Useful for pre-registering the
KV Caches in the KVConnector (e.g. for NIXL).
Args:
kv_caches: dictionary of layer names, kv cache
"""
return
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
"""
Set the xPU-specific ops for copying KV between host and device.
"""Set the xPU-specific ops for copying KV between host and device.
Needed when host buffer is used for kv transfer (e.g., in NixlConnector)
"""
return
@@ -303,16 +296,14 @@ class KVConnectorBase_V1(ABC):
return
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata):
"""
Handle preempted requests or evicted blocks BEFORE they are overwritten.
"""Handle preempted requests or evicted blocks BEFORE they are overwritten.
Needed for connectors which use async saves (e.g., OffloadingConnector)
"""
return
@abstractmethod
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""
Start loading the KV cache from the connector to vLLM's paged
"""Start loading the KV cache from the connector to vLLM's paged
KV buffer. Loads required by the current forward start before it;
independent asynchronous loads may start after it is submitted.
@@ -329,8 +320,7 @@ class KVConnectorBase_V1(ABC):
@abstractmethod
def wait_for_layer_load(self, layer_name: str) -> None:
"""
Block until the KV for a specific layer is loaded into vLLM's
"""Block until the KV for a specific layer is loaded into vLLM's
paged buffer. This is called from within attention layer to ensure
async copying from start_load_kv is complete.
@@ -338,6 +328,7 @@ class KVConnectorBase_V1(ABC):
Args:
layer_name: the name of that layer
"""
pass
@@ -349,8 +340,7 @@ class KVConnectorBase_V1(ABC):
attn_metadata: "AttentionMetadata",
**kwargs: Any,
) -> None:
"""
Start saving a layer of KV cache from vLLM's paged buffer
"""Start saving a layer of KV cache from vLLM's paged buffer
to the connector. This is called from within attention layer to
enable async copying during execution.
@@ -360,13 +350,13 @@ class KVConnectorBase_V1(ABC):
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
**kwargs: additional arguments for the save operation.
"""
pass
@abstractmethod
def wait_for_save(self):
"""
Block until all the save operations is done. This is called
"""Block until all the save operations is done. This is called
as the forward context exits to ensure that the async saving
from save_kv_layer is complete before finishing the forward.
@@ -377,8 +367,7 @@ class KVConnectorBase_V1(ABC):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
"""Notifies worker-side connector ids of requests that have
finished generating tokens on the worker.
The scheduler process (via the Executors) will use this output
to track which workers are done.
@@ -389,6 +378,7 @@ class KVConnectorBase_V1(ABC):
tuple of (sending/saving ids, recving/loading ids).
The finished saves/sends req ids must belong to a set provided in a
call to this method (this call or a prior one).
"""
return None, None
@@ -403,8 +393,7 @@ class KVConnectorBase_V1(ABC):
)
def get_block_ids_with_load_errors(self) -> set[int]:
"""
Get the set of block IDs that failed to load.
"""Get the set of block IDs that failed to load.
Returns:
Set of block IDs that encountered load errors.
@@ -419,50 +408,47 @@ class KVConnectorBase_V1(ABC):
block IDs must appear here no later than that same pass.
- Sync loading: failed blocks should be reported in the forward
pass in which they are detected.
"""
return set()
def shutdown(self):
"""
Shutdown the connector. This is called when the worker process
"""Shutdown the connector. This is called when the worker process
is shutting down to ensure that all the async operations are
completed and the connector is cleaned up properly.
"""
return None
def get_kv_connector_stats(self) -> "KVConnectorStats | None":
"""
Get the KV connector stats collected during the last interval.
"""
"""Get the KV connector stats collected during the last interval."""
return None
def get_kv_connector_kv_cache_events(self) -> "KVConnectorKVEvents | None":
"""
Get the KV connector kv cache events collected during the last interval.
"""Get the KV connector kv cache events collected during the last interval.
This function should be called by the model runner every time after the
model execution and before cleanup.
"""
return None
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
"""
Get the KVConnector handshake metadata for this connector.
"""Get the KVConnector handshake metadata for this connector.
This metadata is used for out-of-band connector handshake
between P/D workers.
Returns:
KVConnectorHandshakeMetadata: the handshake metadata.
None if no handshake metadata is available.
"""
return None
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
"""
Build the KVConnector worker metadata for this engine step.
"""Build the KVConnector worker metadata for this engine step.
Returns:
KVConnectorWorkerMetadata: the worker metadata.
None if no worker metadata is available.
"""
return None
@@ -476,12 +462,12 @@ class KVConnectorBase_V1(ABC):
self.bind_gpu_block_pool(kv_cache_manager.block_pool)
def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None:
"""
Bind the GPU block pool to the connector for per-GPU block status tracking.
"""Bind the GPU block pool to the connector for per-GPU block status tracking.
For example, inc/dec ref counts, or iterate over the prefix cache blocks.
Args:
gpu_block_pool: the GPU block pool.
"""
return
@@ -491,8 +477,7 @@ class KVConnectorBase_V1(ABC):
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
@@ -517,6 +502,7 @@ class KVConnectorBase_V1(ABC):
call. If the cache cannot be loaded for some tokens (e.g., due to
connectivity issues or eviction), those tokens must not be taken
into account.
"""
pass
@@ -524,8 +510,7 @@ class KVConnectorBase_V1(ABC):
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""
Update KVConnector state after block allocation.
"""Update KVConnector state after block allocation.
If get_num_new_matched_tokens previously returned True for a
request, this function may be called twice for that same request -
@@ -543,6 +528,7 @@ class KVConnectorBase_V1(ABC):
blocks (KVCacheBlocks): the blocks allocated for the request.
num_external_tokens (int): the number of tokens to load from the
external KV cache. 0 means nothing should be loaded.
"""
pass
@@ -550,14 +536,14 @@ class KVConnectorBase_V1(ABC):
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""
Build the connector metadata for this step.
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
pass
@@ -570,12 +556,12 @@ class KVConnectorBase_V1(ABC):
return
def update_connector_output(self, connector_output: KVConnectorOutput):
"""
Update KVConnector state from worker-side connectors output.
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
return
@@ -584,8 +570,7 @@ class KVConnectorBase_V1(ABC):
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""
Called exactly once when a request has finished, before its blocks are
"""Called exactly once when a request has finished, before its blocks are
freed.
The connector may assumes responsibility for freeing the blocks
@@ -597,6 +582,7 @@ class KVConnectorBase_V1(ABC):
get_finished().
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
return False, None
@@ -614,11 +600,11 @@ class KVConnectorBase_V1(ABC):
return False
def take_events(self) -> Iterable["KVCacheEvent"]:
"""
Take the KV cache events from the connector.
"""Take the KV cache events from the connector.
Yields:
New KV cache events since the last call.
"""
return ()
@@ -640,16 +626,16 @@ class KVConnectorBase_V1(ABC):
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
"""Get the required KV cache layout for this connector.
Args:
vllm_config (VllmConfig): the vllm config.
Returns:
str: the required KV cache layout. e.g. HND, or NHD.
None if the connector does not require a specific layout.
"""
"""
if cls is KVConnectorBase_V1:
raise TypeError(
"get_required_kvcache_layout should not be called "
@@ -659,8 +645,7 @@ class KVConnectorBase_V1(ABC):
@classmethod
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
"""
Check if this connector requires PIECEWISE CUDA graph mode.
"""Check if this connector requires PIECEWISE CUDA graph mode.
Connectors that use asynchronous layer-by-layer operations
(wait_for_layer_load/save_kv_layer) should override this method
@@ -676,27 +661,26 @@ class KVConnectorBase_V1(ABC):
Returns:
True if this connector requires PIECEWISE CUDA graph mode,
False otherwise.
"""
return False
def get_finished_count(self) -> int | None:
"""
Get the count of requests expected to complete send/receive operations
"""Get the count of requests expected to complete send/receive operations
via this connector. This method is used to initialize the
KVOutputAggregator, overwriting the default world_size.
Returns:
int: expected sending or receiving completion count.
"""
"""
return None
@classmethod
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> "KVConnectorStats | None":
"""
KVConnectorStats resolution method. This method allows dynamically
"""KVConnectorStats resolution method. This method allows dynamically
registered connectors to return their own KVConnectorStats object,
which can implement custom aggregation logic on the data dict.
"""
@@ -705,19 +689,18 @@ class KVConnectorBase_V1(ABC):
def set_xfer_handshake_metadata(
self, metadata: dict[int, KVConnectorHandshakeMetadata]
) -> None:
"""
Set the KV connector handshake metadata for this connector.
"""Set the KV connector handshake metadata for this connector.
Args:
metadata (KVConnectorHandshakeMetadata): the handshake metadata to set.
"""
return None
def set_xfer_handshake_metadata_pp_aware(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
"""
Set handshake metadata keyed by (pp_rank, tp_rank).
"""Set handshake metadata keyed by (pp_rank, tp_rank).
- Default implementation assumes pp_rank is always 0
- PP-aware connectors override this to consume all PP producer shards.
"""
@@ -738,19 +721,18 @@ class KVConnectorBase_V1(ABC):
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> "KVConnectorPromMetrics | None":
"""
Create a KVConnectorPromMetrics subclass which should register
"""Create a KVConnectorPromMetrics subclass which should register
per-connector Prometheus metrics and implement observe() to
expose connector transfer stats via Prometheus.
"""
return None
def reset_cache(self) -> bool | None:
"""
Reset the connector's internal cache.
"""Reset the connector's internal cache.
Returns:
bool: True if the cache was successfully reset, False otherwise.
"""
logger.debug(
"Connector cache reset requested, but %s does not implement reset_cache().",
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
DecodeBenchConnector: A KV Connector for decode instance performance testing.
"""DecodeBenchConnector: A KV Connector for decode instance performance testing.
This connector emulates a prefill-decode disaggregated setting by filling
the KV cache with dummy values, allowing measurement of decoder performance
@@ -10,7 +9,7 @@ under larger input sequence lengths (ISL) in resource-limited environments.
Usage:
To use this connector for benchmarking, configure it in the kv_transfer_config:
Example:
Example:
vllm serve <model> --kv-transfer-config '{
"kv_connector": "DecodeBenchConnector",
"kv_role": "kv_both",
@@ -29,6 +28,7 @@ Usage:
- fill_mean (float): Mean value for random normal fill (default: 0.015)
- fill_std (float): Standard deviation for random fill (default: 0.0)
Set to 0 for constant values, >0 for random sampling
"""
from dataclasses import dataclass
@@ -80,8 +80,7 @@ class DecodeBenchConnectorMetadata(KVConnectorMetadata):
class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA):
"""
A KV Connector for decode instance performance testing.
"""A KV Connector for decode instance performance testing.
This connector fills the KV cache with dummy values to emulate a
prefill-decode disaggregated setting, enabling performance testing of the
@@ -219,8 +218,7 @@ class DecodeBenchConnectorScheduler:
request: "Request",
num_computed_tokens: int,
) -> tuple[int, bool]:
"""
For new requests, return the number of tokens that should be filled
"""For new requests, return the number of tokens that should be filled
with dummy KV cache values.
Returns:
@@ -228,6 +226,7 @@ class DecodeBenchConnectorScheduler:
- num_tokens_to_fill: number of uncomputed tokens minus 1
(we fill everything except the last token for decode)
- is_async: False (synchronous filling)
"""
req_id = request.request_id
@@ -251,8 +250,7 @@ class DecodeBenchConnectorScheduler:
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""
Called after blocks are allocated. Store the block IDs so we can
"""Called after blocks are allocated. Store the block IDs so we can
fill them with dummy values.
Supports both single- and multi-group KV cache configurations.
@@ -334,8 +332,7 @@ class DecodeBenchConnectorScheduler:
def build_connector_meta(
self, scheduler_output: "SchedulerOutput"
) -> KVConnectorMetadata:
"""
Build metadata containing information about which blocks to fill
"""Build metadata containing information about which blocks to fill
with dummy KV values.
"""
meta = DecodeBenchConnectorMetadata(reqs_to_fill=self._pending_fills.copy())
@@ -346,9 +343,7 @@ class DecodeBenchConnectorScheduler:
return meta
def request_finished(self, request: "Request"):
"""
Called when a request has finished. Clean up any state.
"""
"""Called when a request has finished. Clean up any state."""
self._filled_requests.discard(request.request_id)
@@ -389,8 +384,7 @@ class DecodeBenchConnectorWorker:
)
def start_fill_kv(self, metadata: DecodeBenchConnectorMetadata):
"""
Fill the allocated KV cache blocks with dummy values.
"""Fill the allocated KV cache blocks with dummy values.
This simulates having a populated KV cache from a prefill phase,
allowing decode performance testing with larger context sizes.
@@ -419,13 +413,13 @@ class DecodeBenchConnectorWorker:
)
def _fill_blocks(self, group_idx: int, block_ids: list[int], num_tokens: int):
"""
Fill specified blocks with dummy values for a specific KV cache group.
"""Fill specified blocks with dummy values for a specific KV cache group.
Args:
group_idx: The KV cache group index to fill
block_ids: List of block IDs to fill in this group
num_tokens: Total number of tokens to fill across these blocks
"""
if not block_ids:
return
@@ -501,6 +495,7 @@ class DecodeBenchConnectorWorker:
tensor's first dim are ignored.
fill_mean: Mean value for the fill.
fill_std: Standard deviation for the fill.
"""
# Convert block_ids to tensor on device
block_ids_tensor = torch.tensor(
@@ -551,6 +546,7 @@ class DecodeBenchConnectorWorker:
kv_cache: A state tensor to fill in its entirety.
fill_mean: Mean value for the fill.
fill_std: Standard deviation for the fill.
"""
if fill_std > 0:
kv_cache.normal_(mean=fill_mean, std=fill_std)
@@ -118,6 +118,7 @@ class ExampleConnector(KVConnectorBase_V1):
Note:
The number of elements in kv_caches and layer_names should be
the same.
"""
def inject_kv_into_layer(
@@ -134,6 +135,9 @@ class ExampleConnector(KVConnectorBase_V1):
src_kv_cache (torch.Tensor): the source KV cache.
slot_mapping (torch.Tensor): the slot mapping. In shape
[num_tokens].
attn_metadata (AttentionMetadata): attention metadata for the
current forward pass.
"""
# Deliberate bulk transfers in this reference connector.
with gpu_sync_allowed():
@@ -201,6 +205,7 @@ class ExampleConnector(KVConnectorBase_V1):
Args:
layer_name: the name of that layer
"""
return
@@ -220,6 +225,7 @@ class ExampleConnector(KVConnectorBase_V1):
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
**kwargs: additional arguments for the save operation.
"""
def extract_kv_from_layer(
@@ -260,8 +266,7 @@ class ExampleConnector(KVConnectorBase_V1):
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
@@ -272,6 +277,7 @@ class ExampleConnector(KVConnectorBase_V1):
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
# NOTE: in this debug implementation, we assume that the prompt is
# cached_prompt + newly_generated_single_token
@@ -295,8 +301,7 @@ class ExampleConnector(KVConnectorBase_V1):
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""
Update KVConnector state after block allocation.
"""Update KVConnector state after block allocation.
If blocks were allocated, add to _requests_need_load,
such that we load the KVs in the next forward pass.
@@ -315,6 +320,7 @@ class ExampleConnector(KVConnectorBase_V1):
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
meta = ExampleConnectorMetadata()
@@ -57,6 +57,7 @@ def load_hidden_states(path: str) -> dict[str, torch.Tensor]:
Returns:
Dict with "hidden_states" and "token_ids" tensors.
"""
lock_path = path + ".lock"
with open(lock_path) as lf:
@@ -96,8 +97,7 @@ class ExampleHiddenStatesConnectorMetadata(KVConnectorMetadata):
class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
"""
Simple debug implementation of a HiddenStatesConnector.
"""Simple debug implementation of a HiddenStatesConnector.
Simply extracts the hidden states from the kv cache and stores them to disk.
Must be used in conjunction with the `extract_hidden_states` spec decoding method.
@@ -278,7 +278,8 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
def _on_write_done(self, req_id: str, future: Future) -> None:
"""Surface any exception from the disk-write thread and drop the
completed future from the in-flight tracking dict."""
completed future from the in-flight tracking dict.
"""
self._req_futures.pop(req_id, None)
exc = future.exception()
if exc is not None:
@@ -436,8 +437,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
@@ -448,6 +448,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
# This connector is store-only, so we don't need to load any tokens
return 0, False
@@ -471,6 +472,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
meta = ExampleHiddenStatesConnectorMetadata()
@@ -509,8 +511,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""
Called exactly once when a request has finished, before its blocks are
"""Called exactly once when a request has finished, before its blocks are
freed.
Returns True to delay block freeing until get_finished extracts
@@ -587,16 +588,16 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
"""Get the required KV cache layout for this connector.
Args:
vllm_config (VllmConfig): the vllm config.
Returns:
str: the required KV cache layout. e.g. HND, or NHD.
None if the connector does not require a specific layout.
"""
"""
if cls is KVConnectorBase_V1:
raise TypeError(
"get_required_kvcache_layout should not be called "
@@ -96,6 +96,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Args:
forward_context (ForwardContext): the forward context.
**kwargs (Any): additional arguments (unused).
"""
self._flexkv_connector.start_load_kv(forward_context, **kwargs)
@@ -107,6 +108,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Args:
layer_name: the name of the layer (unused).
"""
self._flexkv_connector.wait_for_layer_load(layer_name)
@@ -131,6 +133,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
kv_layer (torch.Tensor): the paged KV buffer (unused).
attn_metadata (AttentionMetadata): the attention metadata (unused).
**kwargs (Any): additional arguments (unused).
"""
self._flexkv_connector.save_kv_layer(
layer_name, kv_layer, attn_metadata, **kwargs
@@ -160,6 +163,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
that have finished asynchronous transfer. The finished saves/sends
req ids must belong to a set provided in a call to this method
(this call or a prior one).
"""
return self._flexkv_connector.get_finished(finished_req_ids)
@@ -169,6 +173,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Args:
kv_caches: dictionary of layer names to kv cache tensors.
"""
self._flexkv_connector.register_kv_caches(kv_caches)
@@ -192,6 +197,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Tuple of (num_external_tokens, is_ready) where
num_external_tokens is the number of additional tokens that
can be loaded from the external KV cache.
"""
return self._flexkv_connector.get_num_new_matched_tokens(
request, num_computed_tokens
@@ -215,6 +221,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
return self._flexkv_connector.build_connector_meta(scheduler_output)
@@ -224,6 +231,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
self._flexkv_connector.update_connector_output(connector_output)
@@ -240,6 +248,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
should not be freed until the request_id is returned from
:meth:`get_finished`. kv_transfer_params is an optional dict of
KVTransferParams to be included in the request outputs.
"""
return self._flexkv_connector.request_finished(request, block_ids)
@@ -248,6 +257,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
Returns:
New KV cache events since the last call.
"""
return self._flexkv_connector.take_events()
@@ -67,6 +67,7 @@ class Hf3fsClient:
size: Total size of the storage file in bytes
bytes_per_page: Size of each page in bytes
entries: Maximum number of concurrent operations
"""
if not HF3FS_AVAILABLE:
raise ImportError(
@@ -154,7 +155,7 @@ class Hf3fsClient:
)
def _release_resources(self) -> None:
"""Release all acquired resources safely"""
"""Release all acquired resources safely."""
# iov must be released before ioring and shm
for attr in ("iov_r", "iov_w", "ior_r", "ior_w"):
obj = getattr(self, attr, None)
@@ -192,6 +193,7 @@ class Hf3fsClient:
Returns:
List of operation results (0 for success, non-zero for error)
"""
self.check(offsets, tensors)
assert self.ior_r is not None
@@ -228,11 +230,12 @@ class Hf3fsClient:
Args:
offsets: List of byte offsets to write to
tensors: List of tensors containing data to write
event: CUDA event to synchronize on before reading the tensors
Returns:
List of operation results (0 for success, non-zero for error)
"""
"""
self.check(offsets, tensors)
assert self.ior_w is not None
assert self.iov_w is not None
@@ -282,6 +285,7 @@ class Hf3fsClient:
Returns:
Size of the file in bytes
"""
return self.size
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
HF3FS KV Connector Implementation for vLLM.
"""HF3FS KV Connector Implementation for vLLM.
This module implements a KV connector that uses
the 3FS for storing and retrieving KV cache data.
@@ -99,9 +98,7 @@ logger = init_logger(__name__)
class AsyncOperationManager:
"""
Manages async save/load operations with background threads.
"""
"""Manages async save/load operations with background threads."""
def __init__(self, connector: "HF3FSKVConnector"):
# Store connector reference and extract commonly used attributes
@@ -667,9 +664,7 @@ class HF3FSKVConnector(KVConnectorBase_V1):
return self._async_manager.get_finished_operations(finished_req_ids)
def get_kv_connector_stats(self) -> Optional["KVConnectorStats"]:
"""
Get the KV connector stats collected during the last interval.
"""
"""Get the KV connector stats collected during the last interval."""
# Clear stats for next iteration
if (
hasattr(self, "_async_manager")
@@ -886,8 +881,7 @@ class HF3FSKVConnector(KVConnectorBase_V1):
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> Optional["KVConnectorStats"]:
"""
KVConnectorStats resolution method. This method allows dynamically
"""KVConnectorStats resolution method. This method allows dynamically
registered connectors to return their own KVConnectorStats object,
which can implement custom aggregation logic on the data dict.
"""
@@ -1021,7 +1015,7 @@ class HF3FSKVConnector(KVConnectorBase_V1):
@dataclass
class HF3FSKVConnectorStats(KVConnectorStats):
"""Container for transfer performance metrics"""
"""Container for transfer performance metrics."""
def __post_init__(self):
if not self.data:
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
HF3FS Metadata Server with key-based organization.
"""
"""HF3FS Metadata Server with key-based organization."""
import argparse
import logging
@@ -119,6 +117,7 @@ class GlobalMetadataState:
Returns:
Dictionary mapping key -> allocated page index
"""
with self.global_lock:
if rank not in self.rank_metadata:
@@ -167,6 +166,7 @@ class GlobalMetadataState:
rank: Rank ID that confirmed the writes
key_confirmations: List of (key, page_index) tuples
pages_to_release: List of page indices to release back to free pool
"""
with self.global_lock:
# Confirm successful writes
@@ -197,6 +197,7 @@ class GlobalMetadataState:
Returns:
List of boolean values indicating key existence and completion
"""
with self.global_lock:
results = []
@@ -218,6 +219,7 @@ class GlobalMetadataState:
Returns:
List of page indices in the same order as input keys (None if key not found)
"""
with self.global_lock:
if rank not in self.rank_metadata:
@@ -128,6 +128,7 @@ def scatter_kv_caches(
num_heads: Size of the H axis
content_size: Size of the C axis
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
"""
num_layers = len(kv_caches_ptrs)
num_tokens_in_block = len(token_indices)
@@ -184,6 +185,7 @@ def gather_kv_caches(
num_heads: Size of the H axis
content_size: Size of the C axis
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
"""
num_layers = kv_caches_ptrs.shape[0]
num_tokens_in_block = len(token_indices)
@@ -32,9 +32,7 @@ logger = init_logger(__name__)
class LMCacheKVEvents(KVConnectorKVEvents):
"""
Concrete implementation of KVConnectorKVEvents using KVEventAggregator.
"""
"""Concrete implementation of KVConnectorKVEvents using KVEventAggregator."""
def __init__(self, num_workers: int) -> None:
self._aggregator = KVEventAggregator(num_workers)
@@ -43,9 +41,7 @@ class LMCacheKVEvents(KVConnectorKVEvents):
self._aggregator.add_events(events)
def aggregate(self) -> "LMCacheKVEvents":
"""
Aggregate KV events and retain only common events.
"""
"""Aggregate KV events and retain only common events."""
common_events = self._aggregator.get_common_events()
self._aggregator.clear_events()
self._aggregator.add_events(common_events)
@@ -72,8 +68,7 @@ class LMCacheKVEvents(KVConnectorKVEvents):
class LMCacheConnectorV1(KVConnectorBase_V1):
@classmethod
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
"""
LMCache requires PIECEWISE CUDA graph mode when layerwise
"""LMCache requires PIECEWISE CUDA graph mode when layerwise
operations are enabled. The wait_for_layer_load and save_kv_layer
methods perform actual async synchronization that cannot be
captured in CUDA graphs.
@@ -118,12 +113,12 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
# Worker-side methods
# ==============================
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""
Initialize with the KV caches. Useful for pre-registering the
"""Initialize with the KV caches. Useful for pre-registering the
KV Caches in the KVConnector (e.g. for NIXL).
Args:
kv_caches: dictionary of layer names, kv cache
"""
if hasattr(self._lmcache_engine, "register_kv_caches"):
self._lmcache_engine.register_kv_caches(kv_caches)
@@ -149,8 +144,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
engine.layerwise_retrievers = []
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""
Start loading the KV cache from the connector to vLLM's paged
"""Start loading the KV cache from the connector to vLLM's paged
KV buffer. This is called from the forward context before the
forward pass to enable async loading during model execution.
@@ -166,8 +160,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
self._lmcache_engine.start_load_kv(forward_context, **kwargs)
def wait_for_layer_load(self, layer_name: str) -> None:
"""
Block until the KV for a specific layer is loaded into vLLM's
"""Block until the KV for a specific layer is loaded into vLLM's
paged buffer. This is called from within attention layer to ensure
async copying from start_load_kv is complete.
@@ -175,6 +168,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
Args:
layer_name: the name of that layer
"""
self._lmcache_engine.wait_for_layer_load(layer_name)
@@ -185,8 +179,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
"""
Start saving the a layer of KV cache from vLLM's paged buffer
"""Start saving the a layer of KV cache from vLLM's paged buffer
to the connector. This is called from within attention layer to
enable async copying during execution.
@@ -196,14 +189,14 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
**kwargs: additional arguments for the save operation.
"""
self._lmcache_engine.save_kv_layer(
layer_name, kv_layer, attn_metadata, **kwargs
)
def wait_for_save(self):
"""
Block until all the save operations is done. This is called
"""Block until all the save operations is done. This is called
as the forward context exits to ensure that the async saving
from save_kv_layer is complete before finishing the forward.
@@ -214,8 +207,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
"""Notifies worker-side connector ids of requests that have
finished generating tokens.
Returns:
@@ -224,16 +216,17 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
tuple of (sending/saving ids, recving/loading ids).
The finished saves/sends req ids must belong to a set provided in a
call to this method (this call or a prior one).
"""
return self._lmcache_engine.get_finished(finished_req_ids)
def get_block_ids_with_load_errors(self) -> set[int]:
"""
Get the set of block IDs that failed to load.
"""Get the set of block IDs that failed to load.
Returns:
Set of block IDs that encountered load errors.
Empty set if no load errors occurred.
"""
method = getattr(self._lmcache_engine, "get_block_ids_with_load_errors", None)
if callable(method):
@@ -243,10 +236,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
return set()
def get_kv_connector_kv_cache_events(self) -> LMCacheKVEvents | None:
"""
Get the KV connector kv cache events collected during the last interval.
"""
"""Get the KV connector kv cache events collected during the last interval."""
events = self._lmcache_engine.get_kv_events() # type: ignore [attr-defined]
if not events:
return None
@@ -276,8 +266,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
@@ -288,6 +277,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
return self._lmcache_engine.get_num_new_matched_tokens(
request, num_computed_tokens
@@ -296,32 +286,30 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""
Update KVConnector state after block allocation.
"""
"""Update KVConnector state after block allocation."""
self._lmcache_engine.update_state_after_alloc(request, num_external_tokens)
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""
Build the connector metadata for this step.
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
return self._lmcache_engine.build_connector_meta(scheduler_output)
def update_connector_output(self, connector_output: KVConnectorOutput):
"""
Update KVConnector state from worker-side connectors output.
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
# Get the KV events
kv_cache_events = connector_output.kv_cache_events
@@ -342,8 +330,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its blocks are freed.
"""Called when a request has finished, before its blocks are freed.
Returns:
True if the request is being saved/sent asynchronously and blocks
@@ -351,15 +338,16 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
get_finished().
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
return self._lmcache_engine.request_finished(request, block_ids)
def take_events(self) -> Iterable["KVCacheEvent"]:
"""
Take the KV cache events from the connector.
"""Take the KV cache events from the connector.
Yields:
New KV cache events since the last call.
"""
if self._kv_cache_events is not None:
self._kv_cache_events.aggregate()
@@ -44,8 +44,7 @@ def send_lmcache_request(
request_type: RequestType,
payloads: list[Any],
) -> MessagingFuture[Any]:
"""
Helper function to send the request to the LMCache multiprocess server
"""Helper function to send the request to the LMCache multiprocess server.
Args:
mq_client: The LMCache multiprocess mode message queue client
@@ -54,8 +53,8 @@ def send_lmcache_request(
Returns:
A messaging future for the request
"""
"""
future = mq_client.submit_request(
request_type, payloads, get_response_class(request_type)
)
@@ -65,14 +64,14 @@ def send_lmcache_request(
def get_lmcache_chunk_size(
mq_client: MessageQueueClient,
) -> int:
"""
Helper function to get the LMCache chunk size from the server
"""Helper function to get the LMCache chunk size from the server.
Args:
mq_client: The LMCache multiprocess mode message queue client
Returns:
An integer representing the LMCache chunk size
"""
future = send_lmcache_request(mq_client, RequestType.GET_CHUNK_SIZE, [])
chunk_size = future.result()
@@ -147,16 +146,16 @@ class LMCacheMPSchedulerAdapter:
vllm_block_size: int,
parallel_strategy: ParallelStrategy,
):
"""
Args:
server_url: The server URL for the LMCache message queue
context: The ZMQ context
"""Args:
server_url: The server URL for the LMCache message queue
context: The ZMQ context
model_name: The model name used for LMCache keys
vllm_block_size: The block size used in vLLM
parallel_strategy:
The parallel strategy, which includes `use_mla`,
`world_size`, `worker_id` and so on
model_name: The model name used for LMCache keys
vllm_block_size: The block size used in vLLM
parallel_strategy:
The parallel strategy, which includes `use_mla`,
`world_size`, `worker_id` and so on
"""
self.mq_client = MessageQueueClient(server_url, context)
@@ -195,8 +194,7 @@ class LMCacheMPSchedulerAdapter:
block_hashes: list[bytes] | None = None,
token_ids: list[int] | None = None,
) -> None:
"""
Submit a new lookup request to LMCache if there is no ongoing request.
"""Submit a new lookup request to LMCache if there is no ongoing request.
Supports both token-based and hash-based vLLM:
- token_ids: token IDs (token-based vLLM) -> single token-mode key
@@ -219,6 +217,7 @@ class LMCacheMPSchedulerAdapter:
for later retrieve operations.
In the meantime, this function will record the lookup request, and the
status of the look up request can be checked by `check_lookup_result`.
"""
if request_id in self.lookup_futures:
# Skip if there is already a lookup request
@@ -260,8 +259,7 @@ class LMCacheMPSchedulerAdapter:
@_lmcache_nvtx_annotate
def check_lookup_result(self, request_id: str) -> int | None:
"""
Check the result of a previously submitted lookup request.
"""Check the result of a previously submitted lookup request.
Args:
request_id: The ID of the lookup request submitted in
@@ -271,6 +269,7 @@ class LMCacheMPSchedulerAdapter:
An integer representing the total number of tokens matched
in LMCache (prefix matching), or
None if the lookup request is not finished yet.
"""
assert request_id in self.lookup_futures, (
f"Lookup request for request_id={request_id} has not been submitted"
@@ -285,25 +284,27 @@ class LMCacheMPSchedulerAdapter:
return num_chunks * self.chunk_size
def num_blocks_per_chunk(self) -> int:
"""
Returns:
The number of vllm blocks in a LMCache data chunk
"""Returns:
The number of vllm blocks in a LMCache data chunk
"""
return self.blocks_in_chunk
def cleanup_lookup_result(self, request_id: str) -> None:
"""
Clean up lookup future for a finished request to prevent memory leak.
"""Clean up lookup future for a finished request to prevent memory leak.
Args:
request_id: The ID of the finished request.
"""
self.lookup_futures.pop(request_id, None)
def end_session(self, request_id: str) -> None:
"""
Notify LMCache server to remove the session for a finished request.
"""Notify LMCache server to remove the session for a finished request.
Args:
request_id: The ID of the finished request.
"""
send_lmcache_request(
self.mq_client,
@@ -319,7 +320,7 @@ class LMCacheMPSchedulerAdapter:
end: int = 0,
request_id: str | None = None,
) -> IPCCacheEngineKey:
"""Convert token IDs to an IPC cache engine key"""
"""Convert token IDs to an IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
@@ -334,7 +335,7 @@ class LMCacheMPSchedulerAdapter:
def _create_hash_key(
self, chunk_hash: bytes, request_id: str | None = None
) -> IPCCacheEngineKey:
"""Create a hash-mode IPC cache engine key"""
"""Create a hash-mode IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
@@ -411,12 +412,12 @@ class LMCacheMPWorkerAdapter:
)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""
Register the kv caches with LMCache server
"""Register the kv caches with LMCache server.
Args:
kv_caches: A dict of kv caches to register. The keys are the
layer names and the values are the corresponding tensors.
"""
# Register kv cache and send the request
self.kv_caches = kv_caches
@@ -432,14 +433,14 @@ class LMCacheMPWorkerAdapter:
def submit_store_request(
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
):
"""
Submit a KV cache store request to LMCache
"""Submit a KV cache store request to LMCache.
Args:
request_id: The ID of the request
op: The LoadStoreOp describing the store operation.
event: The CUDA event that is recorded after the current
model inference step
"""
if op.block_hashes is not None:
# Hash mode
@@ -466,14 +467,14 @@ class LMCacheMPWorkerAdapter:
def submit_retrieve_request(
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
):
"""
Submit a KV cache retrieve request to LMCache
"""Submit a KV cache retrieve request to LMCache.
Args:
request_id: The ID of the request
op: The LoadStoreOp describing the retrieve operation.
event: The CUDA event that is recorded after the current
model inference step
"""
if op.block_hashes is not None:
# Hash mode
@@ -503,8 +504,7 @@ class LMCacheMPWorkerAdapter:
ops: list[LoadStoreOp],
event: torch.cuda.Event,
):
"""
Submit a batched store request to LMCache
"""Submit a batched store request to LMCache.
Args:
request_ids: The IDs of the requests
@@ -512,6 +512,7 @@ class LMCacheMPWorkerAdapter:
the same length as request_ids
event: The CUDA event that is recorded after the current
model inference step
"""
all_keys: list[IPCCacheEngineKey] = []
block_ids: list[int] = []
@@ -552,8 +553,7 @@ class LMCacheMPWorkerAdapter:
ops: list[LoadStoreOp],
event: torch.cuda.Event,
):
"""
Submit a batched retrieve request to LMCache
"""Submit a batched retrieve request to LMCache.
Args:
request_ids: The IDs of the requests
@@ -561,6 +561,7 @@ class LMCacheMPWorkerAdapter:
the same length as request_ids
event: The CUDA event that is recorded after the current
model inference step
"""
all_keys: list[IPCCacheEngineKey] = []
block_ids: list[int] = []
@@ -598,8 +599,7 @@ class LMCacheMPWorkerAdapter:
def get_finished(
self, finished_req_ids_from_engine: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""
Check and get the finished store and retrieve requests.
"""Check and get the finished store and retrieve requests.
Args:
finished_req_ids_from_engine: the set of request ids that are
@@ -617,6 +617,7 @@ class LMCacheMPWorkerAdapter:
multiple times in `finished_req_ids_from_engine`. The adapter should
take care of deduplicating the request IDs and only return the request
IDs that have not been returned before.
"""
finished_stores = set()
finished_retrieves = set()
@@ -675,16 +676,14 @@ class LMCacheMPWorkerAdapter:
return ret_stores, finished_retrieves
def num_blocks_per_chunk(self) -> int:
"""
Returns:
The number of vllm blocks in a LMCache data chunk
"""Returns:
The number of vllm blocks in a LMCache data chunk
"""
return self.blocks_in_chunk
def shutdown(self):
"""
Shutdown the LMCache MP worker adapter
"""
"""Shutdown the LMCache MP worker adapter."""
logger.info("Unregistering kv caches")
send_lmcache_request(
self.mq_client, RequestType.UNREGISTER_KV_CACHE, [self.instance_id]
@@ -712,7 +711,7 @@ class LMCacheMPWorkerAdapter:
end: int = 0,
request_id: str | None = None,
) -> IPCCacheEngineKey:
"""Convert token IDs to an IPC cache engine key"""
"""Convert token IDs to an IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
@@ -726,7 +725,7 @@ class LMCacheMPWorkerAdapter:
def _create_hash_key(
self, chunk_hash: bytes, request_id: str | None = None
) -> IPCCacheEngineKey:
"""Create a hash-mode IPC cache engine key"""
"""Create a hash-mode IPC cache engine key."""
return IPCCacheEngineKey(
model_name=self.model_name,
world_size=self.world_size,
@@ -59,9 +59,7 @@ def lmcache_get_or_create_config() -> V1Config:
def hex_hash_to_int16(s: str) -> int:
"""
Convert a hex hash string to a 16-bit integer.
"""
"""Convert a hex hash string to a 16-bit integer."""
return int(s, 16) & 0xFFFF
@@ -70,8 +68,7 @@ def apply_mm_hashes_to_token_ids(
mm_hashes: list[str],
mm_positions: list["PlaceholderRange"],
) -> torch.Tensor:
"""
Overwrite token_ids in-place for multimodal placeholders using
"""Overwrite token_ids in-place for multimodal placeholders using
efficient slice assignments.
"""
n = token_ids.size(0)
@@ -95,8 +92,7 @@ def mla_enabled(model_config: "ModelConfig") -> bool:
def extract_mm_features(
request: Union["Request", "NewRequestData"], modify: bool = False
) -> tuple[list[str], list["PlaceholderRange"]]:
"""
Normalize multimodal information from a Request into parallel lists.
"""Normalize multimodal information from a Request into parallel lists.
This helper reads either:
1) `request.mm_features` (objects each exposing `.identifier` and
@@ -119,6 +115,7 @@ def extract_mm_features(
Returns:
tuple[list[str], list[PlaceholderRange]]: (`mm_hashes`, `mm_positions`).
May be `([], [])` when no multimodal data is present.
"""
if getattr(request, "mm_features", None):
mm_hashes, mm_positions = zip(
@@ -171,6 +171,7 @@ class RequestTracker:
lmcache_cached_tokens (int): the number of tokens that are
cached in LMCache.
skip_save (bool): whether the request cache should be saved
"""
# vLLM 0.9.0 update: request.block_ids changed from list[int] to
# list[list[int]]
@@ -219,7 +220,6 @@ class RequestTracker:
"""Update the request tracker when a running request is
scheduled again
"""
self.token_ids.extend(new_token_ids)
if new_block_ids is None:
@@ -288,6 +288,7 @@ class ReqMeta:
Returns:
the request metadata if we need to perform load/save
operations, None otherwise.
"""
input_token_ids = tracker.token_ids
input_token_len = len(input_token_ids)
@@ -445,6 +446,7 @@ def _init_lmcache_engine(
Returns:
The initialized LMCache engine
"""
if curr_engine := LMCacheEngineBuilder.get(ENGINE_NAME):
return curr_engine
@@ -563,6 +565,7 @@ class LMCacheConnectorMetadata(KVConnectorMetadata):
Args:
req_meta (ReqMeta): the request metadata.
"""
self.requests.append(req_meta)
@@ -719,6 +722,7 @@ class LMCacheConnectorV1Impl:
Returns:
dict: Dictionary containing inference information
"""
# Get vLLM config information
vllm_config = self._vllm_config
@@ -766,6 +770,7 @@ class LMCacheConnectorV1Impl:
Returns:
str: vLLM version string
"""
return VLLM_VERSION
@@ -816,6 +821,7 @@ class LMCacheConnectorV1Impl:
Note:
The number of elements in kv_caches and layer_names should be
the same.
"""
if len(self.kv_caches) == 0:
self._init_kv_caches_from_forward_context(forward_context)
@@ -920,6 +926,7 @@ class LMCacheConnectorV1Impl:
Args:
layer_name: the name of that layer
"""
if self.layerwise_retrievers:
logger.debug("Waiting for layer %s to be loaded", self.current_layer)
@@ -951,6 +958,7 @@ class LMCacheConnectorV1Impl:
kv_layer (torch.Tensor): the paged KV buffer of the current
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
"""
assert self.lmcache_engine is not None
@@ -1039,7 +1047,6 @@ class LMCacheConnectorV1Impl:
@_lmcache_nvtx_annotate
def wait_for_save(self):
"""Blocking until the KV cache is saved to the connector buffer."""
connector_metadata = self._parent._get_connector_metadata()
assert isinstance(connector_metadata, LMCacheConnectorMetadata)
@@ -1150,8 +1157,7 @@ class LMCacheConnectorV1Impl:
request: "Request",
num_computed_tokens: int,
) -> int | None:
"""
Check for external KV cache hit.
"""Check for external KV cache hit.
Args:
request (Request): the request object.
@@ -1161,6 +1167,7 @@ class LMCacheConnectorV1Impl:
Returns:
the number of tokens that can be loaded from the
external KV cache beyond what is already computed.
"""
if self.kv_role == "kv_producer" and not hasattr(
self.lookup_client, "supports_producer_reuse"
@@ -1236,13 +1243,11 @@ class LMCacheConnectorV1Impl:
@_lmcache_nvtx_annotate
def update_state_after_alloc(self, request: "Request", num_external_tokens: int):
"""
Update KVConnector state after temporary buffer alloc.
"""Update KVConnector state after temporary buffer alloc.
For SharedStorageConnector, update _request_needs_load
if the CacheManager this allocated blocks for us.
"""
# Clear local status in lookup client when a new request is
# successfully scheduled.
self.lookup_client.clear_lookup_status(request.request_id)
@@ -1311,8 +1316,8 @@ class LMCacheConnectorV1Impl:
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
"""
force_skip_save = self.kv_role == "kv_consumer" or self.force_skip_save
meta = LMCacheConnectorMetadata()
@@ -88,9 +88,7 @@ def extract_world_size_and_kv_rank(
rank: int,
vllm_config: VllmConfig,
) -> tuple[int, int]:
"""
Convert the rank for the MLA.
"""
"""Convert the rank for the MLA."""
use_mla = mla_enabled(vllm_config.model_config)
if not use_mla:
return world_size, rank
@@ -174,8 +172,7 @@ def create_worker_adapter(
class LMCacheMPRequestState(enum.Enum):
"""
State machine:
"""State machine:
PREFETCHING -- update_state_after_alloc --> WAITING_FOR_LOAD
WAITING_FOR_LOAD -- process_loading_requests --> READY
"""
@@ -234,7 +231,8 @@ class LMCacheMPRequestTracker:
####
def needs_retrieve(self) -> bool:
"""Check whether the current request needs retrieve, will be used
update_stage_after_alloc"""
update_stage_after_alloc
"""
return (
self.num_lmcache_hit_blocks > self.num_vllm_hit_blocks
and self.state != LMCacheMPRequestState.READY
@@ -242,7 +240,8 @@ class LMCacheMPRequestTracker:
def is_ready_for_retrieving(self) -> bool:
"""Check whether the current request is ready for retrieving,
will be used in process_loading_requests"""
will be used in process_loading_requests
"""
return (
self.state == LMCacheMPRequestState.WAITING_FOR_LOAD
and self.needs_retrieve()
@@ -301,13 +300,13 @@ class LMCacheMPRequestMetadata:
blocks_in_chunk: int,
vllm_block_size: int,
) -> "LMCacheMPRequestMetadata | None":
"""
Generate the store metadata for the current request tracker.
"""Generate the store metadata for the current request tracker.
Args:
tracker: The request tracker to generate the metadata from.
blocks_in_chunk: the number of blocks in a LMCache data chunk
vllm_block_size: the block size used in vLLM
"""
# Store the blocks that has block hashes
# NOTE: the invariant here is that `num_stored_blocks` should
@@ -375,13 +374,13 @@ class LMCacheMPRequestMetadata:
blocks_in_chunk: int,
vllm_block_size: int,
) -> "LMCacheMPRequestMetadata | None":
"""
Generate the retrieve metadata for the current request tracker.
"""Generate the retrieve metadata for the current request tracker.
Args:
tracker: The request tracker to generate the metadata from.
blocks_in_chunk: the number of blocks in a LMCache data chunk
vllm_block_size: the block size used in vLLM
"""
if not tracker.is_ready_for_retrieving():
return None
@@ -462,8 +461,7 @@ class LMCacheMPConnectorMetadata(KVConnectorMetadata):
class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
"""
The connector for LMCache multi-process mode.
"""The connector for LMCache multi-process mode.
Extra configs (kv_transfer_config.extra_config):
- lmcache.mp.host: the host of the LMCache server.
@@ -538,27 +536,26 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
Returns:
ConnectorMetadata: the connector metadata.
"""
"""
# Should only be called while set to valid metadata.
assert self._connector_metadata is not None
return self._connector_metadata
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""
Initialize with the KV caches. Useful for pre-registering the
"""Initialize with the KV caches. Useful for pre-registering the
KV Caches in the KVConnector (e.g. for NIXL).
Args:
kv_caches: dictionary of layer names, kv cache
"""
logger.info("Registering kv caches!")
self.worker_adapter.register_kv_caches(kv_caches)
return
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
"""
Start loading the KV cache from the connector to vLLM's paged
"""Start loading the KV cache from the connector to vLLM's paged
KV buffer. This is called from the forward context before the
forward pass to enable async loading during model execution.
@@ -597,8 +594,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
)
def wait_for_layer_load(self, layer_name: str) -> None:
"""
Block until the KV for a specific layer is loaded into vLLM's
"""Block until the KV for a specific layer is loaded into vLLM's
paged buffer. This is called from within attention layer to ensure
async copying from start_load_kv is complete.
@@ -606,6 +602,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
Args:
layer_name: the name of that layer
"""
return
@@ -616,8 +613,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
attn_metadata: AttentionMetadata,
**kwargs: Any,
) -> None:
"""
Start saving a layer of KV cache from vLLM's paged buffer
"""Start saving a layer of KV cache from vLLM's paged buffer
to the connector. This is called from within attention layer to
enable async copying during execution.
@@ -627,12 +623,12 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
layer in vLLM.
attn_metadata (AttentionMetadata): the attention metadata.
**kwargs: additional arguments for the save operation.
"""
return
def wait_for_save(self):
"""
Block until all the save operations is done. This is called
"""Block until all the save operations is done. This is called
as the forward context exits to ensure that the async saving
from save_kv_layer is complete before finishing the forward.
@@ -673,8 +669,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
"""Notifies worker-side connector ids of requests that have
finished generating tokens on the worker.
The scheduler process (via the Executors) will use this output
to track which workers are done.
@@ -685,14 +680,14 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
tuple of (sending/saving ids, recving/loading ids).
The finished saves/sends req ids must belong to a set provided in a
call to this method (this call or a prior one).
"""
val = self.worker_adapter.get_finished(finished_req_ids)
# logger.error("Finished req ids: %s, %s", val[0], val[1])
return val
def get_block_ids_with_load_errors(self) -> set[int]:
"""
Get the set of block IDs that failed to load.
"""Get the set of block IDs that failed to load.
Returns:
Set of block IDs that encountered load errors.
@@ -707,12 +702,12 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
block IDs must appear here no later than that same pass.
- Sync loading: failed blocks should be reported in the forward
pass in which they are detected.
"""
return self.worker_adapter.get_block_ids_with_load_errors()
def shutdown(self):
"""
Shutdown the connector. This is called when the worker process
"""Shutdown the connector. This is called when the worker process
is shutting down to ensure that all the async operations are
completed and the connector is cleaned up properly.
"""
@@ -721,9 +716,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
return None
def get_kv_connector_stats(self) -> "KVConnectorStats | None":
"""
Get the KV connector stats collected during the last interval.
"""
"""Get the KV connector stats collected during the last interval."""
return None
# ==============================
@@ -735,8 +728,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
request: "Request",
num_computed_tokens: int,
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
"""Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
Args:
@@ -761,6 +753,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
call. If the cache cannot be loaded for some tokens (e.g., due to
connectivity issues or eviction), those tokens must not be taken
into account.
"""
tracker = self._get_or_create_request_tracker(request)
# TODO: support loading KV for preempted requests in the future
@@ -803,8 +796,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
def update_state_after_alloc(
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
):
"""
Update KVConnector state after block allocation.
"""Update KVConnector state after block allocation.
If get_num_new_matched_tokens previously returned True for a
request, this function may be called twice for that same request -
@@ -817,6 +809,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
blocks (KVCacheBlocks): the blocks allocated for the request.
num_external_tokens (int): the number of tokens that will be
loaded from the external KV cache.
"""
# NOTE: `blocks` comes from kv_cache_manager.get_blocks(request_id),
# which returns ALL blocks for the request (not just newly allocated).
@@ -883,14 +876,14 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
"""
Build the connector metadata for this step.
"""Build the connector metadata for this step.
This function should NOT modify fields in the scheduler_output.
Also, calling this function will reset the state of the connector.
Args:
scheduler_output (SchedulerOutput): the scheduler output object.
"""
metadata = LMCacheMPConnectorMetadata()
@@ -907,12 +900,12 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
return metadata
def update_connector_output(self, connector_output: KVConnectorOutput):
"""
Update KVConnector state from worker-side connectors output.
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
return
@@ -921,8 +914,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
request: "Request",
block_ids: list[int],
) -> tuple[bool, dict[str, Any] | None]:
"""
Called exactly once when a request has finished, before its blocks are
"""Called exactly once when a request has finished, before its blocks are
freed.
The connector may assumes responsibility for freeing the blocks
@@ -934,8 +926,8 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
get_finished().
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
"""
params: dict[str, Any] | None = getattr(request, "kv_transfer_params", None)
return_params: dict[str, Any] | None = {} if params is not None else None
@@ -962,26 +954,26 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
return True, return_params
def take_events(self) -> Iterable["KVCacheEvent"]:
"""
Take the KV cache events from the connector.
"""Take the KV cache events from the connector.
Yields:
New KV cache events since the last call.
"""
return ()
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
"""Get the required KV cache layout for this connector.
Args:
vllm_config (VllmConfig): the vllm config.
Returns:
str: the required KV cache layout. e.g. HND, or NHD.
None if the connector does not require a specific layout.
"""
"""
if cls is KVConnectorBase_V1:
raise TypeError(
"get_required_kvcache_layout should not be called "
@@ -990,13 +982,13 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
return None
def get_finished_count(self) -> int | None:
"""
Get the count of requests expected to complete send/receive operations
"""Get the count of requests expected to complete send/receive operations
via this connector. This method is used to initialize the
KVOutputAggregator, overwriting the default world_size.
Returns:
int: expected sending or receiving completion count.
"""
return None
@@ -1004,8 +996,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
def build_kv_connector_stats(
cls, data: dict[str, Any] | None = None
) -> "KVConnectorStats | None":
"""
KVConnectorStats resolution method. This method allows dynamically
"""KVConnectorStats resolution method. This method allows dynamically
registered connectors to return their own KVConnectorStats object,
which can implement custom aggregation logic on the data dict.
"""
@@ -1019,8 +1010,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
) -> "KVConnectorPromMetrics | None":
"""
Create a KVConnectorPromMetrics subclass which should register
"""Create a KVConnectorPromMetrics subclass which should register
per-connector Prometheus metrics and implement observe() to
expose connector transfer stats via Prometheus.
"""
@@ -1183,8 +1173,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
return self.request_trackers[request_id]
def _cleanup_request_tracker(self, request_id: str) -> None:
"""
Clean up request tracker and associated lookup future for a request.
"""Clean up request tracker and associated lookup future for a request.
This should be called when a request is finished to prevent memory leak.
"""
# Clean up request tracker
@@ -17,8 +17,7 @@ logger = init_logger(__name__)
@dataclass
class KVConnectorStats:
"""
Base class for KV Connector Stats, a container for transfer performance
"""Base class for KV Connector Stats, a container for transfer performance
metrics or otherwise important telemetry from the connector.
All sub-classes need to be serializable as stats are sent from worker to
logger process.
@@ -35,14 +34,11 @@ class KVConnectorStats:
raise NotImplementedError
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
"""
Aggregate stats with another `KVConnectorStats` object.
"""
"""Aggregate stats with another `KVConnectorStats` object."""
raise NotImplementedError
def reduce(self) -> dict[str, int | float]:
"""
Reduce the observations collected during a time interval to one or
"""Reduce the observations collected during a time interval to one or
more representative values (eg avg/median/sum of the series).
This is meant to be called by the logger to produce a summary of the
stats for the last time interval.
@@ -95,7 +91,7 @@ class KVConnectorLogging:
)
def log(self, log_fn=logger.info):
"""Log transfer metrics periodically, similar to throughput logging"""
"""Log transfer metrics periodically, similar to throughput logging."""
if (
self.transfer_stats_accumulator
and not self.transfer_stats_accumulator.is_empty()
@@ -111,8 +107,7 @@ class KVConnectorLogging:
class KVConnectorPromMetrics:
"""
A base class for per-connector Prometheus metric registration
"""A base class for per-connector Prometheus metric registration
and recording.
"""
@@ -131,8 +126,7 @@ class KVConnectorPromMetrics:
self.per_engine_labelvalues = per_engine_labelvalues
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
"""
Record the supplied transfer statistics to Prometheus metrics. These
"""Record the supplied transfer statistics to Prometheus metrics. These
statistics are engine-specific, and should be recorded to a metric
with the appropriate 'engine' label. These metric instances can be
created using the create_metric_per_engine() helper method.
@@ -141,8 +135,7 @@ class KVConnectorPromMetrics:
class KVConnectorProm:
"""
Support for registering per-connector Prometheus metrics, and
"""Support for registering per-connector Prometheus metrics, and
recording transfer statistics to those metrics. Uses
KVConnectorBase.build_prom_metrics().
"""
@@ -643,7 +643,7 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
class MooncakeConnectorScheduler:
"""Implementation of Scheduler side methods"""
"""Implementation of Scheduler side methods."""
def __init__(
self,
@@ -718,7 +718,8 @@ class MooncakeConnectorScheduler:
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
"""D-side only. Returns N-1 for Mamba models since the decoder
always recomputes the last token and must start from h(N-1)."""
always recomputes the last token and must start from h(N-1).
"""
if self._has_mamba and num_prompt_tokens > 1:
return num_prompt_tokens - 1
return num_prompt_tokens
@@ -729,7 +730,8 @@ class MooncakeConnectorScheduler:
derive h(N) correctly.
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
request is preempted and rescheduled."""
request is preempted and rescheduled.
"""
params = request.kv_transfer_params
if (
params is not None
@@ -756,8 +758,7 @@ class MooncakeConnectorScheduler:
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int, bool]:
"""
For remote prefill, pull all prompt blocks from remote
"""For remote prefill, pull all prompt blocks from remote
asynchronously relative to engine execution.
Args:
@@ -769,8 +770,8 @@ class MooncakeConnectorScheduler:
external KV cache beyond what is already computed.
* true if the external KV cache tokens will be loaded
asynchronously (between scheduler steps).
"""
"""
params = request.kv_transfer_params
logger.debug(
"MooncakeConnector get_num_new_matched_tokens: "
@@ -881,11 +882,9 @@ class MooncakeConnectorScheduler:
request: "Request",
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
"""
Once a request is finished, determine whether request blocks
"""Once a request is finished, determine whether request blocks
should be freed now or will be sent asynchronously and freed later.
"""
params = request.kv_transfer_params
logger.debug(
"MooncakeConnector request_finished, req_id=%s, request_status=%s, "
@@ -934,7 +933,7 @@ class MooncakeConnectorScheduler:
class MooncakeConnectorWorker:
"""Implementation of Worker side methods"""
"""Implementation of Worker side methods."""
def __init__(
self,
@@ -1176,11 +1175,9 @@ class MooncakeConnectorWorker:
raise e
async def _mooncake_sender_listener(self, ready_event: threading.Event):
"""
Background thread that listens for Mooncake requests, dispatches them
"""Background thread that listens for Mooncake requests, dispatches them
to a thread pool, and sends acknowledgments upon completion.
"""
sock = self.async_zmq_ctx.socket(zmq.ROUTER)
self.side_channel_port = sock.bind_to_random_port(f"tcp://{self.hostname}")
logger.debug(
@@ -1680,7 +1677,8 @@ class MooncakeConnectorWorker:
def _bind_sender_thread_device(self) -> None:
"""ThreadPoolExecutor initializer — binds each pool thread to the
correct CUDA device. CUDA device selection is thread-local, so
without this, NVLink transfers fail for TP ranks > 0."""
without this, NVLink transfers fail for TP ranks > 0.
"""
current_platform.set_device(self.device_id)
def _send_blocks(
@@ -1716,7 +1714,6 @@ class MooncakeConnectorWorker:
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Register the KV Cache data in mooncake."""
logger.info("Registering KV_Caches. use_mla: %s", self.use_mla)
kv_data_ptrs: list[int] = []
@@ -1853,8 +1850,7 @@ class MooncakeConnectorWorker:
return finished_sending_reqs
def get_finished(self) -> tuple[set[str] | None, set[str] | None]:
"""
Get requests that are done sending or recving on this specific worker.
"""Get requests that are done sending or recving on this specific worker.
The scheduler process (via the MultiprocExecutor) will use this output
to track which workers are done.
"""
@@ -1885,8 +1881,7 @@ class MooncakeConnectorWorker:
return finished_sending_reqs or None, finished_recving_reqs or None
def get_transfer_results(self) -> KVConnectorTransferResults:
"""
Get transfers that completed on this specific worker, including
"""Get transfers that completed on this specific worker, including
requests whose remote KV load failed.
The scheduler process (via the MultiprocExecutor) will use this output
@@ -1909,7 +1904,8 @@ class MooncakeConnectorWorker:
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""Return transfer stats collected since the last call, or None
if nothing has been recorded in this interval."""
if nothing has been recorded in this interval.
"""
if self.xfer_stats.is_empty():
return None
return self.xfer_stats.clone_and_reset()
@@ -2321,8 +2317,7 @@ def should_launch_bootstrap_server(vllm_config: VllmConfig) -> bool:
def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]:
"""
Returns the address of the Mooncake bootstrap server.
"""Returns the address of the Mooncake bootstrap server.
This is only used by prefillers to register workers.
Decoders should get addr from kv_transfer_params.
"""
@@ -32,8 +32,7 @@ class EngineEntry:
class MooncakeBootstrapServer:
"""
A centralized server running on the global rank 0 prefiller worker.
"""A centralized server running on the global rank 0 prefiller worker.
Prefiller workers register their connection info (IP, port, ranks) here.
"""
@@ -73,7 +73,8 @@ class ExternalCachedBlockPool:
class MooncakeStoreCoordinator:
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
``ExternalCachedBlockPool``."""
``ExternalCachedBlockPool``.
"""
def __init__(
self,
@@ -630,6 +630,7 @@ class ChunkedTokenDatabase:
``mask_num``. False entries are skipped before hash access.
put_step: Stride for distributing chunks across ranks.
put_step_rank: ``chunk_id % put_step`` value this rank stores.
"""
assert put_step > 0
if not block_hashes:
@@ -662,6 +663,7 @@ class TailKeyBoundary:
``block_hashes[num_tokens // hash_block_size - 1]`` instead of the
hash implied by ``MooncakeLookupResult.hit_length``. This changes
only the load key, not the reusable prefix.
"""
group_id: int
@@ -678,6 +680,7 @@ class MooncakeLookupResult:
tail_key_boundaries: Hash boundary used to store each cache group's
tail block when ``hit_length`` does not identify its store key.
There is one entry per group for every nonzero hit.
"""
hit_length: int
@@ -815,6 +815,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
Returns:
True when no put is needed or every put succeeds, False otherwise.
"""
offloads = req_meta.boundary_state_offloads
if not offloads or not req_meta.block_hashes:
@@ -2628,7 +2629,8 @@ class LookupKeyClient:
non_block: bool = False,
) -> MooncakeLookupResult | None:
"""If non_block is True, will return None until the result is ready,
so the caller retries on a later step."""
so the caller retries on a later step.
"""
future = self.futures.get(req_id)
if future is None:
future = self.executor.submit(self._lookup, num_tokens, list(block_hashes))
@@ -588,8 +588,7 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
@contextlib.contextmanager
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
"""Context manager for a ZMQ socket"""
"""Context manager for a ZMQ socket."""
if socket_type not in (zmq.ROUTER, zmq.REQ, zmq.DEALER):
raise ValueError(f"Unexpected socket type: {socket_type}")
@@ -373,6 +373,7 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
Returns:
bool: True if connector metadata exists, False otherwise.
"""
try:
return self._connector_metadata is not None
@@ -381,7 +382,7 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
class MoRIIOConnectorScheduler:
"""Implementation of Scheduler side methods"""
"""Implementation of Scheduler side methods."""
def __init__(
self,
@@ -547,7 +548,8 @@ class MoRIIOConnectorScheduler:
self, request_id: ReqId, transfer_id: TransferId | None = None
):
"""Unmap request_id/transfer_id. Uses transfer_id for lookup if
exact request_id match fails (handles input_processor mutation)."""
exact request_id match fails (handles input_processor mutation).
"""
if request_id in self.request_id_to_transfer_id:
tid = self.request_id_to_transfer_id[request_id]
del self.request_id_to_transfer_id[request_id]
@@ -579,8 +581,7 @@ class MoRIIOConnectorScheduler:
request: "Request",
num_computed_tokens: int,
) -> tuple[int, bool]:
"""
For remote prefill, pull all prompt blocks from remote
"""For remote prefill, pull all prompt blocks from remote
asynchronously relative to engine execution.
Args:
@@ -592,6 +593,7 @@ class MoRIIOConnectorScheduler:
external KV cache beyond what is already computed.
* true if the external KV cache tokens will be loaded
asynchronously (between scheduler steps).
"""
if self.is_producer:
return 0, False
@@ -1007,11 +1009,9 @@ class MoRIIOConnectorScheduler:
request: "Request",
block_ids: BlockIds,
) -> tuple[bool, dict[str, Any] | None]:
"""
Once a request is finished, determine whether request blocks
"""Once a request is finished, determine whether request blocks
should be freed now or will be sent asynchronously and freed later.
"""
request_id = request.request_id
params = request.kv_transfer_params
# Consumer: can unmap transfer_id<->request_id immediately since done_recving
@@ -1177,7 +1177,7 @@ class MoRIIOConnectorScheduler:
class MoRIIOConnectorWorker:
"""Implementation of Worker side methods"""
"""Implementation of Worker side methods."""
def __init__(
self,
@@ -1435,8 +1435,8 @@ class MoRIIOConnectorWorker:
kv_layer: KV cache tensor
remote_notify_port: Port for completion notification
remote_ip: IP address of remote node
"""
"""
# synchronization to prevent dirty reads between
# transfer and attention operations
# we can consider removing this synchronization after ibgda is enabled.
@@ -1581,7 +1581,6 @@ class MoRIIOConnectorWorker:
layer_name_to_local_kv_cache_metadata: dict,
):
"""Background thread for getting new MoRIIO handshakes."""
encoder = msgspec.msgpack.Encoder()
encoded_data = encoder.encode(metadata)
size_in_bytes = len(encoded_data)
@@ -1637,7 +1636,6 @@ class MoRIIOConnectorWorker:
local-rank mapping _remote_tp_rank -- byte-identical for callers not yet
TP-aware.
"""
start_time = time.perf_counter()
# NOTE(rob): we need each rank to have a unique port. This is
@@ -1856,7 +1854,6 @@ class MoRIIOConnectorWorker:
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Register the KV Cache data in moriio."""
self.kv_caches = kv_caches # layer name to kv cache
self.kv_cache_shapes = {
layer_name: kv_cache.shape for layer_name, kv_cache in kv_caches.items()
@@ -2020,12 +2017,10 @@ class MoRIIOConnectorWorker:
self.moriio_wrapper.async_wait_reqid()
def get_finished(self) -> tuple[set[str], set[str]]:
"""
Get requests that are done sending or recving on this specific worker.
"""Get requests that are done sending or recving on this specific worker.
The scheduler process (via the MultiprocExecutor) will use this output
to track which workers are done.
"""
done_sending, done_recving = set(), set()
if self.is_producer:
@@ -2416,8 +2411,7 @@ class MoRIIOConnectorWorker:
self._eager_handshaked_engines.add(remote_engine_id)
def start_load_kv(self, metadata: MoRIIOConnectorMetadata):
"""
Start loading by triggering non-blocking moriio_xfer.
"""Start loading by triggering non-blocking moriio_xfer.
We check for these trnxs to complete in each step().
"""
self.transfer_id_to_request_id = metadata.transfer_id_to_request_id
@@ -2686,8 +2680,13 @@ class MoRIIOConnectorWorker:
local_block_ids: IDs of local blocks
remote_block_ids: IDs of remote blocks
remote_moriio_meta: Metadata of the remote MoRIIO agent
remote_tp_size: Tensor parallel size of the remote engine. Falls
back to the locally recorded value when None.
remote_engine_id: Id of the remote engine the blocks live on.
Returns:
Tuple of (local_offsets, remote_offsets, transfer_sizes)
"""
validate_moriio_heterogeneous_tp_kv_heads(
local_tp_size=self.world_size,
@@ -87,6 +87,7 @@ class MoRIIOWriter:
Args:
worker: Reference to the parent worker
"""
self._worker_ref: weakref_ref[MoRIIOConnectorWorker] = weakref_ref(worker)
self._write_task_q: Queue[WriteTask] = Queue()
@@ -108,6 +109,7 @@ class MoRIIOWriter:
Raises:
RuntimeError: If worker has been garbage collected
"""
worker = self._worker_ref()
if worker is None:
@@ -131,6 +133,7 @@ class MoRIIOWriter:
Args:
task: The write task to schedule
"""
self.ensure_worker_started()
if self._is_transfer_terminal(task.transfer_id):
@@ -178,7 +181,6 @@ class MoRIIOWriter:
def _write_worker_loop(self) -> None:
"""Main loop for the write worker thread."""
while True:
# Process deferred tasks first
self._process_deferred_tasks()
@@ -275,6 +277,7 @@ class MoRIIOWriter:
Returns:
True if remote blocks are ready
"""
return (
task.transfer_id in self.worker.moriio_wrapper.done_remote_allocate_req_dict
@@ -291,6 +294,7 @@ class MoRIIOWriter:
Raises:
KeyError: If allocation info is missing
"""
try:
return self.worker.moriio_wrapper.done_remote_allocate_req_dict[transfer_id]
@@ -372,9 +376,11 @@ class MoRIIOWriter:
Args:
task: The write task
request_info: Remote allocation information
remote_moriio_meta: Metadata of the remote MoRIIO agent
Returns:
The transfer plan
"""
layer_cache = self.worker.kv_caches[task.layer_name]
key = (task.layer_name, *_get_write_geometry_key(layer_cache))
@@ -412,6 +418,7 @@ class MoRIIOWriter:
Args:
plan: The transfer plan
sessions: List of transfer sessions
"""
if plan.use_batch:
return [
@@ -510,6 +517,7 @@ class MoRIIOWrapper:
moriio_engine: MoRIIO engine instance
tp_rank: Tensor parallel rank
dp_rank: Data parallel rank
"""
def __init__(
@@ -71,8 +71,7 @@ class MultiKVConnectorWorkerMetadata(KVConnectorWorkerMetadata):
@dataclass
class MultiKVConnectorStats(KVConnectorStats):
"""
Maintain a dict of KVConnectorStats objects, one for each connector.
"""Maintain a dict of KVConnectorStats objects, one for each connector.
This is used to aggregate the stats from all connectors separately.
"""
@@ -132,8 +131,7 @@ class MultiKVConnectorPromMetrics(KVConnectorPromMetrics):
class MultiConnector(KVConnectorBase_V1, SupportsHMA):
"""
A wrapper for using multiple KVConnectors at the same time.
"""A wrapper for using multiple KVConnectors at the same time.
The current logic is:
- Load KV from the first connector that advertises available tokens from
@@ -143,8 +141,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
@classmethod
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
"""
MultiConnector requires PIECEWISE CUDA graph mode if any of its
"""MultiConnector requires PIECEWISE CUDA graph mode if any of its
child connectors require it.
"""
connectors_config = extra_config.get("connectors", [])
@@ -480,8 +477,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
connector_output.kv_connector_worker_meta = multi_connector_worker_meta
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
"""
Get the KVConnector handshake metadata from sub-connectors.
"""Get the KVConnector handshake metadata from sub-connectors.
Returns the first non-None metadata from sub-connectors.
"""
for c in self._connectors:
@@ -493,8 +489,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
def set_xfer_handshake_metadata(
self, metadata: dict[int, KVConnectorHandshakeMetadata]
) -> None:
"""
Set the KV connector handshake metadata for all sub-connectors.
"""Set the KV connector handshake metadata for all sub-connectors.
This is needed to start the NIXL listener thread for NixlConnector.
"""
for c in self._connectors:
@@ -590,14 +585,15 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
"""Get the required KV cache layout for this connector.
Args:
vllm_config (VllmConfig): the vllm config.
Returns:
str: the required KV cache layout. e.g. HND, or NHD.
None if the connector does not require a specific layout.
"""
assert vllm_config.kv_transfer_config is not None
layouts: set[str] = set()
@@ -299,11 +299,11 @@ class NixlBaseConnectorScheduler:
def set_xfer_handshake_metadata(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
"""
Set the KV connector handshake metadata for this connector.
"""Set the KV connector handshake metadata for this connector.
Args:
metadata (dict): the handshake metadata to set.
"""
encoded_data: dict[tuple[int, int], bytes] = {}
encoder = msgspec.msgpack.Encoder()
@@ -400,7 +400,8 @@ class NixlBaseConnectorScheduler:
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
"""D-side only. The number of prompt tokens to load from the prefiller.
Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder
will recompute locally."""
will recompute locally.
"""
backoff = self._prefill_backoff()
if backoff and num_prompt_tokens > backoff:
return num_prompt_tokens - backoff
@@ -413,7 +414,8 @@ class NixlBaseConnectorScheduler:
multi-module MTP it is the drafter's whole lookahead window.
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
request is preempted and rescheduled."""
request is preempted and rescheduled.
"""
backoff = self._prefill_backoff()
params = request.kv_transfer_params
if (
@@ -1147,8 +1147,7 @@ class NixlBaseConnectorWorker:
return self.nixl_wrapper.add_remote_agent(metadata.agent_metadata)
def initialize_host_xfer_buffer(self, kv_caches: dict[str, torch.Tensor]) -> None:
"""
Initialize transfer buffer in CPU mem for accelerators
"""Initialize transfer buffer in CPU mem for accelerators
NOT directly supported by NIXL (e.g., tpu)
"""
xfer_buffers: dict[str, torch.Tensor] = {}
@@ -1262,8 +1261,7 @@ class NixlBaseConnectorWorker:
pp_size: int = 1,
notif_agents_only: bool = False,
) -> Future[tuple[dict[tuple[int, int], str], float]] | None:
"""
Ensure a handshake is in-flight (or already done) for *engine_id*.
"""Ensure a handshake is in-flight (or already done) for *engine_id*.
Returns the ``Future`` if a handshake is pending (or was just
started), or ``None`` if the handshake already completed
@@ -1348,7 +1346,6 @@ class NixlBaseConnectorWorker:
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Register the KV Cache data in nixl."""
self.transfer_topo = TransferTopology(
tp_rank=self.transfer_tp_rank,
tp_size=self.transfer_tp_size,
@@ -1876,7 +1873,8 @@ class NixlBaseConnectorWorker:
) -> np.ndarray:
"""Build remote desc regions (conv sub-projections + ssm) per layer.
For hetero-TP, each D rank reads only its sub-projection slice from
the P rank. Returns an Nx3 uint64 array."""
the P rank. Returns an Nx3 uint64 array.
"""
assert nixl_agent_meta.kv_caches_base_addr, (
"Remote KV cache base addresses must not be empty."
)
@@ -2034,8 +2032,7 @@ class NixlBaseConnectorWorker:
self,
block_size: int,
) -> tuple[int, np.ndarray]:
"""
Function used for register local xfer handler with local block_size or
"""Function used for register local xfer handler with local block_size or
Remote block_size.
When local block_size is same as remote block_size, we use local block_size
@@ -2124,8 +2121,7 @@ class NixlBaseConnectorWorker:
remote_tp_size: int = 1,
remote_dcp_size: int = 1,
) -> str:
"""
Add the remote NIXL agent and prepare the descriptors for reading cache
"""Add the remote NIXL agent and prepare the descriptors for reading cache
blocks from remote.
In particular, handle both homogeneous and heterogeneous TP. The former
@@ -2395,8 +2391,7 @@ class NixlBaseConnectorWorker:
remote_tp_size: int,
remote_dcp_size: int = 1,
):
"""
Validate the remote agent handshake metadata ensuring the
"""Validate the remote agent handshake metadata ensuring the
invariants hold true.
"""
remote_engine_id = nixl_agent_meta.engine_id
@@ -2607,7 +2602,7 @@ class NixlBaseConnectorWorker:
assert len(nixl_agent_meta.region_mem_types) == num_remote_regions
def sync_recved_kv_to_device(self, req_id: str, meta: ReqMeta):
"""copy recved kv from host buffer to device."""
"""Copy recved kv from host buffer to device."""
assert self.use_host_buffer
assert self.copy_blocks is not None
@@ -2632,7 +2627,7 @@ class NixlBaseConnectorWorker:
)
def save_kv_to_host(self, metadata: NixlConnectorMetadata):
"""copy kv from device to host buffer."""
"""Copy kv from device to host buffer."""
assert self.use_host_buffer
assert self.copy_blocks is not None
@@ -2662,7 +2657,8 @@ class NixlBaseConnectorWorker:
@cached_property
def _attention_kv_caches(self) -> list[torch.Tensor]:
"""Device KV caches of attention layers (mamba states excluded),
as consumed by the receive post-process."""
as consumed by the receive post-process.
"""
assert self.device_kv_caches, (
"_attention_kv_caches accessed before register_kv_caches"
)
@@ -2681,8 +2677,7 @@ class NixlBaseConnectorWorker:
block_ids_list: list[tuple[list[int], int]],
convert: bool = True,
):
"""
Post process device kv cache after receiving from remote.
"""Post process device kv cache after receiving from remote.
3 types of conversion supported (``convert``):
* kv_cache_postprocess_layout => convert from HND to NHD
@@ -2771,8 +2766,7 @@ class NixlBaseConnectorWorker:
def post_process_device_kv_on_receive_heterogeneous_attn(
self, block_ids: list[int]
):
"""
Post process device kv cache after receiving from remote
"""Post process device kv cache after receiving from remote
for heterogeneous attention.
"""
assert self.enable_heterogeneous_attn_post_process
@@ -2805,8 +2799,7 @@ class NixlBaseConnectorWorker:
pages[block].zero_()
def get_transfer_results(self) -> KVConnectorTransferResults:
"""
Get transfers that completed on this specific worker.
"""Get transfers that completed on this specific worker.
The scheduler process (via the MultiprocExecutor) will use this output
to track which workers are done.
@@ -2984,6 +2977,7 @@ class NixlBaseConnectorWorker:
Args:
payload: comma-separated P-side request IDs, e.g.
"req_abc,req_def".
"""
new_expiry = time.perf_counter() + self._lease_extension
for req_id in payload.split(","):
@@ -3010,6 +3004,7 @@ class NixlBaseConnectorWorker:
Returns:
Requests with no outstanding handles and requests with observed
failures. Failed requests may still have outstanding handles.
"""
done_req_ids: set[str] = set()
failed_req_ids: set[str] = set()
@@ -3091,9 +3086,7 @@ class NixlBaseConnectorWorker:
self.xfer_stats.record_failed_notification()
def _send_heartbeats(self, metadata: NixlConnectorMetadata) -> None:
"""
Send heartbeat notifications to remote engines, extending lease on KV blocks.
"""
"""Send heartbeats to remote engines, extending the lease on KV blocks."""
for engine_id, hb_info in metadata.heartbeat_by_engine.items():
# Proactive handshake (this request may still be in waiting queue) so
# the **next** heartbeat for this remote can go through.
@@ -3126,8 +3119,7 @@ class NixlBaseConnectorWorker:
def get_mapped_blocks(
self, block_ids: np.ndarray, block_size_ratio: int
) -> np.ndarray:
"""
Calculates the new set of block IDs by mapping every element
"""Calculates the new set of block IDs by mapping every element
in the (potentially sparse) input array.
Example: block_ids=[0, 2], block_size_ratio=2
get_mapped_blocks 0 1 [2 3] 4 5
@@ -3184,8 +3176,7 @@ class NixlBaseConnectorWorker:
return mapped_local, mapped_remote
def _logical_to_kernel_block_ids(self, block_ids: BlockIds, ratio: int) -> BlockIds:
"""
Convert block ids to kernel physical block ids.
"""Convert block ids to kernel physical block ids.
This is required when the logical block size (the one set by the user)
does not match the one required by the attn backend.
`ratio` is the number of physical blocks per logical block.
@@ -3239,6 +3230,7 @@ class NixlBaseConnectorWorker:
An empty result (rank 0 above) takes the notification-only path, allowing the
remote to release its blocks without performing a transfer.
"""
local_size, remote_size = local_dcp_size, remote_dcp_size
@@ -3417,17 +3409,14 @@ class NixlBaseConnectorWorker:
return decode_block_ids, prefill_block_ids
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""
Get the KV transfer stats for the connector.
"""
"""Get the KV transfer stats for the connector."""
# Clear stats for next iteration
if not self.xfer_stats.is_empty():
return self.xfer_stats.clone_and_reset()
return None
def get_block_ids_with_load_errors(self) -> set[int]:
"""
Return and clear the set of block IDs that failed to load.
"""Return and clear the set of block IDs that failed to load.
This is called by the scheduler to identify blocks that need
to be retried after a NIXL transfer failure.
@@ -214,12 +214,12 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
def set_xfer_handshake_metadata_pp_aware(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
"""
Set handshake metadata keyed by (pp_rank, tp_rank) so the side
"""Set handshake metadata keyed by (pp_rank, tp_rank) so the side
channel can serve every PP stage's agent metadata.
Args:
metadata (dict): the handshake metadata to set.
"""
assert self.connector_scheduler is not None
self.connector_scheduler.set_xfer_handshake_metadata(metadata)
@@ -317,14 +317,14 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
self.connector_scheduler.shutdown()
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
"""
Get the KVConnector handshake metadata for this connector.
"""Get the KVConnector handshake metadata for this connector.
This metadata is used for out-of-band connector handshake
between P/D workers.
Returns:
KVConnectorHandshakeMetadata: the handshake metadata.
None if no handshake metadata is available.
"""
assert self.connector_worker is not None
if (
@@ -78,8 +78,7 @@ class NixlAgentMetadata:
@dataclass
class NixlHandshakePayload(KVConnectorHandshakeMetadata):
"""
Wrapper for NIXL handshake sent over the wire.
"""Wrapper for NIXL handshake sent over the wire.
Enables two-phase decoding for graceful compatibility checking:
1. Decode NixlHandshakePayload to get compatibility_hash
@@ -142,8 +141,7 @@ def compute_nixl_compatibility_hash(
attn_backend_name: str,
transfer_mode: str = "pull",
) -> str:
"""
Compute compatibility hash for NIXL KV transfer.
"""Compute compatibility hash for NIXL KV transfer.
Hash only the factors that affect whether two NIXL instances can
successfully transfer KV cache data.
@@ -169,6 +167,7 @@ def compute_nixl_compatibility_hash(
Returns:
SHA-256 hex digest
"""
from vllm import __version__ as vllm_version
from vllm.config.utils import hash_factors
@@ -34,8 +34,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int, bool]:
"""
For remote prefill, pull all prompt blocks from remote
"""For remote prefill, pull all prompt blocks from remote
asynchronously relative to engine execution.
Args:
@@ -47,8 +46,8 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
external KV cache beyond what is already computed.
* true if the external KV cache tokens will be loaded
asynchronously (between scheduler steps).
"""
"""
params = request.kv_transfer_params
logger.debug(
"NIXLConnector get_num_new_matched_tokens: "
@@ -194,8 +193,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
request: "Request",
block_ids: "BlockIds",
) -> tuple[bool, dict[str, Any] | None]:
"""
Once a request is finished, determine whether request blocks
"""Once a request is finished, determine whether request blocks
should be freed now or will be sent asynchronously and freed later.
"""
from vllm.v1.request import RequestStatus
@@ -45,8 +45,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
super().__init__(vllm_config, engine_id, kv_cache_config)
def start_load_kv(self, metadata: NixlConnectorMetadata):
"""
Start loading by triggering non-blocking nixl_xfer.
"""Start loading by triggering non-blocking nixl_xfer.
We check for these trnxs to complete in each step().
"""
for req_id, meta in metadata.reqs_to_recv.items():
@@ -373,8 +372,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
expected_consumers: int,
awaiting_kvs: bool,
) -> bool:
"""
Post a READ point-to-point xfer request from a single local worker to
"""Post a READ point-to-point xfer request from a single local worker to
a single remote worker.
Returns True when the read was posted (or was unnecessary), False
@@ -603,8 +601,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
self._recving_transfers[request_id].append(handle)
def _get_new_notifs(self) -> set[str]:
"""
Get req_ids which got a remote xfer message. When multiple consumers
"""Get req_ids which got a remote xfer message. When multiple consumers
are reading from the same producer (heterogeneous TP or DCP
scenario), wait for all consumers to be done pulling.
@@ -315,7 +315,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
executes on the writer; the handshake runs on the background executor
and the request is re-queued onto ``_reg_send_inbox`` once it
completes (at which point ``_ensure_handshake`` returns ``None`` and we
send directly)."""
send directly).
"""
remote_pp_size = reg_data.get("remote_pp_size", 1)
fut = self._ensure_handshake(
reg_data["remote_engine_id"],
@@ -507,7 +508,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
``BlockIds`` is canonically a tuple of per-group lists, but some
registration payloads collapse a single-group case to a flat
list. Re-wrap that case so downstream group-aware helpers see a
consistent shape."""
consistent shape.
"""
if block_ids and not isinstance(block_ids[0], (list, tuple)):
return (list(block_ids),)
return block_ids
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
@dataclass
class NixlKVConnectorStats(KVConnectorStats):
"""Container for transfer performance metrics"""
"""Container for transfer performance metrics."""
def __post_init__(self):
if not self.data:
@@ -34,8 +34,7 @@ _NIXL_SUPPORTED_DEVICE.update(current_platform.get_nixl_supported_devices())
# TODO: merge with vllm.utils.network_utils.zmq_socket_ctx
@contextlib.contextmanager
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
"""Context manager for a ZMQ socket"""
"""Context manager for a ZMQ socket."""
if socket_type not in (zmq.ROUTER, zmq.REQ):
raise ValueError(f"Unexpected socket type: {socket_type}")
@@ -37,7 +37,8 @@ def canonical_format_id(kv_cache_layout: str) -> str:
Canonical pages keep the worker's KV layout family, so the id couples the
format version with that family; consumers must match it exactly. The family
keeps its historical NHD/HND spelling so ids stay stable for KV persisted
before the layout enum existed."""
before the layout enum existed.
"""
layout = KVCacheLayout[kv_cache_layout]
legacy = {KVCacheLayout.LBNHC: "nhd", KVCacheLayout.LBHNC: "hnd"}
family = legacy.get(layout, layout.name.lower())
@@ -121,7 +122,8 @@ def _interleave_cp_tokens(
ctx: _RankContext,
) -> tuple[CopyRun, ...]:
"""Place each region's num_tokens rows at their canonical token positions,
one run per chunk of interleaved tokens."""
one run per chunk of interleaved tokens.
"""
runs: list[CopyRun] = []
for region in regions:
if ctx.cp_size == 1:
@@ -256,7 +258,8 @@ def _attention_byte_regions(
cp_size: int,
) -> list[ByteRegion] | None:
"""Byte regions of an attention page, given this rank's head shard.
None when the physical layout is not recognized (fail closed)."""
None when the physical layout is not recognized (fail closed).
"""
bs, heads, head_size = spec.block_size, spec.num_kv_heads, spec.head_size
if tuple(kv_cache.shape) == (num_blocks, heads, bs, 2 * head_size):
return _packed_kv_regions(kv_cache, spec, head_shard, num_head_shards, cp_size)
@@ -370,7 +373,8 @@ def _is_exact_partition(intervals: list[tuple[int, int]], size: int) -> bool:
def _verify_tiling(layer_name: str, per_rank: list[CanonicalPageMapping]) -> None:
"""Whichever ranks a block elects as writers must tile the canonical page
exactly once, and each rank's runs must cover exactly its local page."""
exactly once, and each rank's runs must cover exactly its local page.
"""
size = per_rank[0].canonical_page_size_bytes
num_writers = per_rank[0].num_writers
for mapping in per_rank:
@@ -79,7 +79,8 @@ def get_offloading_event_group_spec(
class _OffloadEventMetadata:
"""BlockStored payload snapshot for one OffloadKey, captured while the
Request is available and kept until the final matching removal event.
``medium`` and ``ownership`` are forwarded from the OffloadingEvent."""
``medium`` and ``ownership`` are forwarded from the OffloadingEvent.
"""
# The chunk's constituent block hashes; the last one is the OffloadKey.
block_hashes: tuple[BlockHash, ...]
@@ -245,6 +246,7 @@ class OffloadingEventsTracker:
Yields:
``BlockStored`` or ``BlockRemoved`` events corresponding to
the underlying :class:`OffloadingEvent` stream.
"""
for event in events:
if event.removed:
@@ -254,7 +256,8 @@ class OffloadingEventsTracker:
def reset(self) -> None:
"""Drop all tracked state; pending payloads are stale after a
manager cache reset."""
manager cache reset.
"""
self._pending_event_metadata.clear()
def _build_event_metadata(
@@ -265,7 +268,8 @@ class OffloadingEventsTracker:
) -> _OffloadEventMetadata:
"""Build the payload snapshot for one offloaded chunk: its
constituent per-block hashes, the whole chunk's tokens, and the
per-block ``block_size``."""
per-block ``block_size``.
"""
hashes_per_chunk = group_config.hashes_per_chunk
assert hashes_per_chunk > 0
assert chunk_idx >= 0
@@ -170,8 +170,7 @@ class _StatsKey:
@dataclass
class OffloadingConnectorStats(KVConnectorStats):
"""
Offloading connector stats use flat metric names as keys.
"""Offloading connector stats use flat metric names as keys.
The ``data`` dict is structured using ``_StatsKey`` / ``_MetricType``::
@@ -243,8 +242,7 @@ class OffloadingConnectorStats(KVConnectorStats):
return self
def reduce(self) -> dict[str, int | float]:
"""
Reduce the observations collected during a time interval to one or
"""Reduce the observations collected during a time interval to one or
more representative values (eg avg/median/sum of the series).
This is meant to be called by the logger to produce a summary of the
stats for the last time interval.
@@ -676,7 +676,8 @@ class OffloadingConnectorScheduler:
start_chunk_idx: int,
) -> int | None:
"""Return the number of consecutive offloaded chunks from the start,
or None if the backend deferred a lookup."""
or None if the backend deferred a lookup.
"""
hit_count = 0
defer_lookup = False
for local_idx, key in enumerate(keys):
@@ -711,7 +712,8 @@ class OffloadingConnectorScheduler:
"""Return the end index (in `keys`) of the last run of
`sliding_window_size` consecutive hits, scanning from the end.
The first run may need a larger window for a partial rightmost chunk.
Returns 0 on miss, None if the backend deferred a lookup."""
Returns 0 on miss, None if the backend deferred a lookup.
"""
defer_lookup = False
pending_in_window = False
consecutive_hits = 0
@@ -750,8 +752,7 @@ class OffloadingConnectorScheduler:
req_status: RequestOffloadState,
max_num_new_tokens: int | None = None,
) -> int | None:
"""
Find how many tokens beyond num_locally_computed_tokens can be loaded.
"""Find how many tokens beyond num_locally_computed_tokens can be loaded.
Iterates full-attention groups first (prefix lookup), then sliding-window
groups (suffix lookup). Each group may tighten max_hit_size_tokens, which
@@ -1039,14 +1040,15 @@ class OffloadingConnectorScheduler:
num_computed_tokens: int,
max_num_new_tokens: int | None = None,
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded beyond the
"""Get number of new tokens that can be loaded beyond the
num_computed_tokens.
Args:
request (Request): the request object.
num_computed_tokens (int): the number of locally
computed tokens for this request
max_num_new_tokens (int | None): cap on the number of tokens that
may be loaded beyond `num_computed_tokens`, if any.
Returns:
A tuple with the following elements:
@@ -1057,6 +1059,7 @@ class OffloadingConnectorScheduler:
should query for this request again later.
- `True` if tokens will be loaded asynchronously
(between scheduler steps).
"""
req_status = self._req_status[request.request_id]
for group_state in req_status.group_states:
@@ -1208,10 +1211,7 @@ class OffloadingConnectorScheduler:
req_status.partial_tail_boundary = None
def _update_req_states(self, scheduler_output: SchedulerOutput) -> None:
"""
Update request states from the Scheduler's output.
"""
"""Update request states from the Scheduler's output."""
# new_block_ids_end[req_id][i] = end of pre-existing block_ids for
# the i-th sliding window group (before this step's extend).
# Used to detect sliding window blocks that got re-allocated.
@@ -1840,12 +1840,12 @@ class OffloadingConnectorScheduler:
return bool(self._jobs) or self.manager.has_pending_work()
def update_connector_output(self, connector_output: KVConnectorOutput):
"""
Update KVConnector state from worker-side connectors output.
"""Update KVConnector state from worker-side connectors output.
Args:
connector_output (KVConnectorOutput): the worker-side
connectors output.
"""
meta = connector_output.kv_connector_worker_meta
if not isinstance(meta, OffloadingWorkerMetadata):
@@ -1938,8 +1938,7 @@ class OffloadingConnectorScheduler:
self,
request: Request,
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its blocks are freed.
"""Called when a request has finished, before its blocks are freed.
Returns:
True if the request is being saved/sent asynchronously and blocks
@@ -1947,6 +1946,7 @@ class OffloadingConnectorScheduler:
get_finished().
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
req_status = self._req_status.get(request.request_id)
@@ -1985,12 +1985,12 @@ class OffloadingConnectorScheduler:
Yields:
``BlockStored`` or ``BlockRemoved`` events corresponding to
the underlying :class:`OffloadingEvent` stream.
"""
yield from self._events_tracker.take_events(self.manager.take_events())
def reset_cache(self) -> None:
"""Reset the offloading manager cache, evicting all stored chunks."""
# reset_cache cannot be called in the middle of a schedule step
assert not self._current_batch_load_jobs
assert not self._current_batch_jobs_to_flush
@@ -39,7 +39,7 @@ logger = init_logger(__name__)
class OffloadingConnectorWorker:
"""Implementation of Worker side methods"""
"""Implementation of Worker side methods."""
def __init__(
self,
@@ -268,14 +268,14 @@ class OffloadingConnectorWorker:
)
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
"""
Returns:
tuple of (finished_sending, finished_recving). Stores never
emit finished_sending — the scheduler tracks store completion
via kv_connector_worker_meta.completed_jobs and fences any
block reuse via jobs_to_flush. Loads still emit
finished_recving so the base scheduler can resume requests
blocked on remote KV (and free aborted-during-load reqs).
"""Returns:
tuple of (finished_sending, finished_recving). Stores never
emit finished_sending — the scheduler tracks store completion
via kv_connector_worker_meta.completed_jobs and fences any
block reuse via jobs_to_flush. Loads still emit
finished_recving so the base scheduler can resume requests
blocked on remote KV (and free aborted-during-load reqs).
"""
assert self.worker is not None
finished_recving: set[str] = set()
@@ -83,6 +83,7 @@ class MambaConvSplitInfo:
< 0: P_TP > D_TP — P pages are smaller, D reads entire
P page. Local dims are scaled down by |tp_ratio|
to get P-sized offsets.
"""
offsets: list[tuple[int, int]] = []
if tp_ratio >= 1:
@@ -121,6 +122,7 @@ def derive_mamba_conv_split(
Returns:
MambaConvSplitInfo with per-rank sub-projection dims, conv_rows,
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
"""
_supported = (
MambaAttentionBackendEnum.MAMBA1,
@@ -228,5 +230,6 @@ def compute_physical_blocks_per_logical(
Args:
ssm_sizes: (conv_state_bytes, ssm_state_bytes) from NixlAgentMetadata.
block_len: the engine's block_len in bytes (from block_lens[0]).
"""
return math.ceil((ssm_sizes[0] + ssm_sizes[1]) / block_len)
@@ -38,6 +38,7 @@ def is_v1_kv_transfer_group(connector: KVConnectorBaseType | None = None) -> boo
Note:
This function will no-longer be needed after the v1 KV connector
becomes the default.
"""
if connector is None:
connector = _KV_CONNECTOR_AGENT
@@ -72,10 +73,7 @@ def _sync_engine_id_across_tp(vllm_config: "VllmConfig") -> None:
def ensure_kv_transfer_initialized(
vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"
) -> None:
"""
Initialize KV cache transfer parallel group.
"""
"""Initialize KV cache transfer parallel group."""
global _KV_CONNECTOR_AGENT
if _KV_CONNECTOR_AGENT is not None:
+25 -26
View File
@@ -141,9 +141,11 @@ _group_name_counter: dict[str, int] = {}
def _get_unique_name(name: str) -> str:
"""Get a unique name for the group.
Example:
_get_unique_name("tp") -> "tp:0"
_get_unique_name("tp") -> "tp:1"
"""
if name not in _group_name_counter:
_group_name_counter[name] = 0
@@ -420,8 +422,7 @@ direct_register_custom_op(
class GroupCoordinator:
"""
PyTorch ProcessGroup wrapper for a group of processes.
"""PyTorch ProcessGroup wrapper for a group of processes.
PyTorch ProcessGroup is bound to one specific communication backend,
e.g. NCCL, Gloo, MPI, etc.
GroupCoordinator takes charge of all the communication operations among
@@ -634,34 +635,34 @@ class GroupCoordinator:
@property
def first_rank(self):
"""Return the global rank of the first process in the group"""
"""Return the global rank of the first process in the group."""
return self.ranks[0]
@property
def last_rank(self):
"""Return the global rank of the last process in the group"""
"""Return the global rank of the last process in the group."""
return self.ranks[-1]
@property
def is_first_rank(self):
"""Return whether the caller is the first process in the group"""
"""Return whether the caller is the first process in the group."""
return self.rank == self.first_rank
@property
def is_last_rank(self):
"""Return whether the caller is the last process in the group"""
"""Return whether the caller is the last process in the group."""
return self.rank == self.last_rank
@property
def next_rank(self):
"""Return the global rank of the process that follows the caller"""
"""Return the global rank of the process that follows the caller."""
rank_in_group = self.rank_in_group
world_size = self.world_size
return self.ranks[(rank_in_group + 1) % world_size]
@property
def prev_rank(self):
"""Return the global rank of the process that precedes the caller"""
"""Return the global rank of the process that precedes the caller."""
rank_in_group = self.rank_in_group
world_size = self.world_size
return self.ranks[(rank_in_group - 1) % world_size]
@@ -719,8 +720,7 @@ class GroupCoordinator:
yield graph_capture_context
def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
"""
User-facing all-reduce function before we actually call the
"""User-facing all-reduce function before we actually call the
all-reduce operation.
We need this because Dynamo does not support passing an arbitrary
@@ -809,8 +809,7 @@ class GroupCoordinator:
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> torch.Tensor | None:
"""
NOTE: We assume that the input tensor is on the same device across
"""NOTE: We assume that the input tensor is on the same device across
all the ranks.
NOTE: `dst` is the local rank of the destination rank.
"""
@@ -1363,7 +1362,7 @@ class GroupCoordinator:
torch.distributed.barrier(group=self.cpu_group)
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
"""Sends a tensor to the destination rank in a blocking way"""
"""Sends a tensor to the destination rank in a blocking way."""
"""NOTE: `dst` is the local rank of the destination rank."""
if self.device_communicator is None:
raise ValueError("No device communicator found")
@@ -1637,8 +1636,7 @@ def graph_capture(
device: torch.device,
graph_capture_context: GraphCaptureContext | None = None,
):
"""
`graph_capture` is a context manager which should surround the code that
"""`graph_capture` is a context manager which should surround the code that
is capturing the CUDA graph. Its main purpose is to ensure that some
operations will be run after the graph is captured, before the graph
is replayed. It returns a `GraphCaptureContext` object which contains the
@@ -1981,14 +1979,17 @@ def initialize_model_parallel(
decode_context_model_parallel_size: int | None = 1,
backend: str | None = None,
) -> None:
"""
Initialize model parallel groups.
"""Initialize model parallel groups.
Arguments:
tensor_model_parallel_size: number of GPUs used for tensor model
parallelism.
pipeline_model_parallel_size: number of GPUs used for pipeline model
parallelism.
prefill_context_model_parallel_size: number of GPUs used for context
parallelism during prefill.
decode_context_model_parallel_size: number of GPUs used for context
parallelism during decode.
backend: name of torch distributed communication backend.
Let's say we have a total of 8 GPUs denoted by g0 ... g7 and we
@@ -2003,6 +2004,7 @@ def initialize_model_parallel(
are on the same DGX box. For example if we are using 2 DGX-1 boxes
with a total of 16 GPUs, rank 0 to 7 belong to the first box and
ranks 8 to 15 belong to the second box.
"""
# Get world size and rank. Ensure some consistencies.
assert torch.distributed.is_initialized()
@@ -2476,8 +2478,7 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
def in_the_same_node_as(
pg: ProcessGroup | StatelessProcessGroup, source_rank: int = 0
) -> list[bool]:
"""
This is a collective operation that returns if each rank is in the same node
"""This is a collective operation that returns if each rank is in the same node
as the source rank. It tests if processes are attached to the same
memory system (shared access to shared memory).
"""
@@ -2568,8 +2569,7 @@ def in_the_same_node_as(
def is_global_first_rank() -> bool:
"""
Check if the current process is the first rank globally across all
"""Check if the current process is the first rank globally across all
parallelism strategies (PP, TP, DP, EP, etc.).
Unlike group-specific checks like `get_tensor_model_parallel_rank() == 0`
@@ -2579,6 +2579,7 @@ def is_global_first_rank() -> bool:
Returns:
bool: True if this is the global first rank (rank 0), False otherwise.
Returns True if distributed is not initialized (single process).
"""
try:
# If world group is available, use it for the most accurate check
@@ -2599,9 +2600,7 @@ def is_global_first_rank() -> bool:
def is_local_first_rank() -> bool:
"""
Check if the current process is the first local rank (rank 0 on its node).
"""
"""Check if the current process is the first local rank (rank 0 on its node)."""
try:
# prefer the initialized world group if available
global _WORLD
@@ -2622,14 +2621,14 @@ def is_local_first_rank() -> bool:
def _node_count(pg: ProcessGroup | StatelessProcessGroup) -> int:
"""
Returns the total number of nodes in the process group.
"""Returns the total number of nodes in the process group.
Args:
pg: The process group to analyze
Returns:
int: The total number of nodes
"""
if isinstance(pg, ProcessGroup):
world_size = torch.distributed.get_world_size(group=pg)
+1 -2
View File
@@ -59,8 +59,7 @@ def _fetch_group_ports(key: str, coord_store: Store) -> list[int]:
class StatelessGroupCoordinator(GroupCoordinator):
"""
A stateless version of the GroupCoordinator class in parallel_state,
"""A stateless version of the GroupCoordinator class in parallel_state,
It will create CPU, device and TCPStore based communication groups
that are independent of PyTorch's WORLD group. Hence,
communication groups with a different set of participants GPUs
+9 -9
View File
@@ -59,7 +59,8 @@ def ensure_divisibility(numerator, denominator):
def divide(numerator, denominator):
"""Ensure that numerator is divisible by the denominator and return
the division value."""
the division value.
"""
ensure_divisibility(numerator, denominator)
return numerator // denominator
@@ -111,6 +112,7 @@ def split_tensor_along_last_dim(
Returns:
A list of Tensors
"""
# Get the size and dimension.
last_dim = tensor.dim() - 1
@@ -333,7 +335,6 @@ class StatelessProcessGroup:
def barrier(self, timeout: float = 30.0):
"""A robust barrier to synchronize all ranks.
Uses a multi-phase approach to ensure all processes reach the barrier
before proceeding:
@@ -351,6 +352,7 @@ class StatelessProcessGroup:
Raises:
RuntimeError: If coordination fails or times out
"""
# Generate a barrier ID that is globally unique
try:
@@ -549,8 +551,7 @@ def init_gloo_process_group(
group_size: int,
timeout: timedelta,
) -> ProcessGroup:
"""
Stateless init ProcessGroup with gloo backend compatible with
"""Stateless init ProcessGroup with gloo backend compatible with
different torch versions.
"""
with suppress_stdout():
@@ -583,8 +584,7 @@ def stateless_init_torch_distributed_process_group(
return_store: bool = False,
listen_socket: socket.socket | None = None,
) -> ProcessGroup | tuple[ProcessGroup, Store]:
"""
A replacement for `torch.distributed.init_process_group` that does not
"""A replacement for `torch.distributed.init_process_group` that does not
pollute the global state. The created ProcessGroup object can be used for
some operations such as `allreduce`, because it does not depend on the
global rank. However, some operations such as `broadcast` cannot be used
@@ -685,9 +685,8 @@ def stateless_init_torch_distributed_process_group(
def stateless_destroy_torch_distributed_process_group(pg: ProcessGroup) -> None:
"""
Destroy ProcessGroup returned by
stateless_init_torch_distributed_process_group().
"""Destroy ProcessGroup returned by
stateless_init_torch_distributed_process_group().
"""
pg.shutdown()
_unregister_process_group(pg.group_name)
@@ -706,6 +705,7 @@ def get_worker_rank_suffix(global_rank: int | None = None) -> str:
Returns:
A string suffix identifying the worker's position in the
distributed topology.
"""
from vllm.distributed.parallel_state import (
get_dcp_group,
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Weight transfer engines for syncing model weights from trainers
"""Weight transfer engines for syncing model weights from trainers
to inference workers.
"""
+17 -20
View File
@@ -193,6 +193,7 @@ class WeightSource(ABC):
Returns:
The held parameter names, or None to hold every one.
"""
return None
@@ -329,8 +330,7 @@ class WeightTransferUpdateRequest:
class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
"""
Base class for weight transfer engines that handle transport of model weights
"""Base class for weight transfer engines that handle transport of model weights
from a trainer to inference workers.
This abstraction separates weight transfer transport logic from the worker
@@ -386,14 +386,14 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
device: torch.device,
model: torch.nn.Module,
) -> None:
"""
Initialize the weight transfer engine.
"""Initialize the weight transfer engine.
Args:
config: The configuration for the weight transfer engine
vllm_config: The full vLLM config (provides parallel/model config)
device: The device this worker's model lives on
model: The local model instance which will receive the weights
"""
self.config = config
self.vllm_config = vllm_config
@@ -419,8 +419,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
self.model_config = self._default_model_config
def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo:
"""
Construct typed init info from dict with validation.
"""Construct typed init info from dict with validation.
Args:
init_dict: Dictionary containing backend-specific initialization parameters
@@ -430,6 +429,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
Raises:
ValueError: If init_dict is invalid for this backend
"""
try:
return self.init_info_cls(**init_dict)
@@ -439,8 +439,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
) from e
def parse_update_info(self, update_dict: dict[str, Any]) -> TUpdateInfo:
"""
Construct typed update info from dict with validation.
"""Construct typed update info from dict with validation.
Args:
update_dict: Dictionary containing backend-specific update parameters
@@ -450,6 +449,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
Raises:
ValueError: If update_dict is invalid for this backend
"""
try:
return self.update_info_cls(**update_dict)
@@ -460,19 +460,18 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
@abstractmethod
def init_transfer_engine(self, init_info: TInitInfo) -> None:
"""
Initialize the weight transfer mechanism.
"""Initialize the weight transfer mechanism.
This is called once at the beginning of training.
Args:
init_info: Backend-specific initialization info
"""
raise NotImplementedError
@abstractmethod
def start_weight_update(self) -> None:
"""
Prepare the engine for a new weight update.
"""Prepare the engine for a new weight update.
Engines that receive weights in checkpoint format initialize layerwise reloading
here, else this is typically a no-op.
@@ -482,8 +481,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
@abstractmethod
def finish_weight_update(self) -> None:
"""
Finalize the current weight update.
"""Finalize the current weight update.
Checkpoint-format engines finalize layerwise reloading here; engines
that apply weights in place leave this as a no-op.
@@ -491,11 +489,11 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
raise NotImplementedError
def update_weights(self, update_info: dict[str, Any]) -> None:
"""
Receive one weight update chunk and load it into the model.
"""Receive one weight update chunk and load it into the model.
Args:
update_info: Dictionary containing backend-specific update info
"""
typed_update_info = self.parse_update_info(update_info)
self.receive_weights(typed_update_info)
@@ -505,19 +503,18 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
@abstractmethod
def receive_weights(self, update_info: TUpdateInfo) -> None:
"""
Receive weights from the trainer and load them into the model.
"""Receive weights from the trainer and load them into the model.
Args:
update_info: Backend-specific update info containing parameter metadata
and any backend-specific data
"""
raise NotImplementedError
@abstractmethod
def shutdown(self) -> None:
"""
Shutdown the weight transfer engine.
"""Shutdown the weight transfer engine.
This should be called when the worker is shutting down.
"""
raise NotImplementedError
+5 -1
View File
@@ -58,6 +58,7 @@ class WeightTransferEngineFactory:
Raises:
ValueError: If an engine with the same name is already registered
"""
if name in cls._registry:
raise ValueError(f"Weight transfer engine '{name}' is already registered.")
@@ -101,6 +102,7 @@ class WeightTransferEngineFactory:
Raises:
ValueError: If the backend is not registered
"""
backend = config.backend
if backend not in cls._registry:
@@ -139,7 +141,8 @@ class WeightTransferTrainerFactory:
class_name: str | None = None,
) -> None:
"""Register a trainer engine. Same conventions as
`WeightTransferEngineFactory.register_engine`."""
`WeightTransferEngineFactory.register_engine`.
"""
if name in cls._registry:
raise ValueError(
f"Weight transfer trainer engine '{name}' is already registered."
@@ -190,6 +193,7 @@ class WeightTransferTrainerFactory:
Raises:
ValueError: If `init_info.backend` is not registered.
"""
backend = init_info.backend
if backend not in cls._registry:
@@ -40,7 +40,8 @@ class IPCWeightTransferInitInfo(WeightTransferInitInfo):
`packed` is a must-agree wire param: the trainer ships it here at the init
handshake so the worker decodes with the same setting the trainer encoded
with. The consumer rebuilds from the IPC handle + `tensor_sizes`, so it does
not need the buffer size (producer-only)."""
not need the buffer size (producer-only).
"""
packed: bool = False
@@ -53,7 +54,8 @@ class IPCTrainerInitInfo(TrainerInitInfo):
`packed` / `packed_buffer_size_bytes` are the transfer's wire params. The
trainer propagates `packed` to the worker at `trainer_init` so the two sides
cannot disagree. `backend` is the factory dispatch key."""
cannot disagree. `backend` is the factory dispatch key.
"""
backend: ClassVar[str] = "ipc"
packed: bool = False
@@ -121,8 +123,7 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
class IPCWeightTransferEngine(
WeightTransferEngine[IPCWeightTransferInitInfo, IPCWeightTransferUpdateInfo]
):
"""
Weight transfer engine using CUDA IPC for communication between trainer and workers.
"""Weight transfer engine using CUDA IPC between trainer and workers.
This implementation uses CUDA IPC to transfer weights from the trainer (rank 0)
to all inference workers in a process group. IPC handles are used to share
@@ -149,13 +150,13 @@ class IPCWeightTransferEngine(
self._packed_importer = PackedBufferImporter()
def init_transfer_engine(self, init_info: IPCWeightTransferInitInfo) -> None:
"""
Initialize the weight transfer mechanism. No data-plane rendezvous is
"""Initialize the weight transfer mechanism. No data-plane rendezvous is
needed for IPC; this just records the trainer-supplied wire params so the
worker decodes exactly as the trainer encoded.
Args:
init_info: IPC initialization info (carries `packed`).
"""
self.packed = init_info.packed
@@ -182,8 +183,7 @@ class IPCWeightTransferEngine(
self._packed_importer.close()
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
"""
Receive weights from the trainer via CUDA IPC handles and load them.
"""Receive weights from the trainer via CUDA IPC handles and load them.
Whether the transfer is packed is read from `self.packed`, set at the
init handshake from the trainer's init info, so it is guaranteed to
@@ -193,6 +193,7 @@ class IPCWeightTransferEngine(
update_info: IPC update info containing parameter names, dtypes, shapes,
and IPC handles. Each IPC handle is a mapping between physical
GPU UUID and the rebuild_cuda_tensor args tuple.
"""
# Use the worker's assigned device rather than the ambient current
# device: the receive path is no longer wrapped in
@@ -118,7 +118,8 @@ def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
only the mode actually in use. Shared by the dense and sparse trainer
engines so the two cannot drift."""
engines so the two cannot drift.
"""
return {key: value for key, value in asdict(init_info).items() if value is not None}
@@ -142,8 +143,7 @@ def stateless_init_process_group(
world_size: int,
device,
) -> "PyNcclCommunicator":
"""
vLLM provides `StatelessProcessGroup` to create a process group
"""VLLM provides `StatelessProcessGroup` to create a process group
without considering the global process group in torch.distributed.
It is recommended to create `StatelessProcessGroup`, and then initialize
the data-plane communication (NCCL) between external (train processes)
@@ -222,8 +222,7 @@ def worker_init_process_group(
def trainer_init(
init_info: NCCLRendezvous | dict,
) -> "PyNcclCommunicator":
"""
Initialize NCCL process group for trainer-side weight transfer.
"""Initialize NCCL process group for trainer-side weight transfer.
The trainer is always rank 0 in the process group. Uses the current
CUDA device (torch.accelerator.current_device_index()).
@@ -237,6 +236,7 @@ def trainer_init(
Returns:
PyNcclCommunicator for weight transfer.
"""
if isinstance(init_info, dict):
master_address = init_info["master_address"]
@@ -69,7 +69,8 @@ class NCCLTrainerInitInfo(TrainerInitInfo):
propagates them to the worker at `trainer_init` so the two sides cannot
disagree. Note this defaults to packed, unlike the worker-side
`NCCLWeightTransferInitInfo`, whose default only applies when no trainer
ships a value. `backend` is the factory dispatch key."""
ships a value. `backend` is the factory dispatch key.
"""
backend: ClassVar[str] = "nccl"
@@ -113,8 +114,7 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
class NCCLWeightTransferEngine(
WeightTransferEngine[NCCLWeightTransferInitInfo, NCCLWeightTransferUpdateInfo]
):
"""
Weight transfer engine using NCCL for communication between trainer and workers.
"""Weight transfer engine using NCCL for communication between trainer and workers.
This implementation uses NCCL broadcast operations to transfer dense
checkpoint-format weights from the trainer (rank 0) to all inference workers
@@ -142,14 +142,14 @@ class NCCLWeightTransferEngine(
self.packed_num_buffers = DEFAULT_PACKED_NUM_BUFFERS
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
"""
Initialize NCCL process group with the trainer and record the
"""Initialize NCCL process group with the trainer and record the
trainer-supplied wire params so the worker decodes exactly as the
trainer encodes.
Args:
init_info: NCCL initialization info containing master address, port,
rank offset, world size, and the packed wire params
"""
self.packed = init_info.packed
self.packed_buffer_size_bytes = init_info.packed_buffer_size_bytes
@@ -175,8 +175,7 @@ class NCCLWeightTransferEngine(
finalize_layerwise_reload(self.model, self.model_config)
def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None:
"""
Receive weights from trainer via NCCL broadcast.
"""Receive weights from trainer via NCCL broadcast.
Whether to use packed broadcasting (and the buffer geometry) is read
from `self.packed` / `self.packed_*`, set at the init handshake from the
@@ -186,6 +185,7 @@ class NCCLWeightTransferEngine(
Args:
update_info: NCCL update info containing parameter names, dtypes,
and shapes
"""
if self.model_update_group is None:
raise RuntimeError(
@@ -365,7 +365,8 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni
def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None:
"""Iterate the source (materializing each tensor — a collective on all
ranks) and, on the sender, broadcast from rank 0, packed or one-by-one.
Non-sender ranks only replay the iteration to stay in the collective."""
Non-sender ranks only replay the iteration to stay in the collective.
"""
if not self.is_sender:
for _ in source:
pass
@@ -45,6 +45,7 @@ def unpack_tensor(
shapes: List of tensor shapes
dtypes: List of tensor dtypes
tensor_sizes: List of tensor sizes in bytes
"""
unpacked_tensors = packed_tensor.split(tensor_sizes)
@@ -85,6 +86,7 @@ def pack_tensors(
tensor_list: Pre-existing tensor list to append to (for NCCL
multi-buffer reuse). If None, a fresh list is created.
current_size: Byte count already accumulated in tensor_list
"""
if tensor_list is None:
tensor_list = []
@@ -335,6 +337,7 @@ def packed_ipc_producer(
buffer_size_bytes: Exact capacity of the reusable IPC buffer.
Every chunk is guaranteed to fit within this size. A
``ValueError`` is raised if any single tensor exceeds it.
"""
ipc_buffer = torch.empty(buffer_size_bytes, dtype=torch.uint8, device="cuda")
_, ipc_args = reduce_tensor(ipc_buffer)
@@ -463,6 +466,7 @@ def packed_ipc_consumer(
export so the buffer mapping is opened and released exactly once
(see ``PackedBufferImporter``). ``None`` builds an ephemeral one,
which is only safe for single-chunk consumption.
"""
props = torch.cuda.get_device_properties(device_index)
physical_gpu_id = str(props.uuid)
@@ -60,6 +60,7 @@ def check_ray_rdt_version() -> None:
Raises:
ValueError: the installed Ray predates ``RDT_MIN_RAY_VERSION``.
"""
import importlib.metadata
@@ -162,6 +163,7 @@ class RdtRouter:
ValueError: ``workers_per_replica`` does not divide
``num_consumers``, so the deployments are not uniform and two
workers of one deployment would share a block index.
"""
self.num_producers = max(1, num_producers)
self.num_consumers = max(1, num_consumers)
@@ -212,7 +214,8 @@ class RdtRouter:
def class_of(self, name: str) -> int:
"""The name's owner class — the planner's bucketing key, since all names
of a chunk must share one producer."""
of a chunk must share one producer.
"""
return self._class_of.get(name, 0)
def owners(self, name: str) -> list[int]:
@@ -263,6 +266,7 @@ class RdtRouter:
Raises:
ValueError: an owner set is empty or out of range, or a class index
does not resolve.
"""
for c, row in enumerate(self._owner_sets):
if not row:
@@ -282,7 +286,8 @@ class RdtRouter:
def bind(self, actors: list, produce_methods: list, consumer_id: int) -> None:
"""Attach this consumer's producer handles. Rebuilt wholesale per init,
never appended to: every owner index is a position in these lists, so a
rejoining engine that re-inits must not shift them."""
rejoining engine that re-inits must not shift them.
"""
if len(actors) != len(produce_methods):
raise ValueError("actors and produce_methods must be parallel")
self._actors = list(actors)
@@ -345,5 +350,6 @@ def buffer_alloc_bytes(nbytes: int, presize: int = 0) -> int:
optional ``presize`` floor, and a coarse 256MB round-up, so the buffer is
allocated ONCE and never regrows. Regrowth is a correctness hazard, not just a
perf one -- see ``buffer_presize_gb``. Shared by the consumer's receive buffers
and the producer's serve rings."""
and the producer's serve rings.
"""
return max(nbytes, presize, -(-nbytes // (256 << 20)) * (256 << 20))
@@ -134,7 +134,8 @@ class _PendingPull:
``targets``/``blob`` must stay strongly referenced until it completes:
``set_target_for_ref`` stores WEAKREFS, so dropping them silently reroutes the
transfer into a fallback buffer."""
transfer into a fallback buffer.
"""
ref: "Any"
keys: "list[FetchKey]"
@@ -561,7 +562,8 @@ class ShardedRDTWeightTransferEngine(
def _build_static_plan(self, init_info: ShardedRDTWeightTransferInitInfo) -> None:
"""Build the chunk/free plan once. It never changes across syncs, so
``update_weights`` needs no per-sync names."""
``update_weights`` needs no per-sync names.
"""
if not init_info.group_lens:
raise ValueError(
"Sharded RDT engine requires init_info.group_lens (the gather-"
@@ -585,7 +587,8 @@ class ShardedRDTWeightTransferEngine(
Both sides are sized from the static plan: receive buffers are
``ring_depth`` slots at the largest chunk's ``pack_bytes``, and each bound
producer is asked to pre-register a serve ring at the max bytes this
consumer will pull from it. A no-op when this worker has no chunks."""
consumer will pull from it. A no-op when this worker has no chunks.
"""
plan = self._cached_plan
if plan is None or not plan.chunks:
return
@@ -630,7 +633,8 @@ class ShardedRDTWeightTransferEngine(
``data_parallel_index``, not ``data_parallel_rank``: vLLM resets the
latter to 0 in a dense worker but keeps the former as the distinct global
DP rank. Same formula as the sibling ``nccl_engine``, so dense-via-TP and
MoE-via-DP+EP both yield distinct 0..C-1."""
MoE-via-DP+EP both yield distinct 0..C-1.
"""
pc = self.parallel_config
return pc.data_parallel_index * pc.world_size + pc.rank # world_size = TP*PP
@@ -638,14 +642,16 @@ class ShardedRDTWeightTransferEngine(
"""Total inference-worker count. Prefers the driver-supplied
``init_info.num_consumers`` (authoritative -- the driver knows the whole
fleet); else ``world_size_across_dp``, the same stride
``_global_worker_index`` indexes with, so the two agree at any pp."""
``_global_worker_index`` indexes with, so the two agree at any pp.
"""
if self._num_consumers_override > 0:
return self._num_consumers_override
return self.parallel_config.world_size_across_dp
def start_weight_update(self) -> None:
"""Put the model's params on meta so layerwise reload streams them in
as each layer's slices land. Baked replay uses checkpoint format."""
as each layer's slices land. Baked replay uses checkpoint format.
"""
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
@@ -654,7 +660,8 @@ class ShardedRDTWeightTransferEngine(
def finish_weight_update(self) -> None:
"""Drain the deferred pull/process pipeline (so every layer is fully
loaded) before finalizing the layerwise reload."""
loaded) before finalizing the layerwise reload.
"""
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
)
@@ -667,7 +674,8 @@ class ShardedRDTWeightTransferEngine(
device sync: post-processing is deferred to background threads and a
sync here would block on them and serialize the pull/process pipeline.
Completion is guaranteed by ``drain_pending`` in
``finish_weight_update``."""
``finish_weight_update``.
"""
self.receive_weights(self.parse_update_info(update_info))
def receive_weights(
@@ -709,7 +717,8 @@ class ShardedRDTWeightTransferEngine(
device: torch.device,
) -> list[tuple[str, torch.Tensor]]:
"""Zero-storage lazies for ``names``, dtype/shape from the init metadata,
all feeding the bake's recording sink."""
all feeding the bake's recording sink.
"""
return [
(
name,
@@ -977,7 +986,8 @@ class ShardedRDTWeightTransferEngine(
def _ensure_proc_worker(self) -> None:
"""Lazily create the per-slot events, the background CUDA stream, the work
queue, and the single processing thread. Idempotent."""
queue, and the single processing thread. Idempotent.
"""
if self._proc_thread is not None:
return
import queue
@@ -1036,7 +1046,8 @@ class ShardedRDTWeightTransferEngine(
def _mark_slot_done(self, slot: int) -> None:
"""Publish that a queued item's read-done event has been recorded (or the
item failed) so a pull waiting to reuse ``slot`` can proceed to its
CUDA-event synchronize."""
CUDA-event synchronize.
"""
with self._slot_cv:
self._slot_done[slot] += 1
self._slot_cv.notify_all()
@@ -1052,7 +1063,8 @@ class ShardedRDTWeightTransferEngine(
"""Block until the background thread has processed every queued item and
its stream work is complete, then re-raise any error it hit. Called from
the worker's ``finish_weight_update`` before ``finalize_layerwise_reload``
so every baked layer is fully loaded (and ``info.reset()``-ed) first."""
so every baked layer is fully loaded (and ``info.reset()``-ed) first.
"""
if self._proc_queue is not None:
self._proc_queue.join() # every put() item task_done()'d
# The scatter thread feeds the quant thread, so join it SECOND (all
@@ -1104,7 +1116,8 @@ class ShardedRDTWeightTransferEngine(
which is exactly what an owner class is. The cut is a pure function of
the bake and the ownership table. Copy order within a chunk is bake
order, and a module's copies may span chunks (materialize/quant fire on
its first/last chunk; see ``_build_call_plan``)."""
its first/last chunk; see ``_build_call_plan``).
"""
assert self._router is not None
by_class: dict[int, list[_Scatter]] = {}
for copies in modules:
@@ -1235,7 +1248,8 @@ class ShardedRDTWeightTransferEngine(
pulls nothing from. Refs are held and drained in ``drain_pending`` so
every signal has EXECUTED before the sync ends: ``begin_sync`` resets
the counters, and a straggler landing in the next sync would credit a
group it does not belong to."""
group it does not belong to.
"""
assert self._router is not None
self._pending_frees.extend(self._router.free_group(group_idx))
@@ -1370,7 +1384,8 @@ class ShardedRDTWeightTransferEngine(
the modules' scatters via ``ready``; touches only the scattered params
(never a receive slot), so it can overlap subsequent chunks' RDMA and
scatters. ``info.reset()`` is what makes finalize skip the layer —
drain_pending joins the quant queue before finalize runs."""
drain_pending joins the quant queue before finalize runs.
"""
from vllm.model_executor.layers.quantization.base_config import (
QuantizeMethodBase,
)
@@ -1403,7 +1418,8 @@ class ShardedRDTWeightTransferEngine(
def _quant_worker_loop(self) -> None:
"""Dedicated quant thread: drains (completed_modules, scatter-done event)
batches. Errors surface via _proc_error like the scatter thread's."""
batches. Errors surface via _proc_error like the scatter thread's.
"""
torch.cuda.set_device(self.device)
q = self._quant_queue
assert q is not None
@@ -206,7 +206,8 @@ class FakeRDTTensor(torch.Tensor):
def _meta(self) -> torch.Tensor:
"""A zero-storage meta tensor of this fake's shape/dtype, so PyTorch
itself computes post-op geometry rather than us reimplementing it."""
itself computes post-op geometry rather than us reimplementing it.
"""
return torch.empty(self.shape, dtype=self.dtype, device="meta")
@classmethod
@@ -250,7 +251,8 @@ class FakeRDTTensor(torch.Tensor):
):
"""Append the op and return a child, or a tuple of children for
multi-return ops. Each child's geometry comes from running the op on a
meta tensor."""
meta tensor.
"""
meta = self_._meta()
with torch._C.DisableTorchFunctionSubclass():
meta_result = func(meta, *args, **kwargs)
@@ -132,7 +132,8 @@ PRODUCE_METHOD_NAME = "rdt_produce_weights_batched"
@dataclass
class _SharedPack:
"""One generation of one slot-sharing group: the rendezvous state for a
single chunk, identified by the consumers' issue index."""
single chunk, identified by the consumers' issue index.
"""
group: int
slot: int
@@ -466,7 +467,8 @@ class _RDTProducerServer:
def end_sync(self) -> list[int]:
"""Block until every published group has been freed by its consumers;
return the remaining freed keys so the engine drops its last refs."""
return the remaining freed keys so the engine drops its last refs.
"""
with self._cache_cond:
self._wait_for(
lambda: bool(self._inflight_groups),
@@ -479,7 +481,8 @@ class _RDTProducerServer:
def set_gather_error(self, message: str) -> None:
"""Record a trainer-side gather failure so blocked serves / publishes
stop waiting and surface it."""
stop waiting and surface it.
"""
with self._cache_cond:
self._gather_error = RuntimeError(message)
self._cache_cond.notify_all()
@@ -508,7 +511,8 @@ class _RDTProducerServer:
def _new_serve_buffer(self, nbytes: int) -> torch.Tensor:
"""Allocate + NIXL-register one serve slot: the single allocation seam,
so registration cannot be skipped on either of the two paths that make
buffers (the init-time reservation and the serve-path backstop)."""
buffers (the init-time reservation and the serve-path backstop).
"""
t = torch.empty(nbytes, dtype=torch.uint8, device=self._serve_device)
with self._reg_lock:
register_nixl_memory(t)
@@ -534,6 +538,7 @@ class _RDTProducerServer:
Raises:
RuntimeError: two consumers of one sharing group disagree on the
chunks they pull from this producer.
"""
sg = self._share_group(consumer_id)
if plan_digest is not None:
@@ -579,13 +584,15 @@ class _RDTProducerServer:
def _share_group(self, consumer_id: int) -> int:
"""The slot-sharing group ``consumer_id`` belongs to: its index within
its own deployment, since that is what fixes the plan. Without a width
the group is the consumer itself and nothing is shared."""
the group is the consumer itself and nothing is shared.
"""
return consumer_id % self._share_width if self._share_width > 0 else consumer_id
def _sharers_of(self, sg: int) -> frozenset:
"""The LIVE consumers of group ``sg``: the rendezvous width. Derived from
``begin_sync``'s live set, so a degraded sync narrows the barrier instead
of waiting forever on a dead deployment. Caller holds ``_cache_cond``."""
of waiting forever on a dead deployment. Caller holds ``_cache_cond``.
"""
cached = self._sharers.get(sg)
if cached is None:
if not self._sharing_active or self._live_ids is None:
@@ -621,7 +628,8 @@ class _RDTProducerServer:
def _await_shared_pack(self, gen: _SharedPack) -> torch.Tensor:
"""Block until this generation's packer published its blob, and return
it. A failed pack is re-raised here, so every sharer of a bad pack fails
instead of reading a half-written slot."""
instead of reading a half-written slot.
"""
with self._cache_cond:
self._wait_for(
lambda: not gen.done and gen.error is None,
@@ -642,7 +650,8 @@ class _RDTProducerServer:
"""Group ``sg``'s ring slot ``idx``, grown if this chunk outgrew the
reservation. Growing registers memory while the fabric is busy, the
hazard ``reserve_serve_buffer`` exists to avoid, so it is a backstop: the
reservation is sized from the same static plan as this pack."""
reservation is sized from the same static plan as this pack.
"""
with self._serve_lock:
buffer = self._serve_rings.setdefault(sg, [None] * self._nring)[idx]
if buffer is not None and buffer.numel() >= need:
@@ -654,7 +663,8 @@ class _RDTProducerServer:
def _fail_shared_pack(self, gen: _SharedPack, exc: BaseException) -> None:
"""Publish a pack failure so waiting sharers raise instead of hanging on
a generation that will never complete."""
a generation that will never complete.
"""
with self._cache_cond:
gen.error = exc
self._cache_cond.notify_all()
@@ -681,6 +691,7 @@ class _RDTProducerServer:
Raises:
ValueError: ``seq`` was not supplied.
"""
needed = sorted({n for n, _ in specs})
if self._served_names is not None:
@@ -855,7 +866,8 @@ class ShardedRDTTrainerWeightTransferEngine(
def _rpc(self, method: str, *args: Any) -> Any:
"""Call one of the server actor's methods and block for the result.
The single seam through which the engine talks to its server, so tests
can inject a local (non-Ray) fake server."""
can inject a local (non-Ray) fake server.
"""
import ray
return ray.get(getattr(self._server, method).remote(*args))
@@ -932,7 +944,8 @@ class ShardedRDTTrainerWeightTransferEngine(
"""Fire publish_group WITHOUT waiting on the RPC (the gather loop
overlaps the publish's server-side rebuild with the next group's
gather) and return a handle that ``_await_publish`` resolves. Ray actor
handle in production; a plain (non-Ray) fake server runs inline."""
handle in production; a plain (non-Ray) fake server runs inline.
"""
method = self._server.publish_group
remote = getattr(method, "remote", None)
if remote is not None:
@@ -943,7 +956,8 @@ class ShardedRDTTrainerWeightTransferEngine(
"""Resolve one async publish so a server-side rebuild error surfaces at
window depth instead of at end_sync. Publishes carry nothing back —
freed groups flow only through wait_freed/end_sync (one channel; see
publish_group)."""
publish_group).
"""
import ray
if isinstance(ref, ray.ObjectRef):
@@ -1023,6 +1037,7 @@ class ShardedRDTTrainerWeightTransferEngine(
Raises:
RuntimeError: called before ``trainer_init`` cached the server names.
"""
if self._server_names is None:
raise RuntimeError(
@@ -1129,7 +1144,8 @@ class ShardedRDTTrainerWeightTransferEngine(
by stall watchdog 300s later: consumers route pulls here, the pull
passes the served-names guard, and the cache wait never completes. With
it, that is an immediate error naming the weight. One set lookup per
name per sync."""
name per sync.
"""
if self._held_names is None:
return
for name, tensor in zip(names, tensors):
@@ -1330,7 +1346,8 @@ class ShardedRDTTrainerWeightTransferEngine(
credit releases when every live consumer has signaled the group). So
the loop self-paces to the consumers' pull rate with at most
`gather_lookahead + 1` groups resident. Runs on every rank; only the
sender has an `update_future` to fail fast on."""
sender has an `update_future` to fail fast on.
"""
assert self.source is not None # guaranteed by trainer_init
# The live IDS ride along with the count: the free barrier needs only
# the count, but the slot-sharing rendezvous needs to know WHICH
@@ -68,7 +68,8 @@ class SparseNCCLTrainerInitInfo(TrainerInitInfo):
Same rendezvous shape as the dense NCCL backend (the sender opens its
endpoint as NCCL rank 0), but with no packed wire params: sparse transfers
are never packed. `backend` is the factory dispatch key."""
are never packed. `backend` is the factory dispatch key.
"""
backend: ClassVar[str] = "sparse_nccl"
@@ -113,8 +114,7 @@ class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
class SparseNCCLWeightTransferEngine(
WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo]
):
"""
Sparse weight transfer engine using NCCL.
"""Sparse weight transfer engine using NCCL.
Receives checkpoint-coordinate patches broadcast from the trainer and applies
them through the model's native weight loader. Sparse updates modify initialized
@@ -368,7 +368,8 @@ class SparseNCCLTrainerWeightTransferEngine(
rebuild or free the patch tensors as soon as a send method returns rather
than relying on same-stream ordering. See
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
cross-rank barrier."""
cross-rank barrier.
"""
if torch.cuda.is_available():
torch.cuda.current_stream().synchronize()