Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
@@ -22,9 +22,7 @@ class AiterCustomAllreduce:
@classmethod
def effective_max_size(cls) -> int:
"""
Max input byte size eligible for AITER custom allreduce.
"""
"""Max input byte size eligible for AITER custom allreduce."""
return cls.MAX_SIZE // 2
def __init__(
@@ -44,8 +44,7 @@ logger = init_logger(__name__)
class AgRsAll2AllManager(All2AllManagerBase):
"""
An implementation of all2all communication based on
"""An implementation of all2all communication based on
all-gather (dispatch) and reduce-scatter (combine).
"""
@@ -79,9 +78,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Gather hidden_states and router_logits from all dp ranks.
"""
"""Gather hidden_states and router_logits from all dp ranks."""
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
@@ -111,9 +108,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Gather hidden_states and router_logits from all dp ranks.
"""
"""Gather hidden_states and router_logits from all dp ranks."""
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
@@ -140,9 +135,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Reduce-scatter hidden_states across all dp ranks.
"""
"""Reduce-scatter hidden_states across all dp ranks."""
dist_group = self._get_comm_group(is_sequence_parallel)
sizes = self._get_sizes(
hidden_states.shape[0] // dist_group.world_size,
@@ -156,9 +149,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
class DeepEPAll2AllManagerBase(All2AllManagerBase):
"""
All2All communication based on DeepEP High-Throughput kernels.
"""
"""All2All communication based on DeepEP High-Throughput kernels."""
def __init__(self, cpu_group, tcp_store_group=None):
assert has_deep_ep(), (
@@ -210,9 +201,7 @@ class DeepEPAll2AllManagerBase(All2AllManagerBase):
class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
"""
All2All communication based on DeepEP High-Throughput kernels.
"""
"""All2All communication based on DeepEP High-Throughput kernels."""
def __init__(self, cpu_group, tcp_store_group=None):
super().__init__(cpu_group, tcp_store_group)
@@ -271,9 +260,7 @@ class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
"""
All2All communication based on DeepEP Low-Latency kernels.
"""
"""All2All communication based on DeepEP Low-Latency kernels."""
_buffer: Any = None
_mask: torch.Tensor | None = None
@@ -293,8 +280,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
num_global_experts: int,
num_local_experts: int,
) -> dict[Any, Any]:
"""
max_num_tokens_per_dp_rank : the maximum number of tokens a DP rank
"""max_num_tokens_per_dp_rank : the maximum number of tokens a DP rank
can dispatch all the ranks must hold the same value.
token_hidden_size: the hidden dimension of each token.
num_ep_ranks: the number of EP group ranks.
@@ -330,8 +316,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
return kwargs
def get_handle(self, kwargs):
"""
The kwargs for DeepEPLLAll2AllManager is dictated by
"""The kwargs for DeepEPLLAll2AllManager is dictated by
_make_all2all_kwargs.
"""
import deep_ep # type: ignore[import-not-found]
@@ -385,8 +370,7 @@ class _NixlEPBufferState:
class NixlEPAll2AllManager(All2AllManagerBase):
"""
All2All communication based on NIXL EP kernels.
"""All2All communication based on NIXL EP kernels.
This backend supports elastic EP with dynamic rank connection/disconnection.
"""
@@ -612,9 +596,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
"""
All2All communication based on flashinfer all2allv/two-sided NVLink kernels.
"""
"""All2All communication based on flashinfer all2allv/two-sided NVLink kernels."""
# This type lint could be removed after all of the work in
# https://github.com/vllm-project/vllm/issues/26533 done.
@@ -640,7 +622,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
rank: int,
gpus_per_node: int,
):
"""Initialize workspace"""
"""Initialize workspace."""
if self.initialized:
return
@@ -681,7 +663,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
)
def ensure_alltoall_workspace_initialized(self):
"""Ensure workspace is initialized"""
"""Ensure workspace is initialized."""
if not has_flashinfer_nvlink_two_sided():
return False
@@ -700,7 +682,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
return self
def cleanup(self):
"""Clean up workspace"""
"""Clean up workspace."""
if (
self.initialized
and self.workspace_tensor is not None
@@ -719,8 +701,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
"""
All2All communication based on FlashInfer's MoeAlltoAll/One-sided NVLink kernel.
"""All2All communication based on FlashInfer's MoeAlltoAll/One-sided NVLink kernel.
This is a newer kernel from trtllm that should perform better than the kernel
used by flashinfer_nvlink_two_sided.
"""
@@ -1023,8 +1004,7 @@ class MoriAll2AllManager(All2AllManagerBase):
class DeepEPV2All2AllManager(All2AllManagerBase):
"""
All2All communication based on DeepEP v2 ElasticBuffer (unified API).
"""All2All communication based on DeepEP v2 ElasticBuffer (unified API).
Uses NCCL Gin backend with analytical SM calculation.
"""
@@ -119,8 +119,7 @@ NCCL_SYMM_MEM_ALL_REDUCE_CONFIG: dict[str, Any] = {
def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor) -> bool:
"""
Determine if NCCL symmetric memory allreduce should be used.
"""Determine if NCCL symmetric memory allreduce should be used.
Based on H100 and GB200 benchmarks, NCCL symm_mem is preferred for:
- Small tensors (≤16K): Lower latency than custom_AR
@@ -157,7 +156,8 @@ def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor)
def should_nccl_symm_mem_ag_rs() -> bool:
"""Check whether NCCL symmetric memory should be used for
AllGather / ReduceScatter collectives."""
AllGather / ReduceScatter collectives.
"""
from vllm.distributed.device_communicators.pynccl_allocator import (
is_symmetric_memory_enabled,
)
@@ -344,7 +344,6 @@ _gpu_p2p_access_cache: dict[str, bool] | None = None
def gpu_p2p_access_check(src: int, tgt: int) -> bool:
"""Check if GPU src can access GPU tgt."""
# if the cache variable is already calculated,
# read from the cache instead of checking it again
global _gpu_p2p_access_cache
@@ -114,6 +114,7 @@ class All2AllManagerBase:
Returns:
An int32 device tensor where 0 marks a live rank and 1 marks a
masked (dead/unreachable) rank.
"""
raise NotImplementedError
@@ -171,8 +172,7 @@ class All2AllManagerBase:
class DeviceCommunicatorBase:
"""
Base class for device-specific communicator.
"""Base class for device-specific communicator.
It can use the `cpu_group` to initialize the communicator.
If the device has PyTorch integration (PyTorch can recognize its
communication backend), the `device_group` will also be given.
@@ -318,8 +318,7 @@ class DeviceCommunicatorBase:
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> torch.Tensor | None:
"""
NOTE: We assume that the input tensor is on the same device across
"""NOTE: We assume that the input tensor is on the same device across
all the ranks.
NOTE: `dst` is the local rank of the destination rank.
"""
@@ -347,7 +346,7 @@ class DeviceCommunicatorBase:
return output_tensor
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
"""Sends a tensor to the destination rank in a blocking way"""
"""Sends a tensor to the destination rank in a blocking way."""
"""NOTE: `dst` is the local rank of the destination rank."""
if dst is None:
dst = (self.rank_in_group + 1) % self.world_size
@@ -385,8 +384,7 @@ class DeviceCommunicatorBase:
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and router logits to the appropriate device.
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
if extra_tensors is not None:
@@ -404,8 +402,7 @@ class DeviceCommunicatorBase:
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and topk weights/ids to the appropriate device.
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
if extra_tensors is not None:
@@ -415,8 +412,7 @@ class DeviceCommunicatorBase:
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Combine the hidden states and router logits from the appropriate device.
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
return hidden_states
@@ -69,8 +69,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
logger.info("Using allgather_reducescatter all2all manager.")
def _all_group_ranks_share_shm_group_name(self) -> bool:
"""
CPUSHM requires all ranks in this group to agree on one SHM group name.
"""CPUSHM requires all ranks in this group to agree on one SHM group name.
This is a lightweight consistency check for VLLM_DIST_IDENT/name inputs.
"""
local_name = _CPUSHMDistributed.make_group_name(self)
@@ -89,8 +88,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> torch.Tensor | None:
"""
NOTE: We assume that the input tensor is on the same device across
"""NOTE: We assume that the input tensor is on the same device across
all the ranks.
NOTE: `dst` is the local rank of the destination rank.
"""
@@ -180,11 +178,9 @@ class CpuCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and router logits to the appropriate device.
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch_router_logits(
hidden_states,
@@ -204,8 +200,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and topk weights/ids to the appropriate device.
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -220,8 +215,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Combine the hidden states and router logits from the appropriate device.
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -611,7 +611,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
return symm_output
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
"""Sends a tensor to the destination rank in a blocking way"""
"""Sends a tensor to the destination rank in a blocking way."""
"""NOTE: `dst` is the local rank of the destination rank."""
if dst is None:
dst = (self.rank_in_group + 1) % self.world_size
@@ -851,11 +851,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and router logits to the appropriate device.
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch_router_logits(
hidden_states,
@@ -875,8 +873,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and topk weights/ids to the appropriate device.
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -891,8 +888,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Combine the hidden states and router logits from the appropriate device.
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -129,8 +129,7 @@ class CustomAllreduce:
max_mnnvl_reduce_scatter_size=_DEFAULT_MNNVL_REDUCE_SCATTER_MAX_SIZE,
symm_mem_enabled=False,
) -> None:
"""
Args:
"""Args:
group: the process group to work on. If None, it will use the
default process group.
device: the device to bind the CustomAllreduce to. If None,
@@ -138,6 +137,7 @@ class CustomAllreduce:
It is the caller's responsibility to make sure each communicator
is bind to a unique device, and all communicators in this group
are in the same node.
"""
self._IS_CAPTURING = False
self._ptr = 0
@@ -370,8 +370,7 @@ class CustomAllreduce:
@contextmanager
def capture(self):
"""
The main responsibility of this context manager is the
"""The main responsibility of this context manager is the
`register_graph_buffers` call at the end of the context.
It records all the buffer addresses used in the CUDA graph.
"""
@@ -140,6 +140,7 @@ def _resolve_fi_ar_backend() -> tuple[str, bool]:
is True only when ``auto`` selects mnnvl for a single node, so that
workspace creation can fall back to trtllm on single-node topologies
without NVSwitch multicast support (where mnnvl is unavailable).
"""
backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND
if backend != "auto":
@@ -168,8 +169,7 @@ def get_fi_ar_workspace(
dtype: torch.dtype,
group: ProcessGroup,
):
"""
Return the allreduce workspace for non-quant patterns, initializing if needed.
"""Return the allreduce workspace for non-quant patterns, initializing if needed.
Used by AllReduceFusionPass (non-quant patterns) and FlashInferAllReduce
for standalone allreduce. Backend is controlled by
@@ -234,8 +234,7 @@ def get_fi_ar_quant_workspace(
dtype: torch.dtype,
group: ProcessGroup,
):
"""
Return the allreduce workspace for quant patterns, initializing if needed.
"""Return the allreduce workspace for quant patterns, initializing if needed.
Backend is controlled by VLLM_FLASHINFER_ALLREDUCE_BACKEND env var, matching
non-quant fusion. With ``auto`` this prefers mnnvl and falls back to trtllm
@@ -98,8 +98,7 @@ class PyNcclCommunicator:
device: int | str | torch.device,
library_path: str | None = None,
):
"""
Args:
"""Args:
group: the process group to work on. If None, it will use the
default process group.
device: the device to bind the PyNcclCommunicator to. If None,
@@ -108,6 +107,7 @@ class PyNcclCommunicator:
use the default library path.
It is the caller's responsibility to make sure each communicator
is bind to a unique device.
"""
if not isinstance(group, StatelessProcessGroup):
assert dist.is_initialized()
@@ -62,8 +62,7 @@ class QuickAllReduce:
}
def __init__(self, group: ProcessGroup, device: int | str | torch.device) -> None:
"""
Custom allreduce provides non-destructive acceleration and is
"""Custom allreduce provides non-destructive acceleration and is
available for CUDA and ROCm MI300 series.
Custom quick allreduce leverages quantization for further
@@ -86,6 +85,7 @@ class QuickAllReduce:
It is the caller's responsibility to make sure each communicator
is bind to a unique device, and all communicators in this group
are in the same node.
"""
self.disabled = True
if not self._rocm_arch_available():
@@ -301,8 +301,7 @@ class QuickAllReduce:
return False
def create_shared_buffer(self):
"""
Creates a shared buffer for quickreduce.
"""Creates a shared buffer for quickreduce.
Has to be called after init_custom_qr
"""
handle = ops.qr_get_handle(self._ptr)
@@ -312,9 +311,7 @@ class QuickAllReduce:
ops.qr_open_handles(self._ptr, handles)
def should_quick_allreduce(self, inp: torch.Tensor):
"""
Check if quickreduce is available
"""
"""Check if quickreduce is available."""
if self.disabled:
return False
if inp.dtype not in self._SUPPORTED_DTYPES:
@@ -20,8 +20,7 @@ logger = init_logger(__name__)
class RayPPCommunicator(Communicator):
"""
Communicator to be used for pipeline parallelism in Ray Compiled Graph.
"""Communicator to be used for pipeline parallelism in Ray Compiled Graph.
This is wraps around the vLLM _PP GroupCoordinator.
This class is not thread-safe.
@@ -38,8 +37,7 @@ class RayPPCommunicator(Communicator):
cuda_stream: torch.cuda.Stream | None,
use_communication_streams: bool = False,
):
"""
Initialize a RayPPCommunicator that can be used to communicate with
"""Initialize a RayPPCommunicator that can be used to communicate with
other Ray Compiled Graph actors for pipeline parallelism.
Args:
@@ -54,6 +52,7 @@ class RayPPCommunicator(Communicator):
is not supported.
use_communication_streams: Whether to use communication streams.
This is not supported.
"""
self._world_size = world_size
self._rank: int | None = None
@@ -87,8 +86,7 @@ class RayPPCommunicator(Communicator):
self._closed = False
def _build_actor_rank_mapping(self):
"""
Use collective communication to build a mapping from actor IDs to ranks.
"""Use collective communication to build a mapping from actor IDs to ranks.
This should be called once during initialization.
"""
if self._comm is None:
@@ -128,9 +126,7 @@ class RayPPCommunicator(Communicator):
return self._actor_handles
def get_rank(self, actor: ray.actor.ActorHandle) -> int:
"""
Return the given actor's rank using device communicator collective ops.
"""
"""Return the given actor's rank using device communicator collective ops."""
assert hasattr(self, "_actor_id_to_rank"), (
"Actor rank mapping not built. "
"This should have been done during initialization."
@@ -144,20 +140,15 @@ class RayPPCommunicator(Communicator):
raise ValueError(f"Actor {actor} not found in communicator group")
def get_self_rank(self) -> int | None:
"""
Return this actor's rank.
"""
"""Return this actor's rank."""
return self._rank
def get_world_size(self) -> int:
"""
Return the number of ranks in the RayPPCommunicator group.
"""
"""Return the number of ranks in the RayPPCommunicator group."""
return self._world_size
def send(self, buf: "torch.Tensor", peer_rank: int) -> None:
"""
Send a torch.Tensor to a peer.
"""Send a torch.Tensor to a peer.
This returns when the send kernel has been queued, but the kernel may
not have completed. Therefore, the caller should ensure that there are
@@ -170,6 +161,7 @@ class RayPPCommunicator(Communicator):
buf: The torch.Tensor to send. It should already be on this
actor's default device.
peer_rank: The rank of the actor to send to.
"""
if self._closed:
raise RayChannelError("RayPPCommunicator has been destroyed.")
@@ -184,8 +176,7 @@ class RayPPCommunicator(Communicator):
peer_rank: int,
allocator: TorchTensorAllocator,
) -> "torch.Tensor":
"""
Receive a torch.Tensor from a peer and synchronize the current stream.
"""Receive a torch.Tensor from a peer and synchronize the current stream.
After this call returns, the receive buffer is safe to read from
any stream. An RayChannelError will be raised if an error occurred
@@ -197,6 +188,7 @@ class RayPPCommunicator(Communicator):
peer_rank: The rank of the actor to receive from.
allocator: The allocator to use to create the received tensor.
This is ignored for this implementation.
"""
if self._closed:
raise RayChannelError("RayPPCommunicator has been destroyed.")
@@ -77,8 +77,7 @@ _memory_fence_lock = threading.Lock()
def memory_fence():
"""
Full memory barrier for shared memory synchronization.
"""Full memory barrier for shared memory synchronization.
Ensures all prior memory writes are visible to other processes before
any subsequent reads. This is critical for lock-free producer-consumer
@@ -111,8 +110,7 @@ LONG_WAIT_TIME_LOG_MSG = (
class SpinCondition:
"""
This class implements an interface similar to a threading.Condition. It
"""This class implements an interface similar to a threading.Condition. It
allows a writer to notify readers to wake up and read from the shared memory
buffer. This notification is done over a zmq socket.
@@ -241,6 +239,7 @@ def check_shm_free_space(
Raises:
RuntimeError: If the SHM filesystem has insufficient space.
"""
if os.path.isdir(shm_path):
free_bytes = shutil.disk_usage(shm_path).free
@@ -651,7 +650,8 @@ class MessageQueue:
def shutdown(self):
"""If this is an idle reader, wakes it up so it can clean up and shut
down"""
down
"""
self.shutting_down = True
if self._spin_condition is not None:
self._spin_condition.cancel()
@@ -761,7 +761,8 @@ class MessageQueue:
def should_warn(self) -> bool:
"""Returns true if it's time to log a warning for a timeout that is not
indefinite"""
indefinite
"""
if self._should_warn:
elapsed = time.monotonic() - self.started
if elapsed >= VLLM_RINGBUFFER_WARNING_INTERVAL * self.n_warning:
@@ -941,8 +942,7 @@ class MessageQueue:
reader_rank: int = 0,
blocking: bool = False,
) -> tuple["MessageQueue", list[Handle]]:
"""
Creates a MessageQueue for a process group with a single reader.
"""Creates a MessageQueue for a process group with a single reader.
This method is designed for scenarios where only one process (the reader)
will consume messages, and all other processes are writers. It sets up
@@ -962,6 +962,7 @@ class MessageQueue:
tuple[MessageQueue, list[Handle]]:
The MessageQueue instance for the calling process,
and a list of handles (only non-empty for the reader process).
"""
from vllm.platforms.interface import get_assigned_physical_gpu_ids
@@ -994,8 +995,7 @@ class MessageQueue:
external_writer_handle=None,
blocking: bool = True,
) -> "MessageQueue":
"""
Creates a MessageQueue for a distributed process group with one writer and
"""Creates a MessageQueue for a distributed process group with one writer and
multiple readers.
This method is designed for scenarios where one process (the writer) sends
@@ -20,8 +20,7 @@ logger = init_logger(__name__)
class SingleWriterShmRingBuffer:
"""
A single-writer, multiple-reader ring buffer implementation using shared
"""A single-writer, multiple-reader ring buffer implementation using shared
memory. This class provides a thread-safe ring buffer where one process
can write data while multiple processes/threads can read from it.
@@ -190,8 +189,7 @@ class SingleWriterShmRingBuffer:
return int.from_bytes(byte_data, "little", signed=True)
def allocate_buf(self, size: int) -> tuple[int, int]:
"""
Allocate a buffer `MD_SIZE` + `size` bytes in the shared memory.
"""Allocate a buffer `MD_SIZE` + `size` bytes in the shared memory.
Memory layout:
`[4-byte monotonic_id][4-byte size][buffer data...]`
"""
@@ -260,8 +258,7 @@ class SingleWriterShmRingBuffer:
is_free_fn: Callable[[int, memoryview], bool],
nbytes: int | None = None,
) -> Iterable[int]:
"""
Free a buffer of the given size. This is a no-op in shared memory,
"""Free a buffer of the given size. This is a no-op in shared memory,
but we need to keep track of the metadata.
If freed memory spreads across the end and start of the ring buffer,
@@ -269,10 +266,13 @@ class SingleWriterShmRingBuffer:
still might not be a contiguous space of `nbytes` available.
Args:
is_free_fn (Callable[[int, memoryview], bool]): Predicate called with
a monotonic id and the buffer, returning True when that buffer
can be reclaimed.
nbytes (int, optional): The size of the buffer to free. If None,
frees the maximum size of the ring buffer.
"""
"""
assert self.is_writer, "Only the writer can free buffers."
logger.debug(
"Freeing up space in the ring buffer, "
@@ -412,8 +412,7 @@ class ShmObjectStorageHandle:
class SingleWriterShmObjectStorage:
"""
A single-writer, multiple-reader object storage system built on top of a
"""A single-writer, multiple-reader object storage system built on top of a
shared memory ring buffer. Provides key-value storage with automatic memory
management and cross-process serialization support.
@@ -458,8 +457,7 @@ class SingleWriterShmObjectStorage:
serde_class: type[ObjectSerde] = MsgpackSerde,
reader_lock: LockType | None = None,
):
"""
Initialize the object storage.
"""Initialize the object storage.
Args:
max_object_size: Maximum size for a single object in bytes.
@@ -467,10 +465,11 @@ class SingleWriterShmObjectStorage:
ring_buffer: The shared memory ring buffer for storing objects.
serde_class: Serializer/deserializer for objects.
reader_lock: Optional lock for synchronizing reader access.
Raises:
ValueError: If reader_lock is None for readers.
"""
"""
self.max_object_size = max_object_size
self.n_readers = n_readers
self.serde_class = serde_class
@@ -547,22 +546,17 @@ class SingleWriterShmObjectStorage:
del self.writer_flag[freed_id]
def is_cached(self, key: str) -> bool:
"""
Check if the object with the given key is cached.
"""
"""Check if the object with the given key is cached."""
return key in self.key_index
def get_cached(self, key: str) -> tuple[int, int]:
"""
Get the cached object by key if it exists.
"""
"""Get the cached object by key if it exists."""
address, monotonic_id = self.key_index[key]
self.increment_writer_flag(monotonic_id)
return address, monotonic_id
def put(self, key: str, value: Any) -> tuple[int, int]:
"""
Store a key-value pair in the object storage.
"""Store a key-value pair in the object storage.
Attempts to free max_object_size bytes using FIFO order
when the ring buffer runs out of space during a put() operation.
@@ -574,6 +568,7 @@ class SingleWriterShmObjectStorage:
MemoryError: If there's not enough space in the buffer
ValueError: If the serialized object is too large
ValueError: If the key already exists in the storage
"""
if key in self.key_index:
raise ValueError(f"Key '{key}' already exists in the storage.")
@@ -639,8 +634,7 @@ class SingleWriterShmObjectStorage:
address: int = 0,
monotonic_id: int = 0,
) -> None:
"""
Touch an existing cached item to update its eviction status.
"""Touch an existing cached item to update its eviction status.
For writers (ShmObjectStoreSenderCache): Increment writer_flag
For readers (ShmObjectStoreReceiverCache): Increment reader_count
@@ -700,8 +694,7 @@ class SingleWriterShmObjectStorage:
)
def default_is_free_check(self, id: int, buf: memoryview) -> bool:
"""
Default is_free function that checks if the first 4 bytes are zero.
"""Default is_free function that checks if the first 4 bytes are zero.
This indicates that the buffer is free.
"""
reader_count = int.from_bytes(buf[0:4], "little", signed=True)
@@ -205,11 +205,9 @@ class XpuCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and router logits to the appropriate device.
"""Dispatch the hidden states and router logits to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
return self.all2all_manager.dispatch_router_logits(
hidden_states,
@@ -229,8 +227,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
):
"""
Dispatch the hidden states and topk weights/ids to the appropriate device.
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None
@@ -245,8 +242,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
def combine(
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
) -> torch.Tensor:
"""
Combine the hidden states and router logits from the appropriate device.
"""Combine the hidden states and router logits from the appropriate device.
This is a no-op in the base class.
"""
assert self.all2all_manager is not None