Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -22,9 +22,7 @@ class AiterCustomAllreduce:
|
||||
|
||||
@classmethod
|
||||
def effective_max_size(cls) -> int:
|
||||
"""
|
||||
Max input byte size eligible for AITER custom allreduce.
|
||||
"""
|
||||
"""Max input byte size eligible for AITER custom allreduce."""
|
||||
return cls.MAX_SIZE // 2
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -44,8 +44,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AgRsAll2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
An implementation of all2all communication based on
|
||||
"""An implementation of all2all communication based on
|
||||
all-gather (dispatch) and reduce-scatter (combine).
|
||||
"""
|
||||
|
||||
@@ -79,9 +78,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Gather hidden_states and router_logits from all dp ranks.
|
||||
"""
|
||||
"""Gather hidden_states and router_logits from all dp ranks."""
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
|
||||
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
|
||||
@@ -111,9 +108,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Gather hidden_states and router_logits from all dp ranks.
|
||||
"""
|
||||
"""Gather hidden_states and router_logits from all dp ranks."""
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
|
||||
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
|
||||
@@ -140,9 +135,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Reduce-scatter hidden_states across all dp ranks.
|
||||
"""
|
||||
"""Reduce-scatter hidden_states across all dp ranks."""
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(
|
||||
hidden_states.shape[0] // dist_group.world_size,
|
||||
@@ -156,9 +149,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
|
||||
|
||||
class DeepEPAll2AllManagerBase(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on DeepEP High-Throughput kernels.
|
||||
"""
|
||||
"""All2All communication based on DeepEP High-Throughput kernels."""
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
assert has_deep_ep(), (
|
||||
@@ -210,9 +201,7 @@ class DeepEPAll2AllManagerBase(All2AllManagerBase):
|
||||
|
||||
|
||||
class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on DeepEP High-Throughput kernels.
|
||||
"""
|
||||
"""All2All communication based on DeepEP High-Throughput kernels."""
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
@@ -271,9 +260,7 @@ class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
|
||||
|
||||
class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on DeepEP Low-Latency kernels.
|
||||
"""
|
||||
"""All2All communication based on DeepEP Low-Latency kernels."""
|
||||
|
||||
_buffer: Any = None
|
||||
_mask: torch.Tensor | None = None
|
||||
@@ -293,8 +280,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
num_global_experts: int,
|
||||
num_local_experts: int,
|
||||
) -> dict[Any, Any]:
|
||||
"""
|
||||
max_num_tokens_per_dp_rank : the maximum number of tokens a DP rank
|
||||
"""max_num_tokens_per_dp_rank : the maximum number of tokens a DP rank
|
||||
can dispatch all the ranks must hold the same value.
|
||||
token_hidden_size: the hidden dimension of each token.
|
||||
num_ep_ranks: the number of EP group ranks.
|
||||
@@ -330,8 +316,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
return kwargs
|
||||
|
||||
def get_handle(self, kwargs):
|
||||
"""
|
||||
The kwargs for DeepEPLLAll2AllManager is dictated by
|
||||
"""The kwargs for DeepEPLLAll2AllManager is dictated by
|
||||
_make_all2all_kwargs.
|
||||
"""
|
||||
import deep_ep # type: ignore[import-not-found]
|
||||
@@ -385,8 +370,7 @@ class _NixlEPBufferState:
|
||||
|
||||
|
||||
class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on NIXL EP kernels.
|
||||
"""All2All communication based on NIXL EP kernels.
|
||||
This backend supports elastic EP with dynamic rank connection/disconnection.
|
||||
"""
|
||||
|
||||
@@ -612,9 +596,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
|
||||
|
||||
class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on flashinfer all2allv/two-sided NVLink kernels.
|
||||
"""
|
||||
"""All2All communication based on flashinfer all2allv/two-sided NVLink kernels."""
|
||||
|
||||
# This type lint could be removed after all of the work in
|
||||
# https://github.com/vllm-project/vllm/issues/26533 done.
|
||||
@@ -640,7 +622,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
rank: int,
|
||||
gpus_per_node: int,
|
||||
):
|
||||
"""Initialize workspace"""
|
||||
"""Initialize workspace."""
|
||||
if self.initialized:
|
||||
return
|
||||
|
||||
@@ -681,7 +663,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
)
|
||||
|
||||
def ensure_alltoall_workspace_initialized(self):
|
||||
"""Ensure workspace is initialized"""
|
||||
"""Ensure workspace is initialized."""
|
||||
if not has_flashinfer_nvlink_two_sided():
|
||||
return False
|
||||
|
||||
@@ -700,7 +682,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
return self
|
||||
|
||||
def cleanup(self):
|
||||
"""Clean up workspace"""
|
||||
"""Clean up workspace."""
|
||||
if (
|
||||
self.initialized
|
||||
and self.workspace_tensor is not None
|
||||
@@ -719,8 +701,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
|
||||
|
||||
class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on FlashInfer's MoeAlltoAll/One-sided NVLink kernel.
|
||||
"""All2All communication based on FlashInfer's MoeAlltoAll/One-sided NVLink kernel.
|
||||
This is a newer kernel from trtllm that should perform better than the kernel
|
||||
used by flashinfer_nvlink_two_sided.
|
||||
"""
|
||||
@@ -1023,8 +1004,7 @@ class MoriAll2AllManager(All2AllManagerBase):
|
||||
|
||||
|
||||
class DeepEPV2All2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on DeepEP v2 ElasticBuffer (unified API).
|
||||
"""All2All communication based on DeepEP v2 ElasticBuffer (unified API).
|
||||
Uses NCCL Gin backend with analytical SM calculation.
|
||||
"""
|
||||
|
||||
|
||||
@@ -119,8 +119,7 @@ NCCL_SYMM_MEM_ALL_REDUCE_CONFIG: dict[str, Any] = {
|
||||
|
||||
|
||||
def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor) -> bool:
|
||||
"""
|
||||
Determine if NCCL symmetric memory allreduce should be used.
|
||||
"""Determine if NCCL symmetric memory allreduce should be used.
|
||||
|
||||
Based on H100 and GB200 benchmarks, NCCL symm_mem is preferred for:
|
||||
- Small tensors (≤16K): Lower latency than custom_AR
|
||||
@@ -157,7 +156,8 @@ def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor)
|
||||
|
||||
def should_nccl_symm_mem_ag_rs() -> bool:
|
||||
"""Check whether NCCL symmetric memory should be used for
|
||||
AllGather / ReduceScatter collectives."""
|
||||
AllGather / ReduceScatter collectives.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import (
|
||||
is_symmetric_memory_enabled,
|
||||
)
|
||||
@@ -344,7 +344,6 @@ _gpu_p2p_access_cache: dict[str, bool] | None = None
|
||||
|
||||
def gpu_p2p_access_check(src: int, tgt: int) -> bool:
|
||||
"""Check if GPU src can access GPU tgt."""
|
||||
|
||||
# if the cache variable is already calculated,
|
||||
# read from the cache instead of checking it again
|
||||
global _gpu_p2p_access_cache
|
||||
|
||||
@@ -114,6 +114,7 @@ class All2AllManagerBase:
|
||||
Returns:
|
||||
An int32 device tensor where 0 marks a live rank and 1 marks a
|
||||
masked (dead/unreachable) rank.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -171,8 +172,7 @@ class All2AllManagerBase:
|
||||
|
||||
|
||||
class DeviceCommunicatorBase:
|
||||
"""
|
||||
Base class for device-specific communicator.
|
||||
"""Base class for device-specific communicator.
|
||||
It can use the `cpu_group` to initialize the communicator.
|
||||
If the device has PyTorch integration (PyTorch can recognize its
|
||||
communication backend), the `device_group` will also be given.
|
||||
@@ -318,8 +318,7 @@ class DeviceCommunicatorBase:
|
||||
def gather(
|
||||
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
NOTE: We assume that the input tensor is on the same device across
|
||||
"""NOTE: We assume that the input tensor is on the same device across
|
||||
all the ranks.
|
||||
NOTE: `dst` is the local rank of the destination rank.
|
||||
"""
|
||||
@@ -347,7 +346,7 @@ class DeviceCommunicatorBase:
|
||||
return output_tensor
|
||||
|
||||
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
|
||||
"""Sends a tensor to the destination rank in a blocking way"""
|
||||
"""Sends a tensor to the destination rank in a blocking way."""
|
||||
"""NOTE: `dst` is the local rank of the destination rank."""
|
||||
if dst is None:
|
||||
dst = (self.rank_in_group + 1) % self.world_size
|
||||
@@ -385,8 +384,7 @@ class DeviceCommunicatorBase:
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and router logits to the appropriate device.
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
if extra_tensors is not None:
|
||||
@@ -404,8 +402,7 @@ class DeviceCommunicatorBase:
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
if extra_tensors is not None:
|
||||
@@ -415,8 +412,7 @@ class DeviceCommunicatorBase:
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Combine the hidden states and router logits from the appropriate device.
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
return hidden_states
|
||||
|
||||
@@ -69,8 +69,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
logger.info("Using allgather_reducescatter all2all manager.")
|
||||
|
||||
def _all_group_ranks_share_shm_group_name(self) -> bool:
|
||||
"""
|
||||
CPUSHM requires all ranks in this group to agree on one SHM group name.
|
||||
"""CPUSHM requires all ranks in this group to agree on one SHM group name.
|
||||
This is a lightweight consistency check for VLLM_DIST_IDENT/name inputs.
|
||||
"""
|
||||
local_name = _CPUSHMDistributed.make_group_name(self)
|
||||
@@ -89,8 +88,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
def gather(
|
||||
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
NOTE: We assume that the input tensor is on the same device across
|
||||
"""NOTE: We assume that the input tensor is on the same device across
|
||||
all the ranks.
|
||||
NOTE: `dst` is the local rank of the destination rank.
|
||||
"""
|
||||
@@ -180,11 +178,9 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and router logits to the appropriate device.
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.dispatch_router_logits(
|
||||
hidden_states,
|
||||
@@ -204,8 +200,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
@@ -220,8 +215,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Combine the hidden states and router logits from the appropriate device.
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
|
||||
@@ -611,7 +611,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
return symm_output
|
||||
|
||||
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
|
||||
"""Sends a tensor to the destination rank in a blocking way"""
|
||||
"""Sends a tensor to the destination rank in a blocking way."""
|
||||
"""NOTE: `dst` is the local rank of the destination rank."""
|
||||
if dst is None:
|
||||
dst = (self.rank_in_group + 1) % self.world_size
|
||||
@@ -851,11 +851,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and router logits to the appropriate device.
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.dispatch_router_logits(
|
||||
hidden_states,
|
||||
@@ -875,8 +873,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
@@ -891,8 +888,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Combine the hidden states and router logits from the appropriate device.
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
|
||||
@@ -129,8 +129,7 @@ class CustomAllreduce:
|
||||
max_mnnvl_reduce_scatter_size=_DEFAULT_MNNVL_REDUCE_SCATTER_MAX_SIZE,
|
||||
symm_mem_enabled=False,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
group: the process group to work on. If None, it will use the
|
||||
default process group.
|
||||
device: the device to bind the CustomAllreduce to. If None,
|
||||
@@ -138,6 +137,7 @@ class CustomAllreduce:
|
||||
It is the caller's responsibility to make sure each communicator
|
||||
is bind to a unique device, and all communicators in this group
|
||||
are in the same node.
|
||||
|
||||
"""
|
||||
self._IS_CAPTURING = False
|
||||
self._ptr = 0
|
||||
@@ -370,8 +370,7 @@ class CustomAllreduce:
|
||||
|
||||
@contextmanager
|
||||
def capture(self):
|
||||
"""
|
||||
The main responsibility of this context manager is the
|
||||
"""The main responsibility of this context manager is the
|
||||
`register_graph_buffers` call at the end of the context.
|
||||
It records all the buffer addresses used in the CUDA graph.
|
||||
"""
|
||||
|
||||
@@ -140,6 +140,7 @@ def _resolve_fi_ar_backend() -> tuple[str, bool]:
|
||||
is True only when ``auto`` selects mnnvl for a single node, so that
|
||||
workspace creation can fall back to trtllm on single-node topologies
|
||||
without NVSwitch multicast support (where mnnvl is unavailable).
|
||||
|
||||
"""
|
||||
backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND
|
||||
if backend != "auto":
|
||||
@@ -168,8 +169,7 @@ def get_fi_ar_workspace(
|
||||
dtype: torch.dtype,
|
||||
group: ProcessGroup,
|
||||
):
|
||||
"""
|
||||
Return the allreduce workspace for non-quant patterns, initializing if needed.
|
||||
"""Return the allreduce workspace for non-quant patterns, initializing if needed.
|
||||
|
||||
Used by AllReduceFusionPass (non-quant patterns) and FlashInferAllReduce
|
||||
for standalone allreduce. Backend is controlled by
|
||||
@@ -234,8 +234,7 @@ def get_fi_ar_quant_workspace(
|
||||
dtype: torch.dtype,
|
||||
group: ProcessGroup,
|
||||
):
|
||||
"""
|
||||
Return the allreduce workspace for quant patterns, initializing if needed.
|
||||
"""Return the allreduce workspace for quant patterns, initializing if needed.
|
||||
|
||||
Backend is controlled by VLLM_FLASHINFER_ALLREDUCE_BACKEND env var, matching
|
||||
non-quant fusion. With ``auto`` this prefers mnnvl and falls back to trtllm
|
||||
|
||||
@@ -98,8 +98,7 @@ class PyNcclCommunicator:
|
||||
device: int | str | torch.device,
|
||||
library_path: str | None = None,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
group: the process group to work on. If None, it will use the
|
||||
default process group.
|
||||
device: the device to bind the PyNcclCommunicator to. If None,
|
||||
@@ -108,6 +107,7 @@ class PyNcclCommunicator:
|
||||
use the default library path.
|
||||
It is the caller's responsibility to make sure each communicator
|
||||
is bind to a unique device.
|
||||
|
||||
"""
|
||||
if not isinstance(group, StatelessProcessGroup):
|
||||
assert dist.is_initialized()
|
||||
|
||||
@@ -62,8 +62,7 @@ class QuickAllReduce:
|
||||
}
|
||||
|
||||
def __init__(self, group: ProcessGroup, device: int | str | torch.device) -> None:
|
||||
"""
|
||||
Custom allreduce provides non-destructive acceleration and is
|
||||
"""Custom allreduce provides non-destructive acceleration and is
|
||||
available for CUDA and ROCm MI300 series.
|
||||
|
||||
Custom quick allreduce leverages quantization for further
|
||||
@@ -86,6 +85,7 @@ class QuickAllReduce:
|
||||
It is the caller's responsibility to make sure each communicator
|
||||
is bind to a unique device, and all communicators in this group
|
||||
are in the same node.
|
||||
|
||||
"""
|
||||
self.disabled = True
|
||||
if not self._rocm_arch_available():
|
||||
@@ -301,8 +301,7 @@ class QuickAllReduce:
|
||||
return False
|
||||
|
||||
def create_shared_buffer(self):
|
||||
"""
|
||||
Creates a shared buffer for quickreduce.
|
||||
"""Creates a shared buffer for quickreduce.
|
||||
Has to be called after init_custom_qr
|
||||
"""
|
||||
handle = ops.qr_get_handle(self._ptr)
|
||||
@@ -312,9 +311,7 @@ class QuickAllReduce:
|
||||
ops.qr_open_handles(self._ptr, handles)
|
||||
|
||||
def should_quick_allreduce(self, inp: torch.Tensor):
|
||||
"""
|
||||
Check if quickreduce is available
|
||||
"""
|
||||
"""Check if quickreduce is available."""
|
||||
if self.disabled:
|
||||
return False
|
||||
if inp.dtype not in self._SUPPORTED_DTYPES:
|
||||
|
||||
@@ -20,8 +20,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class RayPPCommunicator(Communicator):
|
||||
"""
|
||||
Communicator to be used for pipeline parallelism in Ray Compiled Graph.
|
||||
"""Communicator to be used for pipeline parallelism in Ray Compiled Graph.
|
||||
This is wraps around the vLLM _PP GroupCoordinator.
|
||||
|
||||
This class is not thread-safe.
|
||||
@@ -38,8 +37,7 @@ class RayPPCommunicator(Communicator):
|
||||
cuda_stream: torch.cuda.Stream | None,
|
||||
use_communication_streams: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize a RayPPCommunicator that can be used to communicate with
|
||||
"""Initialize a RayPPCommunicator that can be used to communicate with
|
||||
other Ray Compiled Graph actors for pipeline parallelism.
|
||||
|
||||
Args:
|
||||
@@ -54,6 +52,7 @@ class RayPPCommunicator(Communicator):
|
||||
is not supported.
|
||||
use_communication_streams: Whether to use communication streams.
|
||||
This is not supported.
|
||||
|
||||
"""
|
||||
self._world_size = world_size
|
||||
self._rank: int | None = None
|
||||
@@ -87,8 +86,7 @@ class RayPPCommunicator(Communicator):
|
||||
self._closed = False
|
||||
|
||||
def _build_actor_rank_mapping(self):
|
||||
"""
|
||||
Use collective communication to build a mapping from actor IDs to ranks.
|
||||
"""Use collective communication to build a mapping from actor IDs to ranks.
|
||||
This should be called once during initialization.
|
||||
"""
|
||||
if self._comm is None:
|
||||
@@ -128,9 +126,7 @@ class RayPPCommunicator(Communicator):
|
||||
return self._actor_handles
|
||||
|
||||
def get_rank(self, actor: ray.actor.ActorHandle) -> int:
|
||||
"""
|
||||
Return the given actor's rank using device communicator collective ops.
|
||||
"""
|
||||
"""Return the given actor's rank using device communicator collective ops."""
|
||||
assert hasattr(self, "_actor_id_to_rank"), (
|
||||
"Actor rank mapping not built. "
|
||||
"This should have been done during initialization."
|
||||
@@ -144,20 +140,15 @@ class RayPPCommunicator(Communicator):
|
||||
raise ValueError(f"Actor {actor} not found in communicator group")
|
||||
|
||||
def get_self_rank(self) -> int | None:
|
||||
"""
|
||||
Return this actor's rank.
|
||||
"""
|
||||
"""Return this actor's rank."""
|
||||
return self._rank
|
||||
|
||||
def get_world_size(self) -> int:
|
||||
"""
|
||||
Return the number of ranks in the RayPPCommunicator group.
|
||||
"""
|
||||
"""Return the number of ranks in the RayPPCommunicator group."""
|
||||
return self._world_size
|
||||
|
||||
def send(self, buf: "torch.Tensor", peer_rank: int) -> None:
|
||||
"""
|
||||
Send a torch.Tensor to a peer.
|
||||
"""Send a torch.Tensor to a peer.
|
||||
|
||||
This returns when the send kernel has been queued, but the kernel may
|
||||
not have completed. Therefore, the caller should ensure that there are
|
||||
@@ -170,6 +161,7 @@ class RayPPCommunicator(Communicator):
|
||||
buf: The torch.Tensor to send. It should already be on this
|
||||
actor's default device.
|
||||
peer_rank: The rank of the actor to send to.
|
||||
|
||||
"""
|
||||
if self._closed:
|
||||
raise RayChannelError("RayPPCommunicator has been destroyed.")
|
||||
@@ -184,8 +176,7 @@ class RayPPCommunicator(Communicator):
|
||||
peer_rank: int,
|
||||
allocator: TorchTensorAllocator,
|
||||
) -> "torch.Tensor":
|
||||
"""
|
||||
Receive a torch.Tensor from a peer and synchronize the current stream.
|
||||
"""Receive a torch.Tensor from a peer and synchronize the current stream.
|
||||
|
||||
After this call returns, the receive buffer is safe to read from
|
||||
any stream. An RayChannelError will be raised if an error occurred
|
||||
@@ -197,6 +188,7 @@ class RayPPCommunicator(Communicator):
|
||||
peer_rank: The rank of the actor to receive from.
|
||||
allocator: The allocator to use to create the received tensor.
|
||||
This is ignored for this implementation.
|
||||
|
||||
"""
|
||||
if self._closed:
|
||||
raise RayChannelError("RayPPCommunicator has been destroyed.")
|
||||
|
||||
@@ -77,8 +77,7 @@ _memory_fence_lock = threading.Lock()
|
||||
|
||||
|
||||
def memory_fence():
|
||||
"""
|
||||
Full memory barrier for shared memory synchronization.
|
||||
"""Full memory barrier for shared memory synchronization.
|
||||
|
||||
Ensures all prior memory writes are visible to other processes before
|
||||
any subsequent reads. This is critical for lock-free producer-consumer
|
||||
@@ -111,8 +110,7 @@ LONG_WAIT_TIME_LOG_MSG = (
|
||||
|
||||
|
||||
class SpinCondition:
|
||||
"""
|
||||
This class implements an interface similar to a threading.Condition. It
|
||||
"""This class implements an interface similar to a threading.Condition. It
|
||||
allows a writer to notify readers to wake up and read from the shared memory
|
||||
buffer. This notification is done over a zmq socket.
|
||||
|
||||
@@ -241,6 +239,7 @@ def check_shm_free_space(
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the SHM filesystem has insufficient space.
|
||||
|
||||
"""
|
||||
if os.path.isdir(shm_path):
|
||||
free_bytes = shutil.disk_usage(shm_path).free
|
||||
@@ -651,7 +650,8 @@ class MessageQueue:
|
||||
|
||||
def shutdown(self):
|
||||
"""If this is an idle reader, wakes it up so it can clean up and shut
|
||||
down"""
|
||||
down
|
||||
"""
|
||||
self.shutting_down = True
|
||||
if self._spin_condition is not None:
|
||||
self._spin_condition.cancel()
|
||||
@@ -761,7 +761,8 @@ class MessageQueue:
|
||||
|
||||
def should_warn(self) -> bool:
|
||||
"""Returns true if it's time to log a warning for a timeout that is not
|
||||
indefinite"""
|
||||
indefinite
|
||||
"""
|
||||
if self._should_warn:
|
||||
elapsed = time.monotonic() - self.started
|
||||
if elapsed >= VLLM_RINGBUFFER_WARNING_INTERVAL * self.n_warning:
|
||||
@@ -941,8 +942,7 @@ class MessageQueue:
|
||||
reader_rank: int = 0,
|
||||
blocking: bool = False,
|
||||
) -> tuple["MessageQueue", list[Handle]]:
|
||||
"""
|
||||
Creates a MessageQueue for a process group with a single reader.
|
||||
"""Creates a MessageQueue for a process group with a single reader.
|
||||
|
||||
This method is designed for scenarios where only one process (the reader)
|
||||
will consume messages, and all other processes are writers. It sets up
|
||||
@@ -962,6 +962,7 @@ class MessageQueue:
|
||||
tuple[MessageQueue, list[Handle]]:
|
||||
The MessageQueue instance for the calling process,
|
||||
and a list of handles (only non-empty for the reader process).
|
||||
|
||||
"""
|
||||
from vllm.platforms.interface import get_assigned_physical_gpu_ids
|
||||
|
||||
@@ -994,8 +995,7 @@ class MessageQueue:
|
||||
external_writer_handle=None,
|
||||
blocking: bool = True,
|
||||
) -> "MessageQueue":
|
||||
"""
|
||||
Creates a MessageQueue for a distributed process group with one writer and
|
||||
"""Creates a MessageQueue for a distributed process group with one writer and
|
||||
multiple readers.
|
||||
|
||||
This method is designed for scenarios where one process (the writer) sends
|
||||
|
||||
@@ -20,8 +20,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SingleWriterShmRingBuffer:
|
||||
"""
|
||||
A single-writer, multiple-reader ring buffer implementation using shared
|
||||
"""A single-writer, multiple-reader ring buffer implementation using shared
|
||||
memory. This class provides a thread-safe ring buffer where one process
|
||||
can write data while multiple processes/threads can read from it.
|
||||
|
||||
@@ -190,8 +189,7 @@ class SingleWriterShmRingBuffer:
|
||||
return int.from_bytes(byte_data, "little", signed=True)
|
||||
|
||||
def allocate_buf(self, size: int) -> tuple[int, int]:
|
||||
"""
|
||||
Allocate a buffer `MD_SIZE` + `size` bytes in the shared memory.
|
||||
"""Allocate a buffer `MD_SIZE` + `size` bytes in the shared memory.
|
||||
Memory layout:
|
||||
`[4-byte monotonic_id][4-byte size][buffer data...]`
|
||||
"""
|
||||
@@ -260,8 +258,7 @@ class SingleWriterShmRingBuffer:
|
||||
is_free_fn: Callable[[int, memoryview], bool],
|
||||
nbytes: int | None = None,
|
||||
) -> Iterable[int]:
|
||||
"""
|
||||
Free a buffer of the given size. This is a no-op in shared memory,
|
||||
"""Free a buffer of the given size. This is a no-op in shared memory,
|
||||
but we need to keep track of the metadata.
|
||||
|
||||
If freed memory spreads across the end and start of the ring buffer,
|
||||
@@ -269,10 +266,13 @@ class SingleWriterShmRingBuffer:
|
||||
still might not be a contiguous space of `nbytes` available.
|
||||
|
||||
Args:
|
||||
is_free_fn (Callable[[int, memoryview], bool]): Predicate called with
|
||||
a monotonic id and the buffer, returning True when that buffer
|
||||
can be reclaimed.
|
||||
nbytes (int, optional): The size of the buffer to free. If None,
|
||||
frees the maximum size of the ring buffer.
|
||||
"""
|
||||
|
||||
"""
|
||||
assert self.is_writer, "Only the writer can free buffers."
|
||||
logger.debug(
|
||||
"Freeing up space in the ring buffer, "
|
||||
@@ -412,8 +412,7 @@ class ShmObjectStorageHandle:
|
||||
|
||||
|
||||
class SingleWriterShmObjectStorage:
|
||||
"""
|
||||
A single-writer, multiple-reader object storage system built on top of a
|
||||
"""A single-writer, multiple-reader object storage system built on top of a
|
||||
shared memory ring buffer. Provides key-value storage with automatic memory
|
||||
management and cross-process serialization support.
|
||||
|
||||
@@ -458,8 +457,7 @@ class SingleWriterShmObjectStorage:
|
||||
serde_class: type[ObjectSerde] = MsgpackSerde,
|
||||
reader_lock: LockType | None = None,
|
||||
):
|
||||
"""
|
||||
Initialize the object storage.
|
||||
"""Initialize the object storage.
|
||||
|
||||
Args:
|
||||
max_object_size: Maximum size for a single object in bytes.
|
||||
@@ -467,10 +465,11 @@ class SingleWriterShmObjectStorage:
|
||||
ring_buffer: The shared memory ring buffer for storing objects.
|
||||
serde_class: Serializer/deserializer for objects.
|
||||
reader_lock: Optional lock for synchronizing reader access.
|
||||
|
||||
Raises:
|
||||
ValueError: If reader_lock is None for readers.
|
||||
"""
|
||||
|
||||
"""
|
||||
self.max_object_size = max_object_size
|
||||
self.n_readers = n_readers
|
||||
self.serde_class = serde_class
|
||||
@@ -547,22 +546,17 @@ class SingleWriterShmObjectStorage:
|
||||
del self.writer_flag[freed_id]
|
||||
|
||||
def is_cached(self, key: str) -> bool:
|
||||
"""
|
||||
Check if the object with the given key is cached.
|
||||
"""
|
||||
"""Check if the object with the given key is cached."""
|
||||
return key in self.key_index
|
||||
|
||||
def get_cached(self, key: str) -> tuple[int, int]:
|
||||
"""
|
||||
Get the cached object by key if it exists.
|
||||
"""
|
||||
"""Get the cached object by key if it exists."""
|
||||
address, monotonic_id = self.key_index[key]
|
||||
self.increment_writer_flag(monotonic_id)
|
||||
return address, monotonic_id
|
||||
|
||||
def put(self, key: str, value: Any) -> tuple[int, int]:
|
||||
"""
|
||||
Store a key-value pair in the object storage.
|
||||
"""Store a key-value pair in the object storage.
|
||||
Attempts to free max_object_size bytes using FIFO order
|
||||
when the ring buffer runs out of space during a put() operation.
|
||||
|
||||
@@ -574,6 +568,7 @@ class SingleWriterShmObjectStorage:
|
||||
MemoryError: If there's not enough space in the buffer
|
||||
ValueError: If the serialized object is too large
|
||||
ValueError: If the key already exists in the storage
|
||||
|
||||
"""
|
||||
if key in self.key_index:
|
||||
raise ValueError(f"Key '{key}' already exists in the storage.")
|
||||
@@ -639,8 +634,7 @@ class SingleWriterShmObjectStorage:
|
||||
address: int = 0,
|
||||
monotonic_id: int = 0,
|
||||
) -> None:
|
||||
"""
|
||||
Touch an existing cached item to update its eviction status.
|
||||
"""Touch an existing cached item to update its eviction status.
|
||||
|
||||
For writers (ShmObjectStoreSenderCache): Increment writer_flag
|
||||
For readers (ShmObjectStoreReceiverCache): Increment reader_count
|
||||
@@ -700,8 +694,7 @@ class SingleWriterShmObjectStorage:
|
||||
)
|
||||
|
||||
def default_is_free_check(self, id: int, buf: memoryview) -> bool:
|
||||
"""
|
||||
Default is_free function that checks if the first 4 bytes are zero.
|
||||
"""Default is_free function that checks if the first 4 bytes are zero.
|
||||
This indicates that the buffer is free.
|
||||
"""
|
||||
reader_count = int.from_bytes(buf[0:4], "little", signed=True)
|
||||
|
||||
@@ -205,11 +205,9 @@ class XpuCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and router logits to the appropriate device.
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.dispatch_router_logits(
|
||||
hidden_states,
|
||||
@@ -229,8 +227,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
@@ -245,8 +242,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Combine the hidden states and router logits from the appropriate device.
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
|
||||
Reference in New Issue
Block a user