Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -22,9 +22,7 @@ class AiterCustomAllreduce:
|
||||
|
||||
@classmethod
|
||||
def effective_max_size(cls) -> int:
|
||||
"""
|
||||
Max input byte size eligible for AITER custom allreduce.
|
||||
"""
|
||||
"""Max input byte size eligible for AITER custom allreduce."""
|
||||
return cls.MAX_SIZE // 2
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -44,8 +44,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AgRsAll2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
An implementation of all2all communication based on
|
||||
"""An implementation of all2all communication based on
|
||||
all-gather (dispatch) and reduce-scatter (combine).
|
||||
"""
|
||||
|
||||
@@ -79,9 +78,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Gather hidden_states and router_logits from all dp ranks.
|
||||
"""
|
||||
"""Gather hidden_states and router_logits from all dp ranks."""
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
|
||||
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
|
||||
@@ -111,9 +108,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Gather hidden_states and router_logits from all dp ranks.
|
||||
"""
|
||||
"""Gather hidden_states and router_logits from all dp ranks."""
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(hidden_states.shape[0], dist_group)
|
||||
assert sizes[dist_group.rank_in_group] == hidden_states.shape[0]
|
||||
@@ -140,9 +135,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Reduce-scatter hidden_states across all dp ranks.
|
||||
"""
|
||||
"""Reduce-scatter hidden_states across all dp ranks."""
|
||||
dist_group = self._get_comm_group(is_sequence_parallel)
|
||||
sizes = self._get_sizes(
|
||||
hidden_states.shape[0] // dist_group.world_size,
|
||||
@@ -156,9 +149,7 @@ class AgRsAll2AllManager(All2AllManagerBase):
|
||||
|
||||
|
||||
class DeepEPAll2AllManagerBase(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on DeepEP High-Throughput kernels.
|
||||
"""
|
||||
"""All2All communication based on DeepEP High-Throughput kernels."""
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
assert has_deep_ep(), (
|
||||
@@ -210,9 +201,7 @@ class DeepEPAll2AllManagerBase(All2AllManagerBase):
|
||||
|
||||
|
||||
class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on DeepEP High-Throughput kernels.
|
||||
"""
|
||||
"""All2All communication based on DeepEP High-Throughput kernels."""
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None):
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
@@ -271,9 +260,7 @@ class DeepEPHTAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
|
||||
|
||||
class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on DeepEP Low-Latency kernels.
|
||||
"""
|
||||
"""All2All communication based on DeepEP Low-Latency kernels."""
|
||||
|
||||
_buffer: Any = None
|
||||
_mask: torch.Tensor | None = None
|
||||
@@ -293,8 +280,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
num_global_experts: int,
|
||||
num_local_experts: int,
|
||||
) -> dict[Any, Any]:
|
||||
"""
|
||||
max_num_tokens_per_dp_rank : the maximum number of tokens a DP rank
|
||||
"""max_num_tokens_per_dp_rank : the maximum number of tokens a DP rank
|
||||
can dispatch all the ranks must hold the same value.
|
||||
token_hidden_size: the hidden dimension of each token.
|
||||
num_ep_ranks: the number of EP group ranks.
|
||||
@@ -330,8 +316,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
|
||||
return kwargs
|
||||
|
||||
def get_handle(self, kwargs):
|
||||
"""
|
||||
The kwargs for DeepEPLLAll2AllManager is dictated by
|
||||
"""The kwargs for DeepEPLLAll2AllManager is dictated by
|
||||
_make_all2all_kwargs.
|
||||
"""
|
||||
import deep_ep # type: ignore[import-not-found]
|
||||
@@ -385,8 +370,7 @@ class _NixlEPBufferState:
|
||||
|
||||
|
||||
class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on NIXL EP kernels.
|
||||
"""All2All communication based on NIXL EP kernels.
|
||||
This backend supports elastic EP with dynamic rank connection/disconnection.
|
||||
"""
|
||||
|
||||
@@ -612,9 +596,7 @@ class NixlEPAll2AllManager(All2AllManagerBase):
|
||||
|
||||
|
||||
class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on flashinfer all2allv/two-sided NVLink kernels.
|
||||
"""
|
||||
"""All2All communication based on flashinfer all2allv/two-sided NVLink kernels."""
|
||||
|
||||
# This type lint could be removed after all of the work in
|
||||
# https://github.com/vllm-project/vllm/issues/26533 done.
|
||||
@@ -640,7 +622,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
rank: int,
|
||||
gpus_per_node: int,
|
||||
):
|
||||
"""Initialize workspace"""
|
||||
"""Initialize workspace."""
|
||||
if self.initialized:
|
||||
return
|
||||
|
||||
@@ -681,7 +663,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
)
|
||||
|
||||
def ensure_alltoall_workspace_initialized(self):
|
||||
"""Ensure workspace is initialized"""
|
||||
"""Ensure workspace is initialized."""
|
||||
if not has_flashinfer_nvlink_two_sided():
|
||||
return False
|
||||
|
||||
@@ -700,7 +682,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
return self
|
||||
|
||||
def cleanup(self):
|
||||
"""Clean up workspace"""
|
||||
"""Clean up workspace."""
|
||||
if (
|
||||
self.initialized
|
||||
and self.workspace_tensor is not None
|
||||
@@ -719,8 +701,7 @@ class FlashInferNVLinkTwoSidedManager(All2AllManagerBase):
|
||||
|
||||
|
||||
class FlashInferNVLinkOneSidedManager(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on FlashInfer's MoeAlltoAll/One-sided NVLink kernel.
|
||||
"""All2All communication based on FlashInfer's MoeAlltoAll/One-sided NVLink kernel.
|
||||
This is a newer kernel from trtllm that should perform better than the kernel
|
||||
used by flashinfer_nvlink_two_sided.
|
||||
"""
|
||||
@@ -1023,8 +1004,7 @@ class MoriAll2AllManager(All2AllManagerBase):
|
||||
|
||||
|
||||
class DeepEPV2All2AllManager(All2AllManagerBase):
|
||||
"""
|
||||
All2All communication based on DeepEP v2 ElasticBuffer (unified API).
|
||||
"""All2All communication based on DeepEP v2 ElasticBuffer (unified API).
|
||||
Uses NCCL Gin backend with analytical SM calculation.
|
||||
"""
|
||||
|
||||
|
||||
@@ -119,8 +119,7 @@ NCCL_SYMM_MEM_ALL_REDUCE_CONFIG: dict[str, Any] = {
|
||||
|
||||
|
||||
def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor) -> bool:
|
||||
"""
|
||||
Determine if NCCL symmetric memory allreduce should be used.
|
||||
"""Determine if NCCL symmetric memory allreduce should be used.
|
||||
|
||||
Based on H100 and GB200 benchmarks, NCCL symm_mem is preferred for:
|
||||
- Small tensors (≤16K): Lower latency than custom_AR
|
||||
@@ -157,7 +156,8 @@ def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor)
|
||||
|
||||
def should_nccl_symm_mem_ag_rs() -> bool:
|
||||
"""Check whether NCCL symmetric memory should be used for
|
||||
AllGather / ReduceScatter collectives."""
|
||||
AllGather / ReduceScatter collectives.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import (
|
||||
is_symmetric_memory_enabled,
|
||||
)
|
||||
@@ -344,7 +344,6 @@ _gpu_p2p_access_cache: dict[str, bool] | None = None
|
||||
|
||||
def gpu_p2p_access_check(src: int, tgt: int) -> bool:
|
||||
"""Check if GPU src can access GPU tgt."""
|
||||
|
||||
# if the cache variable is already calculated,
|
||||
# read from the cache instead of checking it again
|
||||
global _gpu_p2p_access_cache
|
||||
|
||||
@@ -114,6 +114,7 @@ class All2AllManagerBase:
|
||||
Returns:
|
||||
An int32 device tensor where 0 marks a live rank and 1 marks a
|
||||
masked (dead/unreachable) rank.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -171,8 +172,7 @@ class All2AllManagerBase:
|
||||
|
||||
|
||||
class DeviceCommunicatorBase:
|
||||
"""
|
||||
Base class for device-specific communicator.
|
||||
"""Base class for device-specific communicator.
|
||||
It can use the `cpu_group` to initialize the communicator.
|
||||
If the device has PyTorch integration (PyTorch can recognize its
|
||||
communication backend), the `device_group` will also be given.
|
||||
@@ -318,8 +318,7 @@ class DeviceCommunicatorBase:
|
||||
def gather(
|
||||
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
NOTE: We assume that the input tensor is on the same device across
|
||||
"""NOTE: We assume that the input tensor is on the same device across
|
||||
all the ranks.
|
||||
NOTE: `dst` is the local rank of the destination rank.
|
||||
"""
|
||||
@@ -347,7 +346,7 @@ class DeviceCommunicatorBase:
|
||||
return output_tensor
|
||||
|
||||
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
|
||||
"""Sends a tensor to the destination rank in a blocking way"""
|
||||
"""Sends a tensor to the destination rank in a blocking way."""
|
||||
"""NOTE: `dst` is the local rank of the destination rank."""
|
||||
if dst is None:
|
||||
dst = (self.rank_in_group + 1) % self.world_size
|
||||
@@ -385,8 +384,7 @@ class DeviceCommunicatorBase:
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and router logits to the appropriate device.
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
if extra_tensors is not None:
|
||||
@@ -404,8 +402,7 @@ class DeviceCommunicatorBase:
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
if extra_tensors is not None:
|
||||
@@ -415,8 +412,7 @@ class DeviceCommunicatorBase:
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Combine the hidden states and router logits from the appropriate device.
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
return hidden_states
|
||||
|
||||
@@ -69,8 +69,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
logger.info("Using allgather_reducescatter all2all manager.")
|
||||
|
||||
def _all_group_ranks_share_shm_group_name(self) -> bool:
|
||||
"""
|
||||
CPUSHM requires all ranks in this group to agree on one SHM group name.
|
||||
"""CPUSHM requires all ranks in this group to agree on one SHM group name.
|
||||
This is a lightweight consistency check for VLLM_DIST_IDENT/name inputs.
|
||||
"""
|
||||
local_name = _CPUSHMDistributed.make_group_name(self)
|
||||
@@ -89,8 +88,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
def gather(
|
||||
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
NOTE: We assume that the input tensor is on the same device across
|
||||
"""NOTE: We assume that the input tensor is on the same device across
|
||||
all the ranks.
|
||||
NOTE: `dst` is the local rank of the destination rank.
|
||||
"""
|
||||
@@ -180,11 +178,9 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and router logits to the appropriate device.
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.dispatch_router_logits(
|
||||
hidden_states,
|
||||
@@ -204,8 +200,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
@@ -220,8 +215,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Combine the hidden states and router logits from the appropriate device.
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
|
||||
@@ -611,7 +611,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
return symm_output
|
||||
|
||||
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
|
||||
"""Sends a tensor to the destination rank in a blocking way"""
|
||||
"""Sends a tensor to the destination rank in a blocking way."""
|
||||
"""NOTE: `dst` is the local rank of the destination rank."""
|
||||
if dst is None:
|
||||
dst = (self.rank_in_group + 1) % self.world_size
|
||||
@@ -851,11 +851,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and router logits to the appropriate device.
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.dispatch_router_logits(
|
||||
hidden_states,
|
||||
@@ -875,8 +873,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
@@ -891,8 +888,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Combine the hidden states and router logits from the appropriate device.
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
|
||||
@@ -129,8 +129,7 @@ class CustomAllreduce:
|
||||
max_mnnvl_reduce_scatter_size=_DEFAULT_MNNVL_REDUCE_SCATTER_MAX_SIZE,
|
||||
symm_mem_enabled=False,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
group: the process group to work on. If None, it will use the
|
||||
default process group.
|
||||
device: the device to bind the CustomAllreduce to. If None,
|
||||
@@ -138,6 +137,7 @@ class CustomAllreduce:
|
||||
It is the caller's responsibility to make sure each communicator
|
||||
is bind to a unique device, and all communicators in this group
|
||||
are in the same node.
|
||||
|
||||
"""
|
||||
self._IS_CAPTURING = False
|
||||
self._ptr = 0
|
||||
@@ -370,8 +370,7 @@ class CustomAllreduce:
|
||||
|
||||
@contextmanager
|
||||
def capture(self):
|
||||
"""
|
||||
The main responsibility of this context manager is the
|
||||
"""The main responsibility of this context manager is the
|
||||
`register_graph_buffers` call at the end of the context.
|
||||
It records all the buffer addresses used in the CUDA graph.
|
||||
"""
|
||||
|
||||
@@ -140,6 +140,7 @@ def _resolve_fi_ar_backend() -> tuple[str, bool]:
|
||||
is True only when ``auto`` selects mnnvl for a single node, so that
|
||||
workspace creation can fall back to trtllm on single-node topologies
|
||||
without NVSwitch multicast support (where mnnvl is unavailable).
|
||||
|
||||
"""
|
||||
backend = envs.VLLM_FLASHINFER_ALLREDUCE_BACKEND
|
||||
if backend != "auto":
|
||||
@@ -168,8 +169,7 @@ def get_fi_ar_workspace(
|
||||
dtype: torch.dtype,
|
||||
group: ProcessGroup,
|
||||
):
|
||||
"""
|
||||
Return the allreduce workspace for non-quant patterns, initializing if needed.
|
||||
"""Return the allreduce workspace for non-quant patterns, initializing if needed.
|
||||
|
||||
Used by AllReduceFusionPass (non-quant patterns) and FlashInferAllReduce
|
||||
for standalone allreduce. Backend is controlled by
|
||||
@@ -234,8 +234,7 @@ def get_fi_ar_quant_workspace(
|
||||
dtype: torch.dtype,
|
||||
group: ProcessGroup,
|
||||
):
|
||||
"""
|
||||
Return the allreduce workspace for quant patterns, initializing if needed.
|
||||
"""Return the allreduce workspace for quant patterns, initializing if needed.
|
||||
|
||||
Backend is controlled by VLLM_FLASHINFER_ALLREDUCE_BACKEND env var, matching
|
||||
non-quant fusion. With ``auto`` this prefers mnnvl and falls back to trtllm
|
||||
|
||||
@@ -98,8 +98,7 @@ class PyNcclCommunicator:
|
||||
device: int | str | torch.device,
|
||||
library_path: str | None = None,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
group: the process group to work on. If None, it will use the
|
||||
default process group.
|
||||
device: the device to bind the PyNcclCommunicator to. If None,
|
||||
@@ -108,6 +107,7 @@ class PyNcclCommunicator:
|
||||
use the default library path.
|
||||
It is the caller's responsibility to make sure each communicator
|
||||
is bind to a unique device.
|
||||
|
||||
"""
|
||||
if not isinstance(group, StatelessProcessGroup):
|
||||
assert dist.is_initialized()
|
||||
|
||||
@@ -62,8 +62,7 @@ class QuickAllReduce:
|
||||
}
|
||||
|
||||
def __init__(self, group: ProcessGroup, device: int | str | torch.device) -> None:
|
||||
"""
|
||||
Custom allreduce provides non-destructive acceleration and is
|
||||
"""Custom allreduce provides non-destructive acceleration and is
|
||||
available for CUDA and ROCm MI300 series.
|
||||
|
||||
Custom quick allreduce leverages quantization for further
|
||||
@@ -86,6 +85,7 @@ class QuickAllReduce:
|
||||
It is the caller's responsibility to make sure each communicator
|
||||
is bind to a unique device, and all communicators in this group
|
||||
are in the same node.
|
||||
|
||||
"""
|
||||
self.disabled = True
|
||||
if not self._rocm_arch_available():
|
||||
@@ -301,8 +301,7 @@ class QuickAllReduce:
|
||||
return False
|
||||
|
||||
def create_shared_buffer(self):
|
||||
"""
|
||||
Creates a shared buffer for quickreduce.
|
||||
"""Creates a shared buffer for quickreduce.
|
||||
Has to be called after init_custom_qr
|
||||
"""
|
||||
handle = ops.qr_get_handle(self._ptr)
|
||||
@@ -312,9 +311,7 @@ class QuickAllReduce:
|
||||
ops.qr_open_handles(self._ptr, handles)
|
||||
|
||||
def should_quick_allreduce(self, inp: torch.Tensor):
|
||||
"""
|
||||
Check if quickreduce is available
|
||||
"""
|
||||
"""Check if quickreduce is available."""
|
||||
if self.disabled:
|
||||
return False
|
||||
if inp.dtype not in self._SUPPORTED_DTYPES:
|
||||
|
||||
@@ -20,8 +20,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class RayPPCommunicator(Communicator):
|
||||
"""
|
||||
Communicator to be used for pipeline parallelism in Ray Compiled Graph.
|
||||
"""Communicator to be used for pipeline parallelism in Ray Compiled Graph.
|
||||
This is wraps around the vLLM _PP GroupCoordinator.
|
||||
|
||||
This class is not thread-safe.
|
||||
@@ -38,8 +37,7 @@ class RayPPCommunicator(Communicator):
|
||||
cuda_stream: torch.cuda.Stream | None,
|
||||
use_communication_streams: bool = False,
|
||||
):
|
||||
"""
|
||||
Initialize a RayPPCommunicator that can be used to communicate with
|
||||
"""Initialize a RayPPCommunicator that can be used to communicate with
|
||||
other Ray Compiled Graph actors for pipeline parallelism.
|
||||
|
||||
Args:
|
||||
@@ -54,6 +52,7 @@ class RayPPCommunicator(Communicator):
|
||||
is not supported.
|
||||
use_communication_streams: Whether to use communication streams.
|
||||
This is not supported.
|
||||
|
||||
"""
|
||||
self._world_size = world_size
|
||||
self._rank: int | None = None
|
||||
@@ -87,8 +86,7 @@ class RayPPCommunicator(Communicator):
|
||||
self._closed = False
|
||||
|
||||
def _build_actor_rank_mapping(self):
|
||||
"""
|
||||
Use collective communication to build a mapping from actor IDs to ranks.
|
||||
"""Use collective communication to build a mapping from actor IDs to ranks.
|
||||
This should be called once during initialization.
|
||||
"""
|
||||
if self._comm is None:
|
||||
@@ -128,9 +126,7 @@ class RayPPCommunicator(Communicator):
|
||||
return self._actor_handles
|
||||
|
||||
def get_rank(self, actor: ray.actor.ActorHandle) -> int:
|
||||
"""
|
||||
Return the given actor's rank using device communicator collective ops.
|
||||
"""
|
||||
"""Return the given actor's rank using device communicator collective ops."""
|
||||
assert hasattr(self, "_actor_id_to_rank"), (
|
||||
"Actor rank mapping not built. "
|
||||
"This should have been done during initialization."
|
||||
@@ -144,20 +140,15 @@ class RayPPCommunicator(Communicator):
|
||||
raise ValueError(f"Actor {actor} not found in communicator group")
|
||||
|
||||
def get_self_rank(self) -> int | None:
|
||||
"""
|
||||
Return this actor's rank.
|
||||
"""
|
||||
"""Return this actor's rank."""
|
||||
return self._rank
|
||||
|
||||
def get_world_size(self) -> int:
|
||||
"""
|
||||
Return the number of ranks in the RayPPCommunicator group.
|
||||
"""
|
||||
"""Return the number of ranks in the RayPPCommunicator group."""
|
||||
return self._world_size
|
||||
|
||||
def send(self, buf: "torch.Tensor", peer_rank: int) -> None:
|
||||
"""
|
||||
Send a torch.Tensor to a peer.
|
||||
"""Send a torch.Tensor to a peer.
|
||||
|
||||
This returns when the send kernel has been queued, but the kernel may
|
||||
not have completed. Therefore, the caller should ensure that there are
|
||||
@@ -170,6 +161,7 @@ class RayPPCommunicator(Communicator):
|
||||
buf: The torch.Tensor to send. It should already be on this
|
||||
actor's default device.
|
||||
peer_rank: The rank of the actor to send to.
|
||||
|
||||
"""
|
||||
if self._closed:
|
||||
raise RayChannelError("RayPPCommunicator has been destroyed.")
|
||||
@@ -184,8 +176,7 @@ class RayPPCommunicator(Communicator):
|
||||
peer_rank: int,
|
||||
allocator: TorchTensorAllocator,
|
||||
) -> "torch.Tensor":
|
||||
"""
|
||||
Receive a torch.Tensor from a peer and synchronize the current stream.
|
||||
"""Receive a torch.Tensor from a peer and synchronize the current stream.
|
||||
|
||||
After this call returns, the receive buffer is safe to read from
|
||||
any stream. An RayChannelError will be raised if an error occurred
|
||||
@@ -197,6 +188,7 @@ class RayPPCommunicator(Communicator):
|
||||
peer_rank: The rank of the actor to receive from.
|
||||
allocator: The allocator to use to create the received tensor.
|
||||
This is ignored for this implementation.
|
||||
|
||||
"""
|
||||
if self._closed:
|
||||
raise RayChannelError("RayPPCommunicator has been destroyed.")
|
||||
|
||||
@@ -77,8 +77,7 @@ _memory_fence_lock = threading.Lock()
|
||||
|
||||
|
||||
def memory_fence():
|
||||
"""
|
||||
Full memory barrier for shared memory synchronization.
|
||||
"""Full memory barrier for shared memory synchronization.
|
||||
|
||||
Ensures all prior memory writes are visible to other processes before
|
||||
any subsequent reads. This is critical for lock-free producer-consumer
|
||||
@@ -111,8 +110,7 @@ LONG_WAIT_TIME_LOG_MSG = (
|
||||
|
||||
|
||||
class SpinCondition:
|
||||
"""
|
||||
This class implements an interface similar to a threading.Condition. It
|
||||
"""This class implements an interface similar to a threading.Condition. It
|
||||
allows a writer to notify readers to wake up and read from the shared memory
|
||||
buffer. This notification is done over a zmq socket.
|
||||
|
||||
@@ -241,6 +239,7 @@ def check_shm_free_space(
|
||||
|
||||
Raises:
|
||||
RuntimeError: If the SHM filesystem has insufficient space.
|
||||
|
||||
"""
|
||||
if os.path.isdir(shm_path):
|
||||
free_bytes = shutil.disk_usage(shm_path).free
|
||||
@@ -651,7 +650,8 @@ class MessageQueue:
|
||||
|
||||
def shutdown(self):
|
||||
"""If this is an idle reader, wakes it up so it can clean up and shut
|
||||
down"""
|
||||
down
|
||||
"""
|
||||
self.shutting_down = True
|
||||
if self._spin_condition is not None:
|
||||
self._spin_condition.cancel()
|
||||
@@ -761,7 +761,8 @@ class MessageQueue:
|
||||
|
||||
def should_warn(self) -> bool:
|
||||
"""Returns true if it's time to log a warning for a timeout that is not
|
||||
indefinite"""
|
||||
indefinite
|
||||
"""
|
||||
if self._should_warn:
|
||||
elapsed = time.monotonic() - self.started
|
||||
if elapsed >= VLLM_RINGBUFFER_WARNING_INTERVAL * self.n_warning:
|
||||
@@ -941,8 +942,7 @@ class MessageQueue:
|
||||
reader_rank: int = 0,
|
||||
blocking: bool = False,
|
||||
) -> tuple["MessageQueue", list[Handle]]:
|
||||
"""
|
||||
Creates a MessageQueue for a process group with a single reader.
|
||||
"""Creates a MessageQueue for a process group with a single reader.
|
||||
|
||||
This method is designed for scenarios where only one process (the reader)
|
||||
will consume messages, and all other processes are writers. It sets up
|
||||
@@ -962,6 +962,7 @@ class MessageQueue:
|
||||
tuple[MessageQueue, list[Handle]]:
|
||||
The MessageQueue instance for the calling process,
|
||||
and a list of handles (only non-empty for the reader process).
|
||||
|
||||
"""
|
||||
from vllm.platforms.interface import get_assigned_physical_gpu_ids
|
||||
|
||||
@@ -994,8 +995,7 @@ class MessageQueue:
|
||||
external_writer_handle=None,
|
||||
blocking: bool = True,
|
||||
) -> "MessageQueue":
|
||||
"""
|
||||
Creates a MessageQueue for a distributed process group with one writer and
|
||||
"""Creates a MessageQueue for a distributed process group with one writer and
|
||||
multiple readers.
|
||||
|
||||
This method is designed for scenarios where one process (the writer) sends
|
||||
|
||||
@@ -20,8 +20,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SingleWriterShmRingBuffer:
|
||||
"""
|
||||
A single-writer, multiple-reader ring buffer implementation using shared
|
||||
"""A single-writer, multiple-reader ring buffer implementation using shared
|
||||
memory. This class provides a thread-safe ring buffer where one process
|
||||
can write data while multiple processes/threads can read from it.
|
||||
|
||||
@@ -190,8 +189,7 @@ class SingleWriterShmRingBuffer:
|
||||
return int.from_bytes(byte_data, "little", signed=True)
|
||||
|
||||
def allocate_buf(self, size: int) -> tuple[int, int]:
|
||||
"""
|
||||
Allocate a buffer `MD_SIZE` + `size` bytes in the shared memory.
|
||||
"""Allocate a buffer `MD_SIZE` + `size` bytes in the shared memory.
|
||||
Memory layout:
|
||||
`[4-byte monotonic_id][4-byte size][buffer data...]`
|
||||
"""
|
||||
@@ -260,8 +258,7 @@ class SingleWriterShmRingBuffer:
|
||||
is_free_fn: Callable[[int, memoryview], bool],
|
||||
nbytes: int | None = None,
|
||||
) -> Iterable[int]:
|
||||
"""
|
||||
Free a buffer of the given size. This is a no-op in shared memory,
|
||||
"""Free a buffer of the given size. This is a no-op in shared memory,
|
||||
but we need to keep track of the metadata.
|
||||
|
||||
If freed memory spreads across the end and start of the ring buffer,
|
||||
@@ -269,10 +266,13 @@ class SingleWriterShmRingBuffer:
|
||||
still might not be a contiguous space of `nbytes` available.
|
||||
|
||||
Args:
|
||||
is_free_fn (Callable[[int, memoryview], bool]): Predicate called with
|
||||
a monotonic id and the buffer, returning True when that buffer
|
||||
can be reclaimed.
|
||||
nbytes (int, optional): The size of the buffer to free. If None,
|
||||
frees the maximum size of the ring buffer.
|
||||
"""
|
||||
|
||||
"""
|
||||
assert self.is_writer, "Only the writer can free buffers."
|
||||
logger.debug(
|
||||
"Freeing up space in the ring buffer, "
|
||||
@@ -412,8 +412,7 @@ class ShmObjectStorageHandle:
|
||||
|
||||
|
||||
class SingleWriterShmObjectStorage:
|
||||
"""
|
||||
A single-writer, multiple-reader object storage system built on top of a
|
||||
"""A single-writer, multiple-reader object storage system built on top of a
|
||||
shared memory ring buffer. Provides key-value storage with automatic memory
|
||||
management and cross-process serialization support.
|
||||
|
||||
@@ -458,8 +457,7 @@ class SingleWriterShmObjectStorage:
|
||||
serde_class: type[ObjectSerde] = MsgpackSerde,
|
||||
reader_lock: LockType | None = None,
|
||||
):
|
||||
"""
|
||||
Initialize the object storage.
|
||||
"""Initialize the object storage.
|
||||
|
||||
Args:
|
||||
max_object_size: Maximum size for a single object in bytes.
|
||||
@@ -467,10 +465,11 @@ class SingleWriterShmObjectStorage:
|
||||
ring_buffer: The shared memory ring buffer for storing objects.
|
||||
serde_class: Serializer/deserializer for objects.
|
||||
reader_lock: Optional lock for synchronizing reader access.
|
||||
|
||||
Raises:
|
||||
ValueError: If reader_lock is None for readers.
|
||||
"""
|
||||
|
||||
"""
|
||||
self.max_object_size = max_object_size
|
||||
self.n_readers = n_readers
|
||||
self.serde_class = serde_class
|
||||
@@ -547,22 +546,17 @@ class SingleWriterShmObjectStorage:
|
||||
del self.writer_flag[freed_id]
|
||||
|
||||
def is_cached(self, key: str) -> bool:
|
||||
"""
|
||||
Check if the object with the given key is cached.
|
||||
"""
|
||||
"""Check if the object with the given key is cached."""
|
||||
return key in self.key_index
|
||||
|
||||
def get_cached(self, key: str) -> tuple[int, int]:
|
||||
"""
|
||||
Get the cached object by key if it exists.
|
||||
"""
|
||||
"""Get the cached object by key if it exists."""
|
||||
address, monotonic_id = self.key_index[key]
|
||||
self.increment_writer_flag(monotonic_id)
|
||||
return address, monotonic_id
|
||||
|
||||
def put(self, key: str, value: Any) -> tuple[int, int]:
|
||||
"""
|
||||
Store a key-value pair in the object storage.
|
||||
"""Store a key-value pair in the object storage.
|
||||
Attempts to free max_object_size bytes using FIFO order
|
||||
when the ring buffer runs out of space during a put() operation.
|
||||
|
||||
@@ -574,6 +568,7 @@ class SingleWriterShmObjectStorage:
|
||||
MemoryError: If there's not enough space in the buffer
|
||||
ValueError: If the serialized object is too large
|
||||
ValueError: If the key already exists in the storage
|
||||
|
||||
"""
|
||||
if key in self.key_index:
|
||||
raise ValueError(f"Key '{key}' already exists in the storage.")
|
||||
@@ -639,8 +634,7 @@ class SingleWriterShmObjectStorage:
|
||||
address: int = 0,
|
||||
monotonic_id: int = 0,
|
||||
) -> None:
|
||||
"""
|
||||
Touch an existing cached item to update its eviction status.
|
||||
"""Touch an existing cached item to update its eviction status.
|
||||
|
||||
For writers (ShmObjectStoreSenderCache): Increment writer_flag
|
||||
For readers (ShmObjectStoreReceiverCache): Increment reader_count
|
||||
@@ -700,8 +694,7 @@ class SingleWriterShmObjectStorage:
|
||||
)
|
||||
|
||||
def default_is_free_check(self, id: int, buf: memoryview) -> bool:
|
||||
"""
|
||||
Default is_free function that checks if the first 4 bytes are zero.
|
||||
"""Default is_free function that checks if the first 4 bytes are zero.
|
||||
This indicates that the buffer is free.
|
||||
"""
|
||||
reader_count = int.from_bytes(buf[0:4], "little", signed=True)
|
||||
|
||||
@@ -205,11 +205,9 @@ class XpuCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and router logits to the appropriate device.
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.dispatch_router_logits(
|
||||
hidden_states,
|
||||
@@ -229,8 +227,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""
|
||||
Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
@@ -245,8 +242,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Combine the hidden states and router logits from the appropriate device.
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
ECConnectorBase Class for Distributed Encoder Cache &
|
||||
"""ECConnectorBase Class for Distributed Encoder Cache &
|
||||
P2P Encoder cache communication in V1
|
||||
|
||||
The class provides the following primitives:
|
||||
@@ -50,8 +49,7 @@ class ECConnectorRole(enum.Enum):
|
||||
|
||||
|
||||
class ECConnectorMetadata(ABC): # noqa: B024
|
||||
"""
|
||||
Abstract Metadata used to communicate between the
|
||||
"""Abstract Metadata used to communicate between the
|
||||
Scheduler ECConnector and Worker ECConnector.
|
||||
"""
|
||||
|
||||
@@ -59,8 +57,7 @@ class ECConnectorMetadata(ABC): # noqa: B024
|
||||
|
||||
|
||||
class ECConnectorWorkerMetadata(ABC):
|
||||
"""
|
||||
Abstract Metadata used to communicate back
|
||||
"""Abstract Metadata used to communicate back
|
||||
Worker ECConnector -> Scheduler ECConnector.
|
||||
|
||||
Each worker can output its own metadata.
|
||||
@@ -73,9 +70,7 @@ class ECConnectorWorkerMetadata(ABC):
|
||||
def aggregate(
|
||||
self, other: "ECConnectorWorkerMetadata"
|
||||
) -> "ECConnectorWorkerMetadata":
|
||||
"""
|
||||
Aggregate metadata with another `ECConnectorWorkerMetadata` object.
|
||||
"""
|
||||
"""Aggregate metadata with another `ECConnectorWorkerMetadata` object."""
|
||||
pass
|
||||
|
||||
|
||||
@@ -103,8 +98,7 @@ class ECConnectorBase(ABC):
|
||||
return self._is_consumer
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""
|
||||
Shutdown the connector. This is called when the process
|
||||
"""Shutdown the connector. This is called when the process
|
||||
is shutting down to ensure that all the async operations are
|
||||
completed and the connector is cleaned up properly.
|
||||
"""
|
||||
@@ -123,6 +117,7 @@ class ECConnectorBase(ABC):
|
||||
|
||||
Args:
|
||||
connector_metadata (dict): the connector metadata.
|
||||
|
||||
"""
|
||||
self._connector_metadata = connector_metadata
|
||||
|
||||
@@ -141,8 +136,8 @@ class ECConnectorBase(ABC):
|
||||
|
||||
Returns:
|
||||
ConnectorMetadata: the connector metadata.
|
||||
"""
|
||||
|
||||
"""
|
||||
# Should only be called while set to valid metadata.
|
||||
assert self._connector_metadata is not None
|
||||
return self._connector_metadata
|
||||
@@ -151,10 +146,11 @@ class ECConnectorBase(ABC):
|
||||
self,
|
||||
ec_caches: dict[str, torch.Tensor],
|
||||
):
|
||||
"""
|
||||
Initialize with the EC caches.
|
||||
"""Initialize with the EC caches.
|
||||
|
||||
Args:
|
||||
ec_caches: dictionary of encoder cache
|
||||
|
||||
"""
|
||||
# TODO: Implement this later for P2P feature
|
||||
return
|
||||
@@ -163,8 +159,7 @@ class ECConnectorBase(ABC):
|
||||
def start_load_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], **kwargs
|
||||
) -> None:
|
||||
"""
|
||||
Start loading the cache from the connector into vLLM's encoder cache.
|
||||
"""Start loading the cache from the connector into vLLM's encoder cache.
|
||||
|
||||
This method loads the encoder cache based on metadata provided by the scheduler.
|
||||
It is called before `_gather_mm_embeddings` for the EC Connector. For EC,
|
||||
@@ -174,6 +169,7 @@ class ECConnectorBase(ABC):
|
||||
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -181,8 +177,7 @@ class ECConnectorBase(ABC):
|
||||
def save_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs
|
||||
) -> None:
|
||||
"""
|
||||
Save the encoder cache to the connector.
|
||||
"""Save the encoder cache to the connector.
|
||||
|
||||
This method saves the encoder cache from the worker's local storage
|
||||
to shared storage or another external connector.
|
||||
@@ -192,14 +187,14 @@ class ECConnectorBase(ABC):
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
mm_hash (str): The hash of the multimodal data whose cache is being saved.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""
|
||||
Notifies worker-side connector ids of requests that have
|
||||
"""Notifies worker-side connector ids of requests that have
|
||||
finished generating tokens on the worker.
|
||||
The scheduler process (via the Executors) will use this output
|
||||
to track which workers are done.
|
||||
@@ -210,16 +205,17 @@ class ECConnectorBase(ABC):
|
||||
tuple of (sending/saving ids, recving/loading ids).
|
||||
The finished saves/sends req ids must belong to a set provided in a
|
||||
call to this method (this call or a prior one).
|
||||
|
||||
"""
|
||||
return None, None
|
||||
|
||||
def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None:
|
||||
"""
|
||||
Build the ECConnector worker metadata for this engine step.
|
||||
"""Build the ECConnector worker metadata for this engine step.
|
||||
|
||||
Returns:
|
||||
ECConnectorWorkerMetadata: the worker metadata.
|
||||
None if no worker metadata is available.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -232,8 +228,7 @@ class ECConnectorBase(ABC):
|
||||
self,
|
||||
identifier: str,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if a single encoder cache exists
|
||||
"""Check if a single encoder cache exists.
|
||||
|
||||
Args:
|
||||
identifier (str): the identifier of the media.
|
||||
@@ -241,14 +236,14 @@ class ECConnectorBase(ABC):
|
||||
Returns:
|
||||
A bool where value is True if cache exist for
|
||||
the media
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
def ensure_cache_available(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
"""
|
||||
Ensure encoder cache items are available for the given request.
|
||||
"""Ensure encoder cache items are available for the given request.
|
||||
May initiate asynchronous transfers for items not yet local.
|
||||
|
||||
Args:
|
||||
@@ -258,6 +253,7 @@ class ECConnectorBase(ABC):
|
||||
Returns:
|
||||
True if all items are ready or no transfer is needed.
|
||||
False if any items are still in transit (request should be deferred).
|
||||
|
||||
"""
|
||||
return True
|
||||
|
||||
@@ -282,11 +278,12 @@ class ECConnectorBase(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def update_state_after_alloc(self, request: "Request", index: int):
|
||||
"""
|
||||
Update ECConnector state to decide allocate cache for requests
|
||||
"""Update ECConnector state to decide allocate cache for requests.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
index (int): index of the encoder cache entry being allocated.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -294,37 +291,37 @@ class ECConnectorBase(ABC):
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> ECConnectorMetadata:
|
||||
"""
|
||||
Build the connector metadata for this step.
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
def update_connector_output(self, connector_output: ECConnectorOutput):
|
||||
"""
|
||||
Update ECConnector state from worker-side connectors output.
|
||||
"""Update ECConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (ECConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
def request_finished(
|
||||
self, request: "Request"
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called when a request has finished, before its encoder cache is freed.
|
||||
"""Called when a request has finished, before its encoder cache is freed.
|
||||
|
||||
Returns:
|
||||
True if the request is being saved/sent asynchronously and cached
|
||||
should not be freed until the request_id is returned from
|
||||
get_finished().
|
||||
|
||||
"""
|
||||
return False, None
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Lightweight mmap-backed shared memory region for encoder cache (EC) data.
|
||||
"""Lightweight mmap-backed shared memory region for encoder cache (EC) data.
|
||||
|
||||
Modeled after SharedOffloadRegion (vllm/v1/kv_offload/cpu/) but simplified
|
||||
for EC: flat shared layout, no multi-tensor cursor, no block_size_factor.
|
||||
|
||||
@@ -229,6 +229,7 @@ class ECCPUScheduler:
|
||||
Returns:
|
||||
True if no item needs to wait for a remote encoding. False if any
|
||||
item is pending or the request must be aborted.
|
||||
|
||||
"""
|
||||
if not request.ec_transfer_params:
|
||||
return True
|
||||
@@ -259,6 +260,7 @@ class ECCPUScheduler:
|
||||
Raises:
|
||||
_RemoteUnavailable: The remote source is invalid, the read failed,
|
||||
or the item's total wait budget expired.
|
||||
|
||||
"""
|
||||
pos = feature.mm_position
|
||||
mm_hash = feature.identifier
|
||||
@@ -334,6 +336,7 @@ class ECCPUScheduler:
|
||||
item's remote announcement so subsequent steps do not retry it.
|
||||
False if only placeholder metadata (or no input) is available,
|
||||
recording the request for take_unavailable_requests() to drain.
|
||||
|
||||
"""
|
||||
mm_hash = feature.identifier
|
||||
self._deferred_since.pop((request.request_id, mm_hash), None)
|
||||
|
||||
@@ -15,7 +15,8 @@ from collections import OrderedDict
|
||||
|
||||
class CacheEntry:
|
||||
"""A single cache entry. Read `.block_ids` and `.ready` freely;
|
||||
mutations only through EmbeddingCache methods."""
|
||||
mutations only through EmbeddingCache methods.
|
||||
"""
|
||||
|
||||
__slots__ = ("block_ids", "_pin_count")
|
||||
|
||||
|
||||
@@ -67,8 +67,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
raise ValueError("ec_transfer_config must be set for ECConnectorBase")
|
||||
|
||||
def start_load_caches(self, encoder_cache, **kwargs) -> None:
|
||||
"""
|
||||
Start loading the cache from the connector into vLLM's encoder cache.
|
||||
"""Start loading the cache from the connector into vLLM's encoder cache.
|
||||
|
||||
This method loads the encoder cache based on metadata provided by the scheduler.
|
||||
It is called before `_gather_mm_embeddings` for the EC Connector. For EC,
|
||||
@@ -78,6 +77,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -108,8 +108,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash)
|
||||
|
||||
def save_caches(self, encoder_cache, mm_hash, **kwargs) -> None:
|
||||
"""
|
||||
Save the encoder cache to the connector.
|
||||
"""Save the encoder cache to the connector.
|
||||
|
||||
This method saves the encoder cache from the worker's local storage
|
||||
to shared storage or another external connector.
|
||||
@@ -119,6 +118,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
mm_hash (str): The hash of the multimodal data whose cache is being saved.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
# Return if it is PD Instance
|
||||
if not self.is_producer:
|
||||
@@ -134,14 +134,14 @@ class ECExampleConnector(ECConnectorBase):
|
||||
self,
|
||||
identifier: str,
|
||||
) -> bool:
|
||||
"""
|
||||
Check if cache exist externally for the media
|
||||
"""Check if cache exist externally for the media.
|
||||
|
||||
Args:
|
||||
identifier (str): the identifier of the media.
|
||||
|
||||
Returns:
|
||||
Bool indicate that media exists in cache or not
|
||||
|
||||
"""
|
||||
return self._found_match_for_mm_data(identifier)
|
||||
|
||||
@@ -150,9 +150,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
request: "Request",
|
||||
index: int,
|
||||
) -> None:
|
||||
"""
|
||||
Update ECConnector state after encoder cache allocation.
|
||||
"""
|
||||
"""Update ECConnector state after encoder cache allocation."""
|
||||
mm_hash = request.mm_features[index].identifier
|
||||
# Only load cache if it is consumer and cache exists
|
||||
if not self.is_consumer or not self.has_cache_item(mm_hash):
|
||||
@@ -212,8 +210,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
mm_hash: str,
|
||||
create_folder: bool = True, # <- now defaults to True
|
||||
) -> str:
|
||||
"""
|
||||
Return the folder in which the cache for this mm_hash lives.
|
||||
"""Return the folder in which the cache for this mm_hash lives.
|
||||
If `create_folder` is True (default) the directory is created
|
||||
recursively the first time it is needed.
|
||||
"""
|
||||
@@ -223,8 +220,7 @@ class ECExampleConnector(ECConnectorBase):
|
||||
return foldername
|
||||
|
||||
def _generate_filename_debug(self, mm_hash: str) -> str:
|
||||
"""
|
||||
Return the full path of the safetensors file for this mm_hash.
|
||||
"""Return the full path of the safetensors file for this mm_hash.
|
||||
Ensures the parent directory exists because
|
||||
`_generate_foldername_debug` is called with its default
|
||||
(`create_folder=True`).
|
||||
|
||||
@@ -24,10 +24,7 @@ def has_ec_transfer() -> bool:
|
||||
|
||||
|
||||
def ensure_ec_transfer_initialized(vllm_config: "VllmConfig") -> None:
|
||||
"""
|
||||
Initialize EC cache connector.
|
||||
"""
|
||||
|
||||
"""Initialize EC cache connector."""
|
||||
global _EC_CONNECTOR_AGENT
|
||||
|
||||
if vllm_config.ec_transfer_config is None:
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
The async worker that transfers experts in the background.
|
||||
"""
|
||||
"""The async worker that transfers experts in the background."""
|
||||
|
||||
import threading
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
EPLB communicator implementations and factory.
|
||||
"""
|
||||
"""EPLB communicator implementations and factory."""
|
||||
|
||||
import contextlib
|
||||
import time
|
||||
@@ -86,7 +84,8 @@ class EplbCommunicator(ABC):
|
||||
@property
|
||||
def needs_profile_buffer_reservation(self) -> bool:
|
||||
"""Whether the profile path must run a dummy collective operation to reserve
|
||||
communication buffers."""
|
||||
communication buffers.
|
||||
"""
|
||||
return True
|
||||
|
||||
def set_stream(self, cuda_stream: torch.cuda.Stream | None) -> None:
|
||||
@@ -258,6 +257,7 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
cpu_group: CPU process group for metadata exchange.
|
||||
all_expert_weights: Expert weight tensors for all MoE layers.
|
||||
expert_buffer: Pre-allocated receive buffer tensors.
|
||||
|
||||
"""
|
||||
assert all_expert_weights, (
|
||||
"NixlEplbCommunicator requires non-empty all_expert_weights."
|
||||
@@ -443,7 +443,8 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
|
||||
def _exchange_remote_send_meta(self) -> None:
|
||||
"""Exchange per-layer per-tensor metadata so receivers can compute
|
||||
remote RDMA addresses at transfer time."""
|
||||
remote RDMA addresses at transfer time.
|
||||
"""
|
||||
local_meta: dict[tuple[int, int], tuple[int, int, int]] = {}
|
||||
for layer_idx, layer_tensors in enumerate(self._all_expert_weights):
|
||||
for t_idx, t in enumerate(layer_tensors):
|
||||
@@ -666,6 +667,7 @@ def create_eplb_communicator(
|
||||
zero-copy RDMA reads.
|
||||
expert_buffer: Pre-allocated receive buffer tensors (one per
|
||||
weight tensor in a single layer).
|
||||
|
||||
"""
|
||||
first_layer = expert_weights[0] if expert_weights else []
|
||||
tensor_device_type = first_layer[0].device.type if first_layer else "cpu"
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Expert parallelism load balancer (EPLB) metrics and states.
|
||||
"""Expert parallelism load balancer (EPLB) metrics and states.
|
||||
|
||||
# Glossary
|
||||
|
||||
@@ -74,9 +73,7 @@ def _compute_eplb_load_stats(
|
||||
|
||||
@dataclass
|
||||
class EplbStats:
|
||||
"""
|
||||
Model stats used in EPLB rebalancing algorithm.
|
||||
"""
|
||||
"""Model stats used in EPLB rebalancing algorithm."""
|
||||
|
||||
global_expert_load_window: torch.Tensor
|
||||
"""
|
||||
@@ -233,9 +230,7 @@ class EplbModelState:
|
||||
|
||||
|
||||
class EplbState:
|
||||
"""
|
||||
EplbState of each expert parallel model. Key is the model config hash.
|
||||
"""
|
||||
"""EplbState of each expert parallel model. Key is the model config hash."""
|
||||
|
||||
def __init__(self, parallel_config: ParallelConfig, device: torch.device):
|
||||
self.parallel_config = parallel_config
|
||||
@@ -305,14 +300,14 @@ class EplbState:
|
||||
num_routed_experts: int,
|
||||
num_redundant_experts: int,
|
||||
) -> Sequence[int]:
|
||||
"""
|
||||
Build an initial expert arrangement using the following structure:
|
||||
"""Build an initial expert arrangement using the following structure:
|
||||
[original routed experts, redundant experts]
|
||||
|
||||
Returns:
|
||||
physical_to_logical_map (Sequence[int]): A list of integers,
|
||||
where each integer is the index of the logical expert
|
||||
that the corresponding physical expert maps to.
|
||||
|
||||
"""
|
||||
global_physical_to_logical_map = list(range(num_routed_experts))
|
||||
global_physical_to_logical_map += [
|
||||
@@ -321,8 +316,7 @@ class EplbState:
|
||||
return global_physical_to_logical_map
|
||||
|
||||
def validate_ep_configuration(self, new_model: MixtureOfExperts):
|
||||
"""
|
||||
Validate that the expert parallel configuration of
|
||||
"""Validate that the expert parallel configuration of
|
||||
the new model is the same as the existing models.
|
||||
"""
|
||||
if len(self.model_states) > 0:
|
||||
@@ -361,9 +355,7 @@ class EplbState:
|
||||
model: MixtureOfExperts,
|
||||
model_config: ModelConfig,
|
||||
):
|
||||
"""
|
||||
Build the initial EPLB state.
|
||||
"""
|
||||
"""Build the initial EPLB state."""
|
||||
self.validate_ep_configuration(model)
|
||||
self.is_async = self.parallel_config.eplb_config.use_async
|
||||
|
||||
@@ -534,6 +526,7 @@ class EplbState:
|
||||
ubatch_slices: When DBO is active, a list of
|
||||
``UBatchSlice`` objects describing each micro-batch's
|
||||
token range. When ``None``, only ``tensors[0]`` is filled.
|
||||
|
||||
"""
|
||||
model_state = self.model_states.get(compute_hash_cached(model_config))
|
||||
if model_state is None or model_state.num_unpadded_tokens_tensors is None:
|
||||
@@ -556,8 +549,7 @@ class EplbState:
|
||||
is_profile: bool = False,
|
||||
log_stats: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Step the EPLB state.
|
||||
"""Step the EPLB state.
|
||||
|
||||
Args:
|
||||
is_dummy (bool): If `True`, this is a dummy step and the load
|
||||
@@ -574,6 +566,7 @@ class EplbState:
|
||||
- `avg_tokens`: The average load across ranks.
|
||||
- `max_tokens`: The maximum load across ranks.
|
||||
- `balancedness`: The ratio of average load to maximum load.
|
||||
|
||||
"""
|
||||
ep_group = get_ep_group().device_group
|
||||
if is_profile:
|
||||
@@ -752,8 +745,7 @@ class EplbState:
|
||||
is_profile: bool = False,
|
||||
rank_mapping: dict[int, int] | None = None,
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
Rearrange the experts according to the current load.
|
||||
"""Rearrange the experts according to the current load.
|
||||
|
||||
Args:
|
||||
is_profile (bool): If `True`, perform a dummy rearrangement.
|
||||
@@ -761,8 +753,8 @@ class EplbState:
|
||||
no memory movement will be performed. Default is False.
|
||||
rank_mapping (dict[int, int] | None): The rank mapping
|
||||
when scaling is done in EEP.
|
||||
"""
|
||||
|
||||
"""
|
||||
ep_group = get_ep_group().device_group
|
||||
ep_rank = ep_group.rank()
|
||||
|
||||
@@ -1033,9 +1025,7 @@ class EplbState:
|
||||
return int(flag.item()) == device_group.size()
|
||||
|
||||
def _allreduce_list(self, tensor_list: list[torch.Tensor]) -> list[torch.Tensor]:
|
||||
"""
|
||||
All-reduce a list of tensors.
|
||||
"""
|
||||
"""All-reduce a list of tensors."""
|
||||
ep_group = get_ep_group().device_group
|
||||
if len(tensor_list) == 1:
|
||||
all_reduce(tensor_list[0], group=ep_group)
|
||||
@@ -1060,8 +1050,7 @@ class EplbState:
|
||||
return all_reduce_list
|
||||
|
||||
def _sync_load_pass(self) -> list[torch.Tensor]:
|
||||
"""
|
||||
Sync the expert load pass across all ranks for log stats.
|
||||
"""Sync the expert load pass across all ranks for log stats.
|
||||
Doesn't update the expert load pass in eplb_model_state.
|
||||
"""
|
||||
load_pass_list = []
|
||||
@@ -1250,8 +1239,7 @@ def compute_logical_maps(
|
||||
physical_to_logical_map: torch.Tensor,
|
||||
num_logical_experts: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Derive logical_to_physical_map and logical_replica_count from
|
||||
"""Derive logical_to_physical_map and logical_replica_count from
|
||||
physical_to_logical_map.
|
||||
|
||||
Args:
|
||||
@@ -1264,6 +1252,7 @@ def compute_logical_maps(
|
||||
physical slots per logical expert; -1 where unused
|
||||
logical_replica_count: [num_layers, num_logical_experts], number of
|
||||
physical replicas per logical expert
|
||||
|
||||
"""
|
||||
device = physical_to_logical_map.device
|
||||
assert physical_to_logical_map.device.type == "cpu"
|
||||
@@ -1339,13 +1328,11 @@ def _commit_eplb_maps_for_layer(
|
||||
new_physical_to_logical_map: torch.Tensor,
|
||||
layer: int,
|
||||
) -> None:
|
||||
"""
|
||||
Per-layer version of _commit_eplb_maps that's used by the sync portion of EPLB
|
||||
"""Per-layer version of _commit_eplb_maps that's used by the sync portion of EPLB
|
||||
when running async EPLB. Copies all of the new_* maps into model_state. After this
|
||||
function completes, the new mappings will become the current mappings and will be
|
||||
visible to the model.
|
||||
"""
|
||||
|
||||
# Commit physical_to_logical_map
|
||||
src = new_physical_to_logical_map
|
||||
dst = model_state.physical_to_logical_map[layer]
|
||||
@@ -1376,12 +1363,10 @@ def _commit_eplb_maps(
|
||||
model_state: EplbModelState,
|
||||
new_physical_to_logical_map: torch.Tensor,
|
||||
) -> None:
|
||||
"""
|
||||
Copies all of the new_* maps into model_state. After this function completes,
|
||||
"""Copies all of the new_* maps into model_state. After this function completes,
|
||||
the new mappings will become the current mappings and will be visible to the
|
||||
model.
|
||||
"""
|
||||
|
||||
# Commit physical_to_logical_map
|
||||
src = new_physical_to_logical_map
|
||||
dst = model_state.physical_to_logical_map
|
||||
|
||||
@@ -14,8 +14,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CpuGpuEvent:
|
||||
"""
|
||||
Combines a CUDA event with a CPU threading event to enforce record->wait
|
||||
"""Combines a CUDA event with a CPU threading event to enforce record->wait
|
||||
ordering across two threads.
|
||||
|
||||
This class is designed for exactly two threads: one producer that calls
|
||||
@@ -35,8 +34,7 @@ class CpuGpuEvent:
|
||||
self._recorded = threading.Event()
|
||||
|
||||
def wait(self, stream: torch.cuda.Stream | None = None):
|
||||
"""
|
||||
Blocks the calling thread until record finishes. Used to guarantee that the
|
||||
"""Blocks the calling thread until record finishes. Used to guarantee that the
|
||||
record kernel is called before wait.
|
||||
|
||||
Should only be called by the Async Eplb thread.
|
||||
@@ -46,8 +44,7 @@ class CpuGpuEvent:
|
||||
self._recorded.clear()
|
||||
|
||||
def record(self, stream: torch.cuda.Stream | None = None):
|
||||
"""
|
||||
Unblocks the waiting thread after calling event.record().
|
||||
"""Unblocks the waiting thread after calling event.record().
|
||||
|
||||
Should only be called by the main thread.
|
||||
"""
|
||||
@@ -65,12 +62,12 @@ def override_envs_for_eplb(
|
||||
parallel_config: ParallelConfig,
|
||||
moe_backend: str | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Override environment variables for EPLB when specific conditions are met.
|
||||
"""Override environment variables for EPLB when specific conditions are met.
|
||||
|
||||
Args:
|
||||
parallel_config: The parallel configuration object.
|
||||
moe_backend: The configured MoE backend (e.g. ``deep_gemm_mega_moe``).
|
||||
|
||||
"""
|
||||
is_data_parallel = parallel_config.data_parallel_size > 1
|
||||
is_eplb_enabled = parallel_config.enable_eplb
|
||||
|
||||
@@ -18,10 +18,9 @@ class AbstractEplbPolicy(ABC):
|
||||
num_ranks: int,
|
||||
old_global_expert_indices: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Entry point for expert-parallelism load balancer.
|
||||
"""Entry point for expert-parallelism load balancer.
|
||||
|
||||
Parameters:
|
||||
Args:
|
||||
weight: [layers, num_logical_experts], the load statistics
|
||||
for all logical experts
|
||||
num_replicas: number of physical experts, must be a multiple of
|
||||
@@ -32,8 +31,10 @@ class AbstractEplbPolicy(ABC):
|
||||
old_global_expert_indices: [layers, num_logical_experts], the old global
|
||||
expert indices. Used to avoid unnecessary weight copying
|
||||
for experts moving within one rank.
|
||||
|
||||
Returns:
|
||||
physical_to_logical_map: [layers, num_replicas], the expert
|
||||
index of each replica
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Expert parallelism load balancer (EPLB) for vLLM.
|
||||
"""Expert parallelism load balancer (EPLB) for vLLM.
|
||||
|
||||
This module implements the core rearrangement algorithm.
|
||||
|
||||
@@ -23,17 +22,17 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
def balanced_packing(
|
||||
cls, weight: np.ndarray, num_packs: int
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Pack n weighted objects to m packs, such that each bin contains exactly
|
||||
"""Pack n weighted objects to m packs, such that each bin contains exactly
|
||||
n/m objects and the weights of all packs are as balanced as possible.
|
||||
|
||||
Parameters:
|
||||
Args:
|
||||
weight: [X, n], the weight of each item
|
||||
num_packs: number of packs
|
||||
|
||||
Returns:
|
||||
pack_index: [X, n], the pack index of each item
|
||||
rank_in_pack: [X, n], the rank of the item in the pack
|
||||
|
||||
"""
|
||||
num_layers, num_groups = weight.shape
|
||||
assert num_groups % num_packs == 0
|
||||
@@ -76,17 +75,17 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
def replicate_experts(
|
||||
cls, weight: np.ndarray, num_phy: int
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Replicate `num_log` experts to `num_phy` replicas, such that the maximum
|
||||
"""Replicate `num_log` experts to `num_phy` replicas, such that the maximum
|
||||
load of all replicas is minimized.
|
||||
|
||||
Parameters:
|
||||
Args:
|
||||
weight: [X, num_log]
|
||||
num_phy: total number of experts after replication
|
||||
|
||||
Returns:
|
||||
phy2log: [X, num_phy], logical expert id of each physical expert
|
||||
logcnt: [X, num_log], number of replicas for each logical expert
|
||||
|
||||
"""
|
||||
n, num_log = weight.shape
|
||||
num_redundant = num_phy - num_log
|
||||
@@ -109,8 +108,7 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
num_nodes: int,
|
||||
num_gpus: int,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Parameters:
|
||||
"""Parameters
|
||||
weight: [num_moe_layers, num_logical_experts]
|
||||
num_physical_experts: number of physical experts after replication
|
||||
num_groups: number of expert groups
|
||||
@@ -121,6 +119,7 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
Returns:
|
||||
phy2log: [layers, num_replicas], the expert
|
||||
index of each replica
|
||||
|
||||
"""
|
||||
num_layers, num_logical_experts = weight.shape
|
||||
assert num_logical_experts % num_groups == 0
|
||||
@@ -195,8 +194,7 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
num_ranks: int,
|
||||
old_phy2log: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Reorder the new mapping per GPU so that experts that remain on the same GPU
|
||||
"""Reorder the new mapping per GPU so that experts that remain on the same GPU
|
||||
keep their previous slot positions when possible. Incoming experts to that GPU
|
||||
fill any remaining available slots. This is applied only when the number of GPUs
|
||||
is unchanged and the slots per GPU remain the same between
|
||||
@@ -281,10 +279,9 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
num_ranks: int,
|
||||
old_global_expert_indices: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Entry point for expert-parallelism load balancer.
|
||||
"""Entry point for expert-parallelism load balancer.
|
||||
|
||||
Parameters:
|
||||
Args:
|
||||
weight: [layers, num_logical_experts], the load statistics for all
|
||||
logical experts
|
||||
num_replicas: number of physical experts, must be a multiple of
|
||||
@@ -296,9 +293,12 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
old_global_expert_indices: [layers, num_logical_experts], the old global
|
||||
expert indices. Used to avoid unnecessary weight copying
|
||||
for experts moving within one rank.
|
||||
|
||||
Returns:
|
||||
-------
|
||||
phy2log: [layers, num_replicas], the expert
|
||||
index of each replica
|
||||
|
||||
"""
|
||||
weight_np = weight.float().cpu().numpy()
|
||||
old_phy2log_np = (
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
The actual execution of the rearrangement.
|
||||
"""The actual execution of the rearrangement.
|
||||
|
||||
This involves the exchange of expert weights between GPUs.
|
||||
"""
|
||||
@@ -41,9 +40,7 @@ class TransferMetadata:
|
||||
|
||||
@dataclass
|
||||
class AsyncEplbLayerResult:
|
||||
"""
|
||||
The result of one completed async EPLB layer transfer.
|
||||
"""
|
||||
"""The result of one completed async EPLB layer transfer."""
|
||||
|
||||
layer_idx: int
|
||||
"""Index of the MoE layer that was transferred."""
|
||||
@@ -69,8 +66,7 @@ def get_ep_ranks_with_experts_batch(
|
||||
old_indices: np.ndarray,
|
||||
new_indices: np.ndarray,
|
||||
) -> tuple[dict[int, list[int]], dict[int, list[int]]]:
|
||||
"""
|
||||
Get the ranks of the experts that need to be exchanged.
|
||||
"""Get the ranks of the experts that need to be exchanged.
|
||||
|
||||
Args:
|
||||
expert_ids: 1D array of expert indices to query.
|
||||
@@ -82,6 +78,7 @@ def get_ep_ranks_with_experts_batch(
|
||||
A tuple of two dictionaries mapping expert_id to:
|
||||
- ranks_to_send: The ranks that have this expert and need to send.
|
||||
- ranks_to_recv: The ranks that need to receive this expert.
|
||||
|
||||
"""
|
||||
ranks_to_send_map: dict[int, list[int]] = {}
|
||||
ranks_to_recv_map: dict[int, list[int]] = {}
|
||||
@@ -181,8 +178,7 @@ def move_to_buffer(
|
||||
communicator: EplbCommunicator,
|
||||
layer_idx: int = 0,
|
||||
) -> TransferMetadata:
|
||||
"""
|
||||
Rearranges expert weights during EPLB rebalancing.
|
||||
"""Rearranges expert weights during EPLB rebalancing.
|
||||
|
||||
Args:
|
||||
num_local_experts: Number of local experts.
|
||||
@@ -199,6 +195,7 @@ def move_to_buffer(
|
||||
|
||||
Returns:
|
||||
TransferMetadata: Metadata needed for completing remote weight transfers.
|
||||
|
||||
"""
|
||||
assert old_indices.shape == new_indices.shape
|
||||
recv_primary_mask = np.zeros((num_local_experts,), dtype=np.bool_)
|
||||
@@ -355,9 +352,8 @@ def move_from_buffer(
|
||||
new_indices: np.ndarray,
|
||||
ep_rank: int,
|
||||
) -> None:
|
||||
"""
|
||||
Copies expert weights from communication buffers back to the target weight tensors
|
||||
after EPLB rebalancing.
|
||||
"""Copies expert weights from communication buffers back to the target weight
|
||||
tensors after EPLB rebalancing.
|
||||
|
||||
Args:
|
||||
expert_weights: List of the actual MoE layer weights used in the execution.
|
||||
@@ -367,6 +363,7 @@ def move_from_buffer(
|
||||
new_indices: (num_experts_total,) mapping from local rows to desired
|
||||
(possibly global) expert id, after rebalance.
|
||||
ep_rank: Rank of the process in the expert parallel group.
|
||||
|
||||
"""
|
||||
is_unchanged = transfer_metadata.is_unchanged
|
||||
is_received_locally = transfer_metadata.is_received_locally
|
||||
@@ -437,8 +434,7 @@ def transfer_layer(
|
||||
rank_mapping: dict[int, int] | None = None,
|
||||
layer_idx: int = 0,
|
||||
) -> TransferMetadata:
|
||||
"""
|
||||
Rearranges the expert weights in place according to the new expert indices.
|
||||
"""Rearranges the expert weights in place according to the new expert indices.
|
||||
|
||||
The value of the indices arguments are logical indices of the experts,
|
||||
while keys are physical.
|
||||
@@ -462,6 +458,7 @@ def transfer_layer(
|
||||
Returns:
|
||||
TransferMetadata: Metadata needed for completing remote weight transfers,
|
||||
including is_unchanged and is_received_locally masks.
|
||||
|
||||
"""
|
||||
ep_size = ep_group.size()
|
||||
if rank_mapping is not None:
|
||||
@@ -519,8 +516,7 @@ def rearrange_expert_weights_inplace(
|
||||
is_profile: bool = False,
|
||||
rank_mapping: dict[int, int] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Rearranges the expert weights in place according to the new expert indices.
|
||||
"""Rearranges the expert weights in place according to the new expert indices.
|
||||
|
||||
The value of the indices arguments are logical indices of the experts,
|
||||
while keys are physical.
|
||||
@@ -540,6 +536,7 @@ def rearrange_expert_weights_inplace(
|
||||
This is used during profile run, where we only perform dummy
|
||||
communications to reserve enough memory for the buffers.
|
||||
rank_mapping: A dictionary mapping old rank to new rank.
|
||||
|
||||
"""
|
||||
if rank_mapping is not None:
|
||||
if len(rank_mapping) == ep_group.size():
|
||||
@@ -624,8 +621,7 @@ def _map_old_expert_indices_with_rank_mapping(
|
||||
rank_mapping: dict[int, int],
|
||||
new_ep_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Map the old global expert indices to the new global expert indices.
|
||||
"""Map the old global expert indices to the new global expert indices.
|
||||
|
||||
Args:
|
||||
old_global_expert_indices:
|
||||
@@ -636,6 +632,7 @@ def _map_old_expert_indices_with_rank_mapping(
|
||||
Returns:
|
||||
Mapped expert indices with shape
|
||||
(num_layers, new_ep_size * num_local_physical_experts).
|
||||
|
||||
"""
|
||||
num_layers, old_num_physical_experts = old_global_expert_indices.shape
|
||||
assert rank_mapping, "Rank mapping is required"
|
||||
|
||||
@@ -39,7 +39,7 @@ class KVCacheEvent(
|
||||
gc=False, # type: ignore[call-arg]
|
||||
tag=True,
|
||||
):
|
||||
"""Base class for all KV cache-related events"""
|
||||
"""Base class for all KV cache-related events."""
|
||||
|
||||
|
||||
MEDIUM_GPU = "GPU"
|
||||
@@ -135,8 +135,7 @@ class KVEventBatch(EventBatch):
|
||||
|
||||
|
||||
class KVEventAggregator:
|
||||
"""
|
||||
Aggregates KV events across multiple workers.
|
||||
"""Aggregates KV events across multiple workers.
|
||||
Tracks how many times each event appears and returns only those
|
||||
that were emitted by all workers.
|
||||
"""
|
||||
@@ -150,22 +149,22 @@ class KVEventAggregator:
|
||||
self._num_workers: int = num_workers
|
||||
|
||||
def add_events(self, events: list[KVCacheEvent]) -> None:
|
||||
"""
|
||||
Add events from a worker batch.
|
||||
"""Add events from a worker batch.
|
||||
|
||||
Args:
|
||||
events: List of KVCacheEvent objects.
|
||||
|
||||
"""
|
||||
if not isinstance(events, list):
|
||||
raise TypeError("events must be a list of KVCacheEvent.")
|
||||
self._event_counter.update(events)
|
||||
|
||||
def get_common_events(self) -> list[KVCacheEvent]:
|
||||
"""
|
||||
Return events that appeared in all workers.
|
||||
"""Return events that appeared in all workers.
|
||||
|
||||
Returns:
|
||||
List of events present in all workers.
|
||||
|
||||
"""
|
||||
return [
|
||||
event
|
||||
@@ -174,43 +173,39 @@ class KVEventAggregator:
|
||||
]
|
||||
|
||||
def get_all_events(self) -> list[KVCacheEvent]:
|
||||
"""
|
||||
Return all events for all workers.
|
||||
"""Return all events for all workers.
|
||||
|
||||
Returns:
|
||||
List of events for all workers.
|
||||
|
||||
"""
|
||||
return list(self._event_counter.elements())
|
||||
|
||||
def clear_events(self) -> None:
|
||||
"""
|
||||
Clear all tracked events.
|
||||
"""
|
||||
"""Clear all tracked events."""
|
||||
self._event_counter.clear()
|
||||
|
||||
def increment_workers(self, count: int = 1) -> None:
|
||||
"""
|
||||
Increment the number of workers contributing events.
|
||||
"""Increment the number of workers contributing events.
|
||||
|
||||
Args:
|
||||
count: Number to increment the workers by.
|
||||
|
||||
"""
|
||||
if count <= 0:
|
||||
raise ValueError("count must be positive.")
|
||||
self._num_workers += count
|
||||
|
||||
def reset_workers(self) -> None:
|
||||
"""
|
||||
Reset the number of workers to 1.
|
||||
"""
|
||||
"""Reset the number of workers to 1."""
|
||||
self._num_workers = 1
|
||||
|
||||
def get_number_of_workers(self) -> int:
|
||||
"""
|
||||
Return the number of workers.
|
||||
"""Return the number of workers.
|
||||
|
||||
Returns:
|
||||
int number of workers.
|
||||
|
||||
"""
|
||||
return self._num_workers
|
||||
|
||||
@@ -222,8 +217,7 @@ class KVEventAggregator:
|
||||
|
||||
|
||||
class KVConnectorKVEvents(ABC):
|
||||
"""
|
||||
Abstract base class for KV events.
|
||||
"""Abstract base class for KV events.
|
||||
Acts as a container for KV events from the connector.
|
||||
"""
|
||||
|
||||
@@ -323,6 +317,7 @@ class ZmqEventPublisher(EventPublisher):
|
||||
Maximum number of events to buffer in memory.
|
||||
topic:
|
||||
Topic to publish events to.
|
||||
|
||||
"""
|
||||
|
||||
SHUTDOWN_TIMEOUT: float = 1.0
|
||||
@@ -520,6 +515,7 @@ class ZmqEventPublisher(EventPublisher):
|
||||
Returns:
|
||||
The endpoint with the port offset by data_parallel_rank
|
||||
or suffix appended
|
||||
|
||||
"""
|
||||
# Do nothing if input is None or data_parallel_rank is 0
|
||||
if not endpoint or data_parallel_rank == 0:
|
||||
|
||||
@@ -87,6 +87,7 @@ class KVConnectorFactory:
|
||||
|
||||
Returns:
|
||||
The connector class.
|
||||
|
||||
"""
|
||||
if connector_name not in cls._registry:
|
||||
raise ValueError(f"Connector '{connector_name}' is not registered.")
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
KV cache helper for store.
|
||||
"""
|
||||
"""KV cache helper for store."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
@@ -53,7 +51,8 @@ def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
|
||||
|
||||
class KVOutputAggregator:
|
||||
"""Utility class to aggregate the output of all workers into a single
|
||||
output corresponding to Rank 0 for scheduler."""
|
||||
output corresponding to Rank 0 for scheduler.
|
||||
"""
|
||||
|
||||
def __init__(self, expected_finished_count: int):
|
||||
# Complete transfer tracker. Used to track finished requests
|
||||
@@ -233,11 +232,10 @@ def copy_kv_blocks(
|
||||
|
||||
|
||||
def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio):
|
||||
"""
|
||||
Transforms the layout of received KV cache blocks to the local block_size.
|
||||
"""Transforms the layout of received KV cache blocks to the local block_size.
|
||||
(Only works for local blocksize > remote blocksize)
|
||||
|
||||
example:
|
||||
Example:
|
||||
local blocksize = 16 tokens, remote blocksize = 4 tokens
|
||||
local block[0] = remote block[0, 1, 2, 3]
|
||||
remote is |h0-b0|h1-b0|h2-b0|h3-b0|h0-b1|h1-b1|h2-b1|h3-b1|...
|
||||
@@ -246,6 +244,7 @@ def kv_postprocess_blksize_on_receive(cache, indices, block_size_ratio):
|
||||
1. view => view remote as n_blocks * remote_shape(H,remoteN,D)
|
||||
2. permute => (H, nblocks, remoteN, D)
|
||||
3. flatten => (H, localN, D)
|
||||
|
||||
"""
|
||||
blocks_to_update = cache.index_select(0, indices)
|
||||
# use physical order
|
||||
@@ -293,8 +292,7 @@ def kv_postprocess_layout_on_receive(cache, indices):
|
||||
|
||||
|
||||
def kv_postprocess_blksize_and_layout_on_receive(cache, indices, block_size_ratio):
|
||||
"""
|
||||
Transforms the layout of received KV cache to the local block_size and LBHNC.
|
||||
"""Transforms the layout of received KV cache to the local block_size and LBHNC.
|
||||
(Only works for local blocksize > remote blocksize)
|
||||
|
||||
prefill is LBHNC, smaller block_size
|
||||
@@ -317,9 +315,9 @@ def kv_postprocess_blksize_and_layout_on_receive(cache, indices, block_size_rati
|
||||
def yield_req_data(
|
||||
scheduler_output,
|
||||
) -> Iterator[tuple[str, tuple[list[int], ...] | None, bool]]:
|
||||
"""
|
||||
Yields:
|
||||
(req_id, new_block_id_groups, preempted)
|
||||
"""Yields:
|
||||
(req_id, new_block_id_groups, preempted)
|
||||
|
||||
"""
|
||||
# new requests
|
||||
for req_data in scheduler_output.scheduled_new_reqs:
|
||||
@@ -346,6 +344,7 @@ def get_current_attn_backends(
|
||||
|
||||
Returns:
|
||||
Deduplicated list of attention backend classes.
|
||||
|
||||
"""
|
||||
layer_type = cast(type[Any], AttentionLayerBase)
|
||||
layers = get_layers_from_vllm_config(vllm_config, layer_type, layer_names)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
KVConnectorBase_V1 Class for Distributed KV Cache & Hidden State
|
||||
"""KVConnectorBase_V1 Class for Distributed KV Cache & Hidden State
|
||||
communication in vLLM v1
|
||||
|
||||
The class provides the following primitives:
|
||||
@@ -97,8 +96,7 @@ class KVConnectorTransferResults:
|
||||
|
||||
|
||||
class SupportsHMA(ABC):
|
||||
"""
|
||||
The class that indicates the corresponding connector supports hybrid memory
|
||||
"""The class that indicates the corresponding connector supports hybrid memory
|
||||
allocator (HMA).
|
||||
This is required to use the connector together with hybrid memory allocator.
|
||||
"""
|
||||
@@ -109,8 +107,7 @@ class SupportsHMA(ABC):
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called exactly once when a request has finished for all kv cache groups,
|
||||
"""Called exactly once when a request has finished for all kv cache groups,
|
||||
before its blocks are freed for each group.
|
||||
|
||||
NOTE(Kuntai): This function is only supported by connectors that support HMA.
|
||||
@@ -124,6 +121,7 @@ class SupportsHMA(ABC):
|
||||
get_finished().
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -144,8 +142,7 @@ class KVConnectorRole(enum.Enum):
|
||||
|
||||
|
||||
class KVConnectorHandshakeMetadata(ABC): # noqa: B024
|
||||
"""
|
||||
Metadata used for out of band connector handshake between
|
||||
"""Metadata used for out of band connector handshake between
|
||||
P/D workers. This needs to serializable.
|
||||
"""
|
||||
|
||||
@@ -153,8 +150,7 @@ class KVConnectorHandshakeMetadata(ABC): # noqa: B024
|
||||
|
||||
|
||||
class KVConnectorMetadata(ABC): # noqa: B024
|
||||
"""
|
||||
Abstract Metadata used to communicate
|
||||
"""Abstract Metadata used to communicate
|
||||
Scheduler KVConnector -> Worker KVConnector.
|
||||
"""
|
||||
|
||||
@@ -162,8 +158,7 @@ class KVConnectorMetadata(ABC): # noqa: B024
|
||||
|
||||
|
||||
class KVConnectorWorkerMetadata(ABC):
|
||||
"""
|
||||
Abstract Metadata used to communicate back
|
||||
"""Abstract Metadata used to communicate back
|
||||
Worker KVConnector -> Scheduler KVConnector.
|
||||
|
||||
Each worker can output its own metadata.
|
||||
@@ -176,16 +171,12 @@ class KVConnectorWorkerMetadata(ABC):
|
||||
def aggregate(
|
||||
self, other: "KVConnectorWorkerMetadata"
|
||||
) -> "KVConnectorWorkerMetadata":
|
||||
"""
|
||||
Aggregate metadata with another `KVConnectorWorkerMetadata` object.
|
||||
"""
|
||||
"""Aggregate metadata with another `KVConnectorWorkerMetadata` object."""
|
||||
pass
|
||||
|
||||
|
||||
class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
Base class for KV connectors.
|
||||
"""
|
||||
"""Base class for KV connectors."""
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
@@ -245,6 +236,7 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
Args:
|
||||
connector_metadata (dict): the connector metadata.
|
||||
|
||||
"""
|
||||
self._connector_metadata = connector_metadata
|
||||
|
||||
@@ -263,6 +255,7 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
Returns:
|
||||
ConnectorMetadata: the connector metadata.
|
||||
|
||||
"""
|
||||
# Should only be called while set to valid metadata.
|
||||
assert self._connector_metadata is not None
|
||||
@@ -273,22 +266,22 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
Returns:
|
||||
bool: True if connector metadata exists, False otherwise.
|
||||
|
||||
"""
|
||||
return self._connector_metadata is not None
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""
|
||||
Initialize with the KV caches. Useful for pre-registering the
|
||||
"""Initialize with the KV caches. Useful for pre-registering the
|
||||
KV Caches in the KVConnector (e.g. for NIXL).
|
||||
|
||||
Args:
|
||||
kv_caches: dictionary of layer names, kv cache
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
|
||||
"""
|
||||
Set the xPU-specific ops for copying KV between host and device.
|
||||
"""Set the xPU-specific ops for copying KV between host and device.
|
||||
Needed when host buffer is used for kv transfer (e.g., in NixlConnector)
|
||||
"""
|
||||
return
|
||||
@@ -303,16 +296,14 @@ class KVConnectorBase_V1(ABC):
|
||||
return
|
||||
|
||||
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata):
|
||||
"""
|
||||
Handle preempted requests or evicted blocks BEFORE they are overwritten.
|
||||
"""Handle preempted requests or evicted blocks BEFORE they are overwritten.
|
||||
Needed for connectors which use async saves (e.g., OffloadingConnector)
|
||||
"""
|
||||
return
|
||||
|
||||
@abstractmethod
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""
|
||||
Start loading the KV cache from the connector to vLLM's paged
|
||||
"""Start loading the KV cache from the connector to vLLM's paged
|
||||
KV buffer. Loads required by the current forward start before it;
|
||||
independent asynchronous loads may start after it is submitted.
|
||||
|
||||
@@ -329,8 +320,7 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""
|
||||
Block until the KV for a specific layer is loaded into vLLM's
|
||||
"""Block until the KV for a specific layer is loaded into vLLM's
|
||||
paged buffer. This is called from within attention layer to ensure
|
||||
async copying from start_load_kv is complete.
|
||||
|
||||
@@ -338,6 +328,7 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -349,8 +340,7 @@ class KVConnectorBase_V1(ABC):
|
||||
attn_metadata: "AttentionMetadata",
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Start saving a layer of KV cache from vLLM's paged buffer
|
||||
"""Start saving a layer of KV cache from vLLM's paged buffer
|
||||
to the connector. This is called from within attention layer to
|
||||
enable async copying during execution.
|
||||
|
||||
@@ -360,13 +350,13 @@ class KVConnectorBase_V1(ABC):
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
**kwargs: additional arguments for the save operation.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def wait_for_save(self):
|
||||
"""
|
||||
Block until all the save operations is done. This is called
|
||||
"""Block until all the save operations is done. This is called
|
||||
as the forward context exits to ensure that the async saving
|
||||
from save_kv_layer is complete before finishing the forward.
|
||||
|
||||
@@ -377,8 +367,7 @@ class KVConnectorBase_V1(ABC):
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""
|
||||
Notifies worker-side connector ids of requests that have
|
||||
"""Notifies worker-side connector ids of requests that have
|
||||
finished generating tokens on the worker.
|
||||
The scheduler process (via the Executors) will use this output
|
||||
to track which workers are done.
|
||||
@@ -389,6 +378,7 @@ class KVConnectorBase_V1(ABC):
|
||||
tuple of (sending/saving ids, recving/loading ids).
|
||||
The finished saves/sends req ids must belong to a set provided in a
|
||||
call to this method (this call or a prior one).
|
||||
|
||||
"""
|
||||
return None, None
|
||||
|
||||
@@ -403,8 +393,7 @@ class KVConnectorBase_V1(ABC):
|
||||
)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""
|
||||
Get the set of block IDs that failed to load.
|
||||
"""Get the set of block IDs that failed to load.
|
||||
|
||||
Returns:
|
||||
Set of block IDs that encountered load errors.
|
||||
@@ -419,50 +408,47 @@ class KVConnectorBase_V1(ABC):
|
||||
block IDs must appear here no later than that same pass.
|
||||
- Sync loading: failed blocks should be reported in the forward
|
||||
pass in which they are detected.
|
||||
|
||||
"""
|
||||
return set()
|
||||
|
||||
def shutdown(self):
|
||||
"""
|
||||
Shutdown the connector. This is called when the worker process
|
||||
"""Shutdown the connector. This is called when the worker process
|
||||
is shutting down to ensure that all the async operations are
|
||||
completed and the connector is cleaned up properly.
|
||||
"""
|
||||
return None
|
||||
|
||||
def get_kv_connector_stats(self) -> "KVConnectorStats | None":
|
||||
"""
|
||||
Get the KV connector stats collected during the last interval.
|
||||
"""
|
||||
"""Get the KV connector stats collected during the last interval."""
|
||||
return None
|
||||
|
||||
def get_kv_connector_kv_cache_events(self) -> "KVConnectorKVEvents | None":
|
||||
"""
|
||||
Get the KV connector kv cache events collected during the last interval.
|
||||
"""Get the KV connector kv cache events collected during the last interval.
|
||||
This function should be called by the model runner every time after the
|
||||
model execution and before cleanup.
|
||||
"""
|
||||
return None
|
||||
|
||||
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
|
||||
"""
|
||||
Get the KVConnector handshake metadata for this connector.
|
||||
"""Get the KVConnector handshake metadata for this connector.
|
||||
This metadata is used for out-of-band connector handshake
|
||||
between P/D workers.
|
||||
|
||||
Returns:
|
||||
KVConnectorHandshakeMetadata: the handshake metadata.
|
||||
None if no handshake metadata is available.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
|
||||
"""
|
||||
Build the KVConnector worker metadata for this engine step.
|
||||
"""Build the KVConnector worker metadata for this engine step.
|
||||
|
||||
Returns:
|
||||
KVConnectorWorkerMetadata: the worker metadata.
|
||||
None if no worker metadata is available.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -476,12 +462,12 @@ class KVConnectorBase_V1(ABC):
|
||||
self.bind_gpu_block_pool(kv_cache_manager.block_pool)
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None:
|
||||
"""
|
||||
Bind the GPU block pool to the connector for per-GPU block status tracking.
|
||||
"""Bind the GPU block pool to the connector for per-GPU block status tracking.
|
||||
For example, inc/dec ref counts, or iterate over the prefix cache blocks.
|
||||
|
||||
Args:
|
||||
gpu_block_pool: the GPU block pool.
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
@@ -491,8 +477,7 @@ class KVConnectorBase_V1(ABC):
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""
|
||||
Get number of new tokens that can be loaded from the
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
@@ -517,6 +502,7 @@ class KVConnectorBase_V1(ABC):
|
||||
call. If the cache cannot be loaded for some tokens (e.g., due to
|
||||
connectivity issues or eviction), those tokens must not be taken
|
||||
into account.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -524,8 +510,7 @@ class KVConnectorBase_V1(ABC):
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""
|
||||
Update KVConnector state after block allocation.
|
||||
"""Update KVConnector state after block allocation.
|
||||
|
||||
If get_num_new_matched_tokens previously returned True for a
|
||||
request, this function may be called twice for that same request -
|
||||
@@ -543,6 +528,7 @@ class KVConnectorBase_V1(ABC):
|
||||
blocks (KVCacheBlocks): the blocks allocated for the request.
|
||||
num_external_tokens (int): the number of tokens to load from the
|
||||
external KV cache. 0 means nothing should be loaded.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -550,14 +536,14 @@ class KVConnectorBase_V1(ABC):
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
"""
|
||||
Build the connector metadata for this step.
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -570,12 +556,12 @@ class KVConnectorBase_V1(ABC):
|
||||
return
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""
|
||||
Update KVConnector state from worker-side connectors output.
|
||||
"""Update KVConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
@@ -584,8 +570,7 @@ class KVConnectorBase_V1(ABC):
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called exactly once when a request has finished, before its blocks are
|
||||
"""Called exactly once when a request has finished, before its blocks are
|
||||
freed.
|
||||
|
||||
The connector may assumes responsibility for freeing the blocks
|
||||
@@ -597,6 +582,7 @@ class KVConnectorBase_V1(ABC):
|
||||
get_finished().
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
|
||||
"""
|
||||
return False, None
|
||||
|
||||
@@ -614,11 +600,11 @@ class KVConnectorBase_V1(ABC):
|
||||
return False
|
||||
|
||||
def take_events(self) -> Iterable["KVCacheEvent"]:
|
||||
"""
|
||||
Take the KV cache events from the connector.
|
||||
"""Take the KV cache events from the connector.
|
||||
|
||||
Yields:
|
||||
New KV cache events since the last call.
|
||||
|
||||
"""
|
||||
return ()
|
||||
|
||||
@@ -640,16 +626,16 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
|
||||
"""
|
||||
Get the required KV cache layout for this connector.
|
||||
"""Get the required KV cache layout for this connector.
|
||||
|
||||
Args:
|
||||
vllm_config (VllmConfig): the vllm config.
|
||||
|
||||
Returns:
|
||||
str: the required KV cache layout. e.g. HND, or NHD.
|
||||
None if the connector does not require a specific layout.
|
||||
"""
|
||||
|
||||
"""
|
||||
if cls is KVConnectorBase_V1:
|
||||
raise TypeError(
|
||||
"get_required_kvcache_layout should not be called "
|
||||
@@ -659,8 +645,7 @@ class KVConnectorBase_V1(ABC):
|
||||
|
||||
@classmethod
|
||||
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
|
||||
"""
|
||||
Check if this connector requires PIECEWISE CUDA graph mode.
|
||||
"""Check if this connector requires PIECEWISE CUDA graph mode.
|
||||
|
||||
Connectors that use asynchronous layer-by-layer operations
|
||||
(wait_for_layer_load/save_kv_layer) should override this method
|
||||
@@ -676,27 +661,26 @@ class KVConnectorBase_V1(ABC):
|
||||
Returns:
|
||||
True if this connector requires PIECEWISE CUDA graph mode,
|
||||
False otherwise.
|
||||
|
||||
"""
|
||||
return False
|
||||
|
||||
def get_finished_count(self) -> int | None:
|
||||
"""
|
||||
Get the count of requests expected to complete send/receive operations
|
||||
"""Get the count of requests expected to complete send/receive operations
|
||||
via this connector. This method is used to initialize the
|
||||
KVOutputAggregator, overwriting the default world_size.
|
||||
|
||||
Returns:
|
||||
int: expected sending or receiving completion count.
|
||||
"""
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> "KVConnectorStats | None":
|
||||
"""
|
||||
KVConnectorStats resolution method. This method allows dynamically
|
||||
"""KVConnectorStats resolution method. This method allows dynamically
|
||||
registered connectors to return their own KVConnectorStats object,
|
||||
which can implement custom aggregation logic on the data dict.
|
||||
"""
|
||||
@@ -705,19 +689,18 @@ class KVConnectorBase_V1(ABC):
|
||||
def set_xfer_handshake_metadata(
|
||||
self, metadata: dict[int, KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""
|
||||
Set the KV connector handshake metadata for this connector.
|
||||
"""Set the KV connector handshake metadata for this connector.
|
||||
|
||||
Args:
|
||||
metadata (KVConnectorHandshakeMetadata): the handshake metadata to set.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
def set_xfer_handshake_metadata_pp_aware(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""
|
||||
Set handshake metadata keyed by (pp_rank, tp_rank).
|
||||
"""Set handshake metadata keyed by (pp_rank, tp_rank).
|
||||
- Default implementation assumes pp_rank is always 0
|
||||
- PP-aware connectors override this to consume all PP producer shards.
|
||||
"""
|
||||
@@ -738,19 +721,18 @@ class KVConnectorBase_V1(ABC):
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> "KVConnectorPromMetrics | None":
|
||||
"""
|
||||
Create a KVConnectorPromMetrics subclass which should register
|
||||
"""Create a KVConnectorPromMetrics subclass which should register
|
||||
per-connector Prometheus metrics and implement observe() to
|
||||
expose connector transfer stats via Prometheus.
|
||||
"""
|
||||
return None
|
||||
|
||||
def reset_cache(self) -> bool | None:
|
||||
"""
|
||||
Reset the connector's internal cache.
|
||||
"""Reset the connector's internal cache.
|
||||
|
||||
Returns:
|
||||
bool: True if the cache was successfully reset, False otherwise.
|
||||
|
||||
"""
|
||||
logger.debug(
|
||||
"Connector cache reset requested, but %s does not implement reset_cache().",
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
DecodeBenchConnector: A KV Connector for decode instance performance testing.
|
||||
"""DecodeBenchConnector: A KV Connector for decode instance performance testing.
|
||||
|
||||
This connector emulates a prefill-decode disaggregated setting by filling
|
||||
the KV cache with dummy values, allowing measurement of decoder performance
|
||||
@@ -10,7 +9,7 @@ under larger input sequence lengths (ISL) in resource-limited environments.
|
||||
Usage:
|
||||
To use this connector for benchmarking, configure it in the kv_transfer_config:
|
||||
|
||||
Example:
|
||||
Example:
|
||||
vllm serve <model> --kv-transfer-config '{
|
||||
"kv_connector": "DecodeBenchConnector",
|
||||
"kv_role": "kv_both",
|
||||
@@ -29,6 +28,7 @@ Usage:
|
||||
- fill_mean (float): Mean value for random normal fill (default: 0.015)
|
||||
- fill_std (float): Standard deviation for random fill (default: 0.0)
|
||||
Set to 0 for constant values, >0 for random sampling
|
||||
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
@@ -80,8 +80,7 @@ class DecodeBenchConnectorMetadata(KVConnectorMetadata):
|
||||
|
||||
|
||||
class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""
|
||||
A KV Connector for decode instance performance testing.
|
||||
"""A KV Connector for decode instance performance testing.
|
||||
|
||||
This connector fills the KV cache with dummy values to emulate a
|
||||
prefill-decode disaggregated setting, enabling performance testing of the
|
||||
@@ -219,8 +218,7 @@ class DecodeBenchConnectorScheduler:
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int, bool]:
|
||||
"""
|
||||
For new requests, return the number of tokens that should be filled
|
||||
"""For new requests, return the number of tokens that should be filled
|
||||
with dummy KV cache values.
|
||||
|
||||
Returns:
|
||||
@@ -228,6 +226,7 @@ class DecodeBenchConnectorScheduler:
|
||||
- num_tokens_to_fill: number of uncomputed tokens minus 1
|
||||
(we fill everything except the last token for decode)
|
||||
- is_async: False (synchronous filling)
|
||||
|
||||
"""
|
||||
req_id = request.request_id
|
||||
|
||||
@@ -251,8 +250,7 @@ class DecodeBenchConnectorScheduler:
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""
|
||||
Called after blocks are allocated. Store the block IDs so we can
|
||||
"""Called after blocks are allocated. Store the block IDs so we can
|
||||
fill them with dummy values.
|
||||
|
||||
Supports both single- and multi-group KV cache configurations.
|
||||
@@ -334,8 +332,7 @@ class DecodeBenchConnectorScheduler:
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> KVConnectorMetadata:
|
||||
"""
|
||||
Build metadata containing information about which blocks to fill
|
||||
"""Build metadata containing information about which blocks to fill
|
||||
with dummy KV values.
|
||||
"""
|
||||
meta = DecodeBenchConnectorMetadata(reqs_to_fill=self._pending_fills.copy())
|
||||
@@ -346,9 +343,7 @@ class DecodeBenchConnectorScheduler:
|
||||
return meta
|
||||
|
||||
def request_finished(self, request: "Request"):
|
||||
"""
|
||||
Called when a request has finished. Clean up any state.
|
||||
"""
|
||||
"""Called when a request has finished. Clean up any state."""
|
||||
self._filled_requests.discard(request.request_id)
|
||||
|
||||
|
||||
@@ -389,8 +384,7 @@ class DecodeBenchConnectorWorker:
|
||||
)
|
||||
|
||||
def start_fill_kv(self, metadata: DecodeBenchConnectorMetadata):
|
||||
"""
|
||||
Fill the allocated KV cache blocks with dummy values.
|
||||
"""Fill the allocated KV cache blocks with dummy values.
|
||||
|
||||
This simulates having a populated KV cache from a prefill phase,
|
||||
allowing decode performance testing with larger context sizes.
|
||||
@@ -419,13 +413,13 @@ class DecodeBenchConnectorWorker:
|
||||
)
|
||||
|
||||
def _fill_blocks(self, group_idx: int, block_ids: list[int], num_tokens: int):
|
||||
"""
|
||||
Fill specified blocks with dummy values for a specific KV cache group.
|
||||
"""Fill specified blocks with dummy values for a specific KV cache group.
|
||||
|
||||
Args:
|
||||
group_idx: The KV cache group index to fill
|
||||
block_ids: List of block IDs to fill in this group
|
||||
num_tokens: Total number of tokens to fill across these blocks
|
||||
|
||||
"""
|
||||
if not block_ids:
|
||||
return
|
||||
@@ -501,6 +495,7 @@ class DecodeBenchConnectorWorker:
|
||||
tensor's first dim are ignored.
|
||||
fill_mean: Mean value for the fill.
|
||||
fill_std: Standard deviation for the fill.
|
||||
|
||||
"""
|
||||
# Convert block_ids to tensor on device
|
||||
block_ids_tensor = torch.tensor(
|
||||
@@ -551,6 +546,7 @@ class DecodeBenchConnectorWorker:
|
||||
kv_cache: A state tensor to fill in its entirety.
|
||||
fill_mean: Mean value for the fill.
|
||||
fill_std: Standard deviation for the fill.
|
||||
|
||||
"""
|
||||
if fill_std > 0:
|
||||
kv_cache.normal_(mean=fill_mean, std=fill_std)
|
||||
|
||||
@@ -118,6 +118,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
Note:
|
||||
The number of elements in kv_caches and layer_names should be
|
||||
the same.
|
||||
|
||||
"""
|
||||
|
||||
def inject_kv_into_layer(
|
||||
@@ -134,6 +135,9 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
src_kv_cache (torch.Tensor): the source KV cache.
|
||||
slot_mapping (torch.Tensor): the slot mapping. In shape
|
||||
[num_tokens].
|
||||
attn_metadata (AttentionMetadata): attention metadata for the
|
||||
current forward pass.
|
||||
|
||||
"""
|
||||
# Deliberate bulk transfers in this reference connector.
|
||||
with gpu_sync_allowed():
|
||||
@@ -201,6 +205,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
@@ -220,6 +225,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
**kwargs: additional arguments for the save operation.
|
||||
|
||||
"""
|
||||
|
||||
def extract_kv_from_layer(
|
||||
@@ -260,8 +266,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""
|
||||
Get number of new tokens that can be loaded from the
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
@@ -272,6 +277,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
Returns:
|
||||
the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
|
||||
"""
|
||||
# NOTE: in this debug implementation, we assume that the prompt is
|
||||
# cached_prompt + newly_generated_single_token
|
||||
@@ -295,8 +301,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""
|
||||
Update KVConnector state after block allocation.
|
||||
"""Update KVConnector state after block allocation.
|
||||
|
||||
If blocks were allocated, add to _requests_need_load,
|
||||
such that we load the KVs in the next forward pass.
|
||||
@@ -315,6 +320,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
meta = ExampleConnectorMetadata()
|
||||
|
||||
|
||||
@@ -57,6 +57,7 @@ def load_hidden_states(path: str) -> dict[str, torch.Tensor]:
|
||||
|
||||
Returns:
|
||||
Dict with "hidden_states" and "token_ids" tensors.
|
||||
|
||||
"""
|
||||
lock_path = path + ".lock"
|
||||
with open(lock_path) as lf:
|
||||
@@ -96,8 +97,7 @@ class ExampleHiddenStatesConnectorMetadata(KVConnectorMetadata):
|
||||
|
||||
|
||||
class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""
|
||||
Simple debug implementation of a HiddenStatesConnector.
|
||||
"""Simple debug implementation of a HiddenStatesConnector.
|
||||
|
||||
Simply extracts the hidden states from the kv cache and stores them to disk.
|
||||
Must be used in conjunction with the `extract_hidden_states` spec decoding method.
|
||||
@@ -278,7 +278,8 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
def _on_write_done(self, req_id: str, future: Future) -> None:
|
||||
"""Surface any exception from the disk-write thread and drop the
|
||||
completed future from the in-flight tracking dict."""
|
||||
completed future from the in-flight tracking dict.
|
||||
"""
|
||||
self._req_futures.pop(req_id, None)
|
||||
exc = future.exception()
|
||||
if exc is not None:
|
||||
@@ -436,8 +437,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""
|
||||
Get number of new tokens that can be loaded from the
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
@@ -448,6 +448,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
Returns:
|
||||
the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
|
||||
"""
|
||||
# This connector is store-only, so we don't need to load any tokens
|
||||
return 0, False
|
||||
@@ -471,6 +472,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
meta = ExampleHiddenStatesConnectorMetadata()
|
||||
|
||||
@@ -509,8 +511,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called exactly once when a request has finished, before its blocks are
|
||||
"""Called exactly once when a request has finished, before its blocks are
|
||||
freed.
|
||||
|
||||
Returns True to delay block freeing until get_finished extracts
|
||||
@@ -587,16 +588,16 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
|
||||
"""
|
||||
Get the required KV cache layout for this connector.
|
||||
"""Get the required KV cache layout for this connector.
|
||||
|
||||
Args:
|
||||
vllm_config (VllmConfig): the vllm config.
|
||||
|
||||
Returns:
|
||||
str: the required KV cache layout. e.g. HND, or NHD.
|
||||
None if the connector does not require a specific layout.
|
||||
"""
|
||||
|
||||
"""
|
||||
if cls is KVConnectorBase_V1:
|
||||
raise TypeError(
|
||||
"get_required_kvcache_layout should not be called "
|
||||
|
||||
@@ -96,6 +96,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
Args:
|
||||
forward_context (ForwardContext): the forward context.
|
||||
**kwargs (Any): additional arguments (unused).
|
||||
|
||||
"""
|
||||
self._flexkv_connector.start_load_kv(forward_context, **kwargs)
|
||||
|
||||
@@ -107,6 +108,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
layer_name: the name of the layer (unused).
|
||||
|
||||
"""
|
||||
self._flexkv_connector.wait_for_layer_load(layer_name)
|
||||
|
||||
@@ -131,6 +133,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
kv_layer (torch.Tensor): the paged KV buffer (unused).
|
||||
attn_metadata (AttentionMetadata): the attention metadata (unused).
|
||||
**kwargs (Any): additional arguments (unused).
|
||||
|
||||
"""
|
||||
self._flexkv_connector.save_kv_layer(
|
||||
layer_name, kv_layer, attn_metadata, **kwargs
|
||||
@@ -160,6 +163,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
that have finished asynchronous transfer. The finished saves/sends
|
||||
req ids must belong to a set provided in a call to this method
|
||||
(this call or a prior one).
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.get_finished(finished_req_ids)
|
||||
|
||||
@@ -169,6 +173,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
kv_caches: dictionary of layer names to kv cache tensors.
|
||||
|
||||
"""
|
||||
self._flexkv_connector.register_kv_caches(kv_caches)
|
||||
|
||||
@@ -192,6 +197,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
Tuple of (num_external_tokens, is_ready) where
|
||||
num_external_tokens is the number of additional tokens that
|
||||
can be loaded from the external KV cache.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
@@ -215,6 +221,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.build_connector_meta(scheduler_output)
|
||||
|
||||
@@ -224,6 +231,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
self._flexkv_connector.update_connector_output(connector_output)
|
||||
|
||||
@@ -240,6 +248,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
should not be freed until the request_id is returned from
|
||||
:meth:`get_finished`. kv_transfer_params is an optional dict of
|
||||
KVTransferParams to be included in the request outputs.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.request_finished(request, block_ids)
|
||||
|
||||
@@ -248,6 +257,7 @@ class FlexKVConnectorV1(KVConnectorBase_V1):
|
||||
|
||||
Returns:
|
||||
New KV cache events since the last call.
|
||||
|
||||
"""
|
||||
return self._flexkv_connector.take_events()
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@ class Hf3fsClient:
|
||||
size: Total size of the storage file in bytes
|
||||
bytes_per_page: Size of each page in bytes
|
||||
entries: Maximum number of concurrent operations
|
||||
|
||||
"""
|
||||
if not HF3FS_AVAILABLE:
|
||||
raise ImportError(
|
||||
@@ -154,7 +155,7 @@ class Hf3fsClient:
|
||||
)
|
||||
|
||||
def _release_resources(self) -> None:
|
||||
"""Release all acquired resources safely"""
|
||||
"""Release all acquired resources safely."""
|
||||
# iov must be released before ioring and shm
|
||||
for attr in ("iov_r", "iov_w", "ior_r", "ior_w"):
|
||||
obj = getattr(self, attr, None)
|
||||
@@ -192,6 +193,7 @@ class Hf3fsClient:
|
||||
|
||||
Returns:
|
||||
List of operation results (0 for success, non-zero for error)
|
||||
|
||||
"""
|
||||
self.check(offsets, tensors)
|
||||
assert self.ior_r is not None
|
||||
@@ -228,11 +230,12 @@ class Hf3fsClient:
|
||||
Args:
|
||||
offsets: List of byte offsets to write to
|
||||
tensors: List of tensors containing data to write
|
||||
event: CUDA event to synchronize on before reading the tensors
|
||||
|
||||
Returns:
|
||||
List of operation results (0 for success, non-zero for error)
|
||||
"""
|
||||
|
||||
"""
|
||||
self.check(offsets, tensors)
|
||||
assert self.ior_w is not None
|
||||
assert self.iov_w is not None
|
||||
@@ -282,6 +285,7 @@ class Hf3fsClient:
|
||||
|
||||
Returns:
|
||||
Size of the file in bytes
|
||||
|
||||
"""
|
||||
return self.size
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
HF3FS KV Connector Implementation for vLLM.
|
||||
"""HF3FS KV Connector Implementation for vLLM.
|
||||
|
||||
This module implements a KV connector that uses
|
||||
the 3FS for storing and retrieving KV cache data.
|
||||
@@ -99,9 +98,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AsyncOperationManager:
|
||||
"""
|
||||
Manages async save/load operations with background threads.
|
||||
"""
|
||||
"""Manages async save/load operations with background threads."""
|
||||
|
||||
def __init__(self, connector: "HF3FSKVConnector"):
|
||||
# Store connector reference and extract commonly used attributes
|
||||
@@ -667,9 +664,7 @@ class HF3FSKVConnector(KVConnectorBase_V1):
|
||||
return self._async_manager.get_finished_operations(finished_req_ids)
|
||||
|
||||
def get_kv_connector_stats(self) -> Optional["KVConnectorStats"]:
|
||||
"""
|
||||
Get the KV connector stats collected during the last interval.
|
||||
"""
|
||||
"""Get the KV connector stats collected during the last interval."""
|
||||
# Clear stats for next iteration
|
||||
if (
|
||||
hasattr(self, "_async_manager")
|
||||
@@ -886,8 +881,7 @@ class HF3FSKVConnector(KVConnectorBase_V1):
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> Optional["KVConnectorStats"]:
|
||||
"""
|
||||
KVConnectorStats resolution method. This method allows dynamically
|
||||
"""KVConnectorStats resolution method. This method allows dynamically
|
||||
registered connectors to return their own KVConnectorStats object,
|
||||
which can implement custom aggregation logic on the data dict.
|
||||
"""
|
||||
@@ -1021,7 +1015,7 @@ class HF3FSKVConnector(KVConnectorBase_V1):
|
||||
|
||||
@dataclass
|
||||
class HF3FSKVConnectorStats(KVConnectorStats):
|
||||
"""Container for transfer performance metrics"""
|
||||
"""Container for transfer performance metrics."""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
HF3FS Metadata Server with key-based organization.
|
||||
"""
|
||||
"""HF3FS Metadata Server with key-based organization."""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
@@ -119,6 +117,7 @@ class GlobalMetadataState:
|
||||
|
||||
Returns:
|
||||
Dictionary mapping key -> allocated page index
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
if rank not in self.rank_metadata:
|
||||
@@ -167,6 +166,7 @@ class GlobalMetadataState:
|
||||
rank: Rank ID that confirmed the writes
|
||||
key_confirmations: List of (key, page_index) tuples
|
||||
pages_to_release: List of page indices to release back to free pool
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
# Confirm successful writes
|
||||
@@ -197,6 +197,7 @@ class GlobalMetadataState:
|
||||
|
||||
Returns:
|
||||
List of boolean values indicating key existence and completion
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
results = []
|
||||
@@ -218,6 +219,7 @@ class GlobalMetadataState:
|
||||
|
||||
Returns:
|
||||
List of page indices in the same order as input keys (None if key not found)
|
||||
|
||||
"""
|
||||
with self.global_lock:
|
||||
if rank not in self.rank_metadata:
|
||||
|
||||
@@ -128,6 +128,7 @@ def scatter_kv_caches(
|
||||
num_heads: Size of the H axis
|
||||
content_size: Size of the C axis
|
||||
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
|
||||
|
||||
"""
|
||||
num_layers = len(kv_caches_ptrs)
|
||||
num_tokens_in_block = len(token_indices)
|
||||
@@ -184,6 +185,7 @@ def gather_kv_caches(
|
||||
num_heads: Size of the H axis
|
||||
content_size: Size of the C axis
|
||||
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
|
||||
|
||||
"""
|
||||
num_layers = kv_caches_ptrs.shape[0]
|
||||
num_tokens_in_block = len(token_indices)
|
||||
|
||||
@@ -32,9 +32,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LMCacheKVEvents(KVConnectorKVEvents):
|
||||
"""
|
||||
Concrete implementation of KVConnectorKVEvents using KVEventAggregator.
|
||||
"""
|
||||
"""Concrete implementation of KVConnectorKVEvents using KVEventAggregator."""
|
||||
|
||||
def __init__(self, num_workers: int) -> None:
|
||||
self._aggregator = KVEventAggregator(num_workers)
|
||||
@@ -43,9 +41,7 @@ class LMCacheKVEvents(KVConnectorKVEvents):
|
||||
self._aggregator.add_events(events)
|
||||
|
||||
def aggregate(self) -> "LMCacheKVEvents":
|
||||
"""
|
||||
Aggregate KV events and retain only common events.
|
||||
"""
|
||||
"""Aggregate KV events and retain only common events."""
|
||||
common_events = self._aggregator.get_common_events()
|
||||
self._aggregator.clear_events()
|
||||
self._aggregator.add_events(common_events)
|
||||
@@ -72,8 +68,7 @@ class LMCacheKVEvents(KVConnectorKVEvents):
|
||||
class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
@classmethod
|
||||
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
|
||||
"""
|
||||
LMCache requires PIECEWISE CUDA graph mode when layerwise
|
||||
"""LMCache requires PIECEWISE CUDA graph mode when layerwise
|
||||
operations are enabled. The wait_for_layer_load and save_kv_layer
|
||||
methods perform actual async synchronization that cannot be
|
||||
captured in CUDA graphs.
|
||||
@@ -118,12 +113,12 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
# Worker-side methods
|
||||
# ==============================
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""
|
||||
Initialize with the KV caches. Useful for pre-registering the
|
||||
"""Initialize with the KV caches. Useful for pre-registering the
|
||||
KV Caches in the KVConnector (e.g. for NIXL).
|
||||
|
||||
Args:
|
||||
kv_caches: dictionary of layer names, kv cache
|
||||
|
||||
"""
|
||||
if hasattr(self._lmcache_engine, "register_kv_caches"):
|
||||
self._lmcache_engine.register_kv_caches(kv_caches)
|
||||
@@ -149,8 +144,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
engine.layerwise_retrievers = []
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""
|
||||
Start loading the KV cache from the connector to vLLM's paged
|
||||
"""Start loading the KV cache from the connector to vLLM's paged
|
||||
KV buffer. This is called from the forward context before the
|
||||
forward pass to enable async loading during model execution.
|
||||
|
||||
@@ -166,8 +160,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
self._lmcache_engine.start_load_kv(forward_context, **kwargs)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""
|
||||
Block until the KV for a specific layer is loaded into vLLM's
|
||||
"""Block until the KV for a specific layer is loaded into vLLM's
|
||||
paged buffer. This is called from within attention layer to ensure
|
||||
async copying from start_load_kv is complete.
|
||||
|
||||
@@ -175,6 +168,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
self._lmcache_engine.wait_for_layer_load(layer_name)
|
||||
|
||||
@@ -185,8 +179,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Start saving the a layer of KV cache from vLLM's paged buffer
|
||||
"""Start saving the a layer of KV cache from vLLM's paged buffer
|
||||
to the connector. This is called from within attention layer to
|
||||
enable async copying during execution.
|
||||
|
||||
@@ -196,14 +189,14 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
**kwargs: additional arguments for the save operation.
|
||||
|
||||
"""
|
||||
self._lmcache_engine.save_kv_layer(
|
||||
layer_name, kv_layer, attn_metadata, **kwargs
|
||||
)
|
||||
|
||||
def wait_for_save(self):
|
||||
"""
|
||||
Block until all the save operations is done. This is called
|
||||
"""Block until all the save operations is done. This is called
|
||||
as the forward context exits to ensure that the async saving
|
||||
from save_kv_layer is complete before finishing the forward.
|
||||
|
||||
@@ -214,8 +207,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""
|
||||
Notifies worker-side connector ids of requests that have
|
||||
"""Notifies worker-side connector ids of requests that have
|
||||
finished generating tokens.
|
||||
|
||||
Returns:
|
||||
@@ -224,16 +216,17 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
tuple of (sending/saving ids, recving/loading ids).
|
||||
The finished saves/sends req ids must belong to a set provided in a
|
||||
call to this method (this call or a prior one).
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.get_finished(finished_req_ids)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""
|
||||
Get the set of block IDs that failed to load.
|
||||
"""Get the set of block IDs that failed to load.
|
||||
|
||||
Returns:
|
||||
Set of block IDs that encountered load errors.
|
||||
Empty set if no load errors occurred.
|
||||
|
||||
"""
|
||||
method = getattr(self._lmcache_engine, "get_block_ids_with_load_errors", None)
|
||||
if callable(method):
|
||||
@@ -243,10 +236,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
return set()
|
||||
|
||||
def get_kv_connector_kv_cache_events(self) -> LMCacheKVEvents | None:
|
||||
"""
|
||||
Get the KV connector kv cache events collected during the last interval.
|
||||
"""
|
||||
|
||||
"""Get the KV connector kv cache events collected during the last interval."""
|
||||
events = self._lmcache_engine.get_kv_events() # type: ignore [attr-defined]
|
||||
if not events:
|
||||
return None
|
||||
@@ -276,8 +266,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""
|
||||
Get number of new tokens that can be loaded from the
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
@@ -288,6 +277,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
Returns:
|
||||
the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
@@ -296,32 +286,30 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""
|
||||
Update KVConnector state after block allocation.
|
||||
"""
|
||||
"""Update KVConnector state after block allocation."""
|
||||
self._lmcache_engine.update_state_after_alloc(request, num_external_tokens)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
"""
|
||||
Build the connector metadata for this step.
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.build_connector_meta(scheduler_output)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""
|
||||
Update KVConnector state from worker-side connectors output.
|
||||
"""Update KVConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
# Get the KV events
|
||||
kv_cache_events = connector_output.kv_cache_events
|
||||
@@ -342,8 +330,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called when a request has finished, before its blocks are freed.
|
||||
"""Called when a request has finished, before its blocks are freed.
|
||||
|
||||
Returns:
|
||||
True if the request is being saved/sent asynchronously and blocks
|
||||
@@ -351,15 +338,16 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
|
||||
get_finished().
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
|
||||
"""
|
||||
return self._lmcache_engine.request_finished(request, block_ids)
|
||||
|
||||
def take_events(self) -> Iterable["KVCacheEvent"]:
|
||||
"""
|
||||
Take the KV cache events from the connector.
|
||||
"""Take the KV cache events from the connector.
|
||||
|
||||
Yields:
|
||||
New KV cache events since the last call.
|
||||
|
||||
"""
|
||||
if self._kv_cache_events is not None:
|
||||
self._kv_cache_events.aggregate()
|
||||
|
||||
+46
-47
@@ -44,8 +44,7 @@ def send_lmcache_request(
|
||||
request_type: RequestType,
|
||||
payloads: list[Any],
|
||||
) -> MessagingFuture[Any]:
|
||||
"""
|
||||
Helper function to send the request to the LMCache multiprocess server
|
||||
"""Helper function to send the request to the LMCache multiprocess server.
|
||||
|
||||
Args:
|
||||
mq_client: The LMCache multiprocess mode message queue client
|
||||
@@ -54,8 +53,8 @@ def send_lmcache_request(
|
||||
|
||||
Returns:
|
||||
A messaging future for the request
|
||||
"""
|
||||
|
||||
"""
|
||||
future = mq_client.submit_request(
|
||||
request_type, payloads, get_response_class(request_type)
|
||||
)
|
||||
@@ -65,14 +64,14 @@ def send_lmcache_request(
|
||||
def get_lmcache_chunk_size(
|
||||
mq_client: MessageQueueClient,
|
||||
) -> int:
|
||||
"""
|
||||
Helper function to get the LMCache chunk size from the server
|
||||
"""Helper function to get the LMCache chunk size from the server.
|
||||
|
||||
Args:
|
||||
mq_client: The LMCache multiprocess mode message queue client
|
||||
|
||||
Returns:
|
||||
An integer representing the LMCache chunk size
|
||||
|
||||
"""
|
||||
future = send_lmcache_request(mq_client, RequestType.GET_CHUNK_SIZE, [])
|
||||
chunk_size = future.result()
|
||||
@@ -147,16 +146,16 @@ class LMCacheMPSchedulerAdapter:
|
||||
vllm_block_size: int,
|
||||
parallel_strategy: ParallelStrategy,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
server_url: The server URL for the LMCache message queue
|
||||
context: The ZMQ context
|
||||
"""Args:
|
||||
server_url: The server URL for the LMCache message queue
|
||||
context: The ZMQ context
|
||||
|
||||
model_name: The model name used for LMCache keys
|
||||
vllm_block_size: The block size used in vLLM
|
||||
parallel_strategy:
|
||||
The parallel strategy, which includes `use_mla`,
|
||||
`world_size`, `worker_id` and so on
|
||||
|
||||
model_name: The model name used for LMCache keys
|
||||
vllm_block_size: The block size used in vLLM
|
||||
parallel_strategy:
|
||||
The parallel strategy, which includes `use_mla`,
|
||||
`world_size`, `worker_id` and so on
|
||||
"""
|
||||
self.mq_client = MessageQueueClient(server_url, context)
|
||||
|
||||
@@ -195,8 +194,7 @@ class LMCacheMPSchedulerAdapter:
|
||||
block_hashes: list[bytes] | None = None,
|
||||
token_ids: list[int] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Submit a new lookup request to LMCache if there is no ongoing request.
|
||||
"""Submit a new lookup request to LMCache if there is no ongoing request.
|
||||
|
||||
Supports both token-based and hash-based vLLM:
|
||||
- token_ids: token IDs (token-based vLLM) -> single token-mode key
|
||||
@@ -219,6 +217,7 @@ class LMCacheMPSchedulerAdapter:
|
||||
for later retrieve operations.
|
||||
In the meantime, this function will record the lookup request, and the
|
||||
status of the look up request can be checked by `check_lookup_result`.
|
||||
|
||||
"""
|
||||
if request_id in self.lookup_futures:
|
||||
# Skip if there is already a lookup request
|
||||
@@ -260,8 +259,7 @@ class LMCacheMPSchedulerAdapter:
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def check_lookup_result(self, request_id: str) -> int | None:
|
||||
"""
|
||||
Check the result of a previously submitted lookup request.
|
||||
"""Check the result of a previously submitted lookup request.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the lookup request submitted in
|
||||
@@ -271,6 +269,7 @@ class LMCacheMPSchedulerAdapter:
|
||||
An integer representing the total number of tokens matched
|
||||
in LMCache (prefix matching), or
|
||||
None if the lookup request is not finished yet.
|
||||
|
||||
"""
|
||||
assert request_id in self.lookup_futures, (
|
||||
f"Lookup request for request_id={request_id} has not been submitted"
|
||||
@@ -285,25 +284,27 @@ class LMCacheMPSchedulerAdapter:
|
||||
return num_chunks * self.chunk_size
|
||||
|
||||
def num_blocks_per_chunk(self) -> int:
|
||||
"""
|
||||
Returns:
|
||||
The number of vllm blocks in a LMCache data chunk
|
||||
"""Returns:
|
||||
The number of vllm blocks in a LMCache data chunk
|
||||
|
||||
"""
|
||||
return self.blocks_in_chunk
|
||||
|
||||
def cleanup_lookup_result(self, request_id: str) -> None:
|
||||
"""
|
||||
Clean up lookup future for a finished request to prevent memory leak.
|
||||
"""Clean up lookup future for a finished request to prevent memory leak.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the finished request.
|
||||
|
||||
"""
|
||||
self.lookup_futures.pop(request_id, None)
|
||||
|
||||
def end_session(self, request_id: str) -> None:
|
||||
"""
|
||||
Notify LMCache server to remove the session for a finished request.
|
||||
"""Notify LMCache server to remove the session for a finished request.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the finished request.
|
||||
|
||||
"""
|
||||
send_lmcache_request(
|
||||
self.mq_client,
|
||||
@@ -319,7 +320,7 @@ class LMCacheMPSchedulerAdapter:
|
||||
end: int = 0,
|
||||
request_id: str | None = None,
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Convert token IDs to an IPC cache engine key"""
|
||||
"""Convert token IDs to an IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
@@ -334,7 +335,7 @@ class LMCacheMPSchedulerAdapter:
|
||||
def _create_hash_key(
|
||||
self, chunk_hash: bytes, request_id: str | None = None
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Create a hash-mode IPC cache engine key"""
|
||||
"""Create a hash-mode IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
@@ -411,12 +412,12 @@ class LMCacheMPWorkerAdapter:
|
||||
)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""
|
||||
Register the kv caches with LMCache server
|
||||
"""Register the kv caches with LMCache server.
|
||||
|
||||
Args:
|
||||
kv_caches: A dict of kv caches to register. The keys are the
|
||||
layer names and the values are the corresponding tensors.
|
||||
|
||||
"""
|
||||
# Register kv cache and send the request
|
||||
self.kv_caches = kv_caches
|
||||
@@ -432,14 +433,14 @@ class LMCacheMPWorkerAdapter:
|
||||
def submit_store_request(
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
|
||||
):
|
||||
"""
|
||||
Submit a KV cache store request to LMCache
|
||||
"""Submit a KV cache store request to LMCache.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the request
|
||||
op: The LoadStoreOp describing the store operation.
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
if op.block_hashes is not None:
|
||||
# Hash mode
|
||||
@@ -466,14 +467,14 @@ class LMCacheMPWorkerAdapter:
|
||||
def submit_retrieve_request(
|
||||
self, request_id: str, op: LoadStoreOp, event: torch.cuda.Event
|
||||
):
|
||||
"""
|
||||
Submit a KV cache retrieve request to LMCache
|
||||
"""Submit a KV cache retrieve request to LMCache.
|
||||
|
||||
Args:
|
||||
request_id: The ID of the request
|
||||
op: The LoadStoreOp describing the retrieve operation.
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
if op.block_hashes is not None:
|
||||
# Hash mode
|
||||
@@ -503,8 +504,7 @@ class LMCacheMPWorkerAdapter:
|
||||
ops: list[LoadStoreOp],
|
||||
event: torch.cuda.Event,
|
||||
):
|
||||
"""
|
||||
Submit a batched store request to LMCache
|
||||
"""Submit a batched store request to LMCache.
|
||||
|
||||
Args:
|
||||
request_ids: The IDs of the requests
|
||||
@@ -512,6 +512,7 @@ class LMCacheMPWorkerAdapter:
|
||||
the same length as request_ids
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
all_keys: list[IPCCacheEngineKey] = []
|
||||
block_ids: list[int] = []
|
||||
@@ -552,8 +553,7 @@ class LMCacheMPWorkerAdapter:
|
||||
ops: list[LoadStoreOp],
|
||||
event: torch.cuda.Event,
|
||||
):
|
||||
"""
|
||||
Submit a batched retrieve request to LMCache
|
||||
"""Submit a batched retrieve request to LMCache.
|
||||
|
||||
Args:
|
||||
request_ids: The IDs of the requests
|
||||
@@ -561,6 +561,7 @@ class LMCacheMPWorkerAdapter:
|
||||
the same length as request_ids
|
||||
event: The CUDA event that is recorded after the current
|
||||
model inference step
|
||||
|
||||
"""
|
||||
all_keys: list[IPCCacheEngineKey] = []
|
||||
block_ids: list[int] = []
|
||||
@@ -598,8 +599,7 @@ class LMCacheMPWorkerAdapter:
|
||||
def get_finished(
|
||||
self, finished_req_ids_from_engine: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""
|
||||
Check and get the finished store and retrieve requests.
|
||||
"""Check and get the finished store and retrieve requests.
|
||||
|
||||
Args:
|
||||
finished_req_ids_from_engine: the set of request ids that are
|
||||
@@ -617,6 +617,7 @@ class LMCacheMPWorkerAdapter:
|
||||
multiple times in `finished_req_ids_from_engine`. The adapter should
|
||||
take care of deduplicating the request IDs and only return the request
|
||||
IDs that have not been returned before.
|
||||
|
||||
"""
|
||||
finished_stores = set()
|
||||
finished_retrieves = set()
|
||||
@@ -675,16 +676,14 @@ class LMCacheMPWorkerAdapter:
|
||||
return ret_stores, finished_retrieves
|
||||
|
||||
def num_blocks_per_chunk(self) -> int:
|
||||
"""
|
||||
Returns:
|
||||
The number of vllm blocks in a LMCache data chunk
|
||||
"""Returns:
|
||||
The number of vllm blocks in a LMCache data chunk
|
||||
|
||||
"""
|
||||
return self.blocks_in_chunk
|
||||
|
||||
def shutdown(self):
|
||||
"""
|
||||
Shutdown the LMCache MP worker adapter
|
||||
"""
|
||||
"""Shutdown the LMCache MP worker adapter."""
|
||||
logger.info("Unregistering kv caches")
|
||||
send_lmcache_request(
|
||||
self.mq_client, RequestType.UNREGISTER_KV_CACHE, [self.instance_id]
|
||||
@@ -712,7 +711,7 @@ class LMCacheMPWorkerAdapter:
|
||||
end: int = 0,
|
||||
request_id: str | None = None,
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Convert token IDs to an IPC cache engine key"""
|
||||
"""Convert token IDs to an IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
@@ -726,7 +725,7 @@ class LMCacheMPWorkerAdapter:
|
||||
def _create_hash_key(
|
||||
self, chunk_hash: bytes, request_id: str | None = None
|
||||
) -> IPCCacheEngineKey:
|
||||
"""Create a hash-mode IPC cache engine key"""
|
||||
"""Create a hash-mode IPC cache engine key."""
|
||||
return IPCCacheEngineKey(
|
||||
model_name=self.model_name,
|
||||
world_size=self.world_size,
|
||||
|
||||
@@ -59,9 +59,7 @@ def lmcache_get_or_create_config() -> V1Config:
|
||||
|
||||
|
||||
def hex_hash_to_int16(s: str) -> int:
|
||||
"""
|
||||
Convert a hex hash string to a 16-bit integer.
|
||||
"""
|
||||
"""Convert a hex hash string to a 16-bit integer."""
|
||||
return int(s, 16) & 0xFFFF
|
||||
|
||||
|
||||
@@ -70,8 +68,7 @@ def apply_mm_hashes_to_token_ids(
|
||||
mm_hashes: list[str],
|
||||
mm_positions: list["PlaceholderRange"],
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Overwrite token_ids in-place for multimodal placeholders using
|
||||
"""Overwrite token_ids in-place for multimodal placeholders using
|
||||
efficient slice assignments.
|
||||
"""
|
||||
n = token_ids.size(0)
|
||||
@@ -95,8 +92,7 @@ def mla_enabled(model_config: "ModelConfig") -> bool:
|
||||
def extract_mm_features(
|
||||
request: Union["Request", "NewRequestData"], modify: bool = False
|
||||
) -> tuple[list[str], list["PlaceholderRange"]]:
|
||||
"""
|
||||
Normalize multimodal information from a Request into parallel lists.
|
||||
"""Normalize multimodal information from a Request into parallel lists.
|
||||
|
||||
This helper reads either:
|
||||
1) `request.mm_features` (objects each exposing `.identifier` and
|
||||
@@ -119,6 +115,7 @@ def extract_mm_features(
|
||||
Returns:
|
||||
tuple[list[str], list[PlaceholderRange]]: (`mm_hashes`, `mm_positions`).
|
||||
May be `([], [])` when no multimodal data is present.
|
||||
|
||||
"""
|
||||
if getattr(request, "mm_features", None):
|
||||
mm_hashes, mm_positions = zip(
|
||||
|
||||
+13
-8
@@ -171,6 +171,7 @@ class RequestTracker:
|
||||
lmcache_cached_tokens (int): the number of tokens that are
|
||||
cached in LMCache.
|
||||
skip_save (bool): whether the request cache should be saved
|
||||
|
||||
"""
|
||||
# vLLM 0.9.0 update: request.block_ids changed from list[int] to
|
||||
# list[list[int]]
|
||||
@@ -219,7 +220,6 @@ class RequestTracker:
|
||||
"""Update the request tracker when a running request is
|
||||
scheduled again
|
||||
"""
|
||||
|
||||
self.token_ids.extend(new_token_ids)
|
||||
|
||||
if new_block_ids is None:
|
||||
@@ -288,6 +288,7 @@ class ReqMeta:
|
||||
Returns:
|
||||
the request metadata if we need to perform load/save
|
||||
operations, None otherwise.
|
||||
|
||||
"""
|
||||
input_token_ids = tracker.token_ids
|
||||
input_token_len = len(input_token_ids)
|
||||
@@ -445,6 +446,7 @@ def _init_lmcache_engine(
|
||||
|
||||
Returns:
|
||||
The initialized LMCache engine
|
||||
|
||||
"""
|
||||
if curr_engine := LMCacheEngineBuilder.get(ENGINE_NAME):
|
||||
return curr_engine
|
||||
@@ -563,6 +565,7 @@ class LMCacheConnectorMetadata(KVConnectorMetadata):
|
||||
|
||||
Args:
|
||||
req_meta (ReqMeta): the request metadata.
|
||||
|
||||
"""
|
||||
self.requests.append(req_meta)
|
||||
|
||||
@@ -719,6 +722,7 @@ class LMCacheConnectorV1Impl:
|
||||
|
||||
Returns:
|
||||
dict: Dictionary containing inference information
|
||||
|
||||
"""
|
||||
# Get vLLM config information
|
||||
vllm_config = self._vllm_config
|
||||
@@ -766,6 +770,7 @@ class LMCacheConnectorV1Impl:
|
||||
|
||||
Returns:
|
||||
str: vLLM version string
|
||||
|
||||
"""
|
||||
return VLLM_VERSION
|
||||
|
||||
@@ -816,6 +821,7 @@ class LMCacheConnectorV1Impl:
|
||||
Note:
|
||||
The number of elements in kv_caches and layer_names should be
|
||||
the same.
|
||||
|
||||
"""
|
||||
if len(self.kv_caches) == 0:
|
||||
self._init_kv_caches_from_forward_context(forward_context)
|
||||
@@ -920,6 +926,7 @@ class LMCacheConnectorV1Impl:
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
if self.layerwise_retrievers:
|
||||
logger.debug("Waiting for layer %s to be loaded", self.current_layer)
|
||||
@@ -951,6 +958,7 @@ class LMCacheConnectorV1Impl:
|
||||
kv_layer (torch.Tensor): the paged KV buffer of the current
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
|
||||
"""
|
||||
assert self.lmcache_engine is not None
|
||||
|
||||
@@ -1039,7 +1047,6 @@ class LMCacheConnectorV1Impl:
|
||||
@_lmcache_nvtx_annotate
|
||||
def wait_for_save(self):
|
||||
"""Blocking until the KV cache is saved to the connector buffer."""
|
||||
|
||||
connector_metadata = self._parent._get_connector_metadata()
|
||||
assert isinstance(connector_metadata, LMCacheConnectorMetadata)
|
||||
|
||||
@@ -1150,8 +1157,7 @@ class LMCacheConnectorV1Impl:
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> int | None:
|
||||
"""
|
||||
Check for external KV cache hit.
|
||||
"""Check for external KV cache hit.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
@@ -1161,6 +1167,7 @@ class LMCacheConnectorV1Impl:
|
||||
Returns:
|
||||
the number of tokens that can be loaded from the
|
||||
external KV cache beyond what is already computed.
|
||||
|
||||
"""
|
||||
if self.kv_role == "kv_producer" and not hasattr(
|
||||
self.lookup_client, "supports_producer_reuse"
|
||||
@@ -1236,13 +1243,11 @@ class LMCacheConnectorV1Impl:
|
||||
|
||||
@_lmcache_nvtx_annotate
|
||||
def update_state_after_alloc(self, request: "Request", num_external_tokens: int):
|
||||
"""
|
||||
Update KVConnector state after temporary buffer alloc.
|
||||
"""Update KVConnector state after temporary buffer alloc.
|
||||
|
||||
For SharedStorageConnector, update _request_needs_load
|
||||
if the CacheManager this allocated blocks for us.
|
||||
"""
|
||||
|
||||
# Clear local status in lookup client when a new request is
|
||||
# successfully scheduled.
|
||||
self.lookup_client.clear_lookup_status(request.request_id)
|
||||
@@ -1311,8 +1316,8 @@ class LMCacheConnectorV1Impl:
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
"""
|
||||
|
||||
"""
|
||||
force_skip_save = self.kv_role == "kv_consumer" or self.force_skip_save
|
||||
|
||||
meta = LMCacheConnectorMetadata()
|
||||
|
||||
@@ -88,9 +88,7 @@ def extract_world_size_and_kv_rank(
|
||||
rank: int,
|
||||
vllm_config: VllmConfig,
|
||||
) -> tuple[int, int]:
|
||||
"""
|
||||
Convert the rank for the MLA.
|
||||
"""
|
||||
"""Convert the rank for the MLA."""
|
||||
use_mla = mla_enabled(vllm_config.model_config)
|
||||
if not use_mla:
|
||||
return world_size, rank
|
||||
@@ -174,8 +172,7 @@ def create_worker_adapter(
|
||||
|
||||
|
||||
class LMCacheMPRequestState(enum.Enum):
|
||||
"""
|
||||
State machine:
|
||||
"""State machine:
|
||||
PREFETCHING -- update_state_after_alloc --> WAITING_FOR_LOAD
|
||||
WAITING_FOR_LOAD -- process_loading_requests --> READY
|
||||
"""
|
||||
@@ -234,7 +231,8 @@ class LMCacheMPRequestTracker:
|
||||
####
|
||||
def needs_retrieve(self) -> bool:
|
||||
"""Check whether the current request needs retrieve, will be used
|
||||
update_stage_after_alloc"""
|
||||
update_stage_after_alloc
|
||||
"""
|
||||
return (
|
||||
self.num_lmcache_hit_blocks > self.num_vllm_hit_blocks
|
||||
and self.state != LMCacheMPRequestState.READY
|
||||
@@ -242,7 +240,8 @@ class LMCacheMPRequestTracker:
|
||||
|
||||
def is_ready_for_retrieving(self) -> bool:
|
||||
"""Check whether the current request is ready for retrieving,
|
||||
will be used in process_loading_requests"""
|
||||
will be used in process_loading_requests
|
||||
"""
|
||||
return (
|
||||
self.state == LMCacheMPRequestState.WAITING_FOR_LOAD
|
||||
and self.needs_retrieve()
|
||||
@@ -301,13 +300,13 @@ class LMCacheMPRequestMetadata:
|
||||
blocks_in_chunk: int,
|
||||
vllm_block_size: int,
|
||||
) -> "LMCacheMPRequestMetadata | None":
|
||||
"""
|
||||
Generate the store metadata for the current request tracker.
|
||||
"""Generate the store metadata for the current request tracker.
|
||||
|
||||
Args:
|
||||
tracker: The request tracker to generate the metadata from.
|
||||
blocks_in_chunk: the number of blocks in a LMCache data chunk
|
||||
vllm_block_size: the block size used in vLLM
|
||||
|
||||
"""
|
||||
# Store the blocks that has block hashes
|
||||
# NOTE: the invariant here is that `num_stored_blocks` should
|
||||
@@ -375,13 +374,13 @@ class LMCacheMPRequestMetadata:
|
||||
blocks_in_chunk: int,
|
||||
vllm_block_size: int,
|
||||
) -> "LMCacheMPRequestMetadata | None":
|
||||
"""
|
||||
Generate the retrieve metadata for the current request tracker.
|
||||
"""Generate the retrieve metadata for the current request tracker.
|
||||
|
||||
Args:
|
||||
tracker: The request tracker to generate the metadata from.
|
||||
blocks_in_chunk: the number of blocks in a LMCache data chunk
|
||||
vllm_block_size: the block size used in vLLM
|
||||
|
||||
"""
|
||||
if not tracker.is_ready_for_retrieving():
|
||||
return None
|
||||
@@ -462,8 +461,7 @@ class LMCacheMPConnectorMetadata(KVConnectorMetadata):
|
||||
|
||||
|
||||
class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
"""
|
||||
The connector for LMCache multi-process mode.
|
||||
"""The connector for LMCache multi-process mode.
|
||||
|
||||
Extra configs (kv_transfer_config.extra_config):
|
||||
- lmcache.mp.host: the host of the LMCache server.
|
||||
@@ -538,27 +536,26 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
|
||||
Returns:
|
||||
ConnectorMetadata: the connector metadata.
|
||||
"""
|
||||
|
||||
"""
|
||||
# Should only be called while set to valid metadata.
|
||||
assert self._connector_metadata is not None
|
||||
return self._connector_metadata
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""
|
||||
Initialize with the KV caches. Useful for pre-registering the
|
||||
"""Initialize with the KV caches. Useful for pre-registering the
|
||||
KV Caches in the KVConnector (e.g. for NIXL).
|
||||
|
||||
Args:
|
||||
kv_caches: dictionary of layer names, kv cache
|
||||
|
||||
"""
|
||||
logger.info("Registering kv caches!")
|
||||
self.worker_adapter.register_kv_caches(kv_caches)
|
||||
return
|
||||
|
||||
def start_load_kv(self, forward_context: "ForwardContext", **kwargs: Any) -> None:
|
||||
"""
|
||||
Start loading the KV cache from the connector to vLLM's paged
|
||||
"""Start loading the KV cache from the connector to vLLM's paged
|
||||
KV buffer. This is called from the forward context before the
|
||||
forward pass to enable async loading during model execution.
|
||||
|
||||
@@ -597,8 +594,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
"""
|
||||
Block until the KV for a specific layer is loaded into vLLM's
|
||||
"""Block until the KV for a specific layer is loaded into vLLM's
|
||||
paged buffer. This is called from within attention layer to ensure
|
||||
async copying from start_load_kv is complete.
|
||||
|
||||
@@ -606,6 +602,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
layer_name: the name of that layer
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
@@ -616,8 +613,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Start saving a layer of KV cache from vLLM's paged buffer
|
||||
"""Start saving a layer of KV cache from vLLM's paged buffer
|
||||
to the connector. This is called from within attention layer to
|
||||
enable async copying during execution.
|
||||
|
||||
@@ -627,12 +623,12 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
**kwargs: additional arguments for the save operation.
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
def wait_for_save(self):
|
||||
"""
|
||||
Block until all the save operations is done. This is called
|
||||
"""Block until all the save operations is done. This is called
|
||||
as the forward context exits to ensure that the async saving
|
||||
from save_kv_layer is complete before finishing the forward.
|
||||
|
||||
@@ -673,8 +669,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""
|
||||
Notifies worker-side connector ids of requests that have
|
||||
"""Notifies worker-side connector ids of requests that have
|
||||
finished generating tokens on the worker.
|
||||
The scheduler process (via the Executors) will use this output
|
||||
to track which workers are done.
|
||||
@@ -685,14 +680,14 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
tuple of (sending/saving ids, recving/loading ids).
|
||||
The finished saves/sends req ids must belong to a set provided in a
|
||||
call to this method (this call or a prior one).
|
||||
|
||||
"""
|
||||
val = self.worker_adapter.get_finished(finished_req_ids)
|
||||
# logger.error("Finished req ids: %s, %s", val[0], val[1])
|
||||
return val
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""
|
||||
Get the set of block IDs that failed to load.
|
||||
"""Get the set of block IDs that failed to load.
|
||||
|
||||
Returns:
|
||||
Set of block IDs that encountered load errors.
|
||||
@@ -707,12 +702,12 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
block IDs must appear here no later than that same pass.
|
||||
- Sync loading: failed blocks should be reported in the forward
|
||||
pass in which they are detected.
|
||||
|
||||
"""
|
||||
return self.worker_adapter.get_block_ids_with_load_errors()
|
||||
|
||||
def shutdown(self):
|
||||
"""
|
||||
Shutdown the connector. This is called when the worker process
|
||||
"""Shutdown the connector. This is called when the worker process
|
||||
is shutting down to ensure that all the async operations are
|
||||
completed and the connector is cleaned up properly.
|
||||
"""
|
||||
@@ -721,9 +716,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return None
|
||||
|
||||
def get_kv_connector_stats(self) -> "KVConnectorStats | None":
|
||||
"""
|
||||
Get the KV connector stats collected during the last interval.
|
||||
"""
|
||||
"""Get the KV connector stats collected during the last interval."""
|
||||
return None
|
||||
|
||||
# ==============================
|
||||
@@ -735,8 +728,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""
|
||||
Get number of new tokens that can be loaded from the
|
||||
"""Get number of new tokens that can be loaded from the
|
||||
external KV cache beyond the num_computed_tokens.
|
||||
|
||||
Args:
|
||||
@@ -761,6 +753,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
call. If the cache cannot be loaded for some tokens (e.g., due to
|
||||
connectivity issues or eviction), those tokens must not be taken
|
||||
into account.
|
||||
|
||||
"""
|
||||
tracker = self._get_or_create_request_tracker(request)
|
||||
# TODO: support loading KV for preempted requests in the future
|
||||
@@ -803,8 +796,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
"""
|
||||
Update KVConnector state after block allocation.
|
||||
"""Update KVConnector state after block allocation.
|
||||
|
||||
If get_num_new_matched_tokens previously returned True for a
|
||||
request, this function may be called twice for that same request -
|
||||
@@ -817,6 +809,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
blocks (KVCacheBlocks): the blocks allocated for the request.
|
||||
num_external_tokens (int): the number of tokens that will be
|
||||
loaded from the external KV cache.
|
||||
|
||||
"""
|
||||
# NOTE: `blocks` comes from kv_cache_manager.get_blocks(request_id),
|
||||
# which returns ALL blocks for the request (not just newly allocated).
|
||||
@@ -883,14 +876,14 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
"""
|
||||
Build the connector metadata for this step.
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
|
||||
"""
|
||||
metadata = LMCacheMPConnectorMetadata()
|
||||
|
||||
@@ -907,12 +900,12 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return metadata
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""
|
||||
Update KVConnector state from worker-side connectors output.
|
||||
"""Update KVConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
return
|
||||
|
||||
@@ -921,8 +914,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
request: "Request",
|
||||
block_ids: list[int],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called exactly once when a request has finished, before its blocks are
|
||||
"""Called exactly once when a request has finished, before its blocks are
|
||||
freed.
|
||||
|
||||
The connector may assumes responsibility for freeing the blocks
|
||||
@@ -934,8 +926,8 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
get_finished().
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
"""
|
||||
|
||||
"""
|
||||
params: dict[str, Any] | None = getattr(request, "kv_transfer_params", None)
|
||||
return_params: dict[str, Any] | None = {} if params is not None else None
|
||||
|
||||
@@ -962,26 +954,26 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return True, return_params
|
||||
|
||||
def take_events(self) -> Iterable["KVCacheEvent"]:
|
||||
"""
|
||||
Take the KV cache events from the connector.
|
||||
"""Take the KV cache events from the connector.
|
||||
|
||||
Yields:
|
||||
New KV cache events since the last call.
|
||||
|
||||
"""
|
||||
return ()
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
|
||||
"""
|
||||
Get the required KV cache layout for this connector.
|
||||
"""Get the required KV cache layout for this connector.
|
||||
|
||||
Args:
|
||||
vllm_config (VllmConfig): the vllm config.
|
||||
|
||||
Returns:
|
||||
str: the required KV cache layout. e.g. HND, or NHD.
|
||||
None if the connector does not require a specific layout.
|
||||
"""
|
||||
|
||||
"""
|
||||
if cls is KVConnectorBase_V1:
|
||||
raise TypeError(
|
||||
"get_required_kvcache_layout should not be called "
|
||||
@@ -990,13 +982,13 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return None
|
||||
|
||||
def get_finished_count(self) -> int | None:
|
||||
"""
|
||||
Get the count of requests expected to complete send/receive operations
|
||||
"""Get the count of requests expected to complete send/receive operations
|
||||
via this connector. This method is used to initialize the
|
||||
KVOutputAggregator, overwriting the default world_size.
|
||||
|
||||
Returns:
|
||||
int: expected sending or receiving completion count.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -1004,8 +996,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
def build_kv_connector_stats(
|
||||
cls, data: dict[str, Any] | None = None
|
||||
) -> "KVConnectorStats | None":
|
||||
"""
|
||||
KVConnectorStats resolution method. This method allows dynamically
|
||||
"""KVConnectorStats resolution method. This method allows dynamically
|
||||
registered connectors to return their own KVConnectorStats object,
|
||||
which can implement custom aggregation logic on the data dict.
|
||||
"""
|
||||
@@ -1019,8 +1010,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
labelnames: list[str],
|
||||
per_engine_labelvalues: dict[int, list[object]],
|
||||
) -> "KVConnectorPromMetrics | None":
|
||||
"""
|
||||
Create a KVConnectorPromMetrics subclass which should register
|
||||
"""Create a KVConnectorPromMetrics subclass which should register
|
||||
per-connector Prometheus metrics and implement observe() to
|
||||
expose connector transfer stats via Prometheus.
|
||||
"""
|
||||
@@ -1183,8 +1173,7 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
return self.request_trackers[request_id]
|
||||
|
||||
def _cleanup_request_tracker(self, request_id: str) -> None:
|
||||
"""
|
||||
Clean up request tracker and associated lookup future for a request.
|
||||
"""Clean up request tracker and associated lookup future for a request.
|
||||
This should be called when a request is finished to prevent memory leak.
|
||||
"""
|
||||
# Clean up request tracker
|
||||
|
||||
@@ -17,8 +17,7 @@ logger = init_logger(__name__)
|
||||
|
||||
@dataclass
|
||||
class KVConnectorStats:
|
||||
"""
|
||||
Base class for KV Connector Stats, a container for transfer performance
|
||||
"""Base class for KV Connector Stats, a container for transfer performance
|
||||
metrics or otherwise important telemetry from the connector.
|
||||
All sub-classes need to be serializable as stats are sent from worker to
|
||||
logger process.
|
||||
@@ -35,14 +34,11 @@ class KVConnectorStats:
|
||||
raise NotImplementedError
|
||||
|
||||
def aggregate(self, other: "KVConnectorStats") -> "KVConnectorStats":
|
||||
"""
|
||||
Aggregate stats with another `KVConnectorStats` object.
|
||||
"""
|
||||
"""Aggregate stats with another `KVConnectorStats` object."""
|
||||
raise NotImplementedError
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
"""
|
||||
Reduce the observations collected during a time interval to one or
|
||||
"""Reduce the observations collected during a time interval to one or
|
||||
more representative values (eg avg/median/sum of the series).
|
||||
This is meant to be called by the logger to produce a summary of the
|
||||
stats for the last time interval.
|
||||
@@ -95,7 +91,7 @@ class KVConnectorLogging:
|
||||
)
|
||||
|
||||
def log(self, log_fn=logger.info):
|
||||
"""Log transfer metrics periodically, similar to throughput logging"""
|
||||
"""Log transfer metrics periodically, similar to throughput logging."""
|
||||
if (
|
||||
self.transfer_stats_accumulator
|
||||
and not self.transfer_stats_accumulator.is_empty()
|
||||
@@ -111,8 +107,7 @@ class KVConnectorLogging:
|
||||
|
||||
|
||||
class KVConnectorPromMetrics:
|
||||
"""
|
||||
A base class for per-connector Prometheus metric registration
|
||||
"""A base class for per-connector Prometheus metric registration
|
||||
and recording.
|
||||
"""
|
||||
|
||||
@@ -131,8 +126,7 @@ class KVConnectorPromMetrics:
|
||||
self.per_engine_labelvalues = per_engine_labelvalues
|
||||
|
||||
def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0):
|
||||
"""
|
||||
Record the supplied transfer statistics to Prometheus metrics. These
|
||||
"""Record the supplied transfer statistics to Prometheus metrics. These
|
||||
statistics are engine-specific, and should be recorded to a metric
|
||||
with the appropriate 'engine' label. These metric instances can be
|
||||
created using the create_metric_per_engine() helper method.
|
||||
@@ -141,8 +135,7 @@ class KVConnectorPromMetrics:
|
||||
|
||||
|
||||
class KVConnectorProm:
|
||||
"""
|
||||
Support for registering per-connector Prometheus metrics, and
|
||||
"""Support for registering per-connector Prometheus metrics, and
|
||||
recording transfer statistics to those metrics. Uses
|
||||
KVConnectorBase.build_prom_metrics().
|
||||
"""
|
||||
|
||||
@@ -643,7 +643,7 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
|
||||
class MooncakeConnectorScheduler:
|
||||
"""Implementation of Scheduler side methods"""
|
||||
"""Implementation of Scheduler side methods."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -718,7 +718,8 @@ class MooncakeConnectorScheduler:
|
||||
|
||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||
"""D-side only. Returns N-1 for Mamba models since the decoder
|
||||
always recomputes the last token and must start from h(N-1)."""
|
||||
always recomputes the last token and must start from h(N-1).
|
||||
"""
|
||||
if self._has_mamba and num_prompt_tokens > 1:
|
||||
return num_prompt_tokens - 1
|
||||
return num_prompt_tokens
|
||||
@@ -729,7 +730,8 @@ class MooncakeConnectorScheduler:
|
||||
derive h(N) correctly.
|
||||
|
||||
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
|
||||
request is preempted and rescheduled."""
|
||||
request is preempted and rescheduled.
|
||||
"""
|
||||
params = request.kv_transfer_params
|
||||
if (
|
||||
params is not None
|
||||
@@ -756,8 +758,7 @@ class MooncakeConnectorScheduler:
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int, bool]:
|
||||
"""
|
||||
For remote prefill, pull all prompt blocks from remote
|
||||
"""For remote prefill, pull all prompt blocks from remote
|
||||
asynchronously relative to engine execution.
|
||||
|
||||
Args:
|
||||
@@ -769,8 +770,8 @@ class MooncakeConnectorScheduler:
|
||||
external KV cache beyond what is already computed.
|
||||
* true if the external KV cache tokens will be loaded
|
||||
asynchronously (between scheduler steps).
|
||||
"""
|
||||
|
||||
"""
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"MooncakeConnector get_num_new_matched_tokens: "
|
||||
@@ -881,11 +882,9 @@ class MooncakeConnectorScheduler:
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Once a request is finished, determine whether request blocks
|
||||
"""Once a request is finished, determine whether request blocks
|
||||
should be freed now or will be sent asynchronously and freed later.
|
||||
"""
|
||||
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"MooncakeConnector request_finished, req_id=%s, request_status=%s, "
|
||||
@@ -934,7 +933,7 @@ class MooncakeConnectorScheduler:
|
||||
|
||||
|
||||
class MooncakeConnectorWorker:
|
||||
"""Implementation of Worker side methods"""
|
||||
"""Implementation of Worker side methods."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1176,11 +1175,9 @@ class MooncakeConnectorWorker:
|
||||
raise e
|
||||
|
||||
async def _mooncake_sender_listener(self, ready_event: threading.Event):
|
||||
"""
|
||||
Background thread that listens for Mooncake requests, dispatches them
|
||||
"""Background thread that listens for Mooncake requests, dispatches them
|
||||
to a thread pool, and sends acknowledgments upon completion.
|
||||
"""
|
||||
|
||||
sock = self.async_zmq_ctx.socket(zmq.ROUTER)
|
||||
self.side_channel_port = sock.bind_to_random_port(f"tcp://{self.hostname}")
|
||||
logger.debug(
|
||||
@@ -1680,7 +1677,8 @@ class MooncakeConnectorWorker:
|
||||
def _bind_sender_thread_device(self) -> None:
|
||||
"""ThreadPoolExecutor initializer — binds each pool thread to the
|
||||
correct CUDA device. CUDA device selection is thread-local, so
|
||||
without this, NVLink transfers fail for TP ranks > 0."""
|
||||
without this, NVLink transfers fail for TP ranks > 0.
|
||||
"""
|
||||
current_platform.set_device(self.device_id)
|
||||
|
||||
def _send_blocks(
|
||||
@@ -1716,7 +1714,6 @@ class MooncakeConnectorWorker:
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Register the KV Cache data in mooncake."""
|
||||
|
||||
logger.info("Registering KV_Caches. use_mla: %s", self.use_mla)
|
||||
|
||||
kv_data_ptrs: list[int] = []
|
||||
@@ -1853,8 +1850,7 @@ class MooncakeConnectorWorker:
|
||||
return finished_sending_reqs
|
||||
|
||||
def get_finished(self) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""
|
||||
Get requests that are done sending or recving on this specific worker.
|
||||
"""Get requests that are done sending or recving on this specific worker.
|
||||
The scheduler process (via the MultiprocExecutor) will use this output
|
||||
to track which workers are done.
|
||||
"""
|
||||
@@ -1885,8 +1881,7 @@ class MooncakeConnectorWorker:
|
||||
return finished_sending_reqs or None, finished_recving_reqs or None
|
||||
|
||||
def get_transfer_results(self) -> KVConnectorTransferResults:
|
||||
"""
|
||||
Get transfers that completed on this specific worker, including
|
||||
"""Get transfers that completed on this specific worker, including
|
||||
requests whose remote KV load failed.
|
||||
|
||||
The scheduler process (via the MultiprocExecutor) will use this output
|
||||
@@ -1909,7 +1904,8 @@ class MooncakeConnectorWorker:
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""Return transfer stats collected since the last call, or None
|
||||
if nothing has been recorded in this interval."""
|
||||
if nothing has been recorded in this interval.
|
||||
"""
|
||||
if self.xfer_stats.is_empty():
|
||||
return None
|
||||
return self.xfer_stats.clone_and_reset()
|
||||
@@ -2321,8 +2317,7 @@ def should_launch_bootstrap_server(vllm_config: VllmConfig) -> bool:
|
||||
|
||||
|
||||
def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]:
|
||||
"""
|
||||
Returns the address of the Mooncake bootstrap server.
|
||||
"""Returns the address of the Mooncake bootstrap server.
|
||||
This is only used by prefillers to register workers.
|
||||
Decoders should get addr from kv_transfer_params.
|
||||
"""
|
||||
|
||||
@@ -32,8 +32,7 @@ class EngineEntry:
|
||||
|
||||
|
||||
class MooncakeBootstrapServer:
|
||||
"""
|
||||
A centralized server running on the global rank 0 prefiller worker.
|
||||
"""A centralized server running on the global rank 0 prefiller worker.
|
||||
Prefiller workers register their connection info (IP, port, ranks) here.
|
||||
"""
|
||||
|
||||
|
||||
@@ -73,7 +73,8 @@ class ExternalCachedBlockPool:
|
||||
|
||||
class MooncakeStoreCoordinator:
|
||||
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
|
||||
``ExternalCachedBlockPool``."""
|
||||
``ExternalCachedBlockPool``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -630,6 +630,7 @@ class ChunkedTokenDatabase:
|
||||
``mask_num``. False entries are skipped before hash access.
|
||||
put_step: Stride for distributing chunks across ranks.
|
||||
put_step_rank: ``chunk_id % put_step`` value this rank stores.
|
||||
|
||||
"""
|
||||
assert put_step > 0
|
||||
if not block_hashes:
|
||||
@@ -662,6 +663,7 @@ class TailKeyBoundary:
|
||||
``block_hashes[num_tokens // hash_block_size - 1]`` instead of the
|
||||
hash implied by ``MooncakeLookupResult.hit_length``. This changes
|
||||
only the load key, not the reusable prefix.
|
||||
|
||||
"""
|
||||
|
||||
group_id: int
|
||||
@@ -678,6 +680,7 @@ class MooncakeLookupResult:
|
||||
tail_key_boundaries: Hash boundary used to store each cache group's
|
||||
tail block when ``hit_length`` does not identify its store key.
|
||||
There is one entry per group for every nonzero hit.
|
||||
|
||||
"""
|
||||
|
||||
hit_length: int
|
||||
|
||||
@@ -815,6 +815,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
|
||||
Returns:
|
||||
True when no put is needed or every put succeeds, False otherwise.
|
||||
|
||||
"""
|
||||
offloads = req_meta.boundary_state_offloads
|
||||
if not offloads or not req_meta.block_hashes:
|
||||
@@ -2628,7 +2629,8 @@ class LookupKeyClient:
|
||||
non_block: bool = False,
|
||||
) -> MooncakeLookupResult | None:
|
||||
"""If non_block is True, will return None until the result is ready,
|
||||
so the caller retries on a later step."""
|
||||
so the caller retries on a later step.
|
||||
"""
|
||||
future = self.futures.get(req_id)
|
||||
if future is None:
|
||||
future = self.executor.submit(self._lookup, num_tokens, list(block_hashes))
|
||||
|
||||
@@ -588,8 +588,7 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata):
|
||||
|
||||
@contextlib.contextmanager
|
||||
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
|
||||
"""Context manager for a ZMQ socket"""
|
||||
|
||||
"""Context manager for a ZMQ socket."""
|
||||
if socket_type not in (zmq.ROUTER, zmq.REQ, zmq.DEALER):
|
||||
raise ValueError(f"Unexpected socket type: {socket_type}")
|
||||
|
||||
|
||||
@@ -373,6 +373,7 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
Returns:
|
||||
bool: True if connector metadata exists, False otherwise.
|
||||
|
||||
"""
|
||||
try:
|
||||
return self._connector_metadata is not None
|
||||
@@ -381,7 +382,7 @@ class MoRIIOConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
|
||||
class MoRIIOConnectorScheduler:
|
||||
"""Implementation of Scheduler side methods"""
|
||||
"""Implementation of Scheduler side methods."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -547,7 +548,8 @@ class MoRIIOConnectorScheduler:
|
||||
self, request_id: ReqId, transfer_id: TransferId | None = None
|
||||
):
|
||||
"""Unmap request_id/transfer_id. Uses transfer_id for lookup if
|
||||
exact request_id match fails (handles input_processor mutation)."""
|
||||
exact request_id match fails (handles input_processor mutation).
|
||||
"""
|
||||
if request_id in self.request_id_to_transfer_id:
|
||||
tid = self.request_id_to_transfer_id[request_id]
|
||||
del self.request_id_to_transfer_id[request_id]
|
||||
@@ -579,8 +581,7 @@ class MoRIIOConnectorScheduler:
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int, bool]:
|
||||
"""
|
||||
For remote prefill, pull all prompt blocks from remote
|
||||
"""For remote prefill, pull all prompt blocks from remote
|
||||
asynchronously relative to engine execution.
|
||||
|
||||
Args:
|
||||
@@ -592,6 +593,7 @@ class MoRIIOConnectorScheduler:
|
||||
external KV cache beyond what is already computed.
|
||||
* true if the external KV cache tokens will be loaded
|
||||
asynchronously (between scheduler steps).
|
||||
|
||||
"""
|
||||
if self.is_producer:
|
||||
return 0, False
|
||||
@@ -1007,11 +1009,9 @@ class MoRIIOConnectorScheduler:
|
||||
request: "Request",
|
||||
block_ids: BlockIds,
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Once a request is finished, determine whether request blocks
|
||||
"""Once a request is finished, determine whether request blocks
|
||||
should be freed now or will be sent asynchronously and freed later.
|
||||
"""
|
||||
|
||||
request_id = request.request_id
|
||||
params = request.kv_transfer_params
|
||||
# Consumer: can unmap transfer_id<->request_id immediately since done_recving
|
||||
@@ -1177,7 +1177,7 @@ class MoRIIOConnectorScheduler:
|
||||
|
||||
|
||||
class MoRIIOConnectorWorker:
|
||||
"""Implementation of Worker side methods"""
|
||||
"""Implementation of Worker side methods."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1435,8 +1435,8 @@ class MoRIIOConnectorWorker:
|
||||
kv_layer: KV cache tensor
|
||||
remote_notify_port: Port for completion notification
|
||||
remote_ip: IP address of remote node
|
||||
"""
|
||||
|
||||
"""
|
||||
# synchronization to prevent dirty reads between
|
||||
# transfer and attention operations
|
||||
# we can consider removing this synchronization after ibgda is enabled.
|
||||
@@ -1581,7 +1581,6 @@ class MoRIIOConnectorWorker:
|
||||
layer_name_to_local_kv_cache_metadata: dict,
|
||||
):
|
||||
"""Background thread for getting new MoRIIO handshakes."""
|
||||
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
encoded_data = encoder.encode(metadata)
|
||||
size_in_bytes = len(encoded_data)
|
||||
@@ -1637,7 +1636,6 @@ class MoRIIOConnectorWorker:
|
||||
local-rank mapping _remote_tp_rank -- byte-identical for callers not yet
|
||||
TP-aware.
|
||||
"""
|
||||
|
||||
start_time = time.perf_counter()
|
||||
|
||||
# NOTE(rob): we need each rank to have a unique port. This is
|
||||
@@ -1856,7 +1854,6 @@ class MoRIIOConnectorWorker:
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Register the KV Cache data in moriio."""
|
||||
|
||||
self.kv_caches = kv_caches # layer name to kv cache
|
||||
self.kv_cache_shapes = {
|
||||
layer_name: kv_cache.shape for layer_name, kv_cache in kv_caches.items()
|
||||
@@ -2020,12 +2017,10 @@ class MoRIIOConnectorWorker:
|
||||
self.moriio_wrapper.async_wait_reqid()
|
||||
|
||||
def get_finished(self) -> tuple[set[str], set[str]]:
|
||||
"""
|
||||
Get requests that are done sending or recving on this specific worker.
|
||||
"""Get requests that are done sending or recving on this specific worker.
|
||||
The scheduler process (via the MultiprocExecutor) will use this output
|
||||
to track which workers are done.
|
||||
"""
|
||||
|
||||
done_sending, done_recving = set(), set()
|
||||
|
||||
if self.is_producer:
|
||||
@@ -2416,8 +2411,7 @@ class MoRIIOConnectorWorker:
|
||||
self._eager_handshaked_engines.add(remote_engine_id)
|
||||
|
||||
def start_load_kv(self, metadata: MoRIIOConnectorMetadata):
|
||||
"""
|
||||
Start loading by triggering non-blocking moriio_xfer.
|
||||
"""Start loading by triggering non-blocking moriio_xfer.
|
||||
We check for these trnxs to complete in each step().
|
||||
"""
|
||||
self.transfer_id_to_request_id = metadata.transfer_id_to_request_id
|
||||
@@ -2686,8 +2680,13 @@ class MoRIIOConnectorWorker:
|
||||
local_block_ids: IDs of local blocks
|
||||
remote_block_ids: IDs of remote blocks
|
||||
remote_moriio_meta: Metadata of the remote MoRIIO agent
|
||||
remote_tp_size: Tensor parallel size of the remote engine. Falls
|
||||
back to the locally recorded value when None.
|
||||
remote_engine_id: Id of the remote engine the blocks live on.
|
||||
|
||||
Returns:
|
||||
Tuple of (local_offsets, remote_offsets, transfer_sizes)
|
||||
|
||||
"""
|
||||
validate_moriio_heterogeneous_tp_kv_heads(
|
||||
local_tp_size=self.world_size,
|
||||
|
||||
@@ -87,6 +87,7 @@ class MoRIIOWriter:
|
||||
|
||||
Args:
|
||||
worker: Reference to the parent worker
|
||||
|
||||
"""
|
||||
self._worker_ref: weakref_ref[MoRIIOConnectorWorker] = weakref_ref(worker)
|
||||
self._write_task_q: Queue[WriteTask] = Queue()
|
||||
@@ -108,6 +109,7 @@ class MoRIIOWriter:
|
||||
|
||||
Raises:
|
||||
RuntimeError: If worker has been garbage collected
|
||||
|
||||
"""
|
||||
worker = self._worker_ref()
|
||||
if worker is None:
|
||||
@@ -131,6 +133,7 @@ class MoRIIOWriter:
|
||||
|
||||
Args:
|
||||
task: The write task to schedule
|
||||
|
||||
"""
|
||||
self.ensure_worker_started()
|
||||
if self._is_transfer_terminal(task.transfer_id):
|
||||
@@ -178,7 +181,6 @@ class MoRIIOWriter:
|
||||
|
||||
def _write_worker_loop(self) -> None:
|
||||
"""Main loop for the write worker thread."""
|
||||
|
||||
while True:
|
||||
# Process deferred tasks first
|
||||
self._process_deferred_tasks()
|
||||
@@ -275,6 +277,7 @@ class MoRIIOWriter:
|
||||
|
||||
Returns:
|
||||
True if remote blocks are ready
|
||||
|
||||
"""
|
||||
return (
|
||||
task.transfer_id in self.worker.moriio_wrapper.done_remote_allocate_req_dict
|
||||
@@ -291,6 +294,7 @@ class MoRIIOWriter:
|
||||
|
||||
Raises:
|
||||
KeyError: If allocation info is missing
|
||||
|
||||
"""
|
||||
try:
|
||||
return self.worker.moriio_wrapper.done_remote_allocate_req_dict[transfer_id]
|
||||
@@ -372,9 +376,11 @@ class MoRIIOWriter:
|
||||
Args:
|
||||
task: The write task
|
||||
request_info: Remote allocation information
|
||||
remote_moriio_meta: Metadata of the remote MoRIIO agent
|
||||
|
||||
Returns:
|
||||
The transfer plan
|
||||
|
||||
"""
|
||||
layer_cache = self.worker.kv_caches[task.layer_name]
|
||||
key = (task.layer_name, *_get_write_geometry_key(layer_cache))
|
||||
@@ -412,6 +418,7 @@ class MoRIIOWriter:
|
||||
Args:
|
||||
plan: The transfer plan
|
||||
sessions: List of transfer sessions
|
||||
|
||||
"""
|
||||
if plan.use_batch:
|
||||
return [
|
||||
@@ -510,6 +517,7 @@ class MoRIIOWrapper:
|
||||
moriio_engine: MoRIIO engine instance
|
||||
tp_rank: Tensor parallel rank
|
||||
dp_rank: Data parallel rank
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -71,8 +71,7 @@ class MultiKVConnectorWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
|
||||
@dataclass
|
||||
class MultiKVConnectorStats(KVConnectorStats):
|
||||
"""
|
||||
Maintain a dict of KVConnectorStats objects, one for each connector.
|
||||
"""Maintain a dict of KVConnectorStats objects, one for each connector.
|
||||
This is used to aggregate the stats from all connectors separately.
|
||||
"""
|
||||
|
||||
@@ -132,8 +131,7 @@ class MultiKVConnectorPromMetrics(KVConnectorPromMetrics):
|
||||
|
||||
|
||||
class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""
|
||||
A wrapper for using multiple KVConnectors at the same time.
|
||||
"""A wrapper for using multiple KVConnectors at the same time.
|
||||
|
||||
The current logic is:
|
||||
- Load KV from the first connector that advertises available tokens from
|
||||
@@ -143,8 +141,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
@classmethod
|
||||
def requires_piecewise_for_cudagraph(cls, extra_config: dict[str, Any]) -> bool:
|
||||
"""
|
||||
MultiConnector requires PIECEWISE CUDA graph mode if any of its
|
||||
"""MultiConnector requires PIECEWISE CUDA graph mode if any of its
|
||||
child connectors require it.
|
||||
"""
|
||||
connectors_config = extra_config.get("connectors", [])
|
||||
@@ -480,8 +477,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
connector_output.kv_connector_worker_meta = multi_connector_worker_meta
|
||||
|
||||
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
|
||||
"""
|
||||
Get the KVConnector handshake metadata from sub-connectors.
|
||||
"""Get the KVConnector handshake metadata from sub-connectors.
|
||||
Returns the first non-None metadata from sub-connectors.
|
||||
"""
|
||||
for c in self._connectors:
|
||||
@@ -493,8 +489,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def set_xfer_handshake_metadata(
|
||||
self, metadata: dict[int, KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""
|
||||
Set the KV connector handshake metadata for all sub-connectors.
|
||||
"""Set the KV connector handshake metadata for all sub-connectors.
|
||||
This is needed to start the NIXL listener thread for NixlConnector.
|
||||
"""
|
||||
for c in self._connectors:
|
||||
@@ -590,14 +585,15 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
|
||||
"""
|
||||
Get the required KV cache layout for this connector.
|
||||
"""Get the required KV cache layout for this connector.
|
||||
|
||||
Args:
|
||||
vllm_config (VllmConfig): the vllm config.
|
||||
|
||||
Returns:
|
||||
str: the required KV cache layout. e.g. HND, or NHD.
|
||||
None if the connector does not require a specific layout.
|
||||
|
||||
"""
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
layouts: set[str] = set()
|
||||
|
||||
@@ -299,11 +299,11 @@ class NixlBaseConnectorScheduler:
|
||||
def set_xfer_handshake_metadata(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""
|
||||
Set the KV connector handshake metadata for this connector.
|
||||
"""Set the KV connector handshake metadata for this connector.
|
||||
|
||||
Args:
|
||||
metadata (dict): the handshake metadata to set.
|
||||
|
||||
"""
|
||||
encoded_data: dict[tuple[int, int], bytes] = {}
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
@@ -400,7 +400,8 @@ class NixlBaseConnectorScheduler:
|
||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||
"""D-side only. The number of prompt tokens to load from the prefiller.
|
||||
Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder
|
||||
will recompute locally."""
|
||||
will recompute locally.
|
||||
"""
|
||||
backoff = self._prefill_backoff()
|
||||
if backoff and num_prompt_tokens > backoff:
|
||||
return num_prompt_tokens - backoff
|
||||
@@ -413,7 +414,8 @@ class NixlBaseConnectorScheduler:
|
||||
multi-module MTP it is the drafter's whole lookahead window.
|
||||
|
||||
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
|
||||
request is preempted and rescheduled."""
|
||||
request is preempted and rescheduled.
|
||||
"""
|
||||
backoff = self._prefill_backoff()
|
||||
params = request.kv_transfer_params
|
||||
if (
|
||||
|
||||
@@ -1147,8 +1147,7 @@ class NixlBaseConnectorWorker:
|
||||
return self.nixl_wrapper.add_remote_agent(metadata.agent_metadata)
|
||||
|
||||
def initialize_host_xfer_buffer(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
"""
|
||||
Initialize transfer buffer in CPU mem for accelerators
|
||||
"""Initialize transfer buffer in CPU mem for accelerators
|
||||
NOT directly supported by NIXL (e.g., tpu)
|
||||
"""
|
||||
xfer_buffers: dict[str, torch.Tensor] = {}
|
||||
@@ -1262,8 +1261,7 @@ class NixlBaseConnectorWorker:
|
||||
pp_size: int = 1,
|
||||
notif_agents_only: bool = False,
|
||||
) -> Future[tuple[dict[tuple[int, int], str], float]] | None:
|
||||
"""
|
||||
Ensure a handshake is in-flight (or already done) for *engine_id*.
|
||||
"""Ensure a handshake is in-flight (or already done) for *engine_id*.
|
||||
|
||||
Returns the ``Future`` if a handshake is pending (or was just
|
||||
started), or ``None`` if the handshake already completed
|
||||
@@ -1348,7 +1346,6 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Register the KV Cache data in nixl."""
|
||||
|
||||
self.transfer_topo = TransferTopology(
|
||||
tp_rank=self.transfer_tp_rank,
|
||||
tp_size=self.transfer_tp_size,
|
||||
@@ -1876,7 +1873,8 @@ class NixlBaseConnectorWorker:
|
||||
) -> np.ndarray:
|
||||
"""Build remote desc regions (conv sub-projections + ssm) per layer.
|
||||
For hetero-TP, each D rank reads only its sub-projection slice from
|
||||
the P rank. Returns an Nx3 uint64 array."""
|
||||
the P rank. Returns an Nx3 uint64 array.
|
||||
"""
|
||||
assert nixl_agent_meta.kv_caches_base_addr, (
|
||||
"Remote KV cache base addresses must not be empty."
|
||||
)
|
||||
@@ -2034,8 +2032,7 @@ class NixlBaseConnectorWorker:
|
||||
self,
|
||||
block_size: int,
|
||||
) -> tuple[int, np.ndarray]:
|
||||
"""
|
||||
Function used for register local xfer handler with local block_size or
|
||||
"""Function used for register local xfer handler with local block_size or
|
||||
Remote block_size.
|
||||
|
||||
When local block_size is same as remote block_size, we use local block_size
|
||||
@@ -2124,8 +2121,7 @@ class NixlBaseConnectorWorker:
|
||||
remote_tp_size: int = 1,
|
||||
remote_dcp_size: int = 1,
|
||||
) -> str:
|
||||
"""
|
||||
Add the remote NIXL agent and prepare the descriptors for reading cache
|
||||
"""Add the remote NIXL agent and prepare the descriptors for reading cache
|
||||
blocks from remote.
|
||||
|
||||
In particular, handle both homogeneous and heterogeneous TP. The former
|
||||
@@ -2395,8 +2391,7 @@ class NixlBaseConnectorWorker:
|
||||
remote_tp_size: int,
|
||||
remote_dcp_size: int = 1,
|
||||
):
|
||||
"""
|
||||
Validate the remote agent handshake metadata ensuring the
|
||||
"""Validate the remote agent handshake metadata ensuring the
|
||||
invariants hold true.
|
||||
"""
|
||||
remote_engine_id = nixl_agent_meta.engine_id
|
||||
@@ -2607,7 +2602,7 @@ class NixlBaseConnectorWorker:
|
||||
assert len(nixl_agent_meta.region_mem_types) == num_remote_regions
|
||||
|
||||
def sync_recved_kv_to_device(self, req_id: str, meta: ReqMeta):
|
||||
"""copy recved kv from host buffer to device."""
|
||||
"""Copy recved kv from host buffer to device."""
|
||||
assert self.use_host_buffer
|
||||
assert self.copy_blocks is not None
|
||||
|
||||
@@ -2632,7 +2627,7 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
|
||||
def save_kv_to_host(self, metadata: NixlConnectorMetadata):
|
||||
"""copy kv from device to host buffer."""
|
||||
"""Copy kv from device to host buffer."""
|
||||
assert self.use_host_buffer
|
||||
assert self.copy_blocks is not None
|
||||
|
||||
@@ -2662,7 +2657,8 @@ class NixlBaseConnectorWorker:
|
||||
@cached_property
|
||||
def _attention_kv_caches(self) -> list[torch.Tensor]:
|
||||
"""Device KV caches of attention layers (mamba states excluded),
|
||||
as consumed by the receive post-process."""
|
||||
as consumed by the receive post-process.
|
||||
"""
|
||||
assert self.device_kv_caches, (
|
||||
"_attention_kv_caches accessed before register_kv_caches"
|
||||
)
|
||||
@@ -2681,8 +2677,7 @@ class NixlBaseConnectorWorker:
|
||||
block_ids_list: list[tuple[list[int], int]],
|
||||
convert: bool = True,
|
||||
):
|
||||
"""
|
||||
Post process device kv cache after receiving from remote.
|
||||
"""Post process device kv cache after receiving from remote.
|
||||
|
||||
3 types of conversion supported (``convert``):
|
||||
* kv_cache_postprocess_layout => convert from HND to NHD
|
||||
@@ -2771,8 +2766,7 @@ class NixlBaseConnectorWorker:
|
||||
def post_process_device_kv_on_receive_heterogeneous_attn(
|
||||
self, block_ids: list[int]
|
||||
):
|
||||
"""
|
||||
Post process device kv cache after receiving from remote
|
||||
"""Post process device kv cache after receiving from remote
|
||||
for heterogeneous attention.
|
||||
"""
|
||||
assert self.enable_heterogeneous_attn_post_process
|
||||
@@ -2805,8 +2799,7 @@ class NixlBaseConnectorWorker:
|
||||
pages[block].zero_()
|
||||
|
||||
def get_transfer_results(self) -> KVConnectorTransferResults:
|
||||
"""
|
||||
Get transfers that completed on this specific worker.
|
||||
"""Get transfers that completed on this specific worker.
|
||||
|
||||
The scheduler process (via the MultiprocExecutor) will use this output
|
||||
to track which workers are done.
|
||||
@@ -2984,6 +2977,7 @@ class NixlBaseConnectorWorker:
|
||||
Args:
|
||||
payload: comma-separated P-side request IDs, e.g.
|
||||
"req_abc,req_def".
|
||||
|
||||
"""
|
||||
new_expiry = time.perf_counter() + self._lease_extension
|
||||
for req_id in payload.split(","):
|
||||
@@ -3010,6 +3004,7 @@ class NixlBaseConnectorWorker:
|
||||
Returns:
|
||||
Requests with no outstanding handles and requests with observed
|
||||
failures. Failed requests may still have outstanding handles.
|
||||
|
||||
"""
|
||||
done_req_ids: set[str] = set()
|
||||
failed_req_ids: set[str] = set()
|
||||
@@ -3091,9 +3086,7 @@ class NixlBaseConnectorWorker:
|
||||
self.xfer_stats.record_failed_notification()
|
||||
|
||||
def _send_heartbeats(self, metadata: NixlConnectorMetadata) -> None:
|
||||
"""
|
||||
Send heartbeat notifications to remote engines, extending lease on KV blocks.
|
||||
"""
|
||||
"""Send heartbeats to remote engines, extending the lease on KV blocks."""
|
||||
for engine_id, hb_info in metadata.heartbeat_by_engine.items():
|
||||
# Proactive handshake (this request may still be in waiting queue) so
|
||||
# the **next** heartbeat for this remote can go through.
|
||||
@@ -3126,8 +3119,7 @@ class NixlBaseConnectorWorker:
|
||||
def get_mapped_blocks(
|
||||
self, block_ids: np.ndarray, block_size_ratio: int
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Calculates the new set of block IDs by mapping every element
|
||||
"""Calculates the new set of block IDs by mapping every element
|
||||
in the (potentially sparse) input array.
|
||||
Example: block_ids=[0, 2], block_size_ratio=2
|
||||
get_mapped_blocks 0 1 [2 3] 4 5
|
||||
@@ -3184,8 +3176,7 @@ class NixlBaseConnectorWorker:
|
||||
return mapped_local, mapped_remote
|
||||
|
||||
def _logical_to_kernel_block_ids(self, block_ids: BlockIds, ratio: int) -> BlockIds:
|
||||
"""
|
||||
Convert block ids to kernel physical block ids.
|
||||
"""Convert block ids to kernel physical block ids.
|
||||
This is required when the logical block size (the one set by the user)
|
||||
does not match the one required by the attn backend.
|
||||
`ratio` is the number of physical blocks per logical block.
|
||||
@@ -3239,6 +3230,7 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
An empty result (rank 0 above) takes the notification-only path, allowing the
|
||||
remote to release its blocks without performing a transfer.
|
||||
|
||||
"""
|
||||
local_size, remote_size = local_dcp_size, remote_dcp_size
|
||||
|
||||
@@ -3417,17 +3409,14 @@ class NixlBaseConnectorWorker:
|
||||
return decode_block_ids, prefill_block_ids
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""
|
||||
Get the KV transfer stats for the connector.
|
||||
"""
|
||||
"""Get the KV transfer stats for the connector."""
|
||||
# Clear stats for next iteration
|
||||
if not self.xfer_stats.is_empty():
|
||||
return self.xfer_stats.clone_and_reset()
|
||||
return None
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""
|
||||
Return and clear the set of block IDs that failed to load.
|
||||
"""Return and clear the set of block IDs that failed to load.
|
||||
|
||||
This is called by the scheduler to identify blocks that need
|
||||
to be retried after a NIXL transfer failure.
|
||||
|
||||
@@ -214,12 +214,12 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def set_xfer_handshake_metadata_pp_aware(
|
||||
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
|
||||
) -> None:
|
||||
"""
|
||||
Set handshake metadata keyed by (pp_rank, tp_rank) so the side
|
||||
"""Set handshake metadata keyed by (pp_rank, tp_rank) so the side
|
||||
channel can serve every PP stage's agent metadata.
|
||||
|
||||
Args:
|
||||
metadata (dict): the handshake metadata to set.
|
||||
|
||||
"""
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.set_xfer_handshake_metadata(metadata)
|
||||
@@ -317,14 +317,14 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self.connector_scheduler.shutdown()
|
||||
|
||||
def get_handshake_metadata(self) -> KVConnectorHandshakeMetadata | None:
|
||||
"""
|
||||
Get the KVConnector handshake metadata for this connector.
|
||||
"""Get the KVConnector handshake metadata for this connector.
|
||||
This metadata is used for out-of-band connector handshake
|
||||
between P/D workers.
|
||||
|
||||
Returns:
|
||||
KVConnectorHandshakeMetadata: the handshake metadata.
|
||||
None if no handshake metadata is available.
|
||||
|
||||
"""
|
||||
assert self.connector_worker is not None
|
||||
if (
|
||||
|
||||
@@ -78,8 +78,7 @@ class NixlAgentMetadata:
|
||||
|
||||
@dataclass
|
||||
class NixlHandshakePayload(KVConnectorHandshakeMetadata):
|
||||
"""
|
||||
Wrapper for NIXL handshake sent over the wire.
|
||||
"""Wrapper for NIXL handshake sent over the wire.
|
||||
|
||||
Enables two-phase decoding for graceful compatibility checking:
|
||||
1. Decode NixlHandshakePayload to get compatibility_hash
|
||||
@@ -142,8 +141,7 @@ def compute_nixl_compatibility_hash(
|
||||
attn_backend_name: str,
|
||||
transfer_mode: str = "pull",
|
||||
) -> str:
|
||||
"""
|
||||
Compute compatibility hash for NIXL KV transfer.
|
||||
"""Compute compatibility hash for NIXL KV transfer.
|
||||
|
||||
Hash only the factors that affect whether two NIXL instances can
|
||||
successfully transfer KV cache data.
|
||||
@@ -169,6 +167,7 @@ def compute_nixl_compatibility_hash(
|
||||
|
||||
Returns:
|
||||
SHA-256 hex digest
|
||||
|
||||
"""
|
||||
from vllm import __version__ as vllm_version
|
||||
from vllm.config.utils import hash_factors
|
||||
|
||||
@@ -34,8 +34,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int, bool]:
|
||||
"""
|
||||
For remote prefill, pull all prompt blocks from remote
|
||||
"""For remote prefill, pull all prompt blocks from remote
|
||||
asynchronously relative to engine execution.
|
||||
|
||||
Args:
|
||||
@@ -47,8 +46,8 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
external KV cache beyond what is already computed.
|
||||
* true if the external KV cache tokens will be loaded
|
||||
asynchronously (between scheduler steps).
|
||||
"""
|
||||
|
||||
"""
|
||||
params = request.kv_transfer_params
|
||||
logger.debug(
|
||||
"NIXLConnector get_num_new_matched_tokens: "
|
||||
@@ -194,8 +193,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
request: "Request",
|
||||
block_ids: "BlockIds",
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Once a request is finished, determine whether request blocks
|
||||
"""Once a request is finished, determine whether request blocks
|
||||
should be freed now or will be sent asynchronously and freed later.
|
||||
"""
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
@@ -45,8 +45,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
super().__init__(vllm_config, engine_id, kv_cache_config)
|
||||
|
||||
def start_load_kv(self, metadata: NixlConnectorMetadata):
|
||||
"""
|
||||
Start loading by triggering non-blocking nixl_xfer.
|
||||
"""Start loading by triggering non-blocking nixl_xfer.
|
||||
We check for these trnxs to complete in each step().
|
||||
"""
|
||||
for req_id, meta in metadata.reqs_to_recv.items():
|
||||
@@ -373,8 +372,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
expected_consumers: int,
|
||||
awaiting_kvs: bool,
|
||||
) -> bool:
|
||||
"""
|
||||
Post a READ point-to-point xfer request from a single local worker to
|
||||
"""Post a READ point-to-point xfer request from a single local worker to
|
||||
a single remote worker.
|
||||
|
||||
Returns True when the read was posted (or was unnecessary), False
|
||||
@@ -603,8 +601,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
|
||||
def _get_new_notifs(self) -> set[str]:
|
||||
"""
|
||||
Get req_ids which got a remote xfer message. When multiple consumers
|
||||
"""Get req_ids which got a remote xfer message. When multiple consumers
|
||||
are reading from the same producer (heterogeneous TP or DCP
|
||||
scenario), wait for all consumers to be done pulling.
|
||||
|
||||
|
||||
@@ -315,7 +315,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
executes on the writer; the handshake runs on the background executor
|
||||
and the request is re-queued onto ``_reg_send_inbox`` once it
|
||||
completes (at which point ``_ensure_handshake`` returns ``None`` and we
|
||||
send directly)."""
|
||||
send directly).
|
||||
"""
|
||||
remote_pp_size = reg_data.get("remote_pp_size", 1)
|
||||
fut = self._ensure_handshake(
|
||||
reg_data["remote_engine_id"],
|
||||
@@ -507,7 +508,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
``BlockIds`` is canonically a tuple of per-group lists, but some
|
||||
registration payloads collapse a single-group case to a flat
|
||||
list. Re-wrap that case so downstream group-aware helpers see a
|
||||
consistent shape."""
|
||||
consistent shape.
|
||||
"""
|
||||
if block_ids and not isinstance(block_ids[0], (list, tuple)):
|
||||
return (list(block_ids),)
|
||||
return block_ids
|
||||
|
||||
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
|
||||
|
||||
@dataclass
|
||||
class NixlKVConnectorStats(KVConnectorStats):
|
||||
"""Container for transfer performance metrics"""
|
||||
"""Container for transfer performance metrics."""
|
||||
|
||||
def __post_init__(self):
|
||||
if not self.data:
|
||||
|
||||
@@ -34,8 +34,7 @@ _NIXL_SUPPORTED_DEVICE.update(current_platform.get_nixl_supported_devices())
|
||||
# TODO: merge with vllm.utils.network_utils.zmq_socket_ctx
|
||||
@contextlib.contextmanager
|
||||
def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
|
||||
"""Context manager for a ZMQ socket"""
|
||||
|
||||
"""Context manager for a ZMQ socket."""
|
||||
if socket_type not in (zmq.ROUTER, zmq.REQ):
|
||||
raise ValueError(f"Unexpected socket type: {socket_type}")
|
||||
|
||||
|
||||
@@ -37,7 +37,8 @@ def canonical_format_id(kv_cache_layout: str) -> str:
|
||||
Canonical pages keep the worker's KV layout family, so the id couples the
|
||||
format version with that family; consumers must match it exactly. The family
|
||||
keeps its historical NHD/HND spelling so ids stay stable for KV persisted
|
||||
before the layout enum existed."""
|
||||
before the layout enum existed.
|
||||
"""
|
||||
layout = KVCacheLayout[kv_cache_layout]
|
||||
legacy = {KVCacheLayout.LBNHC: "nhd", KVCacheLayout.LBHNC: "hnd"}
|
||||
family = legacy.get(layout, layout.name.lower())
|
||||
@@ -121,7 +122,8 @@ def _interleave_cp_tokens(
|
||||
ctx: _RankContext,
|
||||
) -> tuple[CopyRun, ...]:
|
||||
"""Place each region's num_tokens rows at their canonical token positions,
|
||||
one run per chunk of interleaved tokens."""
|
||||
one run per chunk of interleaved tokens.
|
||||
"""
|
||||
runs: list[CopyRun] = []
|
||||
for region in regions:
|
||||
if ctx.cp_size == 1:
|
||||
@@ -256,7 +258,8 @@ def _attention_byte_regions(
|
||||
cp_size: int,
|
||||
) -> list[ByteRegion] | None:
|
||||
"""Byte regions of an attention page, given this rank's head shard.
|
||||
None when the physical layout is not recognized (fail closed)."""
|
||||
None when the physical layout is not recognized (fail closed).
|
||||
"""
|
||||
bs, heads, head_size = spec.block_size, spec.num_kv_heads, spec.head_size
|
||||
if tuple(kv_cache.shape) == (num_blocks, heads, bs, 2 * head_size):
|
||||
return _packed_kv_regions(kv_cache, spec, head_shard, num_head_shards, cp_size)
|
||||
@@ -370,7 +373,8 @@ def _is_exact_partition(intervals: list[tuple[int, int]], size: int) -> bool:
|
||||
|
||||
def _verify_tiling(layer_name: str, per_rank: list[CanonicalPageMapping]) -> None:
|
||||
"""Whichever ranks a block elects as writers must tile the canonical page
|
||||
exactly once, and each rank's runs must cover exactly its local page."""
|
||||
exactly once, and each rank's runs must cover exactly its local page.
|
||||
"""
|
||||
size = per_rank[0].canonical_page_size_bytes
|
||||
num_writers = per_rank[0].num_writers
|
||||
for mapping in per_rank:
|
||||
|
||||
@@ -79,7 +79,8 @@ def get_offloading_event_group_spec(
|
||||
class _OffloadEventMetadata:
|
||||
"""BlockStored payload snapshot for one OffloadKey, captured while the
|
||||
Request is available and kept until the final matching removal event.
|
||||
``medium`` and ``ownership`` are forwarded from the OffloadingEvent."""
|
||||
``medium`` and ``ownership`` are forwarded from the OffloadingEvent.
|
||||
"""
|
||||
|
||||
# The chunk's constituent block hashes; the last one is the OffloadKey.
|
||||
block_hashes: tuple[BlockHash, ...]
|
||||
@@ -245,6 +246,7 @@ class OffloadingEventsTracker:
|
||||
Yields:
|
||||
``BlockStored`` or ``BlockRemoved`` events corresponding to
|
||||
the underlying :class:`OffloadingEvent` stream.
|
||||
|
||||
"""
|
||||
for event in events:
|
||||
if event.removed:
|
||||
@@ -254,7 +256,8 @@ class OffloadingEventsTracker:
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Drop all tracked state; pending payloads are stale after a
|
||||
manager cache reset."""
|
||||
manager cache reset.
|
||||
"""
|
||||
self._pending_event_metadata.clear()
|
||||
|
||||
def _build_event_metadata(
|
||||
@@ -265,7 +268,8 @@ class OffloadingEventsTracker:
|
||||
) -> _OffloadEventMetadata:
|
||||
"""Build the payload snapshot for one offloaded chunk: its
|
||||
constituent per-block hashes, the whole chunk's tokens, and the
|
||||
per-block ``block_size``."""
|
||||
per-block ``block_size``.
|
||||
"""
|
||||
hashes_per_chunk = group_config.hashes_per_chunk
|
||||
assert hashes_per_chunk > 0
|
||||
assert chunk_idx >= 0
|
||||
|
||||
@@ -170,8 +170,7 @@ class _StatsKey:
|
||||
|
||||
@dataclass
|
||||
class OffloadingConnectorStats(KVConnectorStats):
|
||||
"""
|
||||
Offloading connector stats use flat metric names as keys.
|
||||
"""Offloading connector stats use flat metric names as keys.
|
||||
|
||||
The ``data`` dict is structured using ``_StatsKey`` / ``_MetricType``::
|
||||
|
||||
@@ -243,8 +242,7 @@ class OffloadingConnectorStats(KVConnectorStats):
|
||||
return self
|
||||
|
||||
def reduce(self) -> dict[str, int | float]:
|
||||
"""
|
||||
Reduce the observations collected during a time interval to one or
|
||||
"""Reduce the observations collected during a time interval to one or
|
||||
more representative values (eg avg/median/sum of the series).
|
||||
This is meant to be called by the logger to produce a summary of the
|
||||
stats for the last time interval.
|
||||
|
||||
@@ -676,7 +676,8 @@ class OffloadingConnectorScheduler:
|
||||
start_chunk_idx: int,
|
||||
) -> int | None:
|
||||
"""Return the number of consecutive offloaded chunks from the start,
|
||||
or None if the backend deferred a lookup."""
|
||||
or None if the backend deferred a lookup.
|
||||
"""
|
||||
hit_count = 0
|
||||
defer_lookup = False
|
||||
for local_idx, key in enumerate(keys):
|
||||
@@ -711,7 +712,8 @@ class OffloadingConnectorScheduler:
|
||||
"""Return the end index (in `keys`) of the last run of
|
||||
`sliding_window_size` consecutive hits, scanning from the end.
|
||||
The first run may need a larger window for a partial rightmost chunk.
|
||||
Returns 0 on miss, None if the backend deferred a lookup."""
|
||||
Returns 0 on miss, None if the backend deferred a lookup.
|
||||
"""
|
||||
defer_lookup = False
|
||||
pending_in_window = False
|
||||
consecutive_hits = 0
|
||||
@@ -750,8 +752,7 @@ class OffloadingConnectorScheduler:
|
||||
req_status: RequestOffloadState,
|
||||
max_num_new_tokens: int | None = None,
|
||||
) -> int | None:
|
||||
"""
|
||||
Find how many tokens beyond num_locally_computed_tokens can be loaded.
|
||||
"""Find how many tokens beyond num_locally_computed_tokens can be loaded.
|
||||
|
||||
Iterates full-attention groups first (prefix lookup), then sliding-window
|
||||
groups (suffix lookup). Each group may tighten max_hit_size_tokens, which
|
||||
@@ -1039,14 +1040,15 @@ class OffloadingConnectorScheduler:
|
||||
num_computed_tokens: int,
|
||||
max_num_new_tokens: int | None = None,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""
|
||||
Get number of new tokens that can be loaded beyond the
|
||||
"""Get number of new tokens that can be loaded beyond the
|
||||
num_computed_tokens.
|
||||
|
||||
Args:
|
||||
request (Request): the request object.
|
||||
num_computed_tokens (int): the number of locally
|
||||
computed tokens for this request
|
||||
max_num_new_tokens (int | None): cap on the number of tokens that
|
||||
may be loaded beyond `num_computed_tokens`, if any.
|
||||
|
||||
Returns:
|
||||
A tuple with the following elements:
|
||||
@@ -1057,6 +1059,7 @@ class OffloadingConnectorScheduler:
|
||||
should query for this request again later.
|
||||
- `True` if tokens will be loaded asynchronously
|
||||
(between scheduler steps).
|
||||
|
||||
"""
|
||||
req_status = self._req_status[request.request_id]
|
||||
for group_state in req_status.group_states:
|
||||
@@ -1208,10 +1211,7 @@ class OffloadingConnectorScheduler:
|
||||
req_status.partial_tail_boundary = None
|
||||
|
||||
def _update_req_states(self, scheduler_output: SchedulerOutput) -> None:
|
||||
"""
|
||||
Update request states from the Scheduler's output.
|
||||
"""
|
||||
|
||||
"""Update request states from the Scheduler's output."""
|
||||
# new_block_ids_end[req_id][i] = end of pre-existing block_ids for
|
||||
# the i-th sliding window group (before this step's extend).
|
||||
# Used to detect sliding window blocks that got re-allocated.
|
||||
@@ -1840,12 +1840,12 @@ class OffloadingConnectorScheduler:
|
||||
return bool(self._jobs) or self.manager.has_pending_work()
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""
|
||||
Update KVConnector state from worker-side connectors output.
|
||||
"""Update KVConnector state from worker-side connectors output.
|
||||
|
||||
Args:
|
||||
connector_output (KVConnectorOutput): the worker-side
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
meta = connector_output.kv_connector_worker_meta
|
||||
if not isinstance(meta, OffloadingWorkerMetadata):
|
||||
@@ -1938,8 +1938,7 @@ class OffloadingConnectorScheduler:
|
||||
self,
|
||||
request: Request,
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""
|
||||
Called when a request has finished, before its blocks are freed.
|
||||
"""Called when a request has finished, before its blocks are freed.
|
||||
|
||||
Returns:
|
||||
True if the request is being saved/sent asynchronously and blocks
|
||||
@@ -1947,6 +1946,7 @@ class OffloadingConnectorScheduler:
|
||||
get_finished().
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
|
||||
"""
|
||||
req_status = self._req_status.get(request.request_id)
|
||||
|
||||
@@ -1985,12 +1985,12 @@ class OffloadingConnectorScheduler:
|
||||
Yields:
|
||||
``BlockStored`` or ``BlockRemoved`` events corresponding to
|
||||
the underlying :class:`OffloadingEvent` stream.
|
||||
|
||||
"""
|
||||
yield from self._events_tracker.take_events(self.manager.take_events())
|
||||
|
||||
def reset_cache(self) -> None:
|
||||
"""Reset the offloading manager cache, evicting all stored chunks."""
|
||||
|
||||
# reset_cache cannot be called in the middle of a schedule step
|
||||
assert not self._current_batch_load_jobs
|
||||
assert not self._current_batch_jobs_to_flush
|
||||
|
||||
@@ -39,7 +39,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class OffloadingConnectorWorker:
|
||||
"""Implementation of Worker side methods"""
|
||||
"""Implementation of Worker side methods."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -268,14 +268,14 @@ class OffloadingConnectorWorker:
|
||||
)
|
||||
|
||||
def get_finished(self, finished_req_ids: set[str]) -> tuple[set[str], set[str]]:
|
||||
"""
|
||||
Returns:
|
||||
tuple of (finished_sending, finished_recving). Stores never
|
||||
emit finished_sending — the scheduler tracks store completion
|
||||
via kv_connector_worker_meta.completed_jobs and fences any
|
||||
block reuse via jobs_to_flush. Loads still emit
|
||||
finished_recving so the base scheduler can resume requests
|
||||
blocked on remote KV (and free aborted-during-load reqs).
|
||||
"""Returns:
|
||||
tuple of (finished_sending, finished_recving). Stores never
|
||||
emit finished_sending — the scheduler tracks store completion
|
||||
via kv_connector_worker_meta.completed_jobs and fences any
|
||||
block reuse via jobs_to_flush. Loads still emit
|
||||
finished_recving so the base scheduler can resume requests
|
||||
blocked on remote KV (and free aborted-during-load reqs).
|
||||
|
||||
"""
|
||||
assert self.worker is not None
|
||||
finished_recving: set[str] = set()
|
||||
|
||||
@@ -83,6 +83,7 @@ class MambaConvSplitInfo:
|
||||
< 0: P_TP > D_TP — P pages are smaller, D reads entire
|
||||
P page. Local dims are scaled down by |tp_ratio|
|
||||
to get P-sized offsets.
|
||||
|
||||
"""
|
||||
offsets: list[tuple[int, int]] = []
|
||||
if tp_ratio >= 1:
|
||||
@@ -121,6 +122,7 @@ def derive_mamba_conv_split(
|
||||
Returns:
|
||||
MambaConvSplitInfo with per-rank sub-projection dims, conv_rows,
|
||||
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
|
||||
|
||||
"""
|
||||
_supported = (
|
||||
MambaAttentionBackendEnum.MAMBA1,
|
||||
@@ -228,5 +230,6 @@ def compute_physical_blocks_per_logical(
|
||||
Args:
|
||||
ssm_sizes: (conv_state_bytes, ssm_state_bytes) from NixlAgentMetadata.
|
||||
block_len: the engine's block_len in bytes (from block_lens[0]).
|
||||
|
||||
"""
|
||||
return math.ceil((ssm_sizes[0] + ssm_sizes[1]) / block_len)
|
||||
|
||||
@@ -38,6 +38,7 @@ def is_v1_kv_transfer_group(connector: KVConnectorBaseType | None = None) -> boo
|
||||
Note:
|
||||
This function will no-longer be needed after the v1 KV connector
|
||||
becomes the default.
|
||||
|
||||
"""
|
||||
if connector is None:
|
||||
connector = _KV_CONNECTOR_AGENT
|
||||
@@ -72,10 +73,7 @@ def _sync_engine_id_across_tp(vllm_config: "VllmConfig") -> None:
|
||||
def ensure_kv_transfer_initialized(
|
||||
vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"
|
||||
) -> None:
|
||||
"""
|
||||
Initialize KV cache transfer parallel group.
|
||||
"""
|
||||
|
||||
"""Initialize KV cache transfer parallel group."""
|
||||
global _KV_CONNECTOR_AGENT
|
||||
|
||||
if _KV_CONNECTOR_AGENT is not None:
|
||||
|
||||
@@ -141,9 +141,11 @@ _group_name_counter: dict[str, int] = {}
|
||||
|
||||
def _get_unique_name(name: str) -> str:
|
||||
"""Get a unique name for the group.
|
||||
|
||||
Example:
|
||||
_get_unique_name("tp") -> "tp:0"
|
||||
_get_unique_name("tp") -> "tp:1"
|
||||
|
||||
"""
|
||||
if name not in _group_name_counter:
|
||||
_group_name_counter[name] = 0
|
||||
@@ -420,8 +422,7 @@ direct_register_custom_op(
|
||||
|
||||
|
||||
class GroupCoordinator:
|
||||
"""
|
||||
PyTorch ProcessGroup wrapper for a group of processes.
|
||||
"""PyTorch ProcessGroup wrapper for a group of processes.
|
||||
PyTorch ProcessGroup is bound to one specific communication backend,
|
||||
e.g. NCCL, Gloo, MPI, etc.
|
||||
GroupCoordinator takes charge of all the communication operations among
|
||||
@@ -634,34 +635,34 @@ class GroupCoordinator:
|
||||
|
||||
@property
|
||||
def first_rank(self):
|
||||
"""Return the global rank of the first process in the group"""
|
||||
"""Return the global rank of the first process in the group."""
|
||||
return self.ranks[0]
|
||||
|
||||
@property
|
||||
def last_rank(self):
|
||||
"""Return the global rank of the last process in the group"""
|
||||
"""Return the global rank of the last process in the group."""
|
||||
return self.ranks[-1]
|
||||
|
||||
@property
|
||||
def is_first_rank(self):
|
||||
"""Return whether the caller is the first process in the group"""
|
||||
"""Return whether the caller is the first process in the group."""
|
||||
return self.rank == self.first_rank
|
||||
|
||||
@property
|
||||
def is_last_rank(self):
|
||||
"""Return whether the caller is the last process in the group"""
|
||||
"""Return whether the caller is the last process in the group."""
|
||||
return self.rank == self.last_rank
|
||||
|
||||
@property
|
||||
def next_rank(self):
|
||||
"""Return the global rank of the process that follows the caller"""
|
||||
"""Return the global rank of the process that follows the caller."""
|
||||
rank_in_group = self.rank_in_group
|
||||
world_size = self.world_size
|
||||
return self.ranks[(rank_in_group + 1) % world_size]
|
||||
|
||||
@property
|
||||
def prev_rank(self):
|
||||
"""Return the global rank of the process that precedes the caller"""
|
||||
"""Return the global rank of the process that precedes the caller."""
|
||||
rank_in_group = self.rank_in_group
|
||||
world_size = self.world_size
|
||||
return self.ranks[(rank_in_group - 1) % world_size]
|
||||
@@ -719,8 +720,7 @@ class GroupCoordinator:
|
||||
yield graph_capture_context
|
||||
|
||||
def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
User-facing all-reduce function before we actually call the
|
||||
"""User-facing all-reduce function before we actually call the
|
||||
all-reduce operation.
|
||||
|
||||
We need this because Dynamo does not support passing an arbitrary
|
||||
@@ -809,8 +809,7 @@ class GroupCoordinator:
|
||||
def gather(
|
||||
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
NOTE: We assume that the input tensor is on the same device across
|
||||
"""NOTE: We assume that the input tensor is on the same device across
|
||||
all the ranks.
|
||||
NOTE: `dst` is the local rank of the destination rank.
|
||||
"""
|
||||
@@ -1363,7 +1362,7 @@ class GroupCoordinator:
|
||||
torch.distributed.barrier(group=self.cpu_group)
|
||||
|
||||
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
|
||||
"""Sends a tensor to the destination rank in a blocking way"""
|
||||
"""Sends a tensor to the destination rank in a blocking way."""
|
||||
"""NOTE: `dst` is the local rank of the destination rank."""
|
||||
if self.device_communicator is None:
|
||||
raise ValueError("No device communicator found")
|
||||
@@ -1637,8 +1636,7 @@ def graph_capture(
|
||||
device: torch.device,
|
||||
graph_capture_context: GraphCaptureContext | None = None,
|
||||
):
|
||||
"""
|
||||
`graph_capture` is a context manager which should surround the code that
|
||||
"""`graph_capture` is a context manager which should surround the code that
|
||||
is capturing the CUDA graph. Its main purpose is to ensure that some
|
||||
operations will be run after the graph is captured, before the graph
|
||||
is replayed. It returns a `GraphCaptureContext` object which contains the
|
||||
@@ -1981,14 +1979,17 @@ def initialize_model_parallel(
|
||||
decode_context_model_parallel_size: int | None = 1,
|
||||
backend: str | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize model parallel groups.
|
||||
"""Initialize model parallel groups.
|
||||
|
||||
Arguments:
|
||||
tensor_model_parallel_size: number of GPUs used for tensor model
|
||||
parallelism.
|
||||
pipeline_model_parallel_size: number of GPUs used for pipeline model
|
||||
parallelism.
|
||||
prefill_context_model_parallel_size: number of GPUs used for context
|
||||
parallelism during prefill.
|
||||
decode_context_model_parallel_size: number of GPUs used for context
|
||||
parallelism during decode.
|
||||
backend: name of torch distributed communication backend.
|
||||
|
||||
Let's say we have a total of 8 GPUs denoted by g0 ... g7 and we
|
||||
@@ -2003,6 +2004,7 @@ def initialize_model_parallel(
|
||||
are on the same DGX box. For example if we are using 2 DGX-1 boxes
|
||||
with a total of 16 GPUs, rank 0 to 7 belong to the first box and
|
||||
ranks 8 to 15 belong to the second box.
|
||||
|
||||
"""
|
||||
# Get world size and rank. Ensure some consistencies.
|
||||
assert torch.distributed.is_initialized()
|
||||
@@ -2476,8 +2478,7 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
|
||||
def in_the_same_node_as(
|
||||
pg: ProcessGroup | StatelessProcessGroup, source_rank: int = 0
|
||||
) -> list[bool]:
|
||||
"""
|
||||
This is a collective operation that returns if each rank is in the same node
|
||||
"""This is a collective operation that returns if each rank is in the same node
|
||||
as the source rank. It tests if processes are attached to the same
|
||||
memory system (shared access to shared memory).
|
||||
"""
|
||||
@@ -2568,8 +2569,7 @@ def in_the_same_node_as(
|
||||
|
||||
|
||||
def is_global_first_rank() -> bool:
|
||||
"""
|
||||
Check if the current process is the first rank globally across all
|
||||
"""Check if the current process is the first rank globally across all
|
||||
parallelism strategies (PP, TP, DP, EP, etc.).
|
||||
|
||||
Unlike group-specific checks like `get_tensor_model_parallel_rank() == 0`
|
||||
@@ -2579,6 +2579,7 @@ def is_global_first_rank() -> bool:
|
||||
Returns:
|
||||
bool: True if this is the global first rank (rank 0), False otherwise.
|
||||
Returns True if distributed is not initialized (single process).
|
||||
|
||||
"""
|
||||
try:
|
||||
# If world group is available, use it for the most accurate check
|
||||
@@ -2599,9 +2600,7 @@ def is_global_first_rank() -> bool:
|
||||
|
||||
|
||||
def is_local_first_rank() -> bool:
|
||||
"""
|
||||
Check if the current process is the first local rank (rank 0 on its node).
|
||||
"""
|
||||
"""Check if the current process is the first local rank (rank 0 on its node)."""
|
||||
try:
|
||||
# prefer the initialized world group if available
|
||||
global _WORLD
|
||||
@@ -2622,14 +2621,14 @@ def is_local_first_rank() -> bool:
|
||||
|
||||
|
||||
def _node_count(pg: ProcessGroup | StatelessProcessGroup) -> int:
|
||||
"""
|
||||
Returns the total number of nodes in the process group.
|
||||
"""Returns the total number of nodes in the process group.
|
||||
|
||||
Args:
|
||||
pg: The process group to analyze
|
||||
|
||||
Returns:
|
||||
int: The total number of nodes
|
||||
|
||||
"""
|
||||
if isinstance(pg, ProcessGroup):
|
||||
world_size = torch.distributed.get_world_size(group=pg)
|
||||
|
||||
@@ -59,8 +59,7 @@ def _fetch_group_ports(key: str, coord_store: Store) -> list[int]:
|
||||
|
||||
|
||||
class StatelessGroupCoordinator(GroupCoordinator):
|
||||
"""
|
||||
A stateless version of the GroupCoordinator class in parallel_state,
|
||||
"""A stateless version of the GroupCoordinator class in parallel_state,
|
||||
It will create CPU, device and TCPStore based communication groups
|
||||
that are independent of PyTorch's WORLD group. Hence,
|
||||
communication groups with a different set of participants GPUs
|
||||
|
||||
@@ -59,7 +59,8 @@ def ensure_divisibility(numerator, denominator):
|
||||
|
||||
def divide(numerator, denominator):
|
||||
"""Ensure that numerator is divisible by the denominator and return
|
||||
the division value."""
|
||||
the division value.
|
||||
"""
|
||||
ensure_divisibility(numerator, denominator)
|
||||
return numerator // denominator
|
||||
|
||||
@@ -111,6 +112,7 @@ def split_tensor_along_last_dim(
|
||||
|
||||
Returns:
|
||||
A list of Tensors
|
||||
|
||||
"""
|
||||
# Get the size and dimension.
|
||||
last_dim = tensor.dim() - 1
|
||||
@@ -333,7 +335,6 @@ class StatelessProcessGroup:
|
||||
def barrier(self, timeout: float = 30.0):
|
||||
"""A robust barrier to synchronize all ranks.
|
||||
|
||||
|
||||
Uses a multi-phase approach to ensure all processes reach the barrier
|
||||
before proceeding:
|
||||
|
||||
@@ -351,6 +352,7 @@ class StatelessProcessGroup:
|
||||
|
||||
Raises:
|
||||
RuntimeError: If coordination fails or times out
|
||||
|
||||
"""
|
||||
# Generate a barrier ID that is globally unique
|
||||
try:
|
||||
@@ -549,8 +551,7 @@ def init_gloo_process_group(
|
||||
group_size: int,
|
||||
timeout: timedelta,
|
||||
) -> ProcessGroup:
|
||||
"""
|
||||
Stateless init ProcessGroup with gloo backend compatible with
|
||||
"""Stateless init ProcessGroup with gloo backend compatible with
|
||||
different torch versions.
|
||||
"""
|
||||
with suppress_stdout():
|
||||
@@ -583,8 +584,7 @@ def stateless_init_torch_distributed_process_group(
|
||||
return_store: bool = False,
|
||||
listen_socket: socket.socket | None = None,
|
||||
) -> ProcessGroup | tuple[ProcessGroup, Store]:
|
||||
"""
|
||||
A replacement for `torch.distributed.init_process_group` that does not
|
||||
"""A replacement for `torch.distributed.init_process_group` that does not
|
||||
pollute the global state. The created ProcessGroup object can be used for
|
||||
some operations such as `allreduce`, because it does not depend on the
|
||||
global rank. However, some operations such as `broadcast` cannot be used
|
||||
@@ -685,9 +685,8 @@ def stateless_init_torch_distributed_process_group(
|
||||
|
||||
|
||||
def stateless_destroy_torch_distributed_process_group(pg: ProcessGroup) -> None:
|
||||
"""
|
||||
Destroy ProcessGroup returned by
|
||||
stateless_init_torch_distributed_process_group().
|
||||
"""Destroy ProcessGroup returned by
|
||||
stateless_init_torch_distributed_process_group().
|
||||
"""
|
||||
pg.shutdown()
|
||||
_unregister_process_group(pg.group_name)
|
||||
@@ -706,6 +705,7 @@ def get_worker_rank_suffix(global_rank: int | None = None) -> str:
|
||||
Returns:
|
||||
A string suffix identifying the worker's position in the
|
||||
distributed topology.
|
||||
|
||||
"""
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_dcp_group,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Weight transfer engines for syncing model weights from trainers
|
||||
"""Weight transfer engines for syncing model weights from trainers
|
||||
to inference workers.
|
||||
"""
|
||||
|
||||
|
||||
@@ -193,6 +193,7 @@ class WeightSource(ABC):
|
||||
|
||||
Returns:
|
||||
The held parameter names, or None to hold every one.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -329,8 +330,7 @@ class WeightTransferUpdateRequest:
|
||||
|
||||
|
||||
class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
"""
|
||||
Base class for weight transfer engines that handle transport of model weights
|
||||
"""Base class for weight transfer engines that handle transport of model weights
|
||||
from a trainer to inference workers.
|
||||
|
||||
This abstraction separates weight transfer transport logic from the worker
|
||||
@@ -386,14 +386,14 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the weight transfer engine.
|
||||
"""Initialize the weight transfer engine.
|
||||
|
||||
Args:
|
||||
config: The configuration for the weight transfer engine
|
||||
vllm_config: The full vLLM config (provides parallel/model config)
|
||||
device: The device this worker's model lives on
|
||||
model: The local model instance which will receive the weights
|
||||
|
||||
"""
|
||||
self.config = config
|
||||
self.vllm_config = vllm_config
|
||||
@@ -419,8 +419,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
self.model_config = self._default_model_config
|
||||
|
||||
def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo:
|
||||
"""
|
||||
Construct typed init info from dict with validation.
|
||||
"""Construct typed init info from dict with validation.
|
||||
|
||||
Args:
|
||||
init_dict: Dictionary containing backend-specific initialization parameters
|
||||
@@ -430,6 +429,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
Raises:
|
||||
ValueError: If init_dict is invalid for this backend
|
||||
|
||||
"""
|
||||
try:
|
||||
return self.init_info_cls(**init_dict)
|
||||
@@ -439,8 +439,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
) from e
|
||||
|
||||
def parse_update_info(self, update_dict: dict[str, Any]) -> TUpdateInfo:
|
||||
"""
|
||||
Construct typed update info from dict with validation.
|
||||
"""Construct typed update info from dict with validation.
|
||||
|
||||
Args:
|
||||
update_dict: Dictionary containing backend-specific update parameters
|
||||
@@ -450,6 +449,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
Raises:
|
||||
ValueError: If update_dict is invalid for this backend
|
||||
|
||||
"""
|
||||
try:
|
||||
return self.update_info_cls(**update_dict)
|
||||
@@ -460,19 +460,18 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
@abstractmethod
|
||||
def init_transfer_engine(self, init_info: TInitInfo) -> None:
|
||||
"""
|
||||
Initialize the weight transfer mechanism.
|
||||
"""Initialize the weight transfer mechanism.
|
||||
This is called once at the beginning of training.
|
||||
|
||||
Args:
|
||||
init_info: Backend-specific initialization info
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def start_weight_update(self) -> None:
|
||||
"""
|
||||
Prepare the engine for a new weight update.
|
||||
"""Prepare the engine for a new weight update.
|
||||
|
||||
Engines that receive weights in checkpoint format initialize layerwise reloading
|
||||
here, else this is typically a no-op.
|
||||
@@ -482,8 +481,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
@abstractmethod
|
||||
def finish_weight_update(self) -> None:
|
||||
"""
|
||||
Finalize the current weight update.
|
||||
"""Finalize the current weight update.
|
||||
|
||||
Checkpoint-format engines finalize layerwise reloading here; engines
|
||||
that apply weights in place leave this as a no-op.
|
||||
@@ -491,11 +489,11 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
raise NotImplementedError
|
||||
|
||||
def update_weights(self, update_info: dict[str, Any]) -> None:
|
||||
"""
|
||||
Receive one weight update chunk and load it into the model.
|
||||
"""Receive one weight update chunk and load it into the model.
|
||||
|
||||
Args:
|
||||
update_info: Dictionary containing backend-specific update info
|
||||
|
||||
"""
|
||||
typed_update_info = self.parse_update_info(update_info)
|
||||
self.receive_weights(typed_update_info)
|
||||
@@ -505,19 +503,18 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
@abstractmethod
|
||||
def receive_weights(self, update_info: TUpdateInfo) -> None:
|
||||
"""
|
||||
Receive weights from the trainer and load them into the model.
|
||||
"""Receive weights from the trainer and load them into the model.
|
||||
|
||||
Args:
|
||||
update_info: Backend-specific update info containing parameter metadata
|
||||
and any backend-specific data
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def shutdown(self) -> None:
|
||||
"""
|
||||
Shutdown the weight transfer engine.
|
||||
"""Shutdown the weight transfer engine.
|
||||
This should be called when the worker is shutting down.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -58,6 +58,7 @@ class WeightTransferEngineFactory:
|
||||
|
||||
Raises:
|
||||
ValueError: If an engine with the same name is already registered
|
||||
|
||||
"""
|
||||
if name in cls._registry:
|
||||
raise ValueError(f"Weight transfer engine '{name}' is already registered.")
|
||||
@@ -101,6 +102,7 @@ class WeightTransferEngineFactory:
|
||||
|
||||
Raises:
|
||||
ValueError: If the backend is not registered
|
||||
|
||||
"""
|
||||
backend = config.backend
|
||||
if backend not in cls._registry:
|
||||
@@ -139,7 +141,8 @@ class WeightTransferTrainerFactory:
|
||||
class_name: str | None = None,
|
||||
) -> None:
|
||||
"""Register a trainer engine. Same conventions as
|
||||
`WeightTransferEngineFactory.register_engine`."""
|
||||
`WeightTransferEngineFactory.register_engine`.
|
||||
"""
|
||||
if name in cls._registry:
|
||||
raise ValueError(
|
||||
f"Weight transfer trainer engine '{name}' is already registered."
|
||||
@@ -190,6 +193,7 @@ class WeightTransferTrainerFactory:
|
||||
|
||||
Raises:
|
||||
ValueError: If `init_info.backend` is not registered.
|
||||
|
||||
"""
|
||||
backend = init_info.backend
|
||||
if backend not in cls._registry:
|
||||
|
||||
@@ -40,7 +40,8 @@ class IPCWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
`packed` is a must-agree wire param: the trainer ships it here at the init
|
||||
handshake so the worker decodes with the same setting the trainer encoded
|
||||
with. The consumer rebuilds from the IPC handle + `tensor_sizes`, so it does
|
||||
not need the buffer size (producer-only)."""
|
||||
not need the buffer size (producer-only).
|
||||
"""
|
||||
|
||||
packed: bool = False
|
||||
|
||||
@@ -53,7 +54,8 @@ class IPCTrainerInitInfo(TrainerInitInfo):
|
||||
|
||||
`packed` / `packed_buffer_size_bytes` are the transfer's wire params. The
|
||||
trainer propagates `packed` to the worker at `trainer_init` so the two sides
|
||||
cannot disagree. `backend` is the factory dispatch key."""
|
||||
cannot disagree. `backend` is the factory dispatch key.
|
||||
"""
|
||||
|
||||
backend: ClassVar[str] = "ipc"
|
||||
packed: bool = False
|
||||
@@ -121,8 +123,7 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
class IPCWeightTransferEngine(
|
||||
WeightTransferEngine[IPCWeightTransferInitInfo, IPCWeightTransferUpdateInfo]
|
||||
):
|
||||
"""
|
||||
Weight transfer engine using CUDA IPC for communication between trainer and workers.
|
||||
"""Weight transfer engine using CUDA IPC between trainer and workers.
|
||||
|
||||
This implementation uses CUDA IPC to transfer weights from the trainer (rank 0)
|
||||
to all inference workers in a process group. IPC handles are used to share
|
||||
@@ -149,13 +150,13 @@ class IPCWeightTransferEngine(
|
||||
self._packed_importer = PackedBufferImporter()
|
||||
|
||||
def init_transfer_engine(self, init_info: IPCWeightTransferInitInfo) -> None:
|
||||
"""
|
||||
Initialize the weight transfer mechanism. No data-plane rendezvous is
|
||||
"""Initialize the weight transfer mechanism. No data-plane rendezvous is
|
||||
needed for IPC; this just records the trainer-supplied wire params so the
|
||||
worker decodes exactly as the trainer encoded.
|
||||
|
||||
Args:
|
||||
init_info: IPC initialization info (carries `packed`).
|
||||
|
||||
"""
|
||||
self.packed = init_info.packed
|
||||
|
||||
@@ -182,8 +183,7 @@ class IPCWeightTransferEngine(
|
||||
self._packed_importer.close()
|
||||
|
||||
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
|
||||
"""
|
||||
Receive weights from the trainer via CUDA IPC handles and load them.
|
||||
"""Receive weights from the trainer via CUDA IPC handles and load them.
|
||||
|
||||
Whether the transfer is packed is read from `self.packed`, set at the
|
||||
init handshake from the trainer's init info, so it is guaranteed to
|
||||
@@ -193,6 +193,7 @@ class IPCWeightTransferEngine(
|
||||
update_info: IPC update info containing parameter names, dtypes, shapes,
|
||||
and IPC handles. Each IPC handle is a mapping between physical
|
||||
GPU UUID and the rebuild_cuda_tensor args tuple.
|
||||
|
||||
"""
|
||||
# Use the worker's assigned device rather than the ambient current
|
||||
# device: the receive path is no longer wrapped in
|
||||
|
||||
@@ -118,7 +118,8 @@ def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
|
||||
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
|
||||
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
|
||||
only the mode actually in use. Shared by the dense and sparse trainer
|
||||
engines so the two cannot drift."""
|
||||
engines so the two cannot drift.
|
||||
"""
|
||||
return {key: value for key, value in asdict(init_info).items() if value is not None}
|
||||
|
||||
|
||||
@@ -142,8 +143,7 @@ def stateless_init_process_group(
|
||||
world_size: int,
|
||||
device,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""
|
||||
vLLM provides `StatelessProcessGroup` to create a process group
|
||||
"""VLLM provides `StatelessProcessGroup` to create a process group
|
||||
without considering the global process group in torch.distributed.
|
||||
It is recommended to create `StatelessProcessGroup`, and then initialize
|
||||
the data-plane communication (NCCL) between external (train processes)
|
||||
@@ -222,8 +222,7 @@ def worker_init_process_group(
|
||||
def trainer_init(
|
||||
init_info: NCCLRendezvous | dict,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""
|
||||
Initialize NCCL process group for trainer-side weight transfer.
|
||||
"""Initialize NCCL process group for trainer-side weight transfer.
|
||||
|
||||
The trainer is always rank 0 in the process group. Uses the current
|
||||
CUDA device (torch.accelerator.current_device_index()).
|
||||
@@ -237,6 +236,7 @@ def trainer_init(
|
||||
|
||||
Returns:
|
||||
PyNcclCommunicator for weight transfer.
|
||||
|
||||
"""
|
||||
if isinstance(init_info, dict):
|
||||
master_address = init_info["master_address"]
|
||||
|
||||
@@ -69,7 +69,8 @@ class NCCLTrainerInitInfo(TrainerInitInfo):
|
||||
propagates them to the worker at `trainer_init` so the two sides cannot
|
||||
disagree. Note this defaults to packed, unlike the worker-side
|
||||
`NCCLWeightTransferInitInfo`, whose default only applies when no trainer
|
||||
ships a value. `backend` is the factory dispatch key."""
|
||||
ships a value. `backend` is the factory dispatch key.
|
||||
"""
|
||||
|
||||
backend: ClassVar[str] = "nccl"
|
||||
|
||||
@@ -113,8 +114,7 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
class NCCLWeightTransferEngine(
|
||||
WeightTransferEngine[NCCLWeightTransferInitInfo, NCCLWeightTransferUpdateInfo]
|
||||
):
|
||||
"""
|
||||
Weight transfer engine using NCCL for communication between trainer and workers.
|
||||
"""Weight transfer engine using NCCL for communication between trainer and workers.
|
||||
|
||||
This implementation uses NCCL broadcast operations to transfer dense
|
||||
checkpoint-format weights from the trainer (rank 0) to all inference workers
|
||||
@@ -142,14 +142,14 @@ class NCCLWeightTransferEngine(
|
||||
self.packed_num_buffers = DEFAULT_PACKED_NUM_BUFFERS
|
||||
|
||||
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
|
||||
"""
|
||||
Initialize NCCL process group with the trainer and record the
|
||||
"""Initialize NCCL process group with the trainer and record the
|
||||
trainer-supplied wire params so the worker decodes exactly as the
|
||||
trainer encodes.
|
||||
|
||||
Args:
|
||||
init_info: NCCL initialization info containing master address, port,
|
||||
rank offset, world size, and the packed wire params
|
||||
|
||||
"""
|
||||
self.packed = init_info.packed
|
||||
self.packed_buffer_size_bytes = init_info.packed_buffer_size_bytes
|
||||
@@ -175,8 +175,7 @@ class NCCLWeightTransferEngine(
|
||||
finalize_layerwise_reload(self.model, self.model_config)
|
||||
|
||||
def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None:
|
||||
"""
|
||||
Receive weights from trainer via NCCL broadcast.
|
||||
"""Receive weights from trainer via NCCL broadcast.
|
||||
|
||||
Whether to use packed broadcasting (and the buffer geometry) is read
|
||||
from `self.packed` / `self.packed_*`, set at the init handshake from the
|
||||
@@ -186,6 +185,7 @@ class NCCLWeightTransferEngine(
|
||||
Args:
|
||||
update_info: NCCL update info containing parameter names, dtypes,
|
||||
and shapes
|
||||
|
||||
"""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
@@ -365,7 +365,8 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni
|
||||
def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None:
|
||||
"""Iterate the source (materializing each tensor — a collective on all
|
||||
ranks) and, on the sender, broadcast from rank 0, packed or one-by-one.
|
||||
Non-sender ranks only replay the iteration to stay in the collective."""
|
||||
Non-sender ranks only replay the iteration to stay in the collective.
|
||||
"""
|
||||
if not self.is_sender:
|
||||
for _ in source:
|
||||
pass
|
||||
|
||||
@@ -45,6 +45,7 @@ def unpack_tensor(
|
||||
shapes: List of tensor shapes
|
||||
dtypes: List of tensor dtypes
|
||||
tensor_sizes: List of tensor sizes in bytes
|
||||
|
||||
"""
|
||||
unpacked_tensors = packed_tensor.split(tensor_sizes)
|
||||
|
||||
@@ -85,6 +86,7 @@ def pack_tensors(
|
||||
tensor_list: Pre-existing tensor list to append to (for NCCL
|
||||
multi-buffer reuse). If None, a fresh list is created.
|
||||
current_size: Byte count already accumulated in tensor_list
|
||||
|
||||
"""
|
||||
if tensor_list is None:
|
||||
tensor_list = []
|
||||
@@ -335,6 +337,7 @@ def packed_ipc_producer(
|
||||
buffer_size_bytes: Exact capacity of the reusable IPC buffer.
|
||||
Every chunk is guaranteed to fit within this size. A
|
||||
``ValueError`` is raised if any single tensor exceeds it.
|
||||
|
||||
"""
|
||||
ipc_buffer = torch.empty(buffer_size_bytes, dtype=torch.uint8, device="cuda")
|
||||
_, ipc_args = reduce_tensor(ipc_buffer)
|
||||
@@ -463,6 +466,7 @@ def packed_ipc_consumer(
|
||||
export so the buffer mapping is opened and released exactly once
|
||||
(see ``PackedBufferImporter``). ``None`` builds an ephemeral one,
|
||||
which is only safe for single-chunk consumption.
|
||||
|
||||
"""
|
||||
props = torch.cuda.get_device_properties(device_index)
|
||||
physical_gpu_id = str(props.uuid)
|
||||
|
||||
@@ -60,6 +60,7 @@ def check_ray_rdt_version() -> None:
|
||||
|
||||
Raises:
|
||||
ValueError: the installed Ray predates ``RDT_MIN_RAY_VERSION``.
|
||||
|
||||
"""
|
||||
import importlib.metadata
|
||||
|
||||
@@ -162,6 +163,7 @@ class RdtRouter:
|
||||
ValueError: ``workers_per_replica`` does not divide
|
||||
``num_consumers``, so the deployments are not uniform and two
|
||||
workers of one deployment would share a block index.
|
||||
|
||||
"""
|
||||
self.num_producers = max(1, num_producers)
|
||||
self.num_consumers = max(1, num_consumers)
|
||||
@@ -212,7 +214,8 @@ class RdtRouter:
|
||||
|
||||
def class_of(self, name: str) -> int:
|
||||
"""The name's owner class — the planner's bucketing key, since all names
|
||||
of a chunk must share one producer."""
|
||||
of a chunk must share one producer.
|
||||
"""
|
||||
return self._class_of.get(name, 0)
|
||||
|
||||
def owners(self, name: str) -> list[int]:
|
||||
@@ -263,6 +266,7 @@ class RdtRouter:
|
||||
Raises:
|
||||
ValueError: an owner set is empty or out of range, or a class index
|
||||
does not resolve.
|
||||
|
||||
"""
|
||||
for c, row in enumerate(self._owner_sets):
|
||||
if not row:
|
||||
@@ -282,7 +286,8 @@ class RdtRouter:
|
||||
def bind(self, actors: list, produce_methods: list, consumer_id: int) -> None:
|
||||
"""Attach this consumer's producer handles. Rebuilt wholesale per init,
|
||||
never appended to: every owner index is a position in these lists, so a
|
||||
rejoining engine that re-inits must not shift them."""
|
||||
rejoining engine that re-inits must not shift them.
|
||||
"""
|
||||
if len(actors) != len(produce_methods):
|
||||
raise ValueError("actors and produce_methods must be parallel")
|
||||
self._actors = list(actors)
|
||||
@@ -345,5 +350,6 @@ def buffer_alloc_bytes(nbytes: int, presize: int = 0) -> int:
|
||||
optional ``presize`` floor, and a coarse 256MB round-up, so the buffer is
|
||||
allocated ONCE and never regrows. Regrowth is a correctness hazard, not just a
|
||||
perf one -- see ``buffer_presize_gb``. Shared by the consumer's receive buffers
|
||||
and the producer's serve rings."""
|
||||
and the producer's serve rings.
|
||||
"""
|
||||
return max(nbytes, presize, -(-nbytes // (256 << 20)) * (256 << 20))
|
||||
|
||||
@@ -134,7 +134,8 @@ class _PendingPull:
|
||||
|
||||
``targets``/``blob`` must stay strongly referenced until it completes:
|
||||
``set_target_for_ref`` stores WEAKREFS, so dropping them silently reroutes the
|
||||
transfer into a fallback buffer."""
|
||||
transfer into a fallback buffer.
|
||||
"""
|
||||
|
||||
ref: "Any"
|
||||
keys: "list[FetchKey]"
|
||||
@@ -561,7 +562,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
|
||||
def _build_static_plan(self, init_info: ShardedRDTWeightTransferInitInfo) -> None:
|
||||
"""Build the chunk/free plan once. It never changes across syncs, so
|
||||
``update_weights`` needs no per-sync names."""
|
||||
``update_weights`` needs no per-sync names.
|
||||
"""
|
||||
if not init_info.group_lens:
|
||||
raise ValueError(
|
||||
"Sharded RDT engine requires init_info.group_lens (the gather-"
|
||||
@@ -585,7 +587,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
Both sides are sized from the static plan: receive buffers are
|
||||
``ring_depth`` slots at the largest chunk's ``pack_bytes``, and each bound
|
||||
producer is asked to pre-register a serve ring at the max bytes this
|
||||
consumer will pull from it. A no-op when this worker has no chunks."""
|
||||
consumer will pull from it. A no-op when this worker has no chunks.
|
||||
"""
|
||||
plan = self._cached_plan
|
||||
if plan is None or not plan.chunks:
|
||||
return
|
||||
@@ -630,7 +633,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
``data_parallel_index``, not ``data_parallel_rank``: vLLM resets the
|
||||
latter to 0 in a dense worker but keeps the former as the distinct global
|
||||
DP rank. Same formula as the sibling ``nccl_engine``, so dense-via-TP and
|
||||
MoE-via-DP+EP both yield distinct 0..C-1."""
|
||||
MoE-via-DP+EP both yield distinct 0..C-1.
|
||||
"""
|
||||
pc = self.parallel_config
|
||||
return pc.data_parallel_index * pc.world_size + pc.rank # world_size = TP*PP
|
||||
|
||||
@@ -638,14 +642,16 @@ class ShardedRDTWeightTransferEngine(
|
||||
"""Total inference-worker count. Prefers the driver-supplied
|
||||
``init_info.num_consumers`` (authoritative -- the driver knows the whole
|
||||
fleet); else ``world_size_across_dp``, the same stride
|
||||
``_global_worker_index`` indexes with, so the two agree at any pp."""
|
||||
``_global_worker_index`` indexes with, so the two agree at any pp.
|
||||
"""
|
||||
if self._num_consumers_override > 0:
|
||||
return self._num_consumers_override
|
||||
return self.parallel_config.world_size_across_dp
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Put the model's params on meta so layerwise reload streams them in
|
||||
as each layer's slices land. Baked replay uses checkpoint format."""
|
||||
as each layer's slices land. Baked replay uses checkpoint format.
|
||||
"""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
@@ -654,7 +660,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Drain the deferred pull/process pipeline (so every layer is fully
|
||||
loaded) before finalizing the layerwise reload."""
|
||||
loaded) before finalizing the layerwise reload.
|
||||
"""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
)
|
||||
@@ -667,7 +674,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
device sync: post-processing is deferred to background threads and a
|
||||
sync here would block on them and serialize the pull/process pipeline.
|
||||
Completion is guaranteed by ``drain_pending`` in
|
||||
``finish_weight_update``."""
|
||||
``finish_weight_update``.
|
||||
"""
|
||||
self.receive_weights(self.parse_update_info(update_info))
|
||||
|
||||
def receive_weights(
|
||||
@@ -709,7 +717,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
device: torch.device,
|
||||
) -> list[tuple[str, torch.Tensor]]:
|
||||
"""Zero-storage lazies for ``names``, dtype/shape from the init metadata,
|
||||
all feeding the bake's recording sink."""
|
||||
all feeding the bake's recording sink.
|
||||
"""
|
||||
return [
|
||||
(
|
||||
name,
|
||||
@@ -977,7 +986,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
|
||||
def _ensure_proc_worker(self) -> None:
|
||||
"""Lazily create the per-slot events, the background CUDA stream, the work
|
||||
queue, and the single processing thread. Idempotent."""
|
||||
queue, and the single processing thread. Idempotent.
|
||||
"""
|
||||
if self._proc_thread is not None:
|
||||
return
|
||||
import queue
|
||||
@@ -1036,7 +1046,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
def _mark_slot_done(self, slot: int) -> None:
|
||||
"""Publish that a queued item's read-done event has been recorded (or the
|
||||
item failed) so a pull waiting to reuse ``slot`` can proceed to its
|
||||
CUDA-event synchronize."""
|
||||
CUDA-event synchronize.
|
||||
"""
|
||||
with self._slot_cv:
|
||||
self._slot_done[slot] += 1
|
||||
self._slot_cv.notify_all()
|
||||
@@ -1052,7 +1063,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
"""Block until the background thread has processed every queued item and
|
||||
its stream work is complete, then re-raise any error it hit. Called from
|
||||
the worker's ``finish_weight_update`` before ``finalize_layerwise_reload``
|
||||
so every baked layer is fully loaded (and ``info.reset()``-ed) first."""
|
||||
so every baked layer is fully loaded (and ``info.reset()``-ed) first.
|
||||
"""
|
||||
if self._proc_queue is not None:
|
||||
self._proc_queue.join() # every put() item task_done()'d
|
||||
# The scatter thread feeds the quant thread, so join it SECOND (all
|
||||
@@ -1104,7 +1116,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
which is exactly what an owner class is. The cut is a pure function of
|
||||
the bake and the ownership table. Copy order within a chunk is bake
|
||||
order, and a module's copies may span chunks (materialize/quant fire on
|
||||
its first/last chunk; see ``_build_call_plan``)."""
|
||||
its first/last chunk; see ``_build_call_plan``).
|
||||
"""
|
||||
assert self._router is not None
|
||||
by_class: dict[int, list[_Scatter]] = {}
|
||||
for copies in modules:
|
||||
@@ -1235,7 +1248,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
pulls nothing from. Refs are held and drained in ``drain_pending`` so
|
||||
every signal has EXECUTED before the sync ends: ``begin_sync`` resets
|
||||
the counters, and a straggler landing in the next sync would credit a
|
||||
group it does not belong to."""
|
||||
group it does not belong to.
|
||||
"""
|
||||
assert self._router is not None
|
||||
self._pending_frees.extend(self._router.free_group(group_idx))
|
||||
|
||||
@@ -1370,7 +1384,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
the modules' scatters via ``ready``; touches only the scattered params
|
||||
(never a receive slot), so it can overlap subsequent chunks' RDMA and
|
||||
scatters. ``info.reset()`` is what makes finalize skip the layer —
|
||||
drain_pending joins the quant queue before finalize runs."""
|
||||
drain_pending joins the quant queue before finalize runs.
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
@@ -1403,7 +1418,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
|
||||
def _quant_worker_loop(self) -> None:
|
||||
"""Dedicated quant thread: drains (completed_modules, scatter-done event)
|
||||
batches. Errors surface via _proc_error like the scatter thread's."""
|
||||
batches. Errors surface via _proc_error like the scatter thread's.
|
||||
"""
|
||||
torch.cuda.set_device(self.device)
|
||||
q = self._quant_queue
|
||||
assert q is not None
|
||||
|
||||
@@ -206,7 +206,8 @@ class FakeRDTTensor(torch.Tensor):
|
||||
|
||||
def _meta(self) -> torch.Tensor:
|
||||
"""A zero-storage meta tensor of this fake's shape/dtype, so PyTorch
|
||||
itself computes post-op geometry rather than us reimplementing it."""
|
||||
itself computes post-op geometry rather than us reimplementing it.
|
||||
"""
|
||||
return torch.empty(self.shape, dtype=self.dtype, device="meta")
|
||||
|
||||
@classmethod
|
||||
@@ -250,7 +251,8 @@ class FakeRDTTensor(torch.Tensor):
|
||||
):
|
||||
"""Append the op and return a child, or a tuple of children for
|
||||
multi-return ops. Each child's geometry comes from running the op on a
|
||||
meta tensor."""
|
||||
meta tensor.
|
||||
"""
|
||||
meta = self_._meta()
|
||||
with torch._C.DisableTorchFunctionSubclass():
|
||||
meta_result = func(meta, *args, **kwargs)
|
||||
|
||||
@@ -132,7 +132,8 @@ PRODUCE_METHOD_NAME = "rdt_produce_weights_batched"
|
||||
@dataclass
|
||||
class _SharedPack:
|
||||
"""One generation of one slot-sharing group: the rendezvous state for a
|
||||
single chunk, identified by the consumers' issue index."""
|
||||
single chunk, identified by the consumers' issue index.
|
||||
"""
|
||||
|
||||
group: int
|
||||
slot: int
|
||||
@@ -466,7 +467,8 @@ class _RDTProducerServer:
|
||||
|
||||
def end_sync(self) -> list[int]:
|
||||
"""Block until every published group has been freed by its consumers;
|
||||
return the remaining freed keys so the engine drops its last refs."""
|
||||
return the remaining freed keys so the engine drops its last refs.
|
||||
"""
|
||||
with self._cache_cond:
|
||||
self._wait_for(
|
||||
lambda: bool(self._inflight_groups),
|
||||
@@ -479,7 +481,8 @@ class _RDTProducerServer:
|
||||
|
||||
def set_gather_error(self, message: str) -> None:
|
||||
"""Record a trainer-side gather failure so blocked serves / publishes
|
||||
stop waiting and surface it."""
|
||||
stop waiting and surface it.
|
||||
"""
|
||||
with self._cache_cond:
|
||||
self._gather_error = RuntimeError(message)
|
||||
self._cache_cond.notify_all()
|
||||
@@ -508,7 +511,8 @@ class _RDTProducerServer:
|
||||
def _new_serve_buffer(self, nbytes: int) -> torch.Tensor:
|
||||
"""Allocate + NIXL-register one serve slot: the single allocation seam,
|
||||
so registration cannot be skipped on either of the two paths that make
|
||||
buffers (the init-time reservation and the serve-path backstop)."""
|
||||
buffers (the init-time reservation and the serve-path backstop).
|
||||
"""
|
||||
t = torch.empty(nbytes, dtype=torch.uint8, device=self._serve_device)
|
||||
with self._reg_lock:
|
||||
register_nixl_memory(t)
|
||||
@@ -534,6 +538,7 @@ class _RDTProducerServer:
|
||||
Raises:
|
||||
RuntimeError: two consumers of one sharing group disagree on the
|
||||
chunks they pull from this producer.
|
||||
|
||||
"""
|
||||
sg = self._share_group(consumer_id)
|
||||
if plan_digest is not None:
|
||||
@@ -579,13 +584,15 @@ class _RDTProducerServer:
|
||||
def _share_group(self, consumer_id: int) -> int:
|
||||
"""The slot-sharing group ``consumer_id`` belongs to: its index within
|
||||
its own deployment, since that is what fixes the plan. Without a width
|
||||
the group is the consumer itself and nothing is shared."""
|
||||
the group is the consumer itself and nothing is shared.
|
||||
"""
|
||||
return consumer_id % self._share_width if self._share_width > 0 else consumer_id
|
||||
|
||||
def _sharers_of(self, sg: int) -> frozenset:
|
||||
"""The LIVE consumers of group ``sg``: the rendezvous width. Derived from
|
||||
``begin_sync``'s live set, so a degraded sync narrows the barrier instead
|
||||
of waiting forever on a dead deployment. Caller holds ``_cache_cond``."""
|
||||
of waiting forever on a dead deployment. Caller holds ``_cache_cond``.
|
||||
"""
|
||||
cached = self._sharers.get(sg)
|
||||
if cached is None:
|
||||
if not self._sharing_active or self._live_ids is None:
|
||||
@@ -621,7 +628,8 @@ class _RDTProducerServer:
|
||||
def _await_shared_pack(self, gen: _SharedPack) -> torch.Tensor:
|
||||
"""Block until this generation's packer published its blob, and return
|
||||
it. A failed pack is re-raised here, so every sharer of a bad pack fails
|
||||
instead of reading a half-written slot."""
|
||||
instead of reading a half-written slot.
|
||||
"""
|
||||
with self._cache_cond:
|
||||
self._wait_for(
|
||||
lambda: not gen.done and gen.error is None,
|
||||
@@ -642,7 +650,8 @@ class _RDTProducerServer:
|
||||
"""Group ``sg``'s ring slot ``idx``, grown if this chunk outgrew the
|
||||
reservation. Growing registers memory while the fabric is busy, the
|
||||
hazard ``reserve_serve_buffer`` exists to avoid, so it is a backstop: the
|
||||
reservation is sized from the same static plan as this pack."""
|
||||
reservation is sized from the same static plan as this pack.
|
||||
"""
|
||||
with self._serve_lock:
|
||||
buffer = self._serve_rings.setdefault(sg, [None] * self._nring)[idx]
|
||||
if buffer is not None and buffer.numel() >= need:
|
||||
@@ -654,7 +663,8 @@ class _RDTProducerServer:
|
||||
|
||||
def _fail_shared_pack(self, gen: _SharedPack, exc: BaseException) -> None:
|
||||
"""Publish a pack failure so waiting sharers raise instead of hanging on
|
||||
a generation that will never complete."""
|
||||
a generation that will never complete.
|
||||
"""
|
||||
with self._cache_cond:
|
||||
gen.error = exc
|
||||
self._cache_cond.notify_all()
|
||||
@@ -681,6 +691,7 @@ class _RDTProducerServer:
|
||||
|
||||
Raises:
|
||||
ValueError: ``seq`` was not supplied.
|
||||
|
||||
"""
|
||||
needed = sorted({n for n, _ in specs})
|
||||
if self._served_names is not None:
|
||||
@@ -855,7 +866,8 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
def _rpc(self, method: str, *args: Any) -> Any:
|
||||
"""Call one of the server actor's methods and block for the result.
|
||||
The single seam through which the engine talks to its server, so tests
|
||||
can inject a local (non-Ray) fake server."""
|
||||
can inject a local (non-Ray) fake server.
|
||||
"""
|
||||
import ray
|
||||
|
||||
return ray.get(getattr(self._server, method).remote(*args))
|
||||
@@ -932,7 +944,8 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
"""Fire publish_group WITHOUT waiting on the RPC (the gather loop
|
||||
overlaps the publish's server-side rebuild with the next group's
|
||||
gather) and return a handle that ``_await_publish`` resolves. Ray actor
|
||||
handle in production; a plain (non-Ray) fake server runs inline."""
|
||||
handle in production; a plain (non-Ray) fake server runs inline.
|
||||
"""
|
||||
method = self._server.publish_group
|
||||
remote = getattr(method, "remote", None)
|
||||
if remote is not None:
|
||||
@@ -943,7 +956,8 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
"""Resolve one async publish so a server-side rebuild error surfaces at
|
||||
window depth instead of at end_sync. Publishes carry nothing back —
|
||||
freed groups flow only through wait_freed/end_sync (one channel; see
|
||||
publish_group)."""
|
||||
publish_group).
|
||||
"""
|
||||
import ray
|
||||
|
||||
if isinstance(ref, ray.ObjectRef):
|
||||
@@ -1023,6 +1037,7 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
|
||||
Raises:
|
||||
RuntimeError: called before ``trainer_init`` cached the server names.
|
||||
|
||||
"""
|
||||
if self._server_names is None:
|
||||
raise RuntimeError(
|
||||
@@ -1129,7 +1144,8 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
by stall watchdog 300s later: consumers route pulls here, the pull
|
||||
passes the served-names guard, and the cache wait never completes. With
|
||||
it, that is an immediate error naming the weight. One set lookup per
|
||||
name per sync."""
|
||||
name per sync.
|
||||
"""
|
||||
if self._held_names is None:
|
||||
return
|
||||
for name, tensor in zip(names, tensors):
|
||||
@@ -1330,7 +1346,8 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
credit releases when every live consumer has signaled the group). So
|
||||
the loop self-paces to the consumers' pull rate with at most
|
||||
`gather_lookahead + 1` groups resident. Runs on every rank; only the
|
||||
sender has an `update_future` to fail fast on."""
|
||||
sender has an `update_future` to fail fast on.
|
||||
"""
|
||||
assert self.source is not None # guaranteed by trainer_init
|
||||
# The live IDS ride along with the count: the free barrier needs only
|
||||
# the count, but the slot-sharing rendezvous needs to know WHICH
|
||||
|
||||
@@ -68,7 +68,8 @@ class SparseNCCLTrainerInitInfo(TrainerInitInfo):
|
||||
|
||||
Same rendezvous shape as the dense NCCL backend (the sender opens its
|
||||
endpoint as NCCL rank 0), but with no packed wire params: sparse transfers
|
||||
are never packed. `backend` is the factory dispatch key."""
|
||||
are never packed. `backend` is the factory dispatch key.
|
||||
"""
|
||||
|
||||
backend: ClassVar[str] = "sparse_nccl"
|
||||
|
||||
@@ -113,8 +114,7 @@ class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
class SparseNCCLWeightTransferEngine(
|
||||
WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo]
|
||||
):
|
||||
"""
|
||||
Sparse weight transfer engine using NCCL.
|
||||
"""Sparse weight transfer engine using NCCL.
|
||||
|
||||
Receives checkpoint-coordinate patches broadcast from the trainer and applies
|
||||
them through the model's native weight loader. Sparse updates modify initialized
|
||||
@@ -368,7 +368,8 @@ class SparseNCCLTrainerWeightTransferEngine(
|
||||
rebuild or free the patch tensors as soon as a send method returns rather
|
||||
than relying on same-stream ordering. See
|
||||
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
|
||||
cross-rank barrier."""
|
||||
cross-rank barrier.
|
||||
"""
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user