Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
+3 -3
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any, Optional, Union
from typing import Any
import torch
import torch.distributed
@@ -30,13 +30,13 @@ def tensor_model_parallel_reduce_scatter(
def tensor_model_parallel_gather(
input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""Gather the input tensor across model parallel group."""
return get_tp_group().gather(input_, dst, dim)
def broadcast_tensor_dict(
tensor_dict: Optional[dict[Any, Union[torch.Tensor, Any]]] = None, src: int = 0
tensor_dict: dict[Any, torch.Tensor | Any] | None = None, src: int = 0
):
if not torch.distributed.is_initialized():
return tensor_dict
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any, Optional
from typing import Any
import torch
import torch.distributed as dist
@@ -368,7 +368,7 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
return handle
# DeepEP LL uses RDMA so no SMs are used for communication
def max_sms_used(self) -> Optional[int]:
def max_sms_used(self) -> int | None:
return 0
@@ -10,7 +10,7 @@ import sys
import tempfile
from collections.abc import Sequence
from itertools import product
from typing import Any, Optional
from typing import Any
import torch
import torch.distributed as dist
@@ -86,7 +86,7 @@ def producer(
producer_queue,
consumer_queue,
result_queue,
cuda_visible_devices: Optional[str] = None,
cuda_visible_devices: str | None = None,
):
if cuda_visible_devices is not None:
update_environment_variables({"CUDA_VISIBLE_DEVICES": cuda_visible_devices})
@@ -120,7 +120,7 @@ def consumer(
producer_queue,
consumer_queue,
result_queue,
cuda_visible_devices: Optional[str] = None,
cuda_visible_devices: str | None = None,
):
if cuda_visible_devices is not None:
update_environment_variables({"CUDA_VISIBLE_DEVICES": cuda_visible_devices})
@@ -253,7 +253,7 @@ def can_actually_p2p(
# e.g. used by different vllm engines. The device id in the cache file is a
# **local** device id, i.e. from 0 to num_dev-1, where num_dev is the number
# of visible devices in the vllm engine.
_gpu_p2p_access_cache: Optional[dict[str, bool]] = None
_gpu_p2p_access_cache: dict[str, bool] | None = None
def gpu_p2p_access_check(src: int, tgt: int) -> bool:
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from typing import Optional, Union
from weakref import WeakValueDictionary
import torch
@@ -75,7 +74,7 @@ class All2AllManagerBase:
def set_num_sms(self, num_sms: int):
pass
def max_sms_used(self) -> Optional[int]:
def max_sms_used(self) -> int | None:
return None # None means it could use the whole GPU
def combine(self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False):
@@ -96,8 +95,8 @@ class DeviceCommunicatorBase:
def __init__(
self,
cpu_group: ProcessGroup,
device: Optional[torch.device] = None,
device_group: Optional[ProcessGroup] = None,
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
):
self.device = device or torch.device("cpu")
@@ -123,7 +122,7 @@ class DeviceCommunicatorBase:
self.is_ep_communicator = "ep" in unique_name
self.use_all2all = self.is_ep_communicator and use_ep
self.all2all_manager: Optional[All2AllManagerBase] = None
self.all2all_manager: All2AllManagerBase | None = None
def all_reduce(self, input_: torch.Tensor) -> torch.Tensor:
dist.all_reduce(input_, group=self.device_group)
@@ -156,10 +155,10 @@ class DeviceCommunicatorBase:
def all_gatherv(
self,
input_: Union[torch.Tensor, list[torch.Tensor]],
input_: torch.Tensor | list[torch.Tensor],
dim: int = 0,
sizes: Optional[list[int]] = None,
) -> Union[torch.Tensor, list[torch.Tensor]]:
sizes: list[int] | None = None,
) -> torch.Tensor | list[torch.Tensor]:
raise NotImplementedError
def reduce_scatter(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
@@ -196,13 +195,13 @@ class DeviceCommunicatorBase:
return output_tensor.movedim(0, dim).contiguous()
def reduce_scatterv(
self, input_: torch.Tensor, dim: int = -1, sizes: Optional[list[int]] = None
self, input_: torch.Tensor, dim: int = -1, sizes: list[int] | None = None
) -> torch.Tensor:
raise NotImplementedError
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""
NOTE: We assume that the input tensor is on the same device across
all the ranks.
@@ -231,7 +230,7 @@ class DeviceCommunicatorBase:
output_tensor = None
return output_tensor
def send(self, tensor: torch.Tensor, dst: Optional[int] = None) -> None:
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
"""Sends a tensor to the destination rank in a blocking way"""
"""NOTE: `dst` is the local rank of the destination rank."""
if dst is None:
@@ -239,7 +238,7 @@ class DeviceCommunicatorBase:
torch.distributed.send(tensor, self.ranks[dst], self.device_group)
def recv(
self, size: torch.Size, dtype: torch.dtype, src: Optional[int] = None
self, size: torch.Size, dtype: torch.dtype, src: int | None = None
) -> torch.Tensor:
"""Receives a tensor from the source rank."""
"""NOTE: `src` is the local rank of the source rank."""
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from typing import Any, Optional, Union
from typing import Any
import torch
from torch.distributed import ProcessGroup
@@ -18,8 +18,8 @@ class CpuCommunicator(DeviceCommunicatorBase):
def __init__(
self,
cpu_group: ProcessGroup,
device: Optional[torch.device] = None,
device_group: Optional[ProcessGroup] = None,
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
):
super().__init__(cpu_group, device, device_group, unique_name)
@@ -38,7 +38,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""
NOTE: We assume that the input tensor is on the same device across
all the ranks.
@@ -99,7 +99,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
def send_tensor_dict(
self,
tensor_dict: dict[str, Union[torch.Tensor, Any]],
tensor_dict: dict[str, torch.Tensor | Any],
dst: int,
) -> None:
return self.dist_module.send_tensor_dict(tensor_dict, dst)
@@ -107,7 +107,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
def recv_tensor_dict(
self,
src: int,
) -> dict[str, Union[torch.Tensor, Any]]:
) -> dict[str, torch.Tensor | Any]:
return self.dist_module.recv_tensor_dict(src)
@@ -140,16 +140,16 @@ class _CPUSHMDistributed:
return handle
def all_reduce(
self, input: torch.Tensor, group: Optional[ProcessGroup] = None
self, input: torch.Tensor, group: ProcessGroup | None = None
) -> None:
torch.ops._C.shm_allreduce(self.handle, input)
def gather(
self,
input: torch.Tensor,
gather_list: Optional[list[torch.Tensor]],
gather_list: list[torch.Tensor] | None,
dst: int = -1,
group: Optional[ProcessGroup] = None,
group: ProcessGroup | None = None,
) -> None:
# Note: different from the torch gather, here we use local dst rank.
torch.ops._C.shm_gather(
@@ -163,13 +163,13 @@ class _CPUSHMDistributed:
self,
output: torch.Tensor,
input: torch.Tensor,
group: Optional[ProcessGroup] = None,
group: ProcessGroup | None = None,
) -> None:
torch.ops._C.shm_all_gather(self.handle, input, output)
def send_tensor_dict(
self,
tensor_dict: dict[str, Union[torch.Tensor, Any]],
tensor_dict: dict[str, torch.Tensor | Any],
dst: int,
) -> None:
key_list = list(tensor_dict.keys())
@@ -191,7 +191,7 @@ class _CPUSHMDistributed:
def recv_tensor_dict(
self,
src: int,
) -> dict[str, Union[torch.Tensor, Any]]:
) -> dict[str, torch.Tensor | Any]:
tensor_list = torch.ops._C.shm_recv_tensor_list(self.handle, src)
value_list: list[torch.Tensor] = tensor_list[:-1]
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional, Union
import torch
from torch.distributed import ProcessGroup
@@ -26,8 +25,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
def __init__(
self,
cpu_group: ProcessGroup,
device: Optional[torch.device] = None,
device_group: Optional[ProcessGroup] = None,
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
):
super().__init__(cpu_group, device, device_group, unique_name)
@@ -54,7 +53,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
)
from vllm.distributed.device_communicators.symm_mem import SymmMemCommunicator
self.pynccl_comm: Optional[PyNcclCommunicator] = None
self.pynccl_comm: PyNcclCommunicator | None = None
if self.world_size > 1:
self.pynccl_comm = PyNcclCommunicator(
group=self.cpu_group,
@@ -63,9 +62,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
if is_symmetric_memory_enabled():
register_nccl_symmetric_ops(self.pynccl_comm)
self.ca_comm: Optional[CustomAllreduce] = None
self.qr_comm: Optional[QuickAllReduce] = None
self.symm_mem_comm: Optional[SymmMemCommunicator] = None
self.ca_comm: CustomAllreduce | None = None
self.qr_comm: QuickAllReduce | None = None
self.symm_mem_comm: SymmMemCommunicator | None = None
if use_torch_symm_mem and current_platform.is_cuda():
self.symm_mem_comm = SymmMemCommunicator(
group=self.cpu_group,
@@ -201,7 +200,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
return output.movedim(0, dim).contiguous()
def reduce_scatterv(
self, input_: torch.Tensor, dim: int = -1, sizes: Optional[list[int]] = None
self, input_: torch.Tensor, dim: int = -1, sizes: list[int] | None = None
):
world_size = self.world_size
pynccl_comm = self.pynccl_comm
@@ -235,7 +234,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
# Reshape before returning
return output.movedim(0, dim).contiguous()
def send(self, tensor: torch.Tensor, dst: Optional[int] = None) -> None:
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
"""Sends a tensor to the destination rank in a blocking way"""
"""NOTE: `dst` is the local rank of the destination rank."""
if dst is None:
@@ -248,7 +247,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
torch.distributed.send(tensor, self.ranks[dst], self.device_group)
def recv(
self, size: torch.Size, dtype: torch.dtype, src: Optional[int] = None
self, size: torch.Size, dtype: torch.dtype, src: int | None = None
) -> torch.Tensor:
"""Receives a tensor from the source rank."""
"""NOTE: `src` is the local rank of the source rank."""
@@ -274,9 +273,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
def all_gatherv(
self,
input_: Union[torch.Tensor, list[torch.Tensor]],
input_: torch.Tensor | list[torch.Tensor],
dim: int = 0,
sizes: Optional[list[int]] = None,
sizes: list[int] | None = None,
):
if dim != 0:
raise NotImplementedError("only dim 0 all-gatherv is supported")
@@ -289,7 +288,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
if sizes is not None and all(s == sizes[0] for s in sizes):
sizes = None
def _all_gather_single(input_: torch.Tensor, sizes: Optional[list[int]] = None):
def _all_gather_single(input_: torch.Tensor, sizes: list[int] | None = None):
input_size = input_.size()
if sizes is not None:
assert len(sizes) == world_size
@@ -7,7 +7,7 @@ convenient for use when we just need to call a few functions.
import ctypes
from dataclasses import dataclass
from typing import Any, Optional
from typing import Any
# this line makes it possible to directly load `libcudart.so` using `ctypes`
import torch # noqa
@@ -36,7 +36,7 @@ class Function:
argtypes: list[Any]
def find_loaded_library(lib_name) -> Optional[str]:
def find_loaded_library(lib_name) -> str | None:
"""
According to according to https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html,
the file `/proc/self/maps` contains the memory maps of the process, which includes the
@@ -113,7 +113,7 @@ class CudaRTLibrary:
# to the corresponding dictionary
path_to_dict_mapping: dict[str, dict[str, Any]] = {}
def __init__(self, so_file: Optional[str] = None):
def __init__(self, so_file: str | None = None):
if so_file is None:
so_file = find_loaded_library("libcudart")
if so_file is None:
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import contextmanager
from typing import Optional, Union
import torch
import torch.distributed as dist
@@ -55,7 +54,7 @@ class CustomAllreduce:
def __init__(
self,
group: ProcessGroup,
device: Union[int, str, torch.device],
device: int | str | torch.device,
max_size=8192 * 1024,
symm_mem_enabled=False,
) -> None:
@@ -260,7 +259,7 @@ class CustomAllreduce:
)
return out
def custom_all_reduce(self, input: torch.Tensor) -> Optional[torch.Tensor]:
def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor | None:
"""The main allreduce API that provides support for cuda graph."""
# When custom allreduce is disabled, this will be None.
if self.disabled or not self.should_custom_ar(input):
@@ -292,8 +291,8 @@ class CustomAllreduce:
@staticmethod
def create_shared_buffer(
size_in_bytes: int,
group: Optional[ProcessGroup] = None,
uncached: Optional[bool] = False,
group: ProcessGroup | None = None,
uncached: bool | None = False,
) -> list[int]:
pointer, handle = ops.allocate_shared_buffer_and_handle(size_in_bytes)
@@ -313,8 +312,8 @@ class CustomAllreduce:
@staticmethod
def free_shared_buffer(
pointers: list[int],
group: Optional[ProcessGroup] = None,
rank: Optional[int] = None,
group: ProcessGroup | None = None,
rank: int | None = None,
) -> None:
if rank is None:
rank = dist.get_rank(group=group)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional, Union
# ===================== import region =====================
import torch
@@ -59,9 +58,9 @@ def register_nccl_symmetric_ops(pynccl_comm):
class PyNcclCommunicator:
def __init__(
self,
group: Union[ProcessGroup, StatelessProcessGroup],
device: Union[int, str, torch.device],
library_path: Optional[str] = None,
group: ProcessGroup | StatelessProcessGroup,
device: int | str | torch.device,
library_path: str | None = None,
):
"""
Args:
@@ -3,7 +3,7 @@
import atexit
import contextlib
import tempfile
from typing import Any, Optional
from typing import Any
import torch
from packaging import version
@@ -141,7 +141,7 @@ class nccl_symm_mem_context:
or version.parse(torch.__version__) < version.parse("2.8.0.a0")
)
if self.disabled:
self.pynccl_comm: Optional[PyNcclCommunicator] = None
self.pynccl_comm: PyNcclCommunicator | None = None
self._mem_pool_ctx: contextlib.AbstractContextManager[Any] = (
contextlib.nullcontext()
)
@@ -25,7 +25,7 @@
import ctypes
import platform
from dataclasses import dataclass
from typing import Any, Optional
from typing import Any
import torch
from torch.distributed import ReduceOp
@@ -305,7 +305,7 @@ class NCCLLibrary:
# to the corresponding dictionary
path_to_dict_mapping: dict[str, dict[str, Any]] = {}
def __init__(self, so_file: Optional[str] = None):
def __init__(self, so_file: str | None = None):
so_file = so_file or find_nccl_library()
try:
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import Union
import torch
import torch.distributed as dist
@@ -58,9 +57,7 @@ class QuickAllReduce:
(torch.bfloat16, 8): [16 * MB, 2048 * MB, 2048 * MB, 2048 * MB],
}
def __init__(
self, group: ProcessGroup, device: Union[int, str, torch.device]
) -> None:
def __init__(self, group: ProcessGroup, device: int | str | torch.device) -> None:
"""
Custom allreduce provides non-destructive acceleration and is
available for CUDA and ROCm MI300 series.
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import uuid
from typing import Any, Optional
from typing import Any
import ray
import torch
@@ -27,15 +27,15 @@ class RayPPCommunicator(Communicator):
This class is not thread-safe.
"""
_comm: Optional[DeviceCommunicatorBase]
_comm: DeviceCommunicatorBase | None
def __init__(
self,
world_size: int,
comm_id: Any,
rank: Optional[int],
rank: int | None,
actor_handles: list["ray.actor.ActorHandle"],
cuda_stream: Optional[torch.cuda.Stream],
cuda_stream: torch.cuda.Stream | None,
use_communication_streams: bool = False,
):
"""
@@ -56,7 +56,7 @@ class RayPPCommunicator(Communicator):
This is not supported.
"""
self._world_size = world_size
self._rank: Optional[int] = None
self._rank: int | None = None
self._actor_handles = actor_handles
if use_communication_streams:
raise NotImplementedError("use_communication_streams is not supported")
@@ -143,7 +143,7 @@ class RayPPCommunicator(Communicator):
else:
raise ValueError(f"Actor {actor} not found in communicator group")
def get_self_rank(self) -> Optional[int]:
def get_self_rank(self) -> int | None:
"""
Return this actor's rank.
"""
@@ -7,7 +7,7 @@ from contextlib import contextmanager
from dataclasses import dataclass, field
from multiprocessing import shared_memory
from threading import Event
from typing import Any, Optional, Union
from typing import Any
from unittest.mock import patch
import torch
@@ -80,7 +80,7 @@ class ShmRingBuffer:
n_reader: int,
max_chunk_bytes: int,
max_chunks: int,
name: Optional[str] = None,
name: str | None = None,
):
"""
A shared memory ring buffer implementation for broadcast communication.
@@ -213,9 +213,9 @@ class ShmRingBuffer:
class Handle:
local_reader_ranks: list[int] = field(default_factory=list)
buffer_handle: Optional[tuple[int, int, int, str]] = None
local_subscribe_addr: Optional[str] = None
remote_subscribe_addr: Optional[str] = None
buffer_handle: tuple[int, int, int, str] | None = None
local_subscribe_addr: str | None = None
remote_subscribe_addr: str | None = None
remote_addr_ipv6: bool = False
@@ -224,10 +224,10 @@ class MessageQueue:
self,
n_reader, # number of all readers
n_local_reader, # number of local readers through shared memory
local_reader_ranks: Optional[list[int]] = None,
local_reader_ranks: list[int] | None = None,
max_chunk_bytes: int = 1024 * 1024 * 10,
max_chunks: int = 10,
connect_ip: Optional[str] = None,
connect_ip: str | None = None,
):
if local_reader_ranks is None:
local_reader_ranks = list(range(n_local_reader))
@@ -384,7 +384,7 @@ class MessageQueue:
assert recv == b"READY"
@contextmanager
def acquire_write(self, timeout: Optional[float] = None):
def acquire_write(self, timeout: float | None = None):
assert self._is_writer, "Only writers can acquire write"
start_time = time.monotonic()
n_warning = 1
@@ -444,8 +444,8 @@ class MessageQueue:
@contextmanager
def acquire_read(
self,
timeout: Optional[float] = None,
cancel: Optional[Event] = None,
timeout: float | None = None,
cancel: Event | None = None,
indefinite: bool = False,
):
assert self._is_local_reader, "Only readers can acquire read"
@@ -502,7 +502,7 @@ class MessageQueue:
self._read_spin_timer.record_activity()
break
def enqueue(self, obj, timeout: Optional[float] = None):
def enqueue(self, obj, timeout: float | None = None):
"""Write to message queue with optional timeout (in seconds)"""
assert self._is_writer, "Only writers can enqueue"
serialized_obj = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL)
@@ -520,8 +520,8 @@ class MessageQueue:
def dequeue(
self,
timeout: Optional[float] = None,
cancel: Optional[Event] = None,
timeout: float | None = None,
cancel: Event | None = None,
indefinite: bool = False,
):
"""Read from message queue with optional timeout (in seconds)"""
@@ -542,7 +542,7 @@ class MessageQueue:
return obj
@staticmethod
def recv(socket: zmq.Socket, timeout: Optional[float]) -> Any:
def recv(socket: zmq.Socket, timeout: float | None) -> Any:
timeout_ms = None if timeout is None else int(timeout * 1000)
if not socket.poll(timeout=timeout_ms):
raise TimeoutError
@@ -558,7 +558,7 @@ class MessageQueue:
@staticmethod
def create_from_process_group(
pg: Union[ProcessGroup, StatelessProcessGroup],
pg: ProcessGroup | StatelessProcessGroup,
max_chunk_bytes,
max_chunks,
writer_rank=0,
@@ -3,13 +3,13 @@
import pickle
from abc import ABC, abstractmethod
from collections.abc import Iterable
from collections.abc import Callable, Iterable
from contextlib import contextmanager
from dataclasses import dataclass
from itertools import chain
from multiprocessing import shared_memory
from multiprocessing.synchronize import Lock as LockType
from typing import Any, Callable, Optional, Union
from typing import Any
from unittest.mock import patch
import torch
@@ -109,7 +109,7 @@ class SingleWriterShmRingBuffer:
def __init__(
self,
data_buffer_size: int,
name: Optional[str] = None,
name: str | None = None,
create: bool = False,
):
self.data_buffer_size = data_buffer_size
@@ -252,7 +252,7 @@ class SingleWriterShmRingBuffer:
def free_buf(
self,
is_free_fn: Callable[[int, memoryview], bool],
nbytes: Optional[int] = None,
nbytes: int | None = None,
) -> Iterable[int]:
"""
Free a buffer of the given size. This is a no-op in shared memory,
@@ -340,9 +340,7 @@ class MsgpackSerde(ObjectSerde):
self.mm_decoder = MsgpackDecoder(MultiModalKwargsItem)
self._mm_kwargs_item_cls = MultiModalKwargsItem
def serialize(
self, value: Any
) -> tuple[Union[bytes, list[bytes]], int, bytes, int]:
def serialize(self, value: Any) -> tuple[bytes | list[bytes], int, bytes, int]:
len_arr = None
if isinstance(value, (torch.Tensor, self._mm_kwargs_item_cls)):
type_name = type(value).__name__
@@ -396,7 +394,7 @@ class ShmObjectStorageHandle:
n_readers: int
ring_buffer_handle: tuple[int, str]
serde_class: type[ObjectSerde]
reader_lock: Optional[LockType]
reader_lock: LockType | None
class SingleWriterShmObjectStorage:
@@ -444,7 +442,7 @@ class SingleWriterShmObjectStorage:
n_readers: int,
ring_buffer: SingleWriterShmRingBuffer,
serde_class: type[ObjectSerde] = MsgpackSerde,
reader_lock: Optional[LockType] = None,
reader_lock: LockType | None = None,
):
"""
Initialize the object storage.
@@ -492,7 +490,7 @@ class SingleWriterShmObjectStorage:
def copy_to_buffer(
self,
data: Union[bytes, list[bytes]],
data: bytes | list[bytes],
data_bytes: int,
metadata: bytes,
md_bytes: int,
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional, Union
import torch
import torch.distributed as dist
@@ -31,10 +30,10 @@ class SymmMemCommunicator:
def __init__(
self,
group: ProcessGroup,
device: Union[int, str, torch.device],
device: int | str | torch.device,
# add options for testing
force_multimem: Optional[bool] = None,
max_size_override: Optional[int] = None,
force_multimem: bool | None = None,
max_size_override: int | None = None,
):
self.disabled = True
@@ -108,8 +107,8 @@ class SymmMemCommunicator:
return inp_size < self.max_size
def all_reduce(
self, inp: torch.Tensor, *, out: Optional[torch.Tensor] = None
) -> Optional[torch.Tensor]:
self, inp: torch.Tensor, *, out: torch.Tensor | None = None
) -> torch.Tensor | None:
if not self.should_use_symm_mem(inp):
return None
if out is None:
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from typing import Optional
import torch
from torch.distributed import ProcessGroup
@@ -39,8 +38,8 @@ class TpuCommunicator(DeviceCommunicatorBase):
def __init__(
self,
cpu_group: ProcessGroup,
device: Optional[torch.device] = None,
device_group: Optional[ProcessGroup] = None,
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
):
super().__init__(cpu_group, device, device_group, unique_name)
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Optional
import torch
import torch.distributed as dist
@@ -19,8 +18,8 @@ class XpuCommunicator(DeviceCommunicatorBase):
def __init__(
self,
cpu_group: ProcessGroup,
device: Optional[torch.device] = None,
device_group: Optional[ProcessGroup] = None,
device: torch.device | None = None,
device_group: ProcessGroup | None = None,
unique_name: str = "",
):
super().__init__(cpu_group, device, device_group, unique_name)
@@ -45,7 +44,7 @@ class XpuCommunicator(DeviceCommunicatorBase):
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
assert -input_.dim() <= dim < input_.dim(), (
f"Invalid dim ({dim}) for input tensor with shape {input_.size()}"
)
+7 -8
View File
@@ -29,7 +29,6 @@ physical experts.
import time
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Optional, Union
import torch
from torch.distributed import ProcessGroup, all_reduce
@@ -186,9 +185,9 @@ class EplbState:
model: MixtureOfExperts,
device: torch.device,
parallel_config: ParallelConfig,
global_expert_load: Optional[torch.Tensor] = None,
old_global_expert_indices: Optional[torch.Tensor] = None,
rank_mapping: Optional[dict[int, int]] = None,
global_expert_load: torch.Tensor | None = None,
old_global_expert_indices: torch.Tensor | None = None,
rank_mapping: dict[int, int] | None = None,
) -> "EplbState":
"""
Build the initial EPLB state.
@@ -439,9 +438,9 @@ class EplbState:
model: MixtureOfExperts,
is_profile: bool = False,
execute_shuffle: bool = True,
global_expert_load: Optional[torch.Tensor] = None,
rank_mapping: Optional[dict[int, int]] = None,
) -> Optional[torch.Tensor]:
global_expert_load: torch.Tensor | None = None,
rank_mapping: dict[int, int] | None = None,
) -> torch.Tensor | None:
"""
Rearrange the experts according to the current load.
"""
@@ -611,7 +610,7 @@ class EplbState:
def _node_count_with_rank_mapping(
pg: Union[ProcessGroup, StatelessProcessGroup],
pg: ProcessGroup | StatelessProcessGroup,
rank_mapping: dict[int, int],
) -> int:
if isinstance(pg, ProcessGroup):
+1 -2
View File
@@ -8,7 +8,6 @@ This involves the exchange of expert weights between GPUs.
from collections.abc import Iterable, MutableSequence, Sequence
from functools import partial
from typing import Optional
import torch
from torch.distributed import (
@@ -253,7 +252,7 @@ def rearrange_expert_weights_inplace(
expert_weights: Sequence[Iterable[torch.Tensor]],
ep_group: ProcessGroup,
is_profile: bool = False,
rank_mapping: Optional[dict[int, int]] = None,
rank_mapping: dict[int, int] | None = None,
) -> None:
"""
Rearranges the expert weights in place according to the new expert indices.
+15 -14
View File
@@ -6,10 +6,11 @@ import threading
import time
from abc import ABC, abstractmethod
from collections import deque
from collections.abc import Callable
from dataclasses import asdict
from itertools import count
from queue import Queue
from typing import Any, Callable, Optional, Union
from typing import Any
import msgspec
import zmq
@@ -29,7 +30,7 @@ class EventBatch(
):
ts: float
events: list[Any]
data_parallel_rank: Optional[int] = None
data_parallel_rank: int | None = None
class KVCacheEvent(
@@ -47,16 +48,16 @@ MEDIUM_GPU = "GPU"
class BlockStored(KVCacheEvent):
block_hashes: list[ExternalBlockHash]
parent_block_hash: Optional[ExternalBlockHash]
parent_block_hash: ExternalBlockHash | None
token_ids: list[int]
block_size: int
lora_id: Optional[int]
medium: Optional[str]
lora_id: int | None
medium: str | None
class BlockRemoved(KVCacheEvent):
block_hashes: list[ExternalBlockHash]
medium: Optional[str]
medium: str | None
class AllBlocksCleared(KVCacheEvent):
@@ -64,7 +65,7 @@ class AllBlocksCleared(KVCacheEvent):
class KVEventBatch(EventBatch):
events: list[Union[BlockStored, BlockRemoved, AllBlocksCleared]]
events: list[BlockStored | BlockRemoved | AllBlocksCleared]
class EventPublisher(ABC):
@@ -139,7 +140,7 @@ class ZmqEventPublisher(EventPublisher):
self,
data_parallel_rank: int,
endpoint: str = "tcp://*:5557",
replay_endpoint: Optional[str] = None,
replay_endpoint: str | None = None,
buffer_steps: int = 10_000,
hwm: int = 100_000,
max_queue_size: int = 100_000,
@@ -147,13 +148,13 @@ class ZmqEventPublisher(EventPublisher):
) -> None:
# Storage
super().__init__(data_parallel_rank)
self._event_queue = Queue[Optional[EventBatch]](maxsize=max_queue_size)
self._event_queue = Queue[EventBatch | None](maxsize=max_queue_size)
self._buffer = deque[tuple[int, bytes]](maxlen=buffer_steps)
# ZMQ sockets
self._ctx = zmq.Context.instance()
self._pub: Optional[zmq.Socket] = None
self._replay: Optional[zmq.Socket] = None
self._pub: zmq.Socket | None = None
self._replay: zmq.Socket | None = None
self._dp_rank = data_parallel_rank
self._endpoint = self.offset_endpoint_port(endpoint, self._dp_rank)
@@ -303,8 +304,8 @@ class ZmqEventPublisher(EventPublisher):
@staticmethod
def offset_endpoint_port(
endpoint: Optional[str], data_parallel_rank: int
) -> Optional[str]:
endpoint: str | None, data_parallel_rank: int
) -> str | None:
"""Helper function to offset the port in an endpoint by
the data parallel rank.
@@ -349,7 +350,7 @@ class EventPublisherFactory:
@classmethod
def create(
cls, config: Optional[KVEventsConfig], data_parallel_rank: int = 0
cls, config: KVEventsConfig | None, data_parallel_rank: int = 0
) -> EventPublisher:
"""Create publisher from a config mapping."""
if not config:
@@ -2,7 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib
from typing import TYPE_CHECKING, Callable
from collections.abc import Callable
from typing import TYPE_CHECKING
import vllm.envs as envs
from vllm.distributed.kv_transfer.kv_connector.base import (
@@ -7,7 +7,7 @@ KV cache helper for store.
from collections import defaultdict
from collections.abc import Sequence
from concurrent.futures import CancelledError, Future
from typing import Literal, Optional, Union, cast
from typing import Literal, cast
import torch
@@ -136,7 +136,7 @@ class KVOutputAggregator:
# Aggregate kv_connector_output from all workers
def update_finished_set(
req_ids: Optional[set[str]],
req_ids: set[str] | None,
remaining_count_dict: dict[str, int],
finished_set: set[str],
) -> None:
@@ -197,7 +197,7 @@ class KVOutputAggregator:
to the respective list of outputs."""
result_future: Future[ModelRunnerOutput] = Future()
outputs: list[Optional[ModelRunnerOutput]] = [None] * len(output_futures)
outputs: list[ModelRunnerOutput | None] = [None] * len(output_futures)
def make_callback(idx):
def callback(fut):
@@ -230,8 +230,8 @@ class KVOutputAggregator:
def _make_src_and_dst_indices(
src_block_ids: list[int],
dst_block_ids: list[int],
src_device: Union[torch.device, str],
dst_device: Union[torch.device, str],
src_device: torch.device | str,
dst_device: torch.device | str,
) -> tuple[torch.Tensor, torch.Tensor]:
src_indices = torch.tensor(src_block_ids, device=src_device, dtype=torch.int64)
dst_indices = torch.tensor(dst_block_ids, device=dst_device, dtype=torch.int64)
@@ -37,8 +37,8 @@ The class provides the following primitives:
import enum
from abc import ABC, abstractmethod
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, Callable, Literal, Optional
from collections.abc import Callable, Iterable
from typing import TYPE_CHECKING, Any, Literal, Optional
import torch
@@ -93,7 +93,7 @@ class KVConnectorBase_V1(ABC):
"Initializing KVConnectorBase_V1. This API is experimental and "
"subject to change in the future as we iterate the design."
)
self._connector_metadata: Optional[KVConnectorMetadata] = None
self._connector_metadata: KVConnectorMetadata | None = None
self._vllm_config = vllm_config
self._role = role
@@ -222,7 +222,7 @@ class KVConnectorBase_V1(ABC):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens on the worker.
@@ -281,7 +281,7 @@ class KVConnectorBase_V1(ABC):
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
@@ -361,7 +361,7 @@ class KVConnectorBase_V1(ABC):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Called exactly once when a request has finished, before its blocks are
freed.
@@ -388,7 +388,7 @@ class KVConnectorBase_V1(ABC):
return ()
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> Optional[str]:
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
Args:
@@ -406,7 +406,7 @@ class KVConnectorBase_V1(ABC):
)
return None
def get_finished_count(self) -> Optional[int]:
def get_finished_count(self) -> int | None:
"""
Get the count of requests expected to complete send/receive operations
via this connector.
@@ -419,7 +419,7 @@ class KVConnectorBase_V1(ABC):
@classmethod
def build_kv_connector_stats(
cls, data: Optional[dict[str, Any]] = None
cls, data: dict[str, Any] | None = None
) -> Optional["KVConnectorStats"]:
"""
KVConnectorStats resolution method. This method allows dynamically
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Any
import torch
from lmcache.integration.vllm.vllm_v1_adapter import LMCacheConnectorV1Impl
@@ -96,7 +96,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens.
@@ -117,7 +117,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
@@ -161,7 +161,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its blocks are freed.
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
from typing import Any, Optional, Union
from typing import Any
from vllm.config.kv_transfer import KVTransferConfig
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
@@ -32,7 +32,7 @@ class KVConnectorStats:
"""
raise NotImplementedError
def reduce(self) -> dict[str, Union[int, float]]:
def reduce(self) -> dict[str, int | float]:
"""
Reduce the observations collected during a time interval to one or
more representative values (eg avg/median/sum of the series).
@@ -58,7 +58,7 @@ class KVConnectorLogging:
self.reset()
def reset(self):
self.transfer_stats_accumulator: Optional[KVConnectorStats] = None
self.transfer_stats_accumulator: KVConnectorStats | None = None
def observe(self, transfer_stats_data: dict[str, Any]):
# Should not be called when a KVConnector is not configured.
@@ -3,7 +3,7 @@
import copy
from collections.abc import Iterable
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Any
import torch
@@ -33,7 +33,7 @@ logger = init_logger(__name__)
@dataclass
class MultiKVConnectorMetadata(KVConnectorMetadata):
metadata: tuple[KVConnectorMetadata, ...]
extra_async_saves: Optional[dict[str, int]] = None
extra_async_saves: dict[str, int] | None = None
@dataclass
@@ -130,7 +130,7 @@ class MultiConnector(KVConnectorBase_V1):
c.clear_connector_metadata()
def shutdown(self):
exception: Optional[Exception] = None
exception: Exception | None = None
for c in self._connectors:
try:
c.shutdown()
@@ -169,7 +169,7 @@ class MultiConnector(KVConnectorBase_V1):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
finished_sending: set[str] = set()
finished_recving: set[str] = set()
for c in self._connectors:
@@ -207,7 +207,7 @@ class MultiConnector(KVConnectorBase_V1):
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
to_return = (0, False)
for i, c in enumerate(self._connectors):
toks, load_async = c.get_num_new_matched_tokens(
@@ -258,7 +258,7 @@ class MultiConnector(KVConnectorBase_V1):
self,
request: "Request",
blocks: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
async_saves = 0
kv_txfer_params = None
for c in self._connectors:
@@ -286,7 +286,7 @@ class MultiConnector(KVConnectorBase_V1):
yield from c.take_events()
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> Optional[str]:
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
Args:
@@ -323,17 +323,17 @@ class MultiConnector(KVConnectorBase_V1):
@classmethod
def build_kv_connector_stats(
cls, data: Optional[dict[str, Any]] = None
) -> Optional[KVConnectorStats]:
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
MultiKVConnectorStats(data=data)
if data is not None
else MultiKVConnectorStats()
)
def get_kv_connector_stats(self) -> Optional[MultiKVConnectorStats]:
def get_kv_connector_stats(self) -> MultiKVConnectorStats | None:
# Group connector stats by connector type.
stats_by_connector: Optional[MultiKVConnectorStats] = None
stats_by_connector: MultiKVConnectorStats | None = None
for c in self._connectors:
stats = c.get_kv_connector_stats()
if stats is None:
@@ -13,7 +13,7 @@ from collections import defaultdict
from collections.abc import Iterator
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Optional, Union
from typing import TYPE_CHECKING, Any
import msgspec
import numpy as np
@@ -153,10 +153,10 @@ class NixlConnector(KVConnectorBase_V1):
self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler: Optional[NixlConnectorScheduler] = (
self.connector_scheduler: NixlConnectorScheduler | None = (
NixlConnectorScheduler(vllm_config, self.engine_id)
)
self.connector_worker: Optional[NixlConnectorWorker] = None
self.connector_worker: NixlConnectorWorker | None = None
elif role == KVConnectorRole.WORKER:
self.connector_scheduler = None
self.connector_worker = NixlConnectorWorker(vllm_config, self.engine_id)
@@ -189,7 +189,7 @@ class NixlConnector(KVConnectorBase_V1):
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
@@ -214,7 +214,7 @@ class NixlConnector(KVConnectorBase_V1):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
@@ -234,14 +234,14 @@ class NixlConnector(KVConnectorBase_V1):
assert self.connector_worker is not None
return self.connector_worker.get_finished()
def get_kv_connector_stats(self) -> Optional[KVConnectorStats]:
def get_kv_connector_stats(self) -> KVConnectorStats | None:
assert self.connector_worker is not None
return self.connector_worker.get_kv_connector_stats()
@classmethod
def build_kv_connector_stats(
cls, data: Optional[dict[str, Any]] = None
) -> Optional[KVConnectorStats]:
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
NixlKVConnectorStats(data=data)
if data is not None
@@ -445,7 +445,7 @@ class NixlConnectorScheduler:
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Once a request is finished, determine whether request blocks
should be freed now or will be sent asynchronously and freed later.
@@ -584,7 +584,7 @@ class NixlConnectorWorker:
)
# Note: host xfer buffer ops when use_host_buffer is True
self.copy_blocks: Optional[CopyBlocksOp] = None
self.copy_blocks: CopyBlocksOp | None = None
# Map of engine_id -> kv_caches_base_addr. For TP case, each local
# rank will still only pull from a single remote TP worker.
@@ -615,7 +615,7 @@ class NixlConnectorWorker:
self._reqs_to_process: set[ReqId] = set()
# Background thread for handling new handshake requests.
self._nixl_handshake_listener_t: Optional[threading.Thread] = None
self._nixl_handshake_listener_t: threading.Thread | None = None
# Background thread for initializing new NIXL handshakes.
self._handshake_initiation_executor = ThreadPoolExecutor(
# NIXL is not guaranteed to be thread-safe, limit 1 worker.
@@ -635,7 +635,7 @@ class NixlConnectorWorker:
# TODO(mgoin): remove this once we have hybrid memory allocator
# Optimization for models with local attention (Llama 4)
# List of block window sizes for each layer for local attention
self.block_window_per_layer: list[Optional[int]] = []
self.block_window_per_layer: list[int | None] = []
self.use_mla = self.model_config.use_mla
backend = get_attn_backend(
@@ -1472,7 +1472,7 @@ class NixlConnectorWorker:
self._recving_transfers[request_id].append((handle, time.perf_counter()))
def _get_block_descs_ids(
self, engine_id: str, block_ids: list[int], layer_idx: Optional[int] = None
self, engine_id: str, block_ids: list[int], layer_idx: int | None = None
) -> np.ndarray:
"""
Get the descs ids for a set of block ids.
@@ -1518,7 +1518,7 @@ class NixlConnectorWorker:
block_len = self.block_len_per_layer[layer_idx]
return block_len
def get_kv_connector_stats(self) -> Optional[KVConnectorStats]:
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""
Get the KV transfer stats for the connector.
"""
@@ -1559,7 +1559,7 @@ def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
if socket_type not in (zmq.ROUTER, zmq.REQ):
raise ValueError(f"Unexpected socket type: {socket_type}")
ctx: Optional[zmq.Context] = None
ctx: zmq.Context | None = None
try:
ctx = zmq.Context() # type: ignore[attr-defined]
yield make_zmq_socket(
@@ -1611,7 +1611,7 @@ class NixlKVConnectorStats(KVConnectorStats):
accumulator.extend(v)
return self
def reduce(self) -> dict[str, Union[int, float]]:
def reduce(self) -> dict[str, int | float]:
# Compute compact representative stats suitable for CLI logging
if self.is_empty():
return {
@@ -4,7 +4,7 @@ from collections import defaultdict
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from itertools import islice
from typing import Any, Optional
from typing import Any
import torch
@@ -46,8 +46,8 @@ class OffloadingConnector(KVConnectorBase_V1):
spec = OffloadingSpecFactory.create_spec(vllm_config)
self.connector_scheduler: Optional[OffloadingConnectorScheduler] = None
self.connector_worker: Optional[OffloadingConnectorWorker] = None
self.connector_scheduler: OffloadingConnectorScheduler | None = None
self.connector_worker: OffloadingConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = OffloadingConnectorScheduler(spec)
elif role == KVConnectorRole.WORKER:
@@ -113,7 +113,7 @@ class OffloadingConnector(KVConnectorBase_V1):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
@@ -148,7 +148,7 @@ class OffloadingConnectorScheduler:
self,
req: Request,
start_idx: int = 0,
end_idx: Optional[int] = None,
end_idx: int | None = None,
) -> Iterable[BlockHash]:
return islice(
req.block_hashes,
@@ -354,7 +354,7 @@ class OffloadingConnectorScheduler:
self,
request: Request,
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its blocks are freed.
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Any
import regex as re
import torch
@@ -304,7 +304,7 @@ class P2pNcclConnector(KVConnectorBase_V1):
def get_finished(
self, finished_req_ids: set[str], **kwargs: Any
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens.
@@ -466,7 +466,7 @@ class P2pNcclConnector(KVConnectorBase_V1):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its blocks are freed.
@@ -5,11 +5,10 @@ import logging
import os
import threading
import time
import typing
from collections import deque
from contextlib import contextmanager
from dataclasses import dataclass
from typing import Any, Optional
from typing import Any
import msgpack
import torch
@@ -77,7 +76,7 @@ class P2pNcclEngine:
config: KVTransferConfig,
hostname: str = "",
port_offset: int = 0,
library_path: Optional[str] = None,
library_path: str | None = None,
) -> None:
self.config = config
self.rank = port_offset
@@ -187,7 +186,7 @@ class P2pNcclEngine:
self.nccl_num_channels,
)
def create_connect(self, remote_address: typing.Optional[str] = None):
def create_connect(self, remote_address: str | None = None):
assert remote_address is not None
if remote_address not in self.socks:
sock = self.context.socket(zmq.DEALER)
@@ -224,7 +223,7 @@ class P2pNcclEngine:
self,
tensor_id: str,
tensor: torch.Tensor,
remote_address: typing.Optional[str] = None,
remote_address: str | None = None,
) -> bool:
if remote_address is None:
with self.recv_store_cv:
@@ -296,7 +295,7 @@ class P2pNcclEngine:
def recv_tensor(
self,
tensor_id: str,
remote_address: typing.Optional[str] = None,
remote_address: str | None = None,
) -> torch.Tensor:
if self.send_type == "PUT" or self.send_type == "PUT_ASYNC":
start_time = time.time()
@@ -527,7 +526,7 @@ class P2pNcclEngine:
def get_finished(
self, finished_req_ids: set[str], no_compile_layers
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens.
@@ -3,7 +3,7 @@
import hashlib
import os
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Any
import safetensors
import torch
@@ -249,7 +249,7 @@ class SharedStorageConnector(KVConnectorBase_V1):
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
@@ -14,7 +14,6 @@ These classes above are abstracted behind class `KVCacheBufferBase`.
"""
from abc import ABC, abstractmethod
from typing import Optional
import torch
@@ -98,8 +97,8 @@ class KVLookupBufferBase(KVCacheBufferBase):
@abstractmethod
def drop_select(
self, input_tokens: Optional[torch.Tensor], roi: Optional[torch.Tensor]
) -> list[Optional[torch.Tensor]]:
self, input_tokens: torch.Tensor | None, roi: torch.Tensor | None
) -> list[torch.Tensor | None]:
"""Select and *drop* KV cache entries from the lookup buffer.
The functionality is similar to the following python statements
@@ -143,7 +142,7 @@ class KVStoreBufferBase(KVCacheBufferBase):
def put(
self,
key: str,
value: Optional[torch.Tensor],
value: torch.Tensor | None,
) -> None:
"""Store a key-value pair in the buffer.
@@ -163,7 +162,7 @@ class KVStoreBufferBase(KVCacheBufferBase):
def get(
self,
key: str,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""Retrieve a value from the buffer by key.
Args:
@@ -10,7 +10,6 @@ from this remote lookup buffer.
import json
import os
from dataclasses import dataclass
from typing import Optional
import torch
from safetensors.torch import load as safetensors_load
@@ -110,7 +109,7 @@ class MooncakeStore(KVStoreBufferBase):
def put(
self,
key: str,
value: Optional[torch.Tensor],
value: torch.Tensor | None,
) -> None:
# A message queue needs to be introduced before making it asynchronous.
if value is not None:
@@ -119,7 +118,7 @@ class MooncakeStore(KVStoreBufferBase):
def get(
self,
key: str,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
# A message queue needs to be introduced before making it asynchronous.
value = self._get_impl(key)
return value
@@ -142,7 +141,7 @@ class MooncakeStore(KVStoreBufferBase):
def _get_impl(
self,
key: str,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""Get KVCache from Mooncake Store"""
try:
data = self.store.get(key)
@@ -13,7 +13,6 @@ Key Features:
import threading
from collections import deque
from typing import Optional, Union
import torch
@@ -46,7 +45,7 @@ class SimpleBuffer(KVLookupBufferBase):
self.buffer_cv = threading.Condition()
self.signal_pipe = signal_pipe
self.data_pipe = data_pipe
self.request_handling_thread: Optional[threading.Thread] = None
self.request_handling_thread: threading.Thread | None = None
self.normal_signal = torch.tensor([0], device="cpu")
self.end_signal = None
@@ -81,14 +80,14 @@ class SimpleBuffer(KVLookupBufferBase):
return 0
def _send_tensor_and_dec_size(self, tensor: Optional[torch.Tensor]) -> None:
def _send_tensor_and_dec_size(self, tensor: torch.Tensor | None) -> None:
assert tensor is not None, "Use self.data_pipe.send(None) instead"
self.buffer_size -= tensor.element_size() * tensor.numel()
if tensor.dtype == torch.bool:
tensor = tensor.float()
self.data_pipe.send_tensor(tensor)
def _get_element_size(self, data: Optional[Union[list, torch.Tensor]]):
def _get_element_size(self, data: list | torch.Tensor | None):
if isinstance(data, torch.Tensor):
return data.element_size() * data.numel()
if not data:
@@ -184,8 +183,8 @@ class SimpleBuffer(KVLookupBufferBase):
logger.debug("Closing drop_select_handler")
def drop_select(
self, input_tokens: Optional[torch.Tensor], roi: Optional[torch.Tensor]
) -> list[Optional[torch.Tensor]]:
self, input_tokens: torch.Tensor | None, roi: torch.Tensor | None
) -> list[torch.Tensor | None]:
assert self.request_handling_thread is None, (
"drop_select should be called by the KV cache consumer "
"(e.g. the decode vLLM instance)"
+2 -3
View File
@@ -12,7 +12,6 @@ you can bypass this interface and directly start from `kv_lookup_buffer`.
"""
from abc import ABC, abstractmethod
from typing import Optional
import torch
@@ -24,7 +23,7 @@ class KVPipeBase(ABC):
"""
@abstractmethod
def send_tensor(self, tensor: Optional[torch.Tensor]) -> None:
def send_tensor(self, tensor: torch.Tensor | None) -> None:
"""Send a tensor, or None, via the pipe.
Need to support sending None -- important for error handling.
@@ -42,7 +41,7 @@ class KVPipeBase(ABC):
raise NotImplementedError
@abstractmethod
def recv_tensor(self) -> Optional[torch.Tensor]:
def recv_tensor(self) -> torch.Tensor | None:
"""Receive a tensor (can be None) from the pipeline.
Returns:
@@ -6,7 +6,6 @@ import os
import struct
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from typing import Optional, Union
import torch
import zmq
@@ -26,7 +25,7 @@ NONE_INT = -150886311
class MooncakeTransferEngineConfig:
prefill_url: str
decode_url: str
metadata_backend: Union[str, None]
metadata_backend: str | None
metadata_server: str
protocol: str
device_name: str
@@ -143,7 +142,7 @@ class MooncakeTransferEngine:
metadata_server: str,
protocol: str,
device_name: str,
metadata_backend: Union[str, None],
metadata_backend: str | None,
) -> None:
"""Initialize the mooncake instance."""
if metadata_backend is None:
@@ -231,7 +230,7 @@ class MooncakePipe(KVPipeBase):
"""MooncakeTransferEngine based Pipe implementation."""
def __init__(
self, local_rank: int, config: KVTransferConfig, device: Optional[str] = None
self, local_rank: int, config: KVTransferConfig, device: str | None = None
):
"""Initialize the mooncake pipe and set related parameters."""
self.config = config
@@ -243,7 +242,7 @@ class MooncakePipe(KVPipeBase):
self.device = self._select_device(device)
self.transfer_engine = MooncakeTransferEngine(self.kv_rank, self.local_rank)
self.transport_thread: Optional[ThreadPoolExecutor] = None
self.transport_thread: ThreadPoolExecutor | None = None
self.none_tensor = torch.tensor([NONE_INT], device=self.device)
def _select_device(self, device: str) -> torch.device:
@@ -267,7 +266,7 @@ class MooncakePipe(KVPipeBase):
data = self.transfer_engine.recv_bytes()
return safetensors_load(data)["tensor"].to(self.device)
def send_tensor(self, tensor: Optional[torch.Tensor]) -> None:
def send_tensor(self, tensor: torch.Tensor | None) -> None:
"""Send tensor to the target process."""
if self.transport_thread is None:
self.transport_thread = ThreadPoolExecutor(max_workers=1)
@@ -275,7 +274,7 @@ class MooncakePipe(KVPipeBase):
assert len(tensor.shape) > 0
self.transport_thread.submit(self._send_impl, tensor)
def recv_tensor(self) -> Optional[torch.Tensor]:
def recv_tensor(self) -> torch.Tensor | None:
"""Receive tensor from other processes."""
if self.transport_thread is None:
self.transport_thread = ThreadPoolExecutor(max_workers=1)
@@ -15,8 +15,8 @@ Key Features:
import threading
import time
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from typing import Callable, Optional
import torch
@@ -35,7 +35,7 @@ class BrokenPipeException(Exception):
super().__init__(self.message)
Metadata = dict[str, Optional[torch.Tensor]]
Metadata = dict[str, torch.Tensor | None]
class PyNcclPipe(KVPipeBase):
@@ -47,7 +47,7 @@ class PyNcclPipe(KVPipeBase):
self,
local_rank: int,
config: KVTransferConfig,
device: Optional[str] = None,
device: str | None = None,
port_offset: int = 0,
):
self.config = config
@@ -77,7 +77,7 @@ class PyNcclPipe(KVPipeBase):
self.target_rank_for_recv = (self.kv_rank - 1) % self.kv_parallel_size
# transportation-related variables
self.transport_thread: Optional[ThreadPoolExecutor] = None
self.transport_thread: ThreadPoolExecutor | None = None
self.buffer_size = 0
self.buffer_size_lock = threading.Lock()
self.buffer_size_thresh = self.config.kv_buffer_size
@@ -115,7 +115,7 @@ class PyNcclPipe(KVPipeBase):
else:
return torch.device("cpu")
def _make_metadata(self, tensor: Optional[torch.Tensor]) -> Metadata:
def _make_metadata(self, tensor: torch.Tensor | None) -> Metadata:
"""
Create the metadata as a dictionary based on the input tensor.
@@ -167,7 +167,7 @@ class PyNcclPipe(KVPipeBase):
"""
return self.group.recv_obj(self.target_rank_for_recv)
def _send_impl(self, tensor: Optional[torch.Tensor]) -> None:
def _send_impl(self, tensor: torch.Tensor | None) -> None:
"""
The actual implementation of sending the tensor and its metadata to the
target rank.
@@ -181,7 +181,7 @@ class PyNcclPipe(KVPipeBase):
if tensor is not None:
self.device_send_func(tensor.to(self.device), self.target_rank_for_send)
def _recv_impl(self) -> Optional[torch.Tensor]:
def _recv_impl(self) -> torch.Tensor | None:
"""
The actual implementation of receiving a tensor and its metadata from
the target rank.
@@ -198,7 +198,7 @@ class PyNcclPipe(KVPipeBase):
return buffer
def send_tensor_wrapper(
self, tensor: Optional[torch.Tensor], tensor_size: int
self, tensor: torch.Tensor | None, tensor_size: int
) -> None:
"""
Wrapper for _send_impl to handle exceptions and update buffer size.
@@ -228,7 +228,7 @@ class PyNcclPipe(KVPipeBase):
logger.debug("KV cache transfer pipe is full. Waiting...")
time.sleep(0.05)
def send_tensor(self, tensor: Optional[torch.Tensor]) -> None:
def send_tensor(self, tensor: torch.Tensor | None) -> None:
"""
Sends a tensor and its metadata to the destination rank in a
non-blocking way.
@@ -251,7 +251,7 @@ class PyNcclPipe(KVPipeBase):
self.transport_thread.submit(self.send_tensor_wrapper, tensor, tensor_size)
def recv_tensor(self) -> Optional[torch.Tensor]:
def recv_tensor(self) -> torch.Tensor | None:
"""
Receives a tensor and its metadata from the source rank. Blocking call.
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Optional
from typing import TYPE_CHECKING
from vllm import envs
from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBaseType
@@ -13,7 +13,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1 import (
if TYPE_CHECKING:
from vllm.config import VllmConfig
_KV_CONNECTOR_AGENT: Optional[KVConnectorBaseType] = None
_KV_CONNECTOR_AGENT: KVConnectorBaseType | None = None
def get_kv_transfer_group() -> KVConnectorBaseType:
@@ -27,7 +27,7 @@ def has_kv_transfer_group() -> bool:
return _KV_CONNECTOR_AGENT is not None
def is_v1_kv_transfer_group(connector: Optional[KVConnectorBaseType] = None) -> bool:
def is_v1_kv_transfer_group(connector: KVConnectorBaseType | None = None) -> bool:
"""Check if the KV connector is the v1 connector.
If the argument is None, it will check the global KV connector
+43 -44
View File
@@ -28,11 +28,12 @@ import gc
import pickle
import weakref
from collections import namedtuple
from collections.abc import Callable
from contextlib import contextmanager, nullcontext
from dataclasses import dataclass
from datetime import timedelta
from multiprocessing import shared_memory
from typing import Any, Callable, Optional, Union
from typing import Any, Optional
from unittest.mock import patch
import torch
@@ -65,7 +66,7 @@ TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
def _split_tensor_dict(
tensor_dict: dict[str, Union[torch.Tensor, Any]],
tensor_dict: dict[str, torch.Tensor | Any],
) -> tuple[list[tuple[str, Any]], list[torch.Tensor]]:
"""Split the tensor dictionary into two parts:
1. A list of (key, value) pairs. If the value is a tensor, it is replaced
@@ -300,17 +301,17 @@ class GroupCoordinator:
cpu_group: ProcessGroup # group for CPU communication
device_group: ProcessGroup # group for device communication
# device communicator (if use_device_communicator=True)
device_communicator: Optional[DeviceCommunicatorBase]
mq_broadcaster: Optional[Any] # shared memory broadcaster
device_communicator: DeviceCommunicatorBase | None
mq_broadcaster: Any | None # shared memory broadcaster
def __init__(
self,
group_ranks: list[list[int]],
local_rank: int,
torch_distributed_backend: Union[str, Backend],
torch_distributed_backend: str | Backend,
use_device_communicator: bool, # whether to use device communicator
use_message_queue_broadcaster: bool = False,
group_name: Optional[str] = None,
group_name: str | None = None,
):
group_name = group_name or "anonymous"
self.unique_name = _get_unique_name(group_name)
@@ -368,7 +369,7 @@ class GroupCoordinator:
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
self.mq_broadcaster: Optional[MessageQueue] = None
self.mq_broadcaster: MessageQueue | None = None
if use_message_queue_broadcaster and self.world_size > 1:
self.mq_broadcaster = MessageQueue.create_from_process_group(
self.cpu_group, 1 << 22, 6
@@ -419,9 +420,7 @@ class GroupCoordinator:
return self.ranks[(rank_in_group - 1) % world_size]
@contextmanager
def graph_capture(
self, graph_capture_context: Optional[GraphCaptureContext] = None
):
def graph_capture(self, graph_capture_context: GraphCaptureContext | None = None):
if graph_capture_context is None:
stream = torch.cuda.Stream()
graph_capture_context = GraphCaptureContext(stream)
@@ -502,9 +501,9 @@ class GroupCoordinator:
def all_gatherv(
self,
input_: Union[torch.Tensor, list[torch.Tensor]],
input_: torch.Tensor | list[torch.Tensor],
dim: int = 0,
sizes: Optional[list[int]] = None,
sizes: list[int] | None = None,
):
if self.device_communicator is None:
raise ValueError("No device communicator found")
@@ -527,7 +526,7 @@ class GroupCoordinator:
return self._reduce_scatter_out_place(input_, dim)
def reduce_scatterv(
self, input_: torch.Tensor, dim: int = -1, sizes: Optional[list[int]] = None
self, input_: torch.Tensor, dim: int = -1, sizes: list[int] | None = None
) -> torch.Tensor:
if self.device_communicator is None:
raise ValueError("No device communicator found")
@@ -540,7 +539,7 @@ class GroupCoordinator:
def gather(
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""
NOTE: We assume that the input tensor is on the same device across
all the ranks.
@@ -569,7 +568,7 @@ class GroupCoordinator:
)
return input_
def broadcast_object(self, obj: Optional[Any] = None, src: int = 0):
def broadcast_object(self, obj: Any | None = None, src: int = 0):
"""Broadcast the input object.
NOTE: `src` is the local rank of the source rank.
"""
@@ -594,7 +593,7 @@ class GroupCoordinator:
return recv[0]
def broadcast_object_list(
self, obj_list: list[Any], src: int = 0, group: Optional[ProcessGroup] = None
self, obj_list: list[Any], src: int = 0, group: ProcessGroup | None = None
):
"""Broadcast the input object list.
NOTE: `src` is the local rank of the source rank.
@@ -675,11 +674,11 @@ class GroupCoordinator:
def broadcast_tensor_dict(
self,
tensor_dict: Optional[dict[str, Union[torch.Tensor, Any]]] = None,
tensor_dict: dict[str, torch.Tensor | Any] | None = None,
src: int = 0,
group: Optional[ProcessGroup] = None,
metadata_group: Optional[ProcessGroup] = None,
) -> Optional[dict[str, Union[torch.Tensor, Any]]]:
group: ProcessGroup | None = None,
metadata_group: ProcessGroup | None = None,
) -> dict[str, torch.Tensor | Any] | None:
"""Broadcast the input tensor dictionary.
NOTE: `src` is the local rank of the source rank.
"""
@@ -757,11 +756,11 @@ class GroupCoordinator:
def send_tensor_dict(
self,
tensor_dict: dict[str, Union[torch.Tensor, Any]],
dst: Optional[int] = None,
tensor_dict: dict[str, torch.Tensor | Any],
dst: int | None = None,
all_gather_group: Optional["GroupCoordinator"] = None,
all_gather_tensors: Optional[dict[str, bool]] = None,
) -> Optional[dict[str, Union[torch.Tensor, Any]]]:
all_gather_tensors: dict[str, bool] | None = None,
) -> dict[str, torch.Tensor | Any] | None:
"""Send the input tensor dictionary.
NOTE: `dst` is the local rank of the source rank.
@@ -845,10 +844,10 @@ class GroupCoordinator:
def recv_tensor_dict(
self,
src: Optional[int] = None,
src: int | None = None,
all_gather_group: Optional["GroupCoordinator"] = None,
all_gather_tensors: Optional[dict[str, bool]] = None,
) -> Optional[dict[str, Union[torch.Tensor, Any]]]:
all_gather_tensors: dict[str, bool] | None = None,
) -> dict[str, torch.Tensor | Any] | None:
"""Recv the input tensor dictionary.
NOTE: `src` is the local rank of the source rank.
@@ -943,7 +942,7 @@ class GroupCoordinator:
"""
torch.distributed.barrier(group=self.cpu_group)
def send(self, tensor: torch.Tensor, dst: Optional[int] = None) -> None:
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
"""Sends a tensor to the destination rank in a blocking way"""
"""NOTE: `dst` is the local rank of the destination rank."""
if self.device_communicator is None:
@@ -951,7 +950,7 @@ class GroupCoordinator:
self.device_communicator.send(tensor, dst)
def recv(
self, size: torch.Size, dtype: torch.dtype, src: Optional[int] = None
self, size: torch.Size, dtype: torch.dtype, src: int | None = None
) -> torch.Tensor:
"""Receives a tensor from the source rank."""
"""NOTE: `src` is the local rank of the source rank."""
@@ -997,8 +996,8 @@ class GroupCoordinator:
return hidden_states
_WORLD: Optional[GroupCoordinator] = None
_NODE_COUNT: Optional[int] = None
_WORLD: GroupCoordinator | None = None
_NODE_COUNT: int | None = None
def get_world_group() -> GroupCoordinator:
@@ -1023,7 +1022,7 @@ def init_model_parallel_group(
local_rank: int,
backend: str,
use_message_queue_broadcaster: bool = False,
group_name: Optional[str] = None,
group_name: str | None = None,
) -> GroupCoordinator:
return GroupCoordinator(
group_ranks=group_ranks,
@@ -1035,7 +1034,7 @@ def init_model_parallel_group(
)
_TP: Optional[GroupCoordinator] = None
_TP: GroupCoordinator | None = None
def get_tp_group() -> GroupCoordinator:
@@ -1052,7 +1051,7 @@ def get_tensor_model_parallel_group():
return get_tp_group()
_DCP: Optional[GroupCoordinator] = None
_DCP: GroupCoordinator | None = None
def get_dcp_group() -> GroupCoordinator:
@@ -1063,9 +1062,9 @@ def get_dcp_group() -> GroupCoordinator:
# kept for backward compatibility
get_context_model_parallel_group = get_dcp_group
_PP: Optional[GroupCoordinator] = None
_PP: GroupCoordinator | None = None
_DP: Optional[GroupCoordinator] = None
_DP: GroupCoordinator | None = None
def get_dp_group() -> GroupCoordinator:
@@ -1073,7 +1072,7 @@ def get_dp_group() -> GroupCoordinator:
return _DP
_EP: Optional[GroupCoordinator] = None
_EP: GroupCoordinator | None = None
def get_ep_group() -> GroupCoordinator:
@@ -1131,7 +1130,7 @@ def init_distributed_environment(
distributed_init_method: str = "env://",
local_rank: int = -1,
backend: str = "nccl",
timeout: Optional[timedelta] = None,
timeout: timedelta | None = None,
):
logger.debug(
"world_size=%d rank=%d local_rank=%d distributed_init_method=%s backend=%s",
@@ -1208,8 +1207,8 @@ def init_distributed_environment(
def initialize_model_parallel(
tensor_model_parallel_size: int = 1,
pipeline_model_parallel_size: int = 1,
decode_context_model_parallel_size: Optional[int] = 1,
backend: Optional[str] = None,
decode_context_model_parallel_size: int | None = 1,
backend: str | None = None,
) -> None:
"""
Initialize model parallel groups.
@@ -1338,8 +1337,8 @@ def initialize_model_parallel(
def ensure_model_parallel_initialized(
tensor_model_parallel_size: int,
pipeline_model_parallel_size: int,
decode_context_model_parallel_size: Optional[int] = 1,
backend: Optional[str] = None,
decode_context_model_parallel_size: int | None = 1,
backend: str | None = None,
) -> None:
"""Helper to initialize model parallel groups if they are not initialized,
or ensure tensor-parallel and pipeline-parallel sizes are equal to expected
@@ -1504,7 +1503,7 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
def in_the_same_node_as(
pg: Union[ProcessGroup, StatelessProcessGroup], source_rank: int = 0
pg: ProcessGroup | StatelessProcessGroup, source_rank: int = 0
) -> list[bool]:
"""
This is a collective operation that returns if each rank is in the same node
@@ -1624,7 +1623,7 @@ def is_global_first_rank() -> bool:
return True
def _node_count(pg: Union[ProcessGroup, StatelessProcessGroup]) -> int:
def _node_count(pg: ProcessGroup | StatelessProcessGroup) -> int:
"""
Returns the total number of nodes in the process group.
+3 -3
View File
@@ -30,9 +30,9 @@ class XlaQKVParallelLinear(nn.Module):
self.q_weight: Parameter
self.k_weight: Parameter
self.v_weight: Parameter
self.q_bias: Optional[Parameter]
self.k_bias: Optional[Parameter]
self.v_bias: Optional[Parameter]
self.q_bias: Parameter | None
self.k_bias: Parameter | None
self.v_bias: Parameter | None
self._load_weights_from_qkv_linear(qkv_linear)
if mesh is not None:
self._shard_weight(mesh)
+3 -3
View File
@@ -15,7 +15,7 @@ import uuid
from collections import deque
from collections.abc import Sequence
from datetime import timedelta
from typing import Any, Optional
from typing import Any
import torch
from torch.distributed import ProcessGroup, TCPStore
@@ -150,7 +150,7 @@ class StatelessProcessGroup:
store: torch._C._distributed_c10d.Store
# stores a reference to the socket so that the file descriptor stays alive
socket: Optional[socket.socket]
socket: socket.socket | None
data_expiration_seconds: int = 3600 # 1 hour
@@ -197,7 +197,7 @@ class StatelessProcessGroup:
self.recv_src_counter[src] += 1
return obj
def broadcast_obj(self, obj: Optional[Any], src: int) -> Any:
def broadcast_obj(self, obj: Any | None, src: int) -> Any:
"""Broadcast an object from a source rank to all other ranks.
It does not clean up after all ranks have received the object.
Use it for limited times, e.g., for initialization.