Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
@@ -2,7 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib
from typing import TYPE_CHECKING, Callable
from collections.abc import Callable
from typing import TYPE_CHECKING
import vllm.envs as envs
from vllm.distributed.kv_transfer.kv_connector.base import (
@@ -7,7 +7,7 @@ KV cache helper for store.
from collections import defaultdict
from collections.abc import Sequence
from concurrent.futures import CancelledError, Future
from typing import Literal, Optional, Union, cast
from typing import Literal, cast
import torch
@@ -136,7 +136,7 @@ class KVOutputAggregator:
# Aggregate kv_connector_output from all workers
def update_finished_set(
req_ids: Optional[set[str]],
req_ids: set[str] | None,
remaining_count_dict: dict[str, int],
finished_set: set[str],
) -> None:
@@ -197,7 +197,7 @@ class KVOutputAggregator:
to the respective list of outputs."""
result_future: Future[ModelRunnerOutput] = Future()
outputs: list[Optional[ModelRunnerOutput]] = [None] * len(output_futures)
outputs: list[ModelRunnerOutput | None] = [None] * len(output_futures)
def make_callback(idx):
def callback(fut):
@@ -230,8 +230,8 @@ class KVOutputAggregator:
def _make_src_and_dst_indices(
src_block_ids: list[int],
dst_block_ids: list[int],
src_device: Union[torch.device, str],
dst_device: Union[torch.device, str],
src_device: torch.device | str,
dst_device: torch.device | str,
) -> tuple[torch.Tensor, torch.Tensor]:
src_indices = torch.tensor(src_block_ids, device=src_device, dtype=torch.int64)
dst_indices = torch.tensor(dst_block_ids, device=dst_device, dtype=torch.int64)
@@ -37,8 +37,8 @@ The class provides the following primitives:
import enum
from abc import ABC, abstractmethod
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, Callable, Literal, Optional
from collections.abc import Callable, Iterable
from typing import TYPE_CHECKING, Any, Literal, Optional
import torch
@@ -93,7 +93,7 @@ class KVConnectorBase_V1(ABC):
"Initializing KVConnectorBase_V1. This API is experimental and "
"subject to change in the future as we iterate the design."
)
self._connector_metadata: Optional[KVConnectorMetadata] = None
self._connector_metadata: KVConnectorMetadata | None = None
self._vllm_config = vllm_config
self._role = role
@@ -222,7 +222,7 @@ class KVConnectorBase_V1(ABC):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens on the worker.
@@ -281,7 +281,7 @@ class KVConnectorBase_V1(ABC):
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
@@ -361,7 +361,7 @@ class KVConnectorBase_V1(ABC):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Called exactly once when a request has finished, before its blocks are
freed.
@@ -388,7 +388,7 @@ class KVConnectorBase_V1(ABC):
return ()
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> Optional[str]:
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
Args:
@@ -406,7 +406,7 @@ class KVConnectorBase_V1(ABC):
)
return None
def get_finished_count(self) -> Optional[int]:
def get_finished_count(self) -> int | None:
"""
Get the count of requests expected to complete send/receive operations
via this connector.
@@ -419,7 +419,7 @@ class KVConnectorBase_V1(ABC):
@classmethod
def build_kv_connector_stats(
cls, data: Optional[dict[str, Any]] = None
cls, data: dict[str, Any] | None = None
) -> Optional["KVConnectorStats"]:
"""
KVConnectorStats resolution method. This method allows dynamically
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Any
import torch
from lmcache.integration.vllm.vllm_v1_adapter import LMCacheConnectorV1Impl
@@ -96,7 +96,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens.
@@ -117,7 +117,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
@@ -161,7 +161,7 @@ class LMCacheConnectorV1(KVConnectorBase_V1):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its blocks are freed.
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
from typing import Any, Optional, Union
from typing import Any
from vllm.config.kv_transfer import KVTransferConfig
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
@@ -32,7 +32,7 @@ class KVConnectorStats:
"""
raise NotImplementedError
def reduce(self) -> dict[str, Union[int, float]]:
def reduce(self) -> dict[str, int | float]:
"""
Reduce the observations collected during a time interval to one or
more representative values (eg avg/median/sum of the series).
@@ -58,7 +58,7 @@ class KVConnectorLogging:
self.reset()
def reset(self):
self.transfer_stats_accumulator: Optional[KVConnectorStats] = None
self.transfer_stats_accumulator: KVConnectorStats | None = None
def observe(self, transfer_stats_data: dict[str, Any]):
# Should not be called when a KVConnector is not configured.
@@ -3,7 +3,7 @@
import copy
from collections.abc import Iterable
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Any
import torch
@@ -33,7 +33,7 @@ logger = init_logger(__name__)
@dataclass
class MultiKVConnectorMetadata(KVConnectorMetadata):
metadata: tuple[KVConnectorMetadata, ...]
extra_async_saves: Optional[dict[str, int]] = None
extra_async_saves: dict[str, int] | None = None
@dataclass
@@ -130,7 +130,7 @@ class MultiConnector(KVConnectorBase_V1):
c.clear_connector_metadata()
def shutdown(self):
exception: Optional[Exception] = None
exception: Exception | None = None
for c in self._connectors:
try:
c.shutdown()
@@ -169,7 +169,7 @@ class MultiConnector(KVConnectorBase_V1):
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
finished_sending: set[str] = set()
finished_recving: set[str] = set()
for c in self._connectors:
@@ -207,7 +207,7 @@ class MultiConnector(KVConnectorBase_V1):
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
to_return = (0, False)
for i, c in enumerate(self._connectors):
toks, load_async = c.get_num_new_matched_tokens(
@@ -258,7 +258,7 @@ class MultiConnector(KVConnectorBase_V1):
self,
request: "Request",
blocks: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
async_saves = 0
kv_txfer_params = None
for c in self._connectors:
@@ -286,7 +286,7 @@ class MultiConnector(KVConnectorBase_V1):
yield from c.take_events()
@classmethod
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> Optional[str]:
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
"""
Get the required KV cache layout for this connector.
Args:
@@ -323,17 +323,17 @@ class MultiConnector(KVConnectorBase_V1):
@classmethod
def build_kv_connector_stats(
cls, data: Optional[dict[str, Any]] = None
) -> Optional[KVConnectorStats]:
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
MultiKVConnectorStats(data=data)
if data is not None
else MultiKVConnectorStats()
)
def get_kv_connector_stats(self) -> Optional[MultiKVConnectorStats]:
def get_kv_connector_stats(self) -> MultiKVConnectorStats | None:
# Group connector stats by connector type.
stats_by_connector: Optional[MultiKVConnectorStats] = None
stats_by_connector: MultiKVConnectorStats | None = None
for c in self._connectors:
stats = c.get_kv_connector_stats()
if stats is None:
@@ -13,7 +13,7 @@ from collections import defaultdict
from collections.abc import Iterator
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Optional, Union
from typing import TYPE_CHECKING, Any
import msgspec
import numpy as np
@@ -153,10 +153,10 @@ class NixlConnector(KVConnectorBase_V1):
self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler: Optional[NixlConnectorScheduler] = (
self.connector_scheduler: NixlConnectorScheduler | None = (
NixlConnectorScheduler(vllm_config, self.engine_id)
)
self.connector_worker: Optional[NixlConnectorWorker] = None
self.connector_worker: NixlConnectorWorker | None = None
elif role == KVConnectorRole.WORKER:
self.connector_scheduler = None
self.connector_worker = NixlConnectorWorker(vllm_config, self.engine_id)
@@ -189,7 +189,7 @@ class NixlConnector(KVConnectorBase_V1):
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
assert self.connector_scheduler is not None
return self.connector_scheduler.get_num_new_matched_tokens(
request, num_computed_tokens
@@ -214,7 +214,7 @@ class NixlConnector(KVConnectorBase_V1):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
@@ -234,14 +234,14 @@ class NixlConnector(KVConnectorBase_V1):
assert self.connector_worker is not None
return self.connector_worker.get_finished()
def get_kv_connector_stats(self) -> Optional[KVConnectorStats]:
def get_kv_connector_stats(self) -> KVConnectorStats | None:
assert self.connector_worker is not None
return self.connector_worker.get_kv_connector_stats()
@classmethod
def build_kv_connector_stats(
cls, data: Optional[dict[str, Any]] = None
) -> Optional[KVConnectorStats]:
cls, data: dict[str, Any] | None = None
) -> KVConnectorStats | None:
return (
NixlKVConnectorStats(data=data)
if data is not None
@@ -445,7 +445,7 @@ class NixlConnectorScheduler:
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Once a request is finished, determine whether request blocks
should be freed now or will be sent asynchronously and freed later.
@@ -584,7 +584,7 @@ class NixlConnectorWorker:
)
# Note: host xfer buffer ops when use_host_buffer is True
self.copy_blocks: Optional[CopyBlocksOp] = None
self.copy_blocks: CopyBlocksOp | None = None
# Map of engine_id -> kv_caches_base_addr. For TP case, each local
# rank will still only pull from a single remote TP worker.
@@ -615,7 +615,7 @@ class NixlConnectorWorker:
self._reqs_to_process: set[ReqId] = set()
# Background thread for handling new handshake requests.
self._nixl_handshake_listener_t: Optional[threading.Thread] = None
self._nixl_handshake_listener_t: threading.Thread | None = None
# Background thread for initializing new NIXL handshakes.
self._handshake_initiation_executor = ThreadPoolExecutor(
# NIXL is not guaranteed to be thread-safe, limit 1 worker.
@@ -635,7 +635,7 @@ class NixlConnectorWorker:
# TODO(mgoin): remove this once we have hybrid memory allocator
# Optimization for models with local attention (Llama 4)
# List of block window sizes for each layer for local attention
self.block_window_per_layer: list[Optional[int]] = []
self.block_window_per_layer: list[int | None] = []
self.use_mla = self.model_config.use_mla
backend = get_attn_backend(
@@ -1472,7 +1472,7 @@ class NixlConnectorWorker:
self._recving_transfers[request_id].append((handle, time.perf_counter()))
def _get_block_descs_ids(
self, engine_id: str, block_ids: list[int], layer_idx: Optional[int] = None
self, engine_id: str, block_ids: list[int], layer_idx: int | None = None
) -> np.ndarray:
"""
Get the descs ids for a set of block ids.
@@ -1518,7 +1518,7 @@ class NixlConnectorWorker:
block_len = self.block_len_per_layer[layer_idx]
return block_len
def get_kv_connector_stats(self) -> Optional[KVConnectorStats]:
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""
Get the KV transfer stats for the connector.
"""
@@ -1559,7 +1559,7 @@ def zmq_ctx(socket_type: Any, addr: str) -> Iterator[zmq.Socket]:
if socket_type not in (zmq.ROUTER, zmq.REQ):
raise ValueError(f"Unexpected socket type: {socket_type}")
ctx: Optional[zmq.Context] = None
ctx: zmq.Context | None = None
try:
ctx = zmq.Context() # type: ignore[attr-defined]
yield make_zmq_socket(
@@ -1611,7 +1611,7 @@ class NixlKVConnectorStats(KVConnectorStats):
accumulator.extend(v)
return self
def reduce(self) -> dict[str, Union[int, float]]:
def reduce(self) -> dict[str, int | float]:
# Compute compact representative stats suitable for CLI logging
if self.is_empty():
return {
@@ -4,7 +4,7 @@ from collections import defaultdict
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from itertools import islice
from typing import Any, Optional
from typing import Any
import torch
@@ -46,8 +46,8 @@ class OffloadingConnector(KVConnectorBase_V1):
spec = OffloadingSpecFactory.create_spec(vllm_config)
self.connector_scheduler: Optional[OffloadingConnectorScheduler] = None
self.connector_worker: Optional[OffloadingConnectorWorker] = None
self.connector_scheduler: OffloadingConnectorScheduler | None = None
self.connector_worker: OffloadingConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = OffloadingConnectorScheduler(spec)
elif role == KVConnectorRole.WORKER:
@@ -113,7 +113,7 @@ class OffloadingConnector(KVConnectorBase_V1):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
@@ -148,7 +148,7 @@ class OffloadingConnectorScheduler:
self,
req: Request,
start_idx: int = 0,
end_idx: Optional[int] = None,
end_idx: int | None = None,
) -> Iterable[BlockHash]:
return islice(
req.block_hashes,
@@ -354,7 +354,7 @@ class OffloadingConnectorScheduler:
self,
request: Request,
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its blocks are freed.
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Any
import regex as re
import torch
@@ -304,7 +304,7 @@ class P2pNcclConnector(KVConnectorBase_V1):
def get_finished(
self, finished_req_ids: set[str], **kwargs: Any
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens.
@@ -466,7 +466,7 @@ class P2pNcclConnector(KVConnectorBase_V1):
self,
request: "Request",
block_ids: list[int],
) -> tuple[bool, Optional[dict[str, Any]]]:
) -> tuple[bool, dict[str, Any] | None]:
"""
Called when a request has finished, before its blocks are freed.
@@ -5,11 +5,10 @@ import logging
import os
import threading
import time
import typing
from collections import deque
from contextlib import contextmanager
from dataclasses import dataclass
from typing import Any, Optional
from typing import Any
import msgpack
import torch
@@ -77,7 +76,7 @@ class P2pNcclEngine:
config: KVTransferConfig,
hostname: str = "",
port_offset: int = 0,
library_path: Optional[str] = None,
library_path: str | None = None,
) -> None:
self.config = config
self.rank = port_offset
@@ -187,7 +186,7 @@ class P2pNcclEngine:
self.nccl_num_channels,
)
def create_connect(self, remote_address: typing.Optional[str] = None):
def create_connect(self, remote_address: str | None = None):
assert remote_address is not None
if remote_address not in self.socks:
sock = self.context.socket(zmq.DEALER)
@@ -224,7 +223,7 @@ class P2pNcclEngine:
self,
tensor_id: str,
tensor: torch.Tensor,
remote_address: typing.Optional[str] = None,
remote_address: str | None = None,
) -> bool:
if remote_address is None:
with self.recv_store_cv:
@@ -296,7 +295,7 @@ class P2pNcclEngine:
def recv_tensor(
self,
tensor_id: str,
remote_address: typing.Optional[str] = None,
remote_address: str | None = None,
) -> torch.Tensor:
if self.send_type == "PUT" or self.send_type == "PUT_ASYNC":
start_time = time.time()
@@ -527,7 +526,7 @@ class P2pNcclEngine:
def get_finished(
self, finished_req_ids: set[str], no_compile_layers
) -> tuple[Optional[set[str]], Optional[set[str]]]:
) -> tuple[set[str] | None, set[str] | None]:
"""
Notifies worker-side connector ids of requests that have
finished generating tokens.
@@ -3,7 +3,7 @@
import hashlib
import os
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Optional
from typing import TYPE_CHECKING, Any
import safetensors
import torch
@@ -249,7 +249,7 @@ class SharedStorageConnector(KVConnectorBase_V1):
self,
request: "Request",
num_computed_tokens: int,
) -> tuple[Optional[int], bool]:
) -> tuple[int | None, bool]:
"""
Get number of new tokens that can be loaded from the
external KV cache beyond the num_computed_tokens.
@@ -14,7 +14,6 @@ These classes above are abstracted behind class `KVCacheBufferBase`.
"""
from abc import ABC, abstractmethod
from typing import Optional
import torch
@@ -98,8 +97,8 @@ class KVLookupBufferBase(KVCacheBufferBase):
@abstractmethod
def drop_select(
self, input_tokens: Optional[torch.Tensor], roi: Optional[torch.Tensor]
) -> list[Optional[torch.Tensor]]:
self, input_tokens: torch.Tensor | None, roi: torch.Tensor | None
) -> list[torch.Tensor | None]:
"""Select and *drop* KV cache entries from the lookup buffer.
The functionality is similar to the following python statements
@@ -143,7 +142,7 @@ class KVStoreBufferBase(KVCacheBufferBase):
def put(
self,
key: str,
value: Optional[torch.Tensor],
value: torch.Tensor | None,
) -> None:
"""Store a key-value pair in the buffer.
@@ -163,7 +162,7 @@ class KVStoreBufferBase(KVCacheBufferBase):
def get(
self,
key: str,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""Retrieve a value from the buffer by key.
Args:
@@ -10,7 +10,6 @@ from this remote lookup buffer.
import json
import os
from dataclasses import dataclass
from typing import Optional
import torch
from safetensors.torch import load as safetensors_load
@@ -110,7 +109,7 @@ class MooncakeStore(KVStoreBufferBase):
def put(
self,
key: str,
value: Optional[torch.Tensor],
value: torch.Tensor | None,
) -> None:
# A message queue needs to be introduced before making it asynchronous.
if value is not None:
@@ -119,7 +118,7 @@ class MooncakeStore(KVStoreBufferBase):
def get(
self,
key: str,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
# A message queue needs to be introduced before making it asynchronous.
value = self._get_impl(key)
return value
@@ -142,7 +141,7 @@ class MooncakeStore(KVStoreBufferBase):
def _get_impl(
self,
key: str,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""Get KVCache from Mooncake Store"""
try:
data = self.store.get(key)
@@ -13,7 +13,6 @@ Key Features:
import threading
from collections import deque
from typing import Optional, Union
import torch
@@ -46,7 +45,7 @@ class SimpleBuffer(KVLookupBufferBase):
self.buffer_cv = threading.Condition()
self.signal_pipe = signal_pipe
self.data_pipe = data_pipe
self.request_handling_thread: Optional[threading.Thread] = None
self.request_handling_thread: threading.Thread | None = None
self.normal_signal = torch.tensor([0], device="cpu")
self.end_signal = None
@@ -81,14 +80,14 @@ class SimpleBuffer(KVLookupBufferBase):
return 0
def _send_tensor_and_dec_size(self, tensor: Optional[torch.Tensor]) -> None:
def _send_tensor_and_dec_size(self, tensor: torch.Tensor | None) -> None:
assert tensor is not None, "Use self.data_pipe.send(None) instead"
self.buffer_size -= tensor.element_size() * tensor.numel()
if tensor.dtype == torch.bool:
tensor = tensor.float()
self.data_pipe.send_tensor(tensor)
def _get_element_size(self, data: Optional[Union[list, torch.Tensor]]):
def _get_element_size(self, data: list | torch.Tensor | None):
if isinstance(data, torch.Tensor):
return data.element_size() * data.numel()
if not data:
@@ -184,8 +183,8 @@ class SimpleBuffer(KVLookupBufferBase):
logger.debug("Closing drop_select_handler")
def drop_select(
self, input_tokens: Optional[torch.Tensor], roi: Optional[torch.Tensor]
) -> list[Optional[torch.Tensor]]:
self, input_tokens: torch.Tensor | None, roi: torch.Tensor | None
) -> list[torch.Tensor | None]:
assert self.request_handling_thread is None, (
"drop_select should be called by the KV cache consumer "
"(e.g. the decode vLLM instance)"
+2 -3
View File
@@ -12,7 +12,6 @@ you can bypass this interface and directly start from `kv_lookup_buffer`.
"""
from abc import ABC, abstractmethod
from typing import Optional
import torch
@@ -24,7 +23,7 @@ class KVPipeBase(ABC):
"""
@abstractmethod
def send_tensor(self, tensor: Optional[torch.Tensor]) -> None:
def send_tensor(self, tensor: torch.Tensor | None) -> None:
"""Send a tensor, or None, via the pipe.
Need to support sending None -- important for error handling.
@@ -42,7 +41,7 @@ class KVPipeBase(ABC):
raise NotImplementedError
@abstractmethod
def recv_tensor(self) -> Optional[torch.Tensor]:
def recv_tensor(self) -> torch.Tensor | None:
"""Receive a tensor (can be None) from the pipeline.
Returns:
@@ -6,7 +6,6 @@ import os
import struct
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from typing import Optional, Union
import torch
import zmq
@@ -26,7 +25,7 @@ NONE_INT = -150886311
class MooncakeTransferEngineConfig:
prefill_url: str
decode_url: str
metadata_backend: Union[str, None]
metadata_backend: str | None
metadata_server: str
protocol: str
device_name: str
@@ -143,7 +142,7 @@ class MooncakeTransferEngine:
metadata_server: str,
protocol: str,
device_name: str,
metadata_backend: Union[str, None],
metadata_backend: str | None,
) -> None:
"""Initialize the mooncake instance."""
if metadata_backend is None:
@@ -231,7 +230,7 @@ class MooncakePipe(KVPipeBase):
"""MooncakeTransferEngine based Pipe implementation."""
def __init__(
self, local_rank: int, config: KVTransferConfig, device: Optional[str] = None
self, local_rank: int, config: KVTransferConfig, device: str | None = None
):
"""Initialize the mooncake pipe and set related parameters."""
self.config = config
@@ -243,7 +242,7 @@ class MooncakePipe(KVPipeBase):
self.device = self._select_device(device)
self.transfer_engine = MooncakeTransferEngine(self.kv_rank, self.local_rank)
self.transport_thread: Optional[ThreadPoolExecutor] = None
self.transport_thread: ThreadPoolExecutor | None = None
self.none_tensor = torch.tensor([NONE_INT], device=self.device)
def _select_device(self, device: str) -> torch.device:
@@ -267,7 +266,7 @@ class MooncakePipe(KVPipeBase):
data = self.transfer_engine.recv_bytes()
return safetensors_load(data)["tensor"].to(self.device)
def send_tensor(self, tensor: Optional[torch.Tensor]) -> None:
def send_tensor(self, tensor: torch.Tensor | None) -> None:
"""Send tensor to the target process."""
if self.transport_thread is None:
self.transport_thread = ThreadPoolExecutor(max_workers=1)
@@ -275,7 +274,7 @@ class MooncakePipe(KVPipeBase):
assert len(tensor.shape) > 0
self.transport_thread.submit(self._send_impl, tensor)
def recv_tensor(self) -> Optional[torch.Tensor]:
def recv_tensor(self) -> torch.Tensor | None:
"""Receive tensor from other processes."""
if self.transport_thread is None:
self.transport_thread = ThreadPoolExecutor(max_workers=1)
@@ -15,8 +15,8 @@ Key Features:
import threading
import time
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from typing import Callable, Optional
import torch
@@ -35,7 +35,7 @@ class BrokenPipeException(Exception):
super().__init__(self.message)
Metadata = dict[str, Optional[torch.Tensor]]
Metadata = dict[str, torch.Tensor | None]
class PyNcclPipe(KVPipeBase):
@@ -47,7 +47,7 @@ class PyNcclPipe(KVPipeBase):
self,
local_rank: int,
config: KVTransferConfig,
device: Optional[str] = None,
device: str | None = None,
port_offset: int = 0,
):
self.config = config
@@ -77,7 +77,7 @@ class PyNcclPipe(KVPipeBase):
self.target_rank_for_recv = (self.kv_rank - 1) % self.kv_parallel_size
# transportation-related variables
self.transport_thread: Optional[ThreadPoolExecutor] = None
self.transport_thread: ThreadPoolExecutor | None = None
self.buffer_size = 0
self.buffer_size_lock = threading.Lock()
self.buffer_size_thresh = self.config.kv_buffer_size
@@ -115,7 +115,7 @@ class PyNcclPipe(KVPipeBase):
else:
return torch.device("cpu")
def _make_metadata(self, tensor: Optional[torch.Tensor]) -> Metadata:
def _make_metadata(self, tensor: torch.Tensor | None) -> Metadata:
"""
Create the metadata as a dictionary based on the input tensor.
@@ -167,7 +167,7 @@ class PyNcclPipe(KVPipeBase):
"""
return self.group.recv_obj(self.target_rank_for_recv)
def _send_impl(self, tensor: Optional[torch.Tensor]) -> None:
def _send_impl(self, tensor: torch.Tensor | None) -> None:
"""
The actual implementation of sending the tensor and its metadata to the
target rank.
@@ -181,7 +181,7 @@ class PyNcclPipe(KVPipeBase):
if tensor is not None:
self.device_send_func(tensor.to(self.device), self.target_rank_for_send)
def _recv_impl(self) -> Optional[torch.Tensor]:
def _recv_impl(self) -> torch.Tensor | None:
"""
The actual implementation of receiving a tensor and its metadata from
the target rank.
@@ -198,7 +198,7 @@ class PyNcclPipe(KVPipeBase):
return buffer
def send_tensor_wrapper(
self, tensor: Optional[torch.Tensor], tensor_size: int
self, tensor: torch.Tensor | None, tensor_size: int
) -> None:
"""
Wrapper for _send_impl to handle exceptions and update buffer size.
@@ -228,7 +228,7 @@ class PyNcclPipe(KVPipeBase):
logger.debug("KV cache transfer pipe is full. Waiting...")
time.sleep(0.05)
def send_tensor(self, tensor: Optional[torch.Tensor]) -> None:
def send_tensor(self, tensor: torch.Tensor | None) -> None:
"""
Sends a tensor and its metadata to the destination rank in a
non-blocking way.
@@ -251,7 +251,7 @@ class PyNcclPipe(KVPipeBase):
self.transport_thread.submit(self.send_tensor_wrapper, tensor, tensor_size)
def recv_tensor(self) -> Optional[torch.Tensor]:
def recv_tensor(self) -> torch.Tensor | None:
"""
Receives a tensor and its metadata from the source rank. Blocking call.
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING, Optional
from typing import TYPE_CHECKING
from vllm import envs
from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBaseType
@@ -13,7 +13,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1 import (
if TYPE_CHECKING:
from vllm.config import VllmConfig
_KV_CONNECTOR_AGENT: Optional[KVConnectorBaseType] = None
_KV_CONNECTOR_AGENT: KVConnectorBaseType | None = None
def get_kv_transfer_group() -> KVConnectorBaseType:
@@ -27,7 +27,7 @@ def has_kv_transfer_group() -> bool:
return _KV_CONNECTOR_AGENT is not None
def is_v1_kv_transfer_group(connector: Optional[KVConnectorBaseType] = None) -> bool:
def is_v1_kv_transfer_group(connector: KVConnectorBaseType | None = None) -> bool:
"""Check if the KV connector is the v1 connector.
If the argument is None, it will check the global KV connector