Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -9,11 +9,11 @@ import uuid
|
||||
import weakref
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Awaitable, Sequence
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from concurrent.futures import Future
|
||||
from dataclasses import dataclass
|
||||
from threading import Thread
|
||||
from typing import Any, Callable, Optional, TypeVar, Union
|
||||
from typing import Any, TypeAlias, TypeVar
|
||||
|
||||
import msgspec.msgpack
|
||||
import zmq
|
||||
@@ -51,7 +51,7 @@ from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder, bytestr
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
AnyFuture = Union[asyncio.Future[Any], Future[Any]]
|
||||
AnyFuture: TypeAlias = asyncio.Future[Any] | Future[Any]
|
||||
|
||||
_R = TypeVar("_R") # Return type for collective_rpc
|
||||
|
||||
@@ -99,7 +99,7 @@ class EngineCoreClient(ABC):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
) -> "MPClient":
|
||||
@@ -144,7 +144,7 @@ class EngineCoreClient(ABC):
|
||||
def sleep(self, level: int = 1) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def wake_up(self, tags: Optional[list[str]] = None) -> None:
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
@@ -172,16 +172,16 @@ class EngineCoreClient(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
def save_sharded_state(
|
||||
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
|
||||
self, path: str, pattern: str | None = None, max_size: int | None = None
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[..., _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -214,7 +214,7 @@ class EngineCoreClient(ABC):
|
||||
async def sleep_async(self, level: int = 1) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def wake_up_async(self, tags: Optional[list[str]] = None) -> None:
|
||||
async def wake_up_async(self, tags: list[str] | None = None) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def is_sleeping_async(self) -> bool:
|
||||
@@ -236,16 +236,16 @@ class EngineCoreClient(ABC):
|
||||
raise NotImplementedError
|
||||
|
||||
async def save_sharded_state_async(
|
||||
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
|
||||
self, path: str, pattern: str | None = None, max_size: int | None = None
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def collective_rpc_async(
|
||||
self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[..., _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -293,7 +293,7 @@ class InprocClient(EngineCoreClient):
|
||||
def sleep(self, level: int = 1) -> None:
|
||||
self.engine_core.sleep(level)
|
||||
|
||||
def wake_up(self, tags: Optional[list[str]] = None) -> None:
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
self.engine_core.wake_up(tags)
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
@@ -315,16 +315,16 @@ class InprocClient(EngineCoreClient):
|
||||
return self.engine_core.pin_lora(lora_id)
|
||||
|
||||
def save_sharded_state(
|
||||
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
|
||||
self, path: str, pattern: str | None = None, max_size: int | None = None
|
||||
) -> None:
|
||||
self.engine_core.save_sharded_state(path, pattern, max_size)
|
||||
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[..., _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
return self.engine_core.collective_rpc(method, timeout, args, kwargs)
|
||||
|
||||
@@ -340,18 +340,16 @@ class BackgroundResources:
|
||||
ctx: zmq.Context
|
||||
# If CoreEngineProcManager, it manages local engines;
|
||||
# if CoreEngineActorManager, it manages all engines.
|
||||
engine_manager: Optional[Union[CoreEngineProcManager, CoreEngineActorManager]] = (
|
||||
None
|
||||
)
|
||||
coordinator: Optional[DPCoordinator] = None
|
||||
output_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None
|
||||
input_socket: Optional[Union[zmq.Socket, zmq.asyncio.Socket]] = None
|
||||
first_req_send_socket: Optional[zmq.asyncio.Socket] = None
|
||||
first_req_rcv_socket: Optional[zmq.asyncio.Socket] = None
|
||||
stats_update_socket: Optional[zmq.asyncio.Socket] = None
|
||||
output_queue_task: Optional[asyncio.Task] = None
|
||||
stats_update_task: Optional[asyncio.Task] = None
|
||||
shutdown_path: Optional[str] = None
|
||||
engine_manager: CoreEngineProcManager | CoreEngineActorManager | None = None
|
||||
coordinator: DPCoordinator | None = None
|
||||
output_socket: zmq.Socket | zmq.asyncio.Socket | None = None
|
||||
input_socket: zmq.Socket | zmq.asyncio.Socket | None = None
|
||||
first_req_send_socket: zmq.asyncio.Socket | None = None
|
||||
first_req_rcv_socket: zmq.asyncio.Socket | None = None
|
||||
stats_update_socket: zmq.asyncio.Socket | None = None
|
||||
output_queue_task: asyncio.Task | None = None
|
||||
stats_update_task: asyncio.Task | None = None
|
||||
shutdown_path: str | None = None
|
||||
|
||||
# Set if any of the engines are dead. Here so that the output
|
||||
# processing threads can access it without holding a ref to the client.
|
||||
@@ -438,7 +436,7 @@ class MPClient(EngineCoreClient):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
# Serialization setup.
|
||||
@@ -459,7 +457,7 @@ class MPClient(EngineCoreClient):
|
||||
# State used for data parallel.
|
||||
self.engines_running = False
|
||||
|
||||
self.stats_update_address: Optional[str] = None
|
||||
self.stats_update_address: str | None = None
|
||||
if client_addresses:
|
||||
# Engines are managed externally to this client.
|
||||
input_address = client_addresses["input_address"]
|
||||
@@ -646,7 +644,7 @@ class SyncMPClient(MPClient):
|
||||
)
|
||||
|
||||
self.is_dp = self.vllm_config.parallel_config.data_parallel_size > 1
|
||||
self.outputs_queue = queue.Queue[Union[EngineCoreOutputs, Exception]]()
|
||||
self.outputs_queue = queue.Queue[EngineCoreOutputs | Exception]()
|
||||
|
||||
# Ensure that the outputs socket processing thread does not have
|
||||
# a ref to the client which prevents gc.
|
||||
@@ -770,7 +768,7 @@ class SyncMPClient(MPClient):
|
||||
def sleep(self, level: int = 1) -> None:
|
||||
self.call_utility("sleep", level)
|
||||
|
||||
def wake_up(self, tags: Optional[list[str]] = None) -> None:
|
||||
def wake_up(self, tags: list[str] | None = None) -> None:
|
||||
self.call_utility("wake_up", tags)
|
||||
|
||||
def is_sleeping(self) -> bool:
|
||||
@@ -781,15 +779,15 @@ class SyncMPClient(MPClient):
|
||||
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[..., _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
return self.call_utility("collective_rpc", method, timeout, args, kwargs)
|
||||
|
||||
def save_sharded_state(
|
||||
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
|
||||
self, path: str, pattern: str | None = None, max_size: int | None = None
|
||||
) -> None:
|
||||
self.call_utility("save_sharded_state", path, pattern, max_size)
|
||||
|
||||
@@ -802,7 +800,7 @@ class AsyncMPClient(MPClient):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
):
|
||||
@@ -816,7 +814,7 @@ class AsyncMPClient(MPClient):
|
||||
|
||||
self.client_count = client_count
|
||||
self.client_index = client_index
|
||||
self.outputs_queue = asyncio.Queue[Union[EngineCoreOutputs, Exception]]()
|
||||
self.outputs_queue = asyncio.Queue[EngineCoreOutputs | Exception]()
|
||||
try:
|
||||
# If we are running in an asyncio event loop, start the queue task.
|
||||
# Otherwise, it will be started lazily. If it is not started here,
|
||||
@@ -837,9 +835,9 @@ class AsyncMPClient(MPClient):
|
||||
decoder = self.decoder
|
||||
utility_results = self.utility_results
|
||||
outputs_queue = self.outputs_queue
|
||||
output_handler: Optional[
|
||||
Callable[[AsyncMPClient, EngineCoreOutputs], Awaitable[None]]
|
||||
] = getattr(self.__class__, "process_engine_outputs", None)
|
||||
output_handler: (
|
||||
Callable[[AsyncMPClient, EngineCoreOutputs], Awaitable[None]] | None
|
||||
) = getattr(self.__class__, "process_engine_outputs", None)
|
||||
_self_ref = weakref.ref(self) if output_handler else None
|
||||
output_socket = resources.output_socket
|
||||
assert output_socket is not None
|
||||
@@ -888,7 +886,7 @@ class AsyncMPClient(MPClient):
|
||||
self,
|
||||
request_type: EngineCoreRequestType,
|
||||
request: Any,
|
||||
engine: Optional[EngineIdentity] = None,
|
||||
engine: EngineIdentity | None = None,
|
||||
) -> Awaitable[Any]:
|
||||
if engine is None:
|
||||
engine = self.core_engine
|
||||
@@ -962,7 +960,7 @@ class AsyncMPClient(MPClient):
|
||||
async def sleep_async(self, level: int = 1) -> None:
|
||||
await self.call_utility_async("sleep", level)
|
||||
|
||||
async def wake_up_async(self, tags: Optional[list[str]] = None) -> None:
|
||||
async def wake_up_async(self, tags: list[str] | None = None) -> None:
|
||||
await self.call_utility_async("wake_up", tags)
|
||||
|
||||
async def is_sleeping_async(self) -> bool:
|
||||
@@ -984,16 +982,16 @@ class AsyncMPClient(MPClient):
|
||||
return await self.call_utility_async("pin_lora", lora_id)
|
||||
|
||||
async def save_sharded_state_async(
|
||||
self, path: str, pattern: Optional[str] = None, max_size: Optional[int] = None
|
||||
self, path: str, pattern: str | None = None, max_size: int | None = None
|
||||
) -> None:
|
||||
await self.call_utility_async("save_sharded_state", path, pattern, max_size)
|
||||
|
||||
async def collective_rpc_async(
|
||||
self,
|
||||
method: Union[str, Callable[..., _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[..., _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
return await self.call_utility_async(
|
||||
"collective_rpc", method, timeout, args, kwargs
|
||||
@@ -1009,7 +1007,7 @@ class DPAsyncMPClient(AsyncMPClient):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
):
|
||||
@@ -1166,7 +1164,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient):
|
||||
vllm_config: VllmConfig,
|
||||
executor_class: type[Executor],
|
||||
log_stats: bool,
|
||||
client_addresses: Optional[dict[str, str]] = None,
|
||||
client_addresses: dict[str, str] | None = None,
|
||||
client_count: int = 1,
|
||||
client_index: int = 0,
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user