Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -4,9 +4,9 @@
|
||||
import asyncio
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Awaitable
|
||||
from collections.abc import Awaitable, Callable
|
||||
from functools import cached_property
|
||||
from typing import Any, Callable, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
@@ -63,10 +63,10 @@ class ExecutorBase(ABC):
|
||||
@abstractmethod
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable[[WorkerBase], _R]],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable[[WorkerBase], _R],
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[_R]:
|
||||
"""
|
||||
Execute an RPC call on all workers.
|
||||
@@ -143,7 +143,7 @@ class ExecutorBase(ABC):
|
||||
|
||||
def execute_model(
|
||||
self, execute_model_req: ExecuteModelRequest
|
||||
) -> Optional[list[Union[SamplerOutput, PoolerOutput]]]:
|
||||
) -> list[SamplerOutput | PoolerOutput] | None:
|
||||
output = self.collective_rpc("execute_model", args=(execute_model_req,))
|
||||
return output[0]
|
||||
|
||||
@@ -192,7 +192,7 @@ class ExecutorBase(ABC):
|
||||
"It took %.6f seconds to fall asleep.", time_after_sleep - time_before_sleep
|
||||
)
|
||||
|
||||
def wake_up(self, tags: Optional[list[str]] = None):
|
||||
def wake_up(self, tags: list[str] | None = None):
|
||||
if not self.is_sleeping:
|
||||
logger.warning("Executor is not sleeping.")
|
||||
return
|
||||
@@ -222,8 +222,8 @@ class ExecutorBase(ABC):
|
||||
def save_sharded_state(
|
||||
self,
|
||||
path: str,
|
||||
pattern: Optional[str] = None,
|
||||
max_size: Optional[int] = None,
|
||||
pattern: str | None = None,
|
||||
max_size: int | None = None,
|
||||
) -> None:
|
||||
self.collective_rpc(
|
||||
"save_sharded_state",
|
||||
@@ -256,7 +256,7 @@ class ExecutorBase(ABC):
|
||||
exception."""
|
||||
self.check_health()
|
||||
|
||||
def init_kv_output_aggregator(self, finished_count: Optional[int]) -> None:
|
||||
def init_kv_output_aggregator(self, finished_count: int | None) -> None:
|
||||
"""Init KVOutputAggregator"""
|
||||
self.kv_output_aggregator = KVOutputAggregator(
|
||||
finished_count or self.parallel_config.world_size
|
||||
@@ -269,7 +269,7 @@ class DistributedExecutorBase(ExecutorBase):
|
||||
def __init__(self, *args, **kwargs):
|
||||
# This is non-None when the execute model loop is running
|
||||
# in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
|
||||
self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
|
||||
self.parallel_worker_tasks: Any | Awaitable[Any] | None = None
|
||||
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
@@ -302,8 +302,8 @@ class DistributedExecutorBase(ExecutorBase):
|
||||
|
||||
@abstractmethod
|
||||
def _driver_execute_model(
|
||||
self, execute_model_req: Optional[ExecuteModelRequest]
|
||||
) -> Optional[list[SamplerOutput]]:
|
||||
self, execute_model_req: ExecuteModelRequest | None
|
||||
) -> list[SamplerOutput] | None:
|
||||
"""Run execute_model in the driver worker.
|
||||
|
||||
Passing None will cause the driver to stop the model execution loop
|
||||
@@ -314,20 +314,20 @@ class DistributedExecutorBase(ExecutorBase):
|
||||
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable,
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict[str, Any]] = None,
|
||||
kwargs: dict[str, Any] | None = None,
|
||||
) -> list[Any]:
|
||||
return self._run_workers(method, *args, **(kwargs or {}))
|
||||
|
||||
@abstractmethod
|
||||
def _run_workers(
|
||||
self,
|
||||
method: Union[str, Callable],
|
||||
method: str | Callable,
|
||||
*args,
|
||||
async_run_tensor_parallel_workers_only: bool = False,
|
||||
max_concurrent_workers: Optional[int] = None,
|
||||
max_concurrent_workers: int | None = None,
|
||||
**kwargs,
|
||||
) -> Any:
|
||||
"""Runs the given method on all workers.
|
||||
@@ -374,7 +374,7 @@ class DistributedExecutorBase(ExecutorBase):
|
||||
@abstractmethod
|
||||
async def _driver_execute_model_async(
|
||||
self,
|
||||
execute_model_req: Optional[ExecuteModelRequest] = None,
|
||||
execute_model_req: ExecuteModelRequest | None = None,
|
||||
) -> list[SamplerOutput]:
|
||||
"""Execute the model asynchronously in the driver worker.
|
||||
|
||||
|
||||
@@ -4,8 +4,9 @@
|
||||
import asyncio
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import cloudpickle
|
||||
import msgspec
|
||||
@@ -71,7 +72,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
|
||||
uses_ray: bool = True
|
||||
|
||||
def _init_executor(self) -> None:
|
||||
self.forward_dag: Optional[ray.dag.CompiledDAG] = None
|
||||
self.forward_dag: ray.dag.CompiledDAG | None = None
|
||||
if envs.VLLM_USE_V1:
|
||||
# V1 uses SPMD worker and compiled DAG
|
||||
os.environ["VLLM_USE_RAY_SPMD_WORKER"] = "1"
|
||||
@@ -114,10 +115,10 @@ class RayDistributedExecutor(DistributedExecutorBase):
|
||||
self._init_workers_ray(placement_group)
|
||||
|
||||
self.input_encoder = msgspec.msgpack.Encoder(enc_hook=encode_hook)
|
||||
self.output_decoder = msgspec.msgpack.Decoder(Optional[list[SamplerOutput]])
|
||||
self.output_decoder = msgspec.msgpack.Decoder(list[SamplerOutput] | None)
|
||||
self.use_v1 = envs.VLLM_USE_V1
|
||||
|
||||
self.pp_locks: Optional[list[asyncio.Lock]] = None
|
||||
self.pp_locks: list[asyncio.Lock] | None = None
|
||||
if not self.use_ray_compiled_dag:
|
||||
self.driver_exec_method = make_async(self.driver_worker.execute_method)
|
||||
|
||||
@@ -162,7 +163,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
|
||||
|
||||
# The driver dummy worker does not actually use any resources.
|
||||
# It holds the resource for the driver worker.
|
||||
self.driver_dummy_worker: Optional[RayWorkerWrapper] = None
|
||||
self.driver_dummy_worker: RayWorkerWrapper | None = None
|
||||
# The remaining workers are the actual ray actors.
|
||||
self.workers: list[RayWorkerWrapper] = []
|
||||
|
||||
@@ -432,8 +433,8 @@ class RayDistributedExecutor(DistributedExecutorBase):
|
||||
self.non_driver_workers.append(worker)
|
||||
|
||||
def _driver_execute_model(
|
||||
self, execute_model_req: Optional[ExecuteModelRequest]
|
||||
) -> Optional[list[SamplerOutput]]:
|
||||
self, execute_model_req: ExecuteModelRequest | None
|
||||
) -> list[SamplerOutput] | None:
|
||||
"""Run execute_model in the driver worker.
|
||||
|
||||
Passing None will cause the driver to stop the model execution
|
||||
@@ -463,10 +464,10 @@ class RayDistributedExecutor(DistributedExecutorBase):
|
||||
|
||||
def _run_workers(
|
||||
self,
|
||||
method: Union[str, Callable],
|
||||
method: str | Callable,
|
||||
*args,
|
||||
async_run_tensor_parallel_workers_only: bool = False,
|
||||
max_concurrent_workers: Optional[int] = None,
|
||||
max_concurrent_workers: int | None = None,
|
||||
**kwargs,
|
||||
) -> Any:
|
||||
"""Runs the given method on all workers. Can be used in the following
|
||||
@@ -683,7 +684,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
|
||||
return self.output_decoder.decode(output)
|
||||
|
||||
async def _driver_execute_model_async(
|
||||
self, execute_model_req: Optional[ExecuteModelRequest] = None
|
||||
self, execute_model_req: ExecuteModelRequest | None = None
|
||||
) -> list[SamplerOutput]:
|
||||
assert not self.use_ray_spmd_worker, (
|
||||
"driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1"
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import os
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Optional, Union
|
||||
from typing import TYPE_CHECKING, Union
|
||||
|
||||
import msgspec
|
||||
|
||||
@@ -72,7 +72,7 @@ try:
|
||||
|
||||
def execute_model_spmd(
|
||||
self,
|
||||
req_or_tuple: Union[bytes, tuple[bytes, Optional[IntermediateTensors]]],
|
||||
req_or_tuple: bytes | tuple[bytes, IntermediateTensors | None],
|
||||
) -> bytes:
|
||||
"""Execute model in SPMD fashion: used only when SPMD worker and
|
||||
compiled DAG are both enabled.
|
||||
@@ -300,7 +300,7 @@ def _wait_until_pg_removed(current_placement_group: "PlacementGroup"):
|
||||
|
||||
def initialize_ray_cluster(
|
||||
parallel_config: ParallelConfig,
|
||||
ray_address: Optional[str] = None,
|
||||
ray_address: str | None = None,
|
||||
):
|
||||
"""Initialize the distributed cluster with Ray.
|
||||
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from functools import cached_property
|
||||
from multiprocessing import Lock
|
||||
from typing import Any, Callable, Optional, Union
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -36,7 +37,7 @@ class UniProcExecutor(ExecutorBase):
|
||||
shared_worker_lock=Lock(),
|
||||
)
|
||||
|
||||
self.async_output_thread: Optional[ThreadPoolExecutor] = None
|
||||
self.async_output_thread: ThreadPoolExecutor | None = None
|
||||
if self.max_concurrent_batches > 1:
|
||||
self.async_output_thread = ThreadPoolExecutor(
|
||||
max_workers=1, thread_name_prefix="WorkerAsyncOutput"
|
||||
@@ -60,10 +61,10 @@ class UniProcExecutor(ExecutorBase):
|
||||
|
||||
def collective_rpc(
|
||||
self,
|
||||
method: Union[str, Callable],
|
||||
timeout: Optional[float] = None,
|
||||
method: str | Callable,
|
||||
timeout: float | None = None,
|
||||
args: tuple = (),
|
||||
kwargs: Optional[dict] = None,
|
||||
kwargs: dict | None = None,
|
||||
non_block: bool = False,
|
||||
) -> list[Any]:
|
||||
if kwargs is None:
|
||||
|
||||
Reference in New Issue
Block a user